From 5f6b130d6342965bfa49beb9413bdf742440b8ab Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 21:24:29 +0100 Subject: * prototype of mirror method added --- methods/http.cc | 13 ++----- methods/http.h | 7 ++-- methods/http_main.cc | 15 ++++++++ methods/makefile | 9 ++++- methods/mirror.cc | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ methods/mirror.h | 40 ++++++++++++++++++++++ 6 files changed, 166 insertions(+), 15 deletions(-) create mode 100644 methods/http_main.cc create mode 100644 methods/mirror.cc create mode 100644 methods/mirror.h (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index c6623c46f..deaa8d0c8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,7 +38,6 @@ #include #include #include -#include #include // Internet stuff @@ -57,7 +56,7 @@ time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; bool Debug = false; - +URI Proxy; unsigned long CircleBuf::BwReadLimit=0; unsigned long CircleBuf::BwTickReadData=0; @@ -990,7 +989,7 @@ void HttpMethod::SigTerm(int) depth. */ bool HttpMethod::Fetch(FetchItem *) { - if (Server == 0) + if (Server == 0) return true; // Queue the requests @@ -1223,13 +1222,5 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - - HttpMethod Mth; - - return Mth.Loop(); -} diff --git a/methods/http.h b/methods/http.h index 541e2952c..5eac11401 100644 --- a/methods/http.h +++ b/methods/http.h @@ -13,7 +13,7 @@ #define MAXLEN 360 -#include + using std::cout; using std::endl; @@ -134,7 +134,6 @@ class HttpMethod : public pkgAcqMethod bool ServerDie(ServerState *Srv); int DealWithHeaders(FetchResult &Res,ServerState *Srv); - virtual bool Fetch(FetchItem *); virtual bool Configuration(string Message); // In the event of a fatal signal this file will be closed and timestamped. @@ -142,6 +141,9 @@ class HttpMethod : public pkgAcqMethod static int FailFd; static time_t FailTime; static void SigTerm(int); + + protected: + virtual bool Fetch(FetchItem *); public: friend class ServerState; @@ -158,6 +160,5 @@ class HttpMethod : public pkgAcqMethod }; }; -URI Proxy; #endif diff --git a/methods/http_main.cc b/methods/http_main.cc new file mode 100644 index 000000000..2c46ab19d --- /dev/null +++ b/methods/http_main.cc @@ -0,0 +1,15 @@ +#include +#include + +#include "connect.h" +#include "rfc2553emu.h" +#include "http.h" + + +int main() +{ + setlocale(LC_ALL, ""); + + HttpMethod Mth; + return Mth.Loop(); +} diff --git a/methods/makefile b/methods/makefile index 1e3b1ef85..12e446a55 100644 --- a/methods/makefile +++ b/methods/makefile @@ -49,7 +49,7 @@ include $(PROGRAM_H) PROGRAM=http SLIBS = -lapt-pkg $(SOCKETLIBS) LIB_MAKES = apt-pkg/makefile -SOURCE = http.cc rfc2553emu.cc connect.cc +SOURCE = http.cc http_main.cc rfc2553emu.cc connect.cc include $(PROGRAM_H) # The ftp method @@ -66,6 +66,13 @@ LIB_MAKES = apt-pkg/makefile SOURCE = rsh.cc include $(PROGRAM_H) +# The mirror method +PROGRAM=mirror +SLIBS = -lapt-pkg $(SOCKETLIBS) +LIB_MAKES = apt-pkg/makefile +SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc +include $(PROGRAM_H) + # SSH and vzip2 method symlink binary: $(BIN)/ssh $(BIN)/bzip2 veryclean: clean-$(BIN)/ssh clean-$(BIN)/bzip2 diff --git a/methods/mirror.cc b/methods/mirror.cc new file mode 100644 index 000000000..fad076e91 --- /dev/null +++ b/methods/mirror.cc @@ -0,0 +1,97 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ +/* ###################################################################### + + Mirror Aquire Method - This is the Mirror aquire method for APT. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include +#include +#include + +#include +#include +using namespace std; + +#include "mirror.h" +#include "http.h" + + /*}}}*/ + +MirrorMethod::MirrorMethod() + : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) +{ +#if 0 + HasMirrorFile=true; + BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + Mirror="http://de.archive.ubuntu.com/ubuntu/"; +#endif +}; + +bool MirrorMethod::GetMirrorFile(string uri) +{ + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + BaseUri = uri.substr(0,uri.find(Marker)); + BaseUri.replace(0,strlen("mirror://"),"http://"); + + MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + + // FIXME: fetch it with curl + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + cerr << "fetch-result: " << res << endl; + + if(res) + HasMirrorFile = true; + Fetcher.Shutdown(); + return true; +} + +bool MirrorMethod::SelectMirror() +{ + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + cerr << "Mirror: " << Mirror << endl; +} + +// MirrorMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool MirrorMethod::Fetch(FetchItem *Itm) +{ + // get mirror information + if(!HasMirrorFile) + { + GetMirrorFile(Itm->Uri); + SelectMirror(); + } + + // change the items in the queue + Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); + cerr << "new Fetch-uri: " << Itm->Uri << endl; + + // FIXME: fetch it with! + +}; + +int main() +{ + setlocale(LC_ALL, ""); + + MirrorMethod Mth; + + return Mth.Run(); +} + + diff --git a/methods/mirror.h b/methods/mirror.h new file mode 100644 index 000000000..ca3428830 --- /dev/null +++ b/methods/mirror.h @@ -0,0 +1,40 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +/* ###################################################################### + + MIRROR Aquire Method - This is the MIRROR aquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_MIRROR_H +#define APT_MIRROR_H + + +#include + +using std::cout; +using std::cerr; +using std::endl; + +#include "http.h" + +class MirrorMethod : public pkgAcqMethod +{ + FetchResult Res; + string Mirror; + string BaseUri; + string MirrorFile; + bool HasMirrorFile; + + protected: + bool GetMirrorFile(string uri); + bool SelectMirror(); + public: + MirrorMethod(); + virtual bool Fetch(FetchItem *Itm); +}; + + +#endif -- cgit v1.2.3-70-g09d2 From 14e097c105c49a102a642c8108979356c5cc3152 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 22:59:35 +0100 Subject: * working mirror implementation based on http method --- apt-pkg/acquire-method.h | 7 +++-- methods/mirror.cc | 72 ++++++++++++++++++++++++++++++++++++++---------- methods/mirror.h | 15 +++++++--- 3 files changed, 73 insertions(+), 21 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index f46209d12..e0e7c1d09 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -63,9 +63,10 @@ class pkgAcqMethod // Outgoing messages void Fail(bool Transient = false); inline void Fail(const char *Why, bool Transient = false) {Fail(string(Why),Transient);}; - void Fail(string Why, bool Transient = false); - void URIStart(FetchResult &Res); - void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + bool MediaFail(string Required,string Drive); virtual void Exit() {}; diff --git a/methods/mirror.cc b/methods/mirror.cc index fad076e91..c4d911a3c 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -17,6 +17,8 @@ #include #include +#include + using namespace std; #include "mirror.h" @@ -25,31 +27,50 @@ using namespace std; /*}}}*/ MirrorMethod::MirrorMethod() - : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) + : HttpMethod(), HasMirrorFile(false) { #if 0 HasMirrorFile=true; - BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; + MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; Mirror="http://de.archive.ubuntu.com/ubuntu/"; #endif }; +// HttpMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* We stash the desired pipeline depth */ +bool MirrorMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + Debug = _config->FindB("Debug::Acquire::mirror",false); + + return true; +} + /*}}}*/ + + bool MirrorMethod::GetMirrorFile(string uri) { string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); BaseUri = uri.substr(0,uri.find(Marker)); - BaseUri.replace(0,strlen("mirror://"),"http://"); + + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; + if(Debug) + { + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + } // FIXME: fetch it with curl pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - cerr << "fetch-result: " << res << endl; if(res) HasMirrorFile = true; @@ -61,7 +82,9 @@ bool MirrorMethod::SelectMirror() { ifstream in(MirrorFile.c_str()); getline(in, Mirror); - cerr << "Mirror: " << Mirror << endl; + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + return true; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -77,21 +100,42 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - // change the items in the queue - Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); - cerr << "new Fetch-uri: " << Itm->Uri << endl; + if(Queue->Uri.find("mirror://") != string::npos) + Queue->Uri.replace(0,BaseUri.size(),Mirror); - // FIXME: fetch it with! - + // now run the real fetcher + return HttpMethod::Fetch(Itm); }; +void MirrorMethod::Fail(string Err,bool Transient) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::Fail(Err, Transient); +} + +void MirrorMethod::URIStart(FetchResult &Res) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIStart(Res); +} + +void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIDone(Res, Alt); +} + + int main() { setlocale(LC_ALL, ""); MirrorMethod Mth; - return Mth.Run(); + return Mth.Loop(); } diff --git a/methods/mirror.h b/methods/mirror.h index ca3428830..91cf8c64f 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -20,17 +20,24 @@ using std::endl; #include "http.h" -class MirrorMethod : public pkgAcqMethod +class MirrorMethod : public HttpMethod { FetchResult Res; - string Mirror; - string BaseUri; - string MirrorFile; + string BaseUri; // the original mirror://... url + string Mirror; // the selected mirror uri (http://...) + string MirrorFile; // bool HasMirrorFile; + bool Debug; + protected: bool GetMirrorFile(string uri); bool SelectMirror(); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual bool Configuration(string Message); + public: MirrorMethod(); virtual bool Fetch(FetchItem *Itm); -- cgit v1.2.3-70-g09d2 From 86c17f0a4bd3c0e3dde5bb58b3df9c41e5a88763 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:09:26 +0100 Subject: * todo added --- methods/mirror.cc | 8 ++++++++ methods/mirror.h | 3 +++ 2 files changed, 11 insertions(+) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c4d911a3c..c49764044 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -26,6 +26,14 @@ using namespace std; /*}}}*/ +/* + * TODO: + * - support keeping the mirror file around (evil listclearer strikes again) + * - better method to download than having a pkgAcquire interface here + * - testing :) + * + */ + MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { diff --git a/methods/mirror.h b/methods/mirror.h index 91cf8c64f..a47b8fbf1 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -23,6 +23,7 @@ using std::endl; class MirrorMethod : public HttpMethod { FetchResult Res; + // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) string MirrorFile; // @@ -33,6 +34,8 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + + // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); virtual void URIStart(FetchResult &Res); virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); -- cgit v1.2.3-70-g09d2 From 3ed91a17d21d0d456a12accc507b82e1c0f1a697 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:21:47 +0100 Subject: * more todo items --- methods/mirror.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c49764044..1911d6115 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -28,10 +28,11 @@ using namespace std; /* * TODO: - * - support keeping the mirror file around (evil listclearer strikes again) * - better method to download than having a pkgAcquire interface here + * - support keeping the mirror file around (evil listclearer strikes again) + * -> /var/lib/apt/mirrors dir? how to cleanup? by time? + * - provide some TTL time until the mirror file is get again (1h? 6h?) * - testing :) - * */ MirrorMethod::MirrorMethod() -- cgit v1.2.3-70-g09d2 From ef1e6d8822bb34527bba5d2318b5768013efd010 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Nov 2006 11:06:09 +0100 Subject: * methods/mirror.cc: - check the complette queue on Fetch() for urls to transform --- methods/mirror.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 1911d6115..4de981522 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,6 +32,7 @@ using namespace std; * - support keeping the mirror file around (evil listclearer strikes again) * -> /var/lib/apt/mirrors dir? how to cleanup? by time? * - provide some TTL time until the mirror file is get again (1h? 6h?) + * - deal with runing as non-root (we can't write to the lists dir then) * - testing :) */ @@ -109,8 +110,11 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - if(Queue->Uri.find("mirror://") != string::npos) - Queue->Uri.replace(0,BaseUri.size(),Mirror); + for (FetchItem *I = Queue; I != 0; I = I->Next) + { + if(I->Uri.find("mirror://") != string::npos) + I->Uri.replace(0,BaseUri.size(),Mirror); + } // now run the real fetcher return HttpMethod::Fetch(Itm); -- cgit v1.2.3-70-g09d2 From d731f9c574138a9cb31c586a8f1e7d7181a44456 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:19:32 +0100 Subject: * apt-pkg/init.cc: - added Dir::State::Mirrors * doc/examples/configure-index: - added Acquire::mirror::RefreshInterval * methods/mirror.{cc,h}: - download the mirror file into Dir::State::Mirrors - added RefreshInterval option to not ask for the mirror file too often --- apt-pkg/init.cc | 1 + debian/apt.dirs | 1 + doc/examples/configure-index | 7 +++++- methods/mirror.cc | 56 ++++++++++++++++++++++++++++++++++++-------- methods/mirror.h | 1 + po/apt-all.pot | 4 ++-- 6 files changed, 57 insertions(+), 13 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index b47378d4a..fe085a75e 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -54,6 +54,7 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.Set("Dir::State::lists","lists/"); Cnf.Set("Dir::State::cdroms","cdroms.list"); + Cnf.Set("Dir::State::mirrors","mirrors/"); // Cache Cnf.Set("Dir::Cache","var/cache/apt/"); diff --git a/debian/apt.dirs b/debian/apt.dirs index e1cb738fa..b81878baa 100644 --- a/debian/apt.dirs +++ b/debian/apt.dirs @@ -5,5 +5,6 @@ etc/apt etc/apt/sources.list.d var/cache/apt/archives/partial var/lib/apt/lists/partial +var/lib/apt/mirrors/partial var/lib/apt/periodic usr/share/bug/apt diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 0f0abc30c..cb20d767b 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -160,6 +160,11 @@ Acquire { Options {"--ignore-time-conflict";} // not very usefull on a normal system }; + + mirror + { + RefreshInterval "360"; // refresh interval in minutes + }; }; // Directory layout @@ -258,9 +263,9 @@ Debug Acquire::Ftp "false"; // Show ftp command traffic Acquire::Http "false"; // Show http command traffic Acquire::gpgv "false"; // Show the gpgv traffic + Acquire::Mirror "false"; // Show debugging of the mirror method aptcdrom "false"; // Show found package files IdentCdrom "false"; - } /* Whatever you do, do not use this configuration file!! Take out ONLY diff --git a/methods/mirror.cc b/methods/mirror.cc index 4de981522..e70a40f55 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -18,6 +18,7 @@ #include #include #include +#include using namespace std; @@ -28,11 +29,16 @@ using namespace std; /* * TODO: + * - send expected checksum to the mirror method so that + some checking/falling back can be done here already + * - keep the mirror file around in /var/lib/apt/mirrors + * can't be put into lists/ because of the listclearer + * cleanup by time (mtime relative to the other mtimes) + * - use a TTL time the mirror file is fetched again (6h?) + * - deal with runing as non-root because we can't write to the lists + dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - support keeping the mirror file around (evil listclearer strikes again) - * -> /var/lib/apt/mirrors dir? how to cleanup? by time? - * - provide some TTL time until the mirror file is get again (1h? 6h?) - * - deal with runing as non-root (we can't write to the lists dir then) + * - magicmarker is (a bit) evil * - testing :) */ @@ -60,6 +66,12 @@ bool MirrorMethod::Configuration(string Message) } /*}}}*/ +// clean the mirrors dir based on ttl information +bool MirrorMethod::Clean(string dir) +{ + +} + bool MirrorMethod::GetMirrorFile(string uri) { @@ -69,7 +81,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); - MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { @@ -77,19 +89,43 @@ bool MirrorMethod::GetMirrorFile(string uri) cerr << "mirror-file: " << MirrorFile << endl; } - // FIXME: fetch it with curl + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + HasMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now pkgAcquire Fetcher; new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - - if(res) + if(res) HasMirrorFile = true; Fetcher.Shutdown(); - return true; + return res; } bool MirrorMethod::SelectMirror() { + // FIXME: make the mirror selection more clever, do not + // just use the first one! ifstream in(MirrorFile.c_str()); getline(in, Mirror); if(Debug) @@ -103,7 +139,7 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // get mirror information + // select mirror only once per session if(!HasMirrorFile) { GetMirrorFile(Itm->Uri); diff --git a/methods/mirror.h b/methods/mirror.h index a47b8fbf1..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -34,6 +34,7 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + bool Clean(string dir); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index b3da2ba9a..42c43fe8d 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -2238,12 +2238,12 @@ msgstr "" msgid "Please insert the disc labeled: '%s' in the drive '%s' and press enter." msgstr "" -#: apt-pkg/init.cc:120 +#: apt-pkg/init.cc:121 #, c-format msgid "Packaging system '%s' is not supported" msgstr "" -#: apt-pkg/init.cc:136 +#: apt-pkg/init.cc:137 msgid "Unable to determine a suitable packaging system type" msgstr "" -- cgit v1.2.3-70-g09d2 From 70288656715d2fe4c2c33598124ae48f7bca9cdf Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:38:55 +0100 Subject: * methods/mirror.cc: - implemented simple Clean() implementation based on the time of the last access for a mirror file --- doc/examples/configure-index | 2 ++ methods/mirror.cc | 55 +++++++++++++++++++++++++++++++++++++++++--- po/apt-all.pot | 3 ++- 3 files changed, 56 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/doc/examples/configure-index b/doc/examples/configure-index index cb20d767b..81bb6b3b0 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -164,6 +164,8 @@ Acquire mirror { RefreshInterval "360"; // refresh interval in minutes + MaxAge "90"; // max age for a mirror file in days before + // it gets deleted }; }; diff --git a/methods/mirror.cc b/methods/mirror.cc index e70a40f55..425a2f7f5 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -19,12 +19,14 @@ #include #include #include +#include +#include using namespace std; #include "mirror.h" #include "http.h" - +#include "apti18n.h" /*}}}*/ /* @@ -67,9 +69,54 @@ bool MirrorMethod::Configuration(string Message) /*}}}*/ // clean the mirrors dir based on ttl information -bool MirrorMethod::Clean(string dir) +bool MirrorMethod::Clean(string Dir) { + // FIXME: it would better to have a global idea of the mirrors + // in the sources.list and use this instead of this time + // based approach. currently apt does not support this :/ + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + string StartDir = SafeGetCWD(); + if (chdir(Dir.c_str()) != 0) + { + closedir(D); + return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str()); + } + + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name,"lock") == 0 || + strcmp(Dir->d_name,"partial") == 0 || + strcmp(Dir->d_name,".") == 0 || + strcmp(Dir->d_name,"..") == 0) + continue; + + // Del everything not touched for MaxAge days + time_t t,now,max; + struct stat buf; + if(stat(Dir->d_name, &buf) != 0) + { + cerr << "Can't stat '" << Dir->d_name << "'" << endl; + continue; + } + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); + if(t + max < now) + { + if(Debug) + clog << "Mirror file is older than MaxAge days, deleting" << endl; + unlink(Dir->d_name); + } + }; + chdir(StartDir.c_str()); + closedir(D); + return true; } @@ -81,6 +128,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); + // get new file MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) @@ -108,7 +156,7 @@ bool MirrorMethod::GetMirrorFile(string uri) return true; } if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; } // not that great to use pkgAcquire here, but we do not have @@ -142,6 +190,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) // select mirror only once per session if(!HasMirrorFile) { + Clean(_config->FindDir("Dir::State::mirrors")); GetMirrorFile(Itm->Uri); SelectMirror(); } diff --git a/po/apt-all.pot b/po/apt-all.pot index 42c43fe8d..d1ce72503 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:76 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,6 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 +#: methods/mirror.cc:82 #, c-format msgid "Unable to change to %s" msgstr "" -- cgit v1.2.3-70-g09d2 From 0c312e0ed24fed69412c181634dd4515e9b1d46f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 12:10:05 +0100 Subject: * implemented proper mirror list cleanup --- methods/mirror.cc | 39 +++++++++++++++++++++------------------ po/apt-all.pot | 4 ++-- 2 files changed, 23 insertions(+), 20 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 425a2f7f5..00f7b7807 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -71,9 +72,14 @@ bool MirrorMethod::Configuration(string Message) // clean the mirrors dir based on ttl information bool MirrorMethod::Clean(string Dir) { - // FIXME: it would better to have a global idea of the mirrors - // in the sources.list and use this instead of this time - // based approach. currently apt does not support this :/ + vector::const_iterator I; + + if(Debug) + clog << "MirrorMethod::Clean(): " << Dir << endl; + + // read sources.list + pkgSourceList list; + list.ReadMainList(); DIR *D = opendir(Dir.c_str()); if (D == 0) @@ -94,24 +100,21 @@ bool MirrorMethod::Clean(string Dir) strcmp(Dir->d_name,".") == 0 || strcmp(Dir->d_name,"..") == 0) continue; - - // Del everything not touched for MaxAge days - time_t t,now,max; - struct stat buf; - if(stat(Dir->d_name, &buf) != 0) + + // see if we have that uri + for(I=list.begin(); I != list.end(); I++) { - cerr << "Can't stat '" << Dir->d_name << "'" << endl; - continue; + string uri = (*I)->GetURI(); + if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + string BaseUri = uri.substr(0,uri.find(Marker)); + if (URItoFileName(BaseUri) == Dir->d_name) + break; } - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); - if(t + max < now) - { - if(Debug) - clog << "Mirror file is older than MaxAge days, deleting" << endl; + // nothing found, nuke it + if (I == list.end()) unlink(Dir->d_name); - } }; chdir(StartDir.c_str()); diff --git a/po/apt-all.pot b/po/apt-all.pot index d1ce72503..52fadb81c 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:76 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:86 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:82 +#: methods/mirror.cc:92 #, c-format msgid "Unable to change to %s" msgstr "" -- cgit v1.2.3-70-g09d2 From 933833c51fbfe3dca3f3a3073d441e5856462605 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 13:07:22 +0100 Subject: * methods/mirror.cc: - updated the TODO --- methods/mirror.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 00f7b7807..f08b324ec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -34,14 +34,18 @@ using namespace std; * TODO: * - send expected checksum to the mirror method so that some checking/falling back can be done here already - * - keep the mirror file around in /var/lib/apt/mirrors - * can't be put into lists/ because of the listclearer - * cleanup by time (mtime relative to the other mtimes) - * - use a TTL time the mirror file is fetched again (6h?) + use pkgAcquire::Custom600Header() for this? what about gpgv + failures? + #OR# + * - implement it at the pkgAcquire::Item::Failed() level but then + we need to send back what uri exactly was failing + * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil + * - magicmarker is (a bit) evil, maybe just use a similar approach as in + clean and read the sources.list and use the GetURI() method to find + the prefix? * - testing :) */ -- cgit v1.2.3-70-g09d2 From cae9cdcefc34eba6d023cb30cbbdb9bbf909b8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 16:35:56 +0100 Subject: * basic error reporting from apt in place now (ReportMirrorFailures()) --- apt-pkg/acquire-item.cc | 10 +++++++++- apt-pkg/acquire-item.h | 1 + apt-pkg/acquire-method.cc | 1 + apt-pkg/acquire-method.h | 1 + methods/mirror.cc | 27 +++++++++++++++++++++------ methods/mirror.h | 1 + po/apt-all.pot | 16 ++++++++-------- 7 files changed, 42 insertions(+), 15 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 8ec4ba2c0..7b2a89763 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -176,7 +176,8 @@ string pkgAcqIndex::Custom600Headers() struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true"; - + if(ExpectedMD5 != "") + return "\nExpectedMD5: " + ExpectedMD5; return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ @@ -1015,6 +1016,13 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } } /*}}}*/ +// --------------------------------------------------------------------- +string pkgAcqArchive::Custom600Headers() +{ + if(MD5 != "") + return "\nExpectedMD5: " + MD5; + return ""; +} // AcqArchive::IsTrusted - Determine whether this archive comes from a // trusted source /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index da1bea801..c9cd75321 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -205,6 +205,7 @@ class pkgAcqArchive : public pkgAcquire::Item virtual string ShortDesc() {return Desc.ShortDesc;}; virtual void Finished(); virtual bool IsTrusted(); + virtual string Custom600Headers(); pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 41b832f3b..638797657 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -368,6 +368,7 @@ int pkgAcqMethod::Run(bool Single) Tmp->Uri = LookupTag(Message,"URI"); Tmp->DestFile = LookupTag(Message,"FileName"); + Tmp->ExpectedMD5 = LookupTag(Message,"ExpectedMD5"); if (StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified) == false) Tmp->LastModified = 0; Tmp->IndexFile = StringToBool(LookupTag(Message,"Index-File"),false); diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index e0e7c1d09..9e908d32c 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -33,6 +33,7 @@ class pkgAcqMethod string DestFile; time_t LastModified; bool IndexFile; + string ExpectedMD5; }; struct FetchResult diff --git a/methods/mirror.cc b/methods/mirror.cc index f08b324ec..428726a3d 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,14 +32,14 @@ using namespace std; /* * TODO: - * - send expected checksum to the mirror method so that - some checking/falling back can be done here already - use pkgAcquire::Custom600Header() for this? what about gpgv - failures? + * - what about gpgv failures? better standard format for errors + to send back to LP #OR# * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing - + we need to send back what uri exactly was failing + [mvo: the problem with this approach is ::Failed() is not really + called for all failures :/ e.g. md5sum mismatch in a archive + is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here @@ -214,6 +214,9 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // FIXME: queue next mirror? + ReportMirrorFailure(Err); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -228,11 +231,23 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { + // FIXME: queue next mirror? + if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) + ReportMirrorFailure("499 Hash mismatch"); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } +void MirrorMethod::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed + std::cerr << "\nReportMirrorFailure: " + << Queue->Uri + << " FailCode: " + << FailCode << std::endl; +} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..3ff9e1a96 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,6 +35,7 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); + void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index 52fadb81c..aee5c09c8 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:86 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:92 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:92 +#: methods/mirror.cc:98 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2362,35 +2362,35 @@ msgstr "" msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:236 apt-pkg/acquire-item.cc:945 +#: apt-pkg/acquire-item.cc:237 apt-pkg/acquire-item.cc:946 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:640 +#: apt-pkg/acquire-item.cc:641 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:753 +#: apt-pkg/acquire-item.cc:754 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:812 +#: apt-pkg/acquire-item.cc:813 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:848 +#: apt-pkg/acquire-item.cc:849 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:935 +#: apt-pkg/acquire-item.cc:936 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 2769f0bcf6c1ac73e883f47857fa227de92f4525 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 17:16:42 +0100 Subject: * mirror-failure.py: example mirror failure cgi * methods/mirror.cc: prepare for the failure submit --- methods/mirror.cc | 6 ++++++ mirror-failure.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 mirror-failure.py (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 428726a3d..6621d47e2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -247,6 +247,12 @@ void MirrorMethod::ReportMirrorFailure(string FailCode) << Queue->Uri << " FailCode: " << FailCode << std::endl; +#if 0 // FIXME: do not use system, make sure to properly encode + // URI/FailCode, do not hardcode the submit url + system("curl -d url=" + Queue->Uri + + " -d FailureCode=" + FailCode + + " http://localhost:8000/ &"); +#endif } int main() diff --git a/mirror-failure.py b/mirror-failure.py new file mode 100644 index 000000000..e7d2bbf54 --- /dev/null +++ b/mirror-failure.py @@ -0,0 +1,23 @@ +# File: cgihttpserver-example-1.py + +import CGIHTTPServer +import BaseHTTPServer + +class Handler(CGIHTTPServer.CGIHTTPRequestHandler): + #cgi_directories = ["/cgi"] + def do_POST(self): + print "do_POST" + #print self.command + #print self.path + #print self.headers + print self.client_address + data = self.rfile.read(int(self.headers["content-length"])) + print data + self.wfile.write("200 Ok\n"); + +PORT = 8000 + +httpd = BaseHTTPServer.HTTPServer(("", PORT), Handler) +print "serving at port", PORT +httpd.serve_forever() + -- cgit v1.2.3-70-g09d2 From 362d29343e5d25248bcd84300aa1b18effe891e7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Jan 2007 14:42:17 +0100 Subject: make the mirror failures actually produce a error message --- cmdline/makefile | 6 ++++++ cmdline/report-mirror-failure | 21 -------------------- configure.in | 2 +- debian/changelog | 2 +- methods/mirror.cc | 46 ++++++++++++++++++++++++++++++++----------- po/apt-all.pot | 6 +++--- 6 files changed, 46 insertions(+), 37 deletions(-) delete mode 100755 cmdline/report-mirror-failure (limited to 'methods') diff --git a/cmdline/makefile b/cmdline/makefile index 882a0e1b5..f07c7da3b 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -52,3 +52,9 @@ SOURCE=apt-key TO=$(BIN) TARGET=program include $(COPY_H) + +# The apt-key program +SOURCE=apt-report-mirror-failure +TO=$(LIB) +TARGET=program +include $(COPY_H) diff --git a/cmdline/report-mirror-failure b/cmdline/report-mirror-failure deleted file mode 100755 index 8301ed079..000000000 --- a/cmdline/report-mirror-failure +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/python - -import sys -import urllib -import apt_pkg - -apt_pkg.init() -url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", - "http://people.ubuntu.com:9000/mirror-failure") - #"http://localhost:9000/mirror-failure") -if not url: - sys.exit(0) - -data = {} -data['url'] = sys.argv[1] -data['error'] = sys.argv[2] -f = urllib.urlopen(url, urllib.urlencode(data)) -f.read() -f.close() - - diff --git a/configure.in b/configure.in index 81f327c37..df452ba34 100644 --- a/configure.in +++ b/configure.in @@ -18,7 +18,7 @@ AC_CONFIG_AUX_DIR(buildlib) AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in) dnl -- SET THIS TO THE RELEASE VERSION -- -AC_DEFINE_UNQUOTED(VERSION,"0.6.46.3") +AC_DEFINE_UNQUOTED(VERSION,"0.6.46.4") PACKAGE="apt" AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE") AC_SUBST(PACKAGE) diff --git a/debian/changelog b/debian/changelog index 24e10dcd5..6c9ada3a5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,7 +2,7 @@ apt (0.6.46.4) unstable; urgency=low * add apt-secure.8 to "See also" section - -- + -- Michael Vogt Wed, 24 Jan 2007 14:34:15 +0100 apt (0.6.46.3) unstable; urgency=low diff --git a/methods/mirror.cc b/methods/mirror.cc index 6621d47e2..8f9b8ed34 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -30,13 +30,18 @@ using namespace std; #include "apti18n.h" /*}}}*/ -/* +/* Done: + * - works with http only + * - always picks the first mirror from the list + * - call out to problem reporting script + * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * * TODO: - * - what about gpgv failures? better standard format for errors - to send back to LP - #OR# - * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing + * what about gpgv failures? this should call-out to the problem reporting + script, but we need to know what mirror was used + * better standard format for errors to send back + * - implement failure reporting at the pkgAcquire::Item::Failed() level + but then we need to send back what uri exactly was failing [mvo: the problem with this approach is ::Failed() is not really called for all failures :/ e.g. md5sum mismatch in a archive is not] @@ -46,6 +51,7 @@ using namespace std; * - magicmarker is (a bit) evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? + * support more than http * - testing :) */ @@ -243,16 +249,34 @@ void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) void MirrorMethod::ReportMirrorFailure(string FailCode) { // report that Queue->Uri failed +#if 0 std::cerr << "\nReportMirrorFailure: " << Queue->Uri << " FailCode: " << FailCode << std::endl; -#if 0 // FIXME: do not use system, make sure to properly encode - // URI/FailCode, do not hardcode the submit url - system("curl -d url=" + Queue->Uri + - " -d FailureCode=" + FailCode + - " http://localhost:8000/ &"); #endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/lib/apt/report-mirror-failure"); + Args[i++] = report.c_str(); + Args[i++] = Queue->Uri.c_str(); + Args[i++] = FailCode.c_str(); + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(report.c_str(), (char**)Args); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Acquire::Mirror::ReportFailures").c_str()); + } } int main() diff --git a/po/apt-all.pot b/po/apt-all.pot index aee5c09c8..dd140bd63 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2006-11-24 16:37+0100\n" +"POT-Creation-Date: 2007-01-24 12:51+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:92 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:96 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:98 +#: methods/mirror.cc:102 #, c-format msgid "Unable to change to %s" msgstr "" -- cgit v1.2.3-70-g09d2 From 36280399db0ae203d3f1ae4d44b946f31e9a38ce Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 1 Feb 2007 12:32:50 +0100 Subject: * commited the latest mirror failure detection code --- apt-pkg/acquire-item.cc | 51 +++++++++++++++++++++++++++++++-------- apt-pkg/acquire-item.h | 5 +++- apt-pkg/acquire-method.cc | 21 ++++++++++------ apt-pkg/acquire-method.h | 8 +++--- cmdline/apt-report-mirror-failure | 23 ++++++++++++++++++ methods/connect.cc | 17 ++++++------- methods/mirror.cc | 45 +++------------------------------- methods/mirror.h | 1 - po/apt-all.pot | 44 ++++++++++++++++----------------- 9 files changed, 120 insertions(+), 95 deletions(-) create mode 100755 cmdline/apt-report-mirror-failure (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7b2a89763..61564c7aa 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -79,6 +79,10 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Status = StatError; Dequeue(); } + + // report mirror failure back to LP if we actually use a mirror + if(!UsedMirror.empty()) + ReportMirrorFailure(ErrorText); } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -100,6 +104,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string, { // We just downloaded something.. string FileName = LookupTag(Message,"Filename"); + UsedMirror = LookupTag(Message,"UsedMirror"); if (Complete == false && FileName == DestFile) { if (Owner->Log != 0) @@ -108,7 +113,6 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string, if (FileSize == 0) FileSize= Size; - Status = StatDone; ErrorText = string(); Owner->Dequeue(this); @@ -131,6 +135,42 @@ void pkgAcquire::Item::Rename(string From,string To) } /*}}}*/ +void pkgAcquire::Item::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed +#if 0 + std::cerr << "\nReportMirrorFailure: " + << UsedMirror + << " FailCode: " + << FailCode << std::endl; +#endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/bin/apt-report-mirror-failure"); + if(!FileExists(report)) + return; + Args[i++] = report.c_str(); + Args[i++] = UsedMirror.c_str(); + Args[i++] = FailCode.c_str(); + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(report.c_str(), (char**)Args); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Acquire::Mirror::ReportFailures").c_str()); + } +} + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is @@ -176,8 +216,6 @@ string pkgAcqIndex::Custom600Headers() struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true"; - if(ExpectedMD5 != "") - return "\nExpectedMD5: " + ExpectedMD5; return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ @@ -1016,13 +1054,6 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } } /*}}}*/ -// --------------------------------------------------------------------- -string pkgAcqArchive::Custom600Headers() -{ - if(MD5 != "") - return "\nExpectedMD5: " + MD5; - return ""; -} // AcqArchive::IsTrusted - Determine whether this archive comes from a // trusted source /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index c9cd75321..9949d0044 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -56,6 +56,7 @@ class pkgAcquire::Item unsigned long ID; bool Complete; bool Local; + string UsedMirror; // Number of queues we are inserted into unsigned int QueueCounter; @@ -78,6 +79,9 @@ class pkgAcquire::Item pkgAcquire *GetOwner() {return Owner;}; virtual bool IsTrusted() {return false;}; + // report mirror problems + void ReportMirrorFailure(string FailCode); + Item(pkgAcquire *Owner); virtual ~Item(); }; @@ -205,7 +209,6 @@ class pkgAcqArchive : public pkgAcquire::Item virtual string ShortDesc() {return Desc.ShortDesc;}; virtual void Finished(); virtual bool IsTrusted(); - virtual string Custom600Headers(); pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 638797657..13201d310 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -99,12 +99,11 @@ void pkgAcqMethod::Fail(string Err,bool Transient) } char S[1024]; + char *End = S; if (Queue != 0) { - snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" - "Message: %s %s\n",Queue->Uri.c_str(),Err.c_str(), - FailExtra.c_str()); - + End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" + "Message: %s %s\n",Queue->Uri.c_str(), Err.c_str(), IP.c_str()); // Dequeue FetchItem *Tmp = Queue; Queue = Queue->Next; @@ -113,10 +112,14 @@ void pkgAcqMethod::Fail(string Err,bool Transient) QueueBack = Queue; } else - snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: \n" - "Message: %s %s\n",Err.c_str(), - FailExtra.c_str()); - + { + End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: \n" + "Message: %s\n",Err.c_str()); + } + if(FailReason.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"FailReason: %s\n",FailReason.c_str()); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); // Set the transient flag if (Transient == true) strcat(S,"Transient-Failure: true\n\n"); @@ -182,6 +185,8 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) End += snprintf(End,sizeof(S)-50 - (End - S),"MD5-Hash: %s\n",Res.MD5Sum.c_str()); if (Res.SHA1Sum.empty() == false) End += snprintf(End,sizeof(S)-50 - (End - S),"SHA1-Hash: %s\n",Res.SHA1Sum.c_str()); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); if (Res.GPGVOutput.size() > 0) End += snprintf(End,sizeof(S)-50 - (End - S),"GPGVOutput:\n"); for (vector::iterator I = Res.GPGVOutput.begin(); diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 9e908d32c..ee68c75f7 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -55,7 +55,9 @@ class pkgAcqMethod vector Messages; FetchItem *Queue; FetchItem *QueueBack; - string FailExtra; + string FailReason; + string UsedMirror; + string IP; // Handlers for messages virtual bool Configuration(string Message); @@ -72,7 +74,6 @@ class pkgAcqMethod virtual void Exit() {}; public: - enum CnfFlags {SingleInstance = (1<<0), Pipeline = (1<<1), SendConfig = (1<<2), LocalOnly = (1<<3), NeedsCleanup = (1<<4), @@ -82,7 +83,8 @@ class pkgAcqMethod void Status(const char *Format,...); int Run(bool Single = false); - inline void SetFailExtraMsg(string Msg) {FailExtra = Msg;}; + inline void SetFailReason(string Msg) {FailReason = Msg;}; + inline void SetIP(string aIP) {IP = aIP;}; pkgAcqMethod(const char *Ver,unsigned long Flags = 0); virtual ~pkgAcqMethod() {}; diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure new file mode 100755 index 000000000..70b16cf4f --- /dev/null +++ b/cmdline/apt-report-mirror-failure @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import sys +import urllib +import apt_pkg + +print "apt-report-mirror-failure" + +apt_pkg.init() +url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", + "http://people.ubuntu.com:9000/mirror-failure") + #"http://localhost:9000/mirror-failure") +if not url: + sys.exit(0) + +data = {} +data['url'] = sys.argv[1] +data['error'] = sys.argv[2] +f = urllib.urlopen(url, urllib.urlencode(data)) +f.read() +f.close() + + diff --git a/methods/connect.cc b/methods/connect.cc index 8c2ac6d56..4e227c3fd 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -18,6 +18,7 @@ #include #include #include +#include // Internet stuff #include @@ -67,12 +68,10 @@ static bool DoConnect(struct addrinfo *Addr,string Host, wrong this will get tacked onto the end of the error message */ if (LastHostAddr->ai_next != 0) { - char Name2[NI_MAXHOST + NI_MAXSERV + 10]; - snprintf(Name2,sizeof(Name2),_("[IP: %s %s]"),Name,Service); - Owner->SetFailExtraMsg(string(Name2)); - } - else - Owner->SetFailExtraMsg(""); + std::stringstream ss; + ioprintf(ss, _("[IP: %s %s]"),Name,Service); + Owner->SetIP(ss.str()); + } // Get a socket if ((Fd = socket(Addr->ai_family,Addr->ai_socktype, @@ -89,7 +88,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, /* This implements a timeout for connect by opening the connection nonblocking */ if (WaitFd(Fd,true,TimeOut) == false) { - Owner->SetFailExtraMsg("\nFailReason: Timeout"); + Owner->SetFailReason("Timeout"); return _error->Error(_("Could not connect to %s:%s (%s), " "connection timed out"),Host.c_str(),Service,Name); } @@ -104,7 +103,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, { errno = Err; if(errno == ECONNREFUSED) - Owner->SetFailExtraMsg("\nFailReason: ConnectionRefused"); + Owner->SetFailReason("ConnectionRefused"); return _error->Errno("connect",_("Could not connect to %s:%s (%s)."),Host.c_str(), Service,Name); } @@ -169,7 +168,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, if (Res == EAI_AGAIN) { - Owner->SetFailExtraMsg("\nFailReason: TmpResolveFailure"); + Owner->SetFailReason("TmpResolveFailure"); return _error->Error(_("Temporary failure resolving '%s'"), Host.c_str()); } diff --git a/methods/mirror.cc b/methods/mirror.cc index 8f9b8ed34..8ccfb8559 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -38,7 +38,7 @@ using namespace std; * * TODO: * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used + script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() * better standard format for errors to send back * - implement failure reporting at the pkgAcquire::Item::Failed() level but then we need to send back what uri exactly was failing @@ -48,7 +48,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil, maybe just use a similar approach as in + * - magicmarker is evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? * support more than http @@ -191,6 +191,8 @@ bool MirrorMethod::SelectMirror() getline(in, Mirror); if(Debug) cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; return true; } @@ -220,9 +222,6 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // FIXME: queue next mirror? - ReportMirrorFailure(Err); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -237,47 +236,11 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - // FIXME: queue next mirror? - if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) - ReportMirrorFailure("499 Hash mismatch"); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } -void MirrorMethod::ReportMirrorFailure(string FailCode) -{ - // report that Queue->Uri failed -#if 0 - std::cerr << "\nReportMirrorFailure: " - << Queue->Uri - << " FailCode: " - << FailCode << std::endl; -#endif - const char *Args[40]; - unsigned int i = 0; - string report = _config->Find("Methods::Mirror::ProblemReporting", - "/usr/lib/apt/report-mirror-failure"); - Args[i++] = report.c_str(); - Args[i++] = Queue->Uri.c_str(); - Args[i++] = FailCode.c_str(); - pid_t pid = ExecFork(); - if(pid < 0) - { - _error->Error("ReportMirrorFailure Fork failed"); - return; - } - else if(pid == 0) - { - execvp(report.c_str(), (char**)Args); - } - if(!ExecWait(pid, "report-mirror-failure")) - { - _error->Warning("Couldn't report problem to '%s'", - _config->Find("Acquire::Mirror::ReportFailures").c_str()); - } -} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 3ff9e1a96..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,7 +35,6 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); - void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index dd140bd63..15b650fda 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2007-01-24 12:51+0100\n" +"POT-Creation-Date: 2007-01-24 17:20+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -950,7 +950,7 @@ msgstr "" msgid "Calculating upgrade... " msgstr "" -#: cmdline/apt-get.cc:1716 methods/ftp.cc:702 methods/connect.cc:101 +#: cmdline/apt-get.cc:1716 methods/ftp.cc:702 methods/connect.cc:100 msgid "Failed" msgstr "" @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:96 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:99 #, c-format msgid "Unable to read %s" msgstr "" @@ -1648,59 +1648,59 @@ msgstr "" msgid "Unable to invoke " msgstr "" -#: methods/connect.cc:64 +#: methods/connect.cc:65 #, c-format msgid "Connecting to %s (%s)" msgstr "" -#: methods/connect.cc:71 +#: methods/connect.cc:72 #, c-format msgid "[IP: %s %s]" msgstr "" -#: methods/connect.cc:80 +#: methods/connect.cc:79 #, c-format msgid "Could not create a socket for %s (f=%u t=%u p=%u)" msgstr "" -#: methods/connect.cc:86 +#: methods/connect.cc:85 #, c-format msgid "Cannot initiate the connection to %s:%s (%s)." msgstr "" -#: methods/connect.cc:93 +#: methods/connect.cc:92 #, c-format msgid "Could not connect to %s:%s (%s), connection timed out" msgstr "" -#: methods/connect.cc:108 +#: methods/connect.cc:107 #, c-format msgid "Could not connect to %s:%s (%s)." msgstr "" #. We say this mainly because the pause here is for the #. ssh connection that is still going -#: methods/connect.cc:136 methods/rsh.cc:425 +#: methods/connect.cc:135 methods/rsh.cc:425 #, c-format msgid "Connecting to %s" msgstr "" -#: methods/connect.cc:167 +#: methods/connect.cc:166 #, c-format msgid "Could not resolve '%s'" msgstr "" -#: methods/connect.cc:173 +#: methods/connect.cc:172 #, c-format msgid "Temporary failure resolving '%s'" msgstr "" -#: methods/connect.cc:176 +#: methods/connect.cc:175 #, c-format msgid "Something wicked happened resolving '%s:%s' (%i)" msgstr "" -#: methods/connect.cc:223 +#: methods/connect.cc:222 #, c-format msgid "Unable to connect to %s %s:" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:102 +#: methods/mirror.cc:105 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2357,40 +2357,40 @@ msgstr "" msgid "IO Error saving source cache" msgstr "" -#: apt-pkg/acquire-item.cc:126 +#: apt-pkg/acquire-item.cc:128 #, c-format msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:237 apt-pkg/acquire-item.cc:946 +#: apt-pkg/acquire-item.cc:275 apt-pkg/acquire-item.cc:984 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:641 +#: apt-pkg/acquire-item.cc:679 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:754 +#: apt-pkg/acquire-item.cc:792 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:813 +#: apt-pkg/acquire-item.cc:851 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:849 +#: apt-pkg/acquire-item.cc:887 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:936 +#: apt-pkg/acquire-item.cc:974 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 59271f62e4a291c8d96e1f6073203c395734b6ca Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 17:52:28 +0100 Subject: * use pkgAcqMethod::FailReason() for consistent error reporting --- apt-pkg/acquire-item.cc | 10 +++++++--- methods/connect.cc | 1 + methods/http.cc | 3 +++ methods/mirror.cc | 12 +++--------- po/apt-all.pot | 34 +++++++++++++++++----------------- 5 files changed, 31 insertions(+), 29 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 0d3d6a083..08a029ff4 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -82,8 +82,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } // report mirror failure back to LP if we actually use a mirror - if(!UsedMirror.empty()) - ReportMirrorFailure(ErrorText); + ReportMirrorFailure(ErrorText); } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -138,10 +137,13 @@ void pkgAcquire::Item::Rename(string From,string To) void pkgAcquire::Item::ReportMirrorFailure(string FailCode) { - // report that Queue->Uri failed + // we only act if a mirror was used at all + if(UsedMirror.empty()) + return; #if 0 std::cerr << "\nReportMirrorFailure: " << UsedMirror + << " Uri: " << DescURI() << " FailCode: " << FailCode << std::endl; #endif @@ -278,6 +280,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5, Status = StatAuthError; ErrorText = _("MD5Sum mismatch"); Rename(DestFile,DestFile + ".FAILED"); + ReportMirrorFailure("HashChecksumFailure"); return; } // Done, move it into position @@ -765,6 +768,7 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } // gpgv method failed + ReportMirrorFailure("GPGFailure"); _error->Warning("GPG error: %s: %s", Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); diff --git a/methods/connect.cc b/methods/connect.cc index 4e227c3fd..145001fb3 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -163,6 +163,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, DefPort = 0; continue; } + Owner->SetFailReason("ResolveFailure"); return _error->Error(_("Could not resolve '%s'"),Host.c_str()); } diff --git a/methods/http.cc b/methods/http.cc index deaa8d0c8..01ad14655 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -914,6 +914,9 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) failure */ if (Srv->Result < 200 || Srv->Result >= 300) { + char err[255]; + snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result); + SetFailReason(err); _error->Error("%u %s",Srv->Result,Srv->Code); if (Srv->HaveContent == true) return 4; diff --git a/methods/mirror.cc b/methods/mirror.cc index 8ccfb8559..b64879bec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -31,20 +31,14 @@ using namespace std; /*}}}*/ /* Done: - * - works with http only + * - works with http (only!) * - always picks the first mirror from the list * - call out to problem reporting script * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * - use pkgAcqMethod::FailReason() to have a string representation + * of the failure that is also send to LP * * TODO: - * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() - * better standard format for errors to send back - * - implement failure reporting at the pkgAcquire::Item::Failed() level - but then we need to send back what uri exactly was failing - [mvo: the problem with this approach is ::Failed() is not really - called for all failures :/ e.g. md5sum mismatch in a archive - is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here diff --git a/po/apt-all.pot b/po/apt-all.pot index 39759f265..f24f19ba7 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:96 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:98 #, c-format msgid "Unable to read %s" msgstr "" @@ -1621,7 +1621,7 @@ msgstr "" msgid "Unable to accept connection" msgstr "" -#: methods/ftp.cc:864 methods/http.cc:957 methods/rsh.cc:303 +#: methods/ftp.cc:864 methods/http.cc:960 methods/rsh.cc:303 msgid "Problem hashing file" msgstr "" @@ -1685,22 +1685,22 @@ msgstr "" msgid "Connecting to %s" msgstr "" -#: methods/connect.cc:166 +#: methods/connect.cc:167 #, c-format msgid "Could not resolve '%s'" msgstr "" -#: methods/connect.cc:172 +#: methods/connect.cc:173 #, c-format msgid "Temporary failure resolving '%s'" msgstr "" -#: methods/connect.cc:175 +#: methods/connect.cc:176 #, c-format msgid "Something wicked happened resolving '%s:%s' (%i)" msgstr "" -#: methods/connect.cc:222 +#: methods/connect.cc:223 #, c-format msgid "Unable to connect to %s %s:" msgstr "" @@ -1813,15 +1813,15 @@ msgstr "" msgid "Error reading from server" msgstr "" -#: methods/http.cc:1106 +#: methods/http.cc:1109 msgid "Bad header data" msgstr "" -#: methods/http.cc:1123 +#: methods/http.cc:1126 msgid "Connection failed" msgstr "" -#: methods/http.cc:1214 +#: methods/http.cc:1217 msgid "Internal error" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:102 +#: methods/mirror.cc:104 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2357,40 +2357,40 @@ msgstr "" msgid "IO Error saving source cache" msgstr "" -#: apt-pkg/acquire-item.cc:131 +#: apt-pkg/acquire-item.cc:130 #, c-format msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:279 apt-pkg/acquire-item.cc:988 +#: apt-pkg/acquire-item.cc:281 apt-pkg/acquire-item.cc:992 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:683 +#: apt-pkg/acquire-item.cc:686 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:796 +#: apt-pkg/acquire-item.cc:800 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:855 +#: apt-pkg/acquire-item.cc:859 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:891 +#: apt-pkg/acquire-item.cc:895 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:978 +#: apt-pkg/acquire-item.cc:982 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 066b53e95ba224f81d078e7d8df3002275f8a092 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 18:46:45 +0100 Subject: * get rid of magic mirror --- methods/mirror.cc | 62 +++++++++++++++++++++++++++++++++++++++++-------------- po/apt-all.pot | 4 ++-- 2 files changed, 48 insertions(+), 18 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b64879bec..ff91130b8 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,22 +42,13 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is evil, maybe just use a similar approach as in - clean and read the sources.list and use the GetURI() method to find - the prefix? - * support more than http + * - support more than http * - testing :) */ MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { -#if 0 - HasMirrorFile=true; - BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; - MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; - Mirror="http://de.archive.ubuntu.com/ubuntu/"; -#endif }; // HttpMethod::Configuration - Handle a configuration message /*{{{*/ @@ -111,8 +102,7 @@ bool MirrorMethod::Clean(string Dir) string uri = (*I)->GetURI(); if(uri.substr(0,strlen("mirror://")) != string("mirror://")) continue; - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - string BaseUri = uri.substr(0,uri.find(Marker)); + string BaseUri = uri.substr(0,uri.size()-1); if (URItoFileName(BaseUri) == Dir->d_name) break; } @@ -127,11 +117,51 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string uri) +bool MirrorMethod::GetMirrorFile(string mirror_uri_str) { - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - BaseUri = uri.substr(0,uri.find(Marker)); - + /* + - a mirror_uri_str looks like this: + mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg + + - the matching source.list entry + deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main + + - we actually want to go after: + http://people.ubuntu.com/~mvo/apt/mirror/mirrors + + And we need to save the BaseUri for later: + - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors + + FIXME: what if we have two similar prefixes? + mirror://people.ubuntu.com/~mvo/mirror + mirror://people.ubuntu.com/~mvo/mirror2 + then mirror_uri_str looks like: + mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg + mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg + we search sources.list and find: + mirror://people.ubuntu.com/~mvo/apt/mirror + in both cases! So we need to apply some domain knowledge here :( and + check for /dists/ or /Release.gpg as suffixes + */ + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + + // read sources.list and find match + vector::const_iterator I; + pkgSourceList list; + list.ReadMainList(); + for(I=list.begin(); I != list.end(); I++) + { + string uristr = (*I)->GetURI(); + std::cerr << "Checking: " << uristr << std::endl; + if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + // find matching uri in sources.list + if(mirror_uri_str.substr(0,uristr.size()) == uristr) + { + std::cerr << "found BaseURI: " << uristr << std::endl; + BaseUri = uristr.substr(0,uristr.size()-1); + } + } string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); diff --git a/po/apt-all.pot b/po/apt-all.pot index f24f19ba7..8fa68d296 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:98 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:90 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:104 +#: methods/mirror.cc:96 #, c-format msgid "Unable to change to %s" msgstr "" -- cgit v1.2.3-70-g09d2 From f0b509cdb44cb5e79e9c5ddd7ebec46965138534 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 19:06:42 +0100 Subject: * apt-pkg/acquire-item.cc: - use FailReason in pkgAcquire::Item::Failed when available * methods/mirror.cc: - move some debug output into if(Debug) --- apt-pkg/acquire-item.cc | 7 ++++++- cmdline/apt-report-mirror-failure | 5 +++-- methods/mirror.cc | 9 ++++++--- po/apt-all.pot | 18 +++++++++--------- 4 files changed, 24 insertions(+), 15 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 08a029ff4..14610d6e9 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -82,7 +82,11 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } // report mirror failure back to LP if we actually use a mirror - ReportMirrorFailure(ErrorText); + string FailReason = LookupTag(Message, "FailReason"); + if(FailReason.size() != 0) + ReportMirrorFailure(FailReason); + else + ReportMirrorFailure(ErrorText); } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -155,6 +159,7 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) return; Args[i++] = report.c_str(); Args[i++] = UsedMirror.c_str(); + Args[i++] = DescURI().c_str(); Args[i++] = FailCode.c_str(); Args[i++] = NULL; pid_t pid = ExecFork(); diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure index 277b23e9a..fb43f0e74 100755 --- a/cmdline/apt-report-mirror-failure +++ b/cmdline/apt-report-mirror-failure @@ -14,8 +14,9 @@ if not url: print "Reporting mirror failure to '%s'" % url data = {} -data['url'] = sys.argv[1] -data['error'] = sys.argv[2] +data['mirror'] = sys.argv[1] +data['failurl'] = sys.argv[2] +data['error'] = sys.argv[3] f = urllib.urlopen(url, urllib.urlencode(data)) f.read() f.close() diff --git a/methods/mirror.cc b/methods/mirror.cc index ff91130b8..c5c0c7461 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -143,7 +143,8 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + if(Debug) + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -152,13 +153,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) for(I=list.begin(); I != list.end(); I++) { string uristr = (*I)->GetURI(); - std::cerr << "Checking: " << uristr << std::endl; + if(Debug) + std::cerr << "Checking: " << uristr << std::endl; if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) continue; // find matching uri in sources.list if(mirror_uri_str.substr(0,uristr.size()) == uristr) { - std::cerr << "found BaseURI: " << uristr << std::endl; + if(Debug) + std::cerr << "found BaseURI: " << uristr << std::endl; BaseUri = uristr.substr(0,uristr.size()-1); } } diff --git a/po/apt-all.pot b/po/apt-all.pot index 8fa68d296..2355eee20 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:90 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:81 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:96 +#: methods/mirror.cc:87 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2357,40 +2357,40 @@ msgstr "" msgid "IO Error saving source cache" msgstr "" -#: apt-pkg/acquire-item.cc:130 +#: apt-pkg/acquire-item.cc:134 #, c-format msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:281 apt-pkg/acquire-item.cc:992 +#: apt-pkg/acquire-item.cc:286 apt-pkg/acquire-item.cc:997 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:686 +#: apt-pkg/acquire-item.cc:691 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:800 +#: apt-pkg/acquire-item.cc:805 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:859 +#: apt-pkg/acquire-item.cc:864 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:895 +#: apt-pkg/acquire-item.cc:900 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:982 +#: apt-pkg/acquire-item.cc:987 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 3f599bb721c4ac58d8ff18991c9704b5f30eaa2b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 10:49:25 +0100 Subject: * apt-pkg/acquire-item.cc: - default to "/usr/lib/apt/apt-report-mirror-failure" * cmdline/apt-report-mirror-failure: - no default comit url for now * debian/rules: - move apt-report-mirror-failure into /usr/lib/apt * doc/examples/configure-index: - more documentation * methods/mirror.cc: - updated TODO --- apt-pkg/acquire-item.cc | 2 +- cmdline/apt-report-mirror-failure | 4 ++-- cmdline/makefile | 2 +- debian/rules | 4 ++++ doc/examples/configure-index | 4 ++++ methods/mirror.cc | 1 + 6 files changed, 13 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 14610d6e9..38530152f 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -154,7 +154,7 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) const char *Args[40]; unsigned int i = 0; string report = _config->Find("Methods::Mirror::ProblemReporting", - "/usr/bin/apt-report-mirror-failure"); + "/usr/lib/apt/apt-report-mirror-failure"); if(!FileExists(report)) return; Args[i++] = report.c_str(); diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure index fb43f0e74..1567e78e3 100755 --- a/cmdline/apt-report-mirror-failure +++ b/cmdline/apt-report-mirror-failure @@ -5,8 +5,8 @@ import urllib import apt_pkg apt_pkg.init() -url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", - "http://people.ubuntu.com:9000/mirror-failure") +url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", None) + #"http://people.ubuntu.com:9000/mirror-failure") #"http://localhost:9000/mirror-failure") if not url: sys.exit(0) diff --git a/cmdline/makefile b/cmdline/makefile index f07c7da3b..8d284c2f3 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -55,6 +55,6 @@ include $(COPY_H) # The apt-key program SOURCE=apt-report-mirror-failure -TO=$(LIB) +TO=$(BIN) TARGET=program include $(COPY_H) diff --git a/debian/rules b/debian/rules index 02ee734bb..9e87b1723 100755 --- a/debian/rules +++ b/debian/rules @@ -214,6 +214,10 @@ apt: build debian/shlibs.local # head -n 500 ChangeLog > debian/ChangeLog + # move the mirror failure script in place + mv debian/$@/usr/bin/apt-report-mirror-failure \ + debian/$@/usr/lib/apt/apt-report-mirror-failure \ + dh_installexamples -p$@ $(BLD)/docs/examples/* dh_installman -p$@ dh_installcron -p$@ diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 81bb6b3b0..81fa4c52f 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -166,6 +166,10 @@ Acquire RefreshInterval "360"; // refresh interval in minutes MaxAge "90"; // max age for a mirror file in days before // it gets deleted + // mirror failure reporting script + ProblemReporting "/usr/lib/apt/apt-report-mirror-failure"; + // mirror failure reporting url + ReportFailures "http://example.com/mirror-failure"; }; }; diff --git a/methods/mirror.cc b/methods/mirror.cc index c5c0c7461..9a86a10c2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,6 +42,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here + * and better error handling there! * - support more than http * - testing :) */ -- cgit v1.2.3-70-g09d2 From 68acfb437f0c6c2e026be6fd3b7da3fc8d64c523 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 16:37:20 +0100 Subject: * ABI version bumped --- apt-pkg/init.h | 2 +- apt-pkg/makefile | 2 +- debian/changelog | 1 + methods/makefile | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.h b/apt-pkg/init.h index b036b8609..0e91f164d 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -18,7 +18,7 @@ // See the makefile #define APT_PKG_MAJOR 3 -#define APT_PKG_MINOR 51 +#define APT_PKG_MINOR 52 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 6cc1414b2..10b012774 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=3.51 +MAJOR=3.52 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/debian/changelog b/debian/changelog index 6fdf646f0..caed5e531 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,7 @@ apt (0.6.46.4ubuntu7) feisty; urgency=low It also supports error reporting to a configurable url for mirror problems/failures. + * Bump ABI version -- Michael Vogt Tue, 6 Feb 2007 11:38:06 +0100 diff --git a/methods/makefile b/methods/makefile index f971e56e8..d66ba11e3 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 3.51 +LIB_APT_PKG_MAJOR = 3.52 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3-70-g09d2 From d715b9c969264eec96913b577f63a3e9e498b2e6 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 21 Feb 2007 09:59:53 +0100 Subject: * fix in the pkgRecords destructor * Bump ABI version --- apt-pkg/init.h | 2 +- apt-pkg/makefile | 2 +- apt-pkg/pkgrecords.cc | 16 +++++++++------- apt-pkg/pkgrecords.h | 5 +++-- configure.in | 2 +- debian/changelog | 7 +++++++ methods/makefile | 2 +- po/apt-all.pot | 48 ++++++++++++++++++++++++------------------------ 8 files changed, 47 insertions(+), 37 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.h b/apt-pkg/init.h index 0e91f164d..905173a7a 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -18,7 +18,7 @@ // See the makefile #define APT_PKG_MAJOR 3 -#define APT_PKG_MINOR 52 +#define APT_PKG_MINOR 53 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 10b012774..d36747d85 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=3.52 +MAJOR=3.53 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/apt-pkg/pkgrecords.cc b/apt-pkg/pkgrecords.cc index b22f3e73f..456b5aef4 100644 --- a/apt-pkg/pkgrecords.cc +++ b/apt-pkg/pkgrecords.cc @@ -23,11 +23,9 @@ // Records::pkgRecords - Constructor /*{{{*/ // --------------------------------------------------------------------- /* This will create the necessary structures to access the status files */ -pkgRecords::pkgRecords(pkgCache &Cache) : Cache(Cache), Files(0) +pkgRecords::pkgRecords(pkgCache &Cache) : Cache(Cache), + Files(Cache.HeaderP->PackageFileCount) { - Files = new Parser *[Cache.HeaderP->PackageFileCount]; - memset(Files,0,sizeof(*Files)*Cache.HeaderP->PackageFileCount); - for (pkgCache::PkgFileIterator I = Cache.FileBegin(); I.end() == false; I++) { @@ -49,9 +47,13 @@ pkgRecords::pkgRecords(pkgCache &Cache) : Cache(Cache), Files(0) /* */ pkgRecords::~pkgRecords() { - for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++) - delete Files[I]; - delete [] Files; + for ( vector::iterator it = Files.begin(); + it != Files.end(); + ++it) + { + delete *it; + } + } /*}}}*/ // Records::Lookup - Get a parser for the package version file /*{{{*/ diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h index 31c444dbf..ad3946c1b 100644 --- a/apt-pkg/pkgrecords.h +++ b/apt-pkg/pkgrecords.h @@ -23,6 +23,7 @@ #include #include +#include class pkgRecords { @@ -32,8 +33,8 @@ class pkgRecords private: pkgCache &Cache; - Parser **Files; - + std::vectorFiles; + public: // Lookup function diff --git a/configure.in b/configure.in index ce166b31e..7d93de955 100644 --- a/configure.in +++ b/configure.in @@ -18,7 +18,7 @@ AC_CONFIG_AUX_DIR(buildlib) AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in) dnl -- SET THIS TO THE RELEASE VERSION -- -AC_DEFINE_UNQUOTED(VERSION,"0.6.46.4ubuntu7") +AC_DEFINE_UNQUOTED(VERSION,"0.6.46.4ubuntu8") PACKAGE="apt" AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE") AC_SUBST(PACKAGE) diff --git a/debian/changelog b/debian/changelog index caed5e531..b926a495f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +apt (0.6.46.4ubuntu8) feisty; urgency=low + + * fix in the pkgRecords destructor + * Bump ABI version + + -- + apt (0.6.46.4ubuntu7) feisty; urgency=low * Merged the apt--mirror branch. This means that a new 'mirror' diff --git a/methods/makefile b/methods/makefile index d66ba11e3..610bd5b49 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 3.52 +LIB_APT_PKG_MAJOR = 3.53 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method diff --git a/po/apt-all.pot b/po/apt-all.pot index f2f151dd3..7233cd617 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2007-02-06 11:14+0100\n" +"POT-Creation-Date: 2007-02-09 16:54+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -251,7 +251,7 @@ msgid "" " -o=? Set an arbitrary configuration option, eg -o dir::cache=/tmp\n" msgstr "" -#: cmdline/apt-extracttemplates.cc:267 apt-pkg/pkgcachegen.cc:819 +#: cmdline/apt-extracttemplates.cc:267 apt-pkg/pkgcachegen.cc:820 #, c-format msgid "Unable to write to %s" msgstr "" @@ -1397,9 +1397,9 @@ msgid "The info and temp directories need to be on the same filesystem" msgstr "" #. Build the status cache -#: apt-inst/deb/dpkgdb.cc:139 apt-pkg/pkgcachegen.cc:752 -#: apt-pkg/pkgcachegen.cc:821 apt-pkg/pkgcachegen.cc:826 -#: apt-pkg/pkgcachegen.cc:949 +#: apt-inst/deb/dpkgdb.cc:139 apt-pkg/pkgcachegen.cc:753 +#: apt-pkg/pkgcachegen.cc:822 apt-pkg/pkgcachegen.cc:827 +#: apt-pkg/pkgcachegen.cc:950 msgid "Reading package lists" msgstr "" @@ -2337,92 +2337,92 @@ msgstr "" msgid "Cache has an incompatible versioning system" msgstr "" -#: apt-pkg/pkgcachegen.cc:119 +#: apt-pkg/pkgcachegen.cc:120 #, c-format msgid "Error occurred while processing %s (NewPackage)" msgstr "" -#: apt-pkg/pkgcachegen.cc:134 +#: apt-pkg/pkgcachegen.cc:135 #, c-format msgid "Error occurred while processing %s (UsePackage1)" msgstr "" -#: apt-pkg/pkgcachegen.cc:157 +#: apt-pkg/pkgcachegen.cc:158 #, c-format msgid "Error occured while processing %s (NewFileDesc1)" msgstr "" -#: apt-pkg/pkgcachegen.cc:182 +#: apt-pkg/pkgcachegen.cc:183 #, c-format msgid "Error occurred while processing %s (UsePackage2)" msgstr "" -#: apt-pkg/pkgcachegen.cc:186 +#: apt-pkg/pkgcachegen.cc:187 #, c-format msgid "Error occurred while processing %s (NewFileVer1)" msgstr "" -#: apt-pkg/pkgcachegen.cc:217 +#: apt-pkg/pkgcachegen.cc:218 #, c-format msgid "Error occurred while processing %s (NewVersion1)" msgstr "" -#: apt-pkg/pkgcachegen.cc:221 +#: apt-pkg/pkgcachegen.cc:222 #, c-format msgid "Error occurred while processing %s (UsePackage3)" msgstr "" -#: apt-pkg/pkgcachegen.cc:225 +#: apt-pkg/pkgcachegen.cc:226 #, c-format msgid "Error occurred while processing %s (NewVersion2)" msgstr "" -#: apt-pkg/pkgcachegen.cc:249 +#: apt-pkg/pkgcachegen.cc:250 #, c-format msgid "Error occured while processing %s (NewFileDesc2)" msgstr "" -#: apt-pkg/pkgcachegen.cc:255 +#: apt-pkg/pkgcachegen.cc:256 msgid "Wow, you exceeded the number of package names this APT is capable of." msgstr "" -#: apt-pkg/pkgcachegen.cc:258 +#: apt-pkg/pkgcachegen.cc:259 msgid "Wow, you exceeded the number of versions this APT is capable of." msgstr "" -#: apt-pkg/pkgcachegen.cc:261 +#: apt-pkg/pkgcachegen.cc:262 msgid "Wow, you exceeded the number of descriptions this APT is capable of." msgstr "" -#: apt-pkg/pkgcachegen.cc:264 +#: apt-pkg/pkgcachegen.cc:265 msgid "Wow, you exceeded the number of dependencies this APT is capable of." msgstr "" -#: apt-pkg/pkgcachegen.cc:292 +#: apt-pkg/pkgcachegen.cc:293 #, c-format msgid "Error occurred while processing %s (FindPkg)" msgstr "" -#: apt-pkg/pkgcachegen.cc:305 +#: apt-pkg/pkgcachegen.cc:306 #, c-format msgid "Error occurred while processing %s (CollectFileProvides)" msgstr "" -#: apt-pkg/pkgcachegen.cc:311 +#: apt-pkg/pkgcachegen.cc:312 #, c-format msgid "Package %s %s was not found while processing file dependencies" msgstr "" -#: apt-pkg/pkgcachegen.cc:682 +#: apt-pkg/pkgcachegen.cc:683 #, c-format msgid "Couldn't stat source package list %s" msgstr "" -#: apt-pkg/pkgcachegen.cc:767 +#: apt-pkg/pkgcachegen.cc:768 msgid "Collecting File Provides" msgstr "" -#: apt-pkg/pkgcachegen.cc:894 apt-pkg/pkgcachegen.cc:901 +#: apt-pkg/pkgcachegen.cc:895 apt-pkg/pkgcachegen.cc:902 msgid "IO Error saving source cache" msgstr "" -- cgit v1.2.3-70-g09d2 From d530b4c24fc452efc82be4b7e1ebc536a00c09b7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 3 Oct 2007 02:21:52 +0200 Subject: * Reset curl options and timestamp between downloaded files. Thanks to Ryan Murray for the patch --- debian/changelog | 2 ++ methods/https.cc | 13 +++++++++++++ 2 files changed, 15 insertions(+) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 87b9dd024..375a6bb1e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,8 @@ apt (0.7.6ubuntu12) gutsy; urgency=low (thanks to Laurent Bigonville, LP: #140019) * apt-pkg/deb/debmetaindex.cc: comparison with string literal results in unspecified behaviour; + * Reset curl options and timestamp between downloaded files. Thanks to + Ryan Murray for the patch [Paul Sladen] * Have 'cron.daily/apt' send D-Bus doesn't exist error messages diff --git a/methods/https.cc b/methods/https.cc index e6717e63a..3b2b0bb19 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -115,6 +115,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // - error checking/reporting // - more debug options? (CURLOPT_DEBUGFUNCTION?) + curl_easy_reset(curl); SetupProxy(); // callbacks @@ -125,6 +126,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FILETIME, true); // FIXME: https: offer various options of verification bool peer_verify = _config->FindB("Acquire::https::Verify-Peer", false); @@ -202,6 +204,9 @@ bool HttpsMethod::Fetch(FetchItem *Itm) CURLcode success = curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &curl_responsecode); + long curl_servdate; + curl_easy_getinfo(curl, CURLINFO_FILETIME, &curl_servdate); + // cleanup if(success != 0) { @@ -215,6 +220,14 @@ bool HttpsMethod::Fetch(FetchItem *Itm) if (Res.Size == 0) Res.Size = File->Size(); + // Timestamp + struct utimbuf UBuf; + if (curl_servdate != -1) { + UBuf.actime = curl_servdate; + UBuf.modtime = curl_servdate; + utime(File->Name().c_str(),&UBuf); + } + // check the downloaded result struct stat Buf; if (stat(File->Name().c_str(),&Buf) == 0) -- cgit v1.2.3-70-g09d2 From 3de6d4585be0b8a9bd98b5fb1d62ff922537c435 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 5 Oct 2007 07:50:24 +0200 Subject: * methods/https.cc: - fix off-by-one error I-M-S handling - cleanup after I-M-S hit - only send LastModified if we actually have a file - delete zero size I-M-S hits --- debian/changelog | 6 +++++- methods/https.cc | 26 +++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index b25e75ef5..30ac4e10d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ apt (0.7.6ubuntu13) gutsy; urgency=low - fix crash in WriteApportReport (LP: #144537) * apt-pkg/acquire-item.cc - fix disappearing local Packages.gz file (LP: #131166) + * methods/https.cc: + - fix off-by-one error I-M-S handling + - cleanup after I-M-S hit -- @@ -124,10 +127,11 @@ apt (0.7.6ubuntu1) gutsy; urgency=low * apt-pkg/acquire-item.cc: - remove zero size files on I-M-S hit * methods/https.cc: - - only send LastModified if we actually have one + - only send LastModified if we actually have a file - send range request with if-range - delete failed downloads (thanks to Thom May for his help here) + - delete zero size I-M-S hits * apt-pkg/deb/dpkgpm.{cc,h}: - merged dpkg-log branch, this lets you specify a Dir::Log::Terminal file to log dpkg output to diff --git a/methods/https.cc b/methods/https.cc index 3b2b0bb19..b2bbbddb1 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -161,13 +161,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - // set time values - if(Itm->LastModified > 0) - { - curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); - curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); - } - // speed limit int dlLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024; if (dlLimit > 0) @@ -183,7 +176,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // error handling curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); - // In this case we send an if-range query with a range header + // if we have the file send an if-range query with a range header if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { char Buf[1000]; @@ -191,11 +184,17 @@ bool HttpsMethod::Fetch(FetchItem *Itm) (long)SBuf.st_size - 1, TimeRFC1123(SBuf.st_mtime).c_str()); headers = curl_slist_append(headers, Buf); + } + else if(Itm->LastModified > 0) + { + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); } // go for it - if the file exists, append on it File = new FileFd(Itm->DestFile, FileFd::WriteAny); - File->Seek(File->Size()); + if (File->Size() > 0) + File->Seek(File->Size() - 1); // keep apt updated Res.Filename = Itm->DestFile; @@ -217,9 +216,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) } File->Close(); - if (Res.Size == 0) - Res.Size = File->Size(); - // Timestamp struct utimbuf UBuf; if (curl_servdate != -1) { @@ -232,15 +228,19 @@ bool HttpsMethod::Fetch(FetchItem *Itm) struct stat Buf; if (stat(File->Name().c_str(),&Buf) == 0) { - Res.Size = Buf.st_size; Res.Filename = File->Name(); Res.LastModified = Buf.st_mtime; Res.IMSHit = false; if (curl_responsecode == 304) { + unlink(File->Name().c_str()); Res.IMSHit = true; Res.LastModified = Itm->LastModified; + Res.Size = 0; + URIDone(Res); + return true; } + Res.Size = Buf.st_size; } // take hashes -- cgit v1.2.3-70-g09d2 From 38eedeb77790c106eb35940e3d7835a7e0f0297e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Nov 2007 15:54:17 +0100 Subject: methods/mirror.{cc,h}: - improve the mirror method so that it only updates the mirror list from the server if we get new indexfiles - code cleanup --- methods/mirror.cc | 152 +++++++++++++++++++++++++++++++++--------------------- methods/mirror.h | 7 +-- 2 files changed, 97 insertions(+), 62 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 9a86a10c2..43e56c71a 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -48,7 +48,7 @@ using namespace std; */ MirrorMethod::MirrorMethod() - : HttpMethod(), HasMirrorFile(false) + : HttpMethod(), DownloadedMirrorFile(false) { }; @@ -73,6 +73,9 @@ bool MirrorMethod::Clean(string Dir) if(Debug) clog << "MirrorMethod::Clean(): " << Dir << endl; + if(Dir == "/") + return _error->Error("will not clean: '/'"); + // read sources.list pkgSourceList list; list.ReadMainList(); @@ -118,7 +121,71 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string mirror_uri_str) +bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) +{ + + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + DownloadedMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); + + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + if(res) + DownloadedMirrorFile = true; + Fetcher.Shutdown(); + return res; +} + +bool MirrorMethod::SelectMirror() +{ + // if we do not have a MirrorFile, fallback + if(!FileExists(MirrorFile)) + { + // FIXME: fallback to a default mirror here instead + // and provide a config option to define that default + return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + } + + // FIXME: make the mirror selection more clever, do not + // just use the first one! + // BUT: we can not make this random, the mirror has to be + // stable accross session, because otherwise we can + // get into sync issues (got indexfiles from mirror A, + // but packages from mirror B - one might be out of date etc) + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +string MirrorMethod::GetMirrorFileName(string mirror_uri_str) { /* - a mirror_uri_str looks like this: @@ -144,8 +211,9 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ + string name; if(Debug) - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -166,62 +234,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) BaseUri = uristr.substr(0,uristr.size()-1); } } - string fetch = BaseUri; - fetch.replace(0,strlen("mirror://"),"http://"); - // get new file - MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); + name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; - } - - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) - { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) - { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - HasMirrorFile = true; - return true; - } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + cerr << "mirror-file: " << name << endl; } - - // not that great to use pkgAcquire here, but we do not have - // any other way right now - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) - HasMirrorFile = true; - Fetcher.Shutdown(); - return res; -} - -bool MirrorMethod::SelectMirror() -{ - // FIXME: make the mirror selection more clever, do not - // just use the first one! - ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; - return true; + return name; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -230,20 +251,33 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // select mirror only once per session - if(!HasMirrorFile) + // the http method uses Fetch(0) as a way to update the pipeline, + // just let it do its work in this case - Fetch() with a valid + // Itm will always run before the first Fetch(0) + if(Itm == NULL) + return HttpMethod::Fetch(Itm); + + // if we don't have the name of the mirror file on disk yet, + // calculate it now (can be derived from the uri) + if(MirrorFile.empty()) + MirrorFile = GetMirrorFileName(Itm->Uri); + + // download mirror file once (if we are after index files) + if(Itm->IndexFile && !DownloadedMirrorFile) { Clean(_config->FindDir("Dir::State::mirrors")); - GetMirrorFile(Itm->Uri); - SelectMirror(); + DownloadMirrorFile(Itm->Uri); } + if(Mirror.empty()) + SelectMirror(); + for (FetchItem *I = Queue; I != 0; I = I->Next) { if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(),Mirror); + I->Uri.replace(0,BaseUri.size(), Mirror); } - + // now run the real fetcher return HttpMethod::Fetch(Itm); }; diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..ed817806b 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,13 +26,14 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) - string MirrorFile; // - bool HasMirrorFile; + string MirrorFile; // the file that contains the list of mirrors + bool DownloadedMirrorFile; // already downloaded this session bool Debug; protected: - bool GetMirrorFile(string uri); + bool DownloadMirrorFile(string uri); + string GetMirrorFileName(string uri); bool SelectMirror(); bool Clean(string dir); -- cgit v1.2.3-70-g09d2 From 0df1d3ac4312a2051b10478cf1c8b242864cb3b1 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 6 Dec 2007 19:47:36 +0100 Subject: we do not break the abi --- apt-pkg/init.h | 2 +- apt-pkg/makefile | 2 +- methods/makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.h b/apt-pkg/init.h index 6d8693be9..23e951eff 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -18,7 +18,7 @@ // See the makefile #define APT_PKG_MAJOR 4 -#define APT_PKG_MINOR 6 +#define APT_PKG_MINOR 5 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 1b78c94f6..b327dbf64 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=4.6 +MAJOR=4.5 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) -lutil APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/methods/makefile b/methods/makefile index 085b357b8..3c12bb974 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.6 +LIB_APT_PKG_MAJOR = 4.5 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3-70-g09d2 From 5148def81fb20b22fc9ec54bc212e34260fbfebb Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 7 Dec 2007 22:54:41 +0100 Subject: update ABI, the homepage change broke it --- apt-pkg/init.h | 2 +- apt-pkg/makefile | 2 +- methods/makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.h b/apt-pkg/init.h index 23e951eff..6d8693be9 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -18,7 +18,7 @@ // See the makefile #define APT_PKG_MAJOR 4 -#define APT_PKG_MINOR 5 +#define APT_PKG_MINOR 6 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index b327dbf64..1b78c94f6 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=4.5 +MAJOR=4.6 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) -lutil APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/methods/makefile b/methods/makefile index 3c12bb974..085b357b8 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.5 +LIB_APT_PKG_MAJOR = 4.6 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3-70-g09d2 From d7bab8f1142e474ac4bafd139e93439aeddfd546 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 3 Jan 2008 15:22:44 +0100 Subject: methods/makefile: remove duplicated entry for https --- methods/makefile | 7 ------- 1 file changed, 7 deletions(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index 085b357b8..f765a4d37 100644 --- a/methods/makefile +++ b/methods/makefile @@ -59,13 +59,6 @@ LIB_MAKES = apt-pkg/makefile SOURCE = https.cc include $(PROGRAM_H) -# The https method -PROGRAM=https -SLIBS = -lapt-pkg -lcurl -LIB_MAKES = apt-pkg/makefile -SOURCE = https.cc -include $(PROGRAM_H) - # The ftp method PROGRAM=ftp SLIBS = -lapt-pkg $(SOCKETLIBS) -- cgit v1.2.3-70-g09d2 From 484befd1518245e795ed92e6c97ad50786f3b4a2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 11 Mar 2008 09:59:32 +0100 Subject: * typo fixes (LP: #107960) --- apt-pkg/depcache.h | 4 ++-- debian/changelog | 1 + methods/http.cc | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/apt-pkg/depcache.h b/apt-pkg/depcache.h index 7adf6fe7f..f41ad17e9 100644 --- a/apt-pkg/depcache.h +++ b/apt-pkg/depcache.h @@ -92,7 +92,7 @@ class pkgDepCache : protected pkgCache::Namespace * \param rootFunc A callback that can be used to add extra * packages to the root set. * - * \return \b false if an error occured. + * \return \b false if an error occurred. */ bool MarkRequired(InRootSetFunc &rootFunc); @@ -103,7 +103,7 @@ class pkgDepCache : protected pkgCache::Namespace * are tested to see whether they are actually garbage. If so, * they are marked as such. * - * \return \b false if an error occured. + * \return \b false if an error occurred. */ bool Sweep(); diff --git a/debian/changelog b/debian/changelog index e531be876..6fe4e5ece 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,7 @@ apt (0.7.9ubuntu13) hardy; urgency=low - updated for hardy (LP: #195879) * cmdline/apt-get.cc: - fix incorrect help output for -f (LP: #57487) + * typo fixes (LP: #107960) -- diff --git a/methods/http.cc b/methods/http.cc index 6aa4261ff..61321c850 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -367,8 +367,8 @@ bool ServerState::Close() /*}}}*/ // ServerState::RunHeaders - Get the headers before the data /*{{{*/ // --------------------------------------------------------------------- -/* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header - parse error occured */ +/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header + parse error occurred */ int ServerState::RunHeaders() { State = Header; -- cgit v1.2.3-70-g09d2 From 236527f0fffe30fc8988e17d926778c23ac1f902 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Jan 2009 17:39:37 +0100 Subject: LP: #263089 --- apt-pkg/deb/debsystem.cc | 10 +++++----- cmdline/apt-get.cc | 2 +- debian/changelog | 2 ++ methods/gpgv.cc | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) (limited to 'methods') diff --git a/apt-pkg/deb/debsystem.cc b/apt-pkg/deb/debsystem.cc index e1aacdbdb..ee2039c75 100644 --- a/apt-pkg/deb/debsystem.cc +++ b/apt-pkg/deb/debsystem.cc @@ -18,7 +18,7 @@ #include #include #include - + #include #include #include @@ -67,11 +67,11 @@ bool debSystem::Lock() if (LockFD == -1) { if (errno == EACCES || errno == EAGAIN) - return _error->Error("Unable to lock the administration directory (%s), " - "is another process using it?",AdminDir.c_str()); + return _error->Error(_("Unable to lock the administration directory (%s), " + "is another process using it?"),AdminDir.c_str()); else - return _error->Error("Unable to lock the administration directory (%s), " - "are you root?",AdminDir.c_str()); + return _error->Error(_("Unable to lock the administration directory (%s), " + "are you root?"),AdminDir.c_str()); } // See if we need to abort with a dirty journal diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index d5869da43..38775f70d 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1297,7 +1297,7 @@ pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, // show name mismatches if (IsMatch == true && Parse->Package() != Src) - ioprintf(c1out, _("No source package '%s' picking '%s' instead"), Parse->Package(), Src); + ioprintf(c1out, _("No source package '%s' picking '%s' instead"), Parse->Package().c_str(), Src.c_str()); if (VerTag.empty() == false) { diff --git a/debian/changelog b/debian/changelog index 2c4802e3b..478638a63 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,8 @@ apt (0.7.19ubuntu3) jaunty; urgency=low * debian/apt.conf.autoremove: - readd "linux-image" (and friends) to the auto-remove blacklist + * fix some i18n issues (thanks to Gabor Kelemen) + LP: #263089 [ Christian Perrier ] * Translations: diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 9f4683e6e..f3277b300 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -264,7 +264,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) // least one bad signature. good signatures and NoPubKey signatures // happen easily when a file is signed with multiple signatures if(GoodSigners.empty() or !BadSigners.empty()) - return _error->Error(errmsg.c_str()); + return _error->Error("%s", errmsg.c_str()); } // Just pass the raw output up, because passing it as a real data -- cgit v1.2.3-70-g09d2 From ebb461fdf02ee3e038d4b3a4ab1a0a60188edf9a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Feb 2009 12:14:36 +0100 Subject: * [ABI break] merge support for http redirects, thanks to Jeff Licquia and Anthony Towns * [ABI break] use int for the package IDs (thanks to Steve Cotton) - Galician updated. Closes: #509151 - Catalan updated. Closes: #509375 - Vietnamese updated. Closes: #509422 - Traditional Chinese added. Closes: #510664 * COPYING: - Actualized. Removed obsolete Qt section, added GPLv2 clause. (Closes: #440049, #509337) * Clarify the --help for 'purge' (LP: #243948) --- apt-pkg/acquire-method.cc | 32 +++++++++++++++++++ apt-pkg/acquire-method.h | 2 ++ apt-pkg/acquire-worker.cc | 14 +++++++++ apt-pkg/makefile | 2 +- apt-pkg/pkgcache.h | 8 ++--- debian/changelog | 44 ++++++++++++-------------- methods/http.cc | 78 ++++++++++++++++++++++++++++++++++++++++++++++- methods/http.h | 3 ++ methods/makefile | 2 +- 9 files changed, 153 insertions(+), 32 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 26f992bcf..150fbb77b 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -452,6 +452,38 @@ void pkgAcqMethod::Status(const char *Format,...) } /*}}}*/ +// AcqMethod::Redirect - Send a redirect message /*{{{*/ +// --------------------------------------------------------------------- +/* This method sends the redirect message and also manipulates the queue + to keep the pipeline synchronized. */ +void pkgAcqMethod::Redirect(const string &NewURI) +{ + string CurrentURI = ""; + if (Queue != 0) + CurrentURI = Queue->Uri; + + char S[1024]; + snprintf(S, sizeof(S)-50, "103 Redirect\nURI: %s\nNew-URI: %s\n\n", + CurrentURI.c_str(), NewURI.c_str()); + + if (write(STDOUT_FILENO,S,strlen(S)) != (ssize_t)strlen(S)) + exit(100); + + // Change the URI for the request. + Queue->Uri = NewURI; + + /* To keep the pipeline synchronized, move the current request to + the end of the queue, past the end of the current pipeline. */ + FetchItem *I; + for (I = Queue; I->Next != 0; I = I->Next) ; + I->Next = Queue; + Queue = Queue->Next; + I->Next->Next = 0; + if (QueueBack == 0) + QueueBack = I->Next; +} + /*}}}*/ + // AcqMethod::FetchResult::FetchResult - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 18c2cf009..99a4605b1 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -86,6 +86,8 @@ class pkgAcqMethod void Log(const char *Format,...); void Status(const char *Format,...); + void Redirect(const string &NewURI); + int Run(bool Single = false); inline void SetFailReason(string Msg) {FailReason = Msg;}; inline void SetIP(string aIP) {IP = aIP;}; diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 1a754dae9..78c68737c 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -220,6 +220,20 @@ bool pkgAcquire::Worker::RunMessages() Status = LookupTag(Message,"Message"); break; + // 103 Redirect + case 103: + { + if (Itm == 0) + { + _error->Error("Method gave invalid 103 Redirect message"); + break; + } + + string NewURI = LookupTag(Message,"New-URI",URI.c_str()); + Itm->URI = NewURI; + break; + } + // 200 URI Start case 200: { diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 1b78c94f6..087f17740 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=4.6 +MAJOR=4.7 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) -lutil APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 59d5003bb..759e9a225 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -214,7 +214,7 @@ struct pkgCache::Package unsigned char InstState; // Flags unsigned char CurrentState; // State - unsigned short ID; + unsigned int ID; unsigned long Flags; }; @@ -235,7 +235,7 @@ struct pkgCache::PackageFile // Linked list map_ptrloc NextFile; // PackageFile - unsigned short ID; + unsigned int ID; time_t mtime; // Modification time for the file }; @@ -272,7 +272,7 @@ struct pkgCache::Version map_ptrloc Size; // These are the .deb size map_ptrloc InstalledSize; unsigned short Hash; - unsigned short ID; + unsigned int ID; unsigned char Priority; }; @@ -289,7 +289,7 @@ struct pkgCache::Description map_ptrloc NextDesc; // Description map_ptrloc ParentPkg; // Package - unsigned short ID; + unsigned int ID; }; struct pkgCache::Dependency diff --git a/debian/changelog b/debian/changelog index 478638a63..bfbe6be5c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,29 +6,11 @@ apt (0.7.19ubuntu3) jaunty; urgency=low * apt-inst/contrib/arfile.cc: - support members ending with '/' as well (thanks to Michal Cihr, closes: #500988) - * debian/apt.conf.autoremove: - - readd "linux-image" (and friends) to the auto-remove + * debian/apt.conf.autoremove: + - readd "linux-image" (and friends) to the auto-remove blacklist - * fix some i18n issues (thanks to Gabor Kelemen) - LP: #263089 - - [ Christian Perrier ] - * Translations: - - Galician updated. Closes: #509151 - - Catalan updated. Closes: #509375 - - Vietnamese updated. Closes: #509422 - - Traditional Chinese added. Closes: #510664 - - [ Eugene V. Lyubimkin ] - * COPYING: - - Actualized. Removed obsolete Qt section, added GPLv2 clause. - (Closes: #440049, #509337) - - -- Michael Vogt Mon, 05 Jan 2009 08:59:20 +0100 - -apt (0.7.19ubuntu2) jaunty; urgency=low - - [ Michael Vogt ] + * fix some i18n issues (thanks to Gabor Kelemen) + LP: #263089 * apt-pkg/deb/dpkgpm.cc: - filter "ENOMEM" errors when creating apport reports * cmdline/apt-get.cc: @@ -40,15 +22,27 @@ apt (0.7.19ubuntu2) jaunty; urgency=low - add new strprintf() function to make i18n strings easier * apt-pkg/dev/debsystem.cc: - add missing apti18n.h header - - [ Dereck Wonnacott ] - * Clarify the --help for 'purge' (LP: #243948) + * [ABI break] merge support for http redirects, thanks to + Jeff Licquia and Anthony Towns + * [ABI break] use int for the package IDs (thanks to Steve Cotton) [ Christian Perrier ] * Translations: + - Galician updated. Closes: #509151 + - Catalan updated. Closes: #509375 + - Vietnamese updated. Closes: #509422 + - Traditional Chinese added. Closes: #510664 - French corrected (remove awful use of first person) - Finnish updated. Closes: #508449 + [ Eugene V. Lyubimkin ] + * COPYING: + - Actualized. Removed obsolete Qt section, added GPLv2 clause. + (Closes: #440049, #509337) + + [ Dereck Wonnacott ] + * Clarify the --help for 'purge' (LP: #243948) + [ Eugene V. Lyubimkin ] * doc/examples/sources.list: - Removed obsolete commented non-us deb-src entry, replaced it with diff --git a/methods/http.cc b/methods/http.cc index b0fb89fda..44274bd78 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include // Internet stuff @@ -56,6 +57,7 @@ int HttpMethod::FailFd = -1; time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; +bool AllowRedirect = false; bool Debug = false; URI Proxy; @@ -627,6 +629,12 @@ bool ServerState::HeaderLine(string Line) return true; } + if (stringcasecmp(Tag,"Location:") == 0) + { + Location = Val; + return true; + } + return true; } /*}}}*/ @@ -899,7 +907,9 @@ bool HttpMethod::ServerDie(ServerState *Srv) 1 - IMS hit 3 - Unrecoverable error 4 - Error with error content page - 5 - Unrecoverable non-server error (close the connection) */ + 5 - Unrecoverable non-server error (close the connection) + 6 - Try again with a new or changed URI + */ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { // Not Modified @@ -911,6 +921,27 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) return 1; } + /* Redirect + * + * Note that it is only OK for us to treat all redirection the same + * because we *always* use GET, not other HTTP methods. There are + * three redirection codes for which it is not appropriate that we + * redirect. Pass on those codes so the error handling kicks in. + */ + if (AllowRedirect + && (Srv->Result > 300 && Srv->Result < 400) + && (Srv->Result != 300 // Multiple Choices + && Srv->Result != 304 // Not Modified + && Srv->Result != 306)) // (Not part of HTTP/1.1, reserved) + { + if (!Srv->Location.empty()) + { + NextURI = Srv->Location; + return 6; + } + /* else pass through for error message */ + } + /* We have a reply we dont handle. This should indicate a perm server failure */ if (Srv->Result < 200 || Srv->Result >= 300) @@ -1028,6 +1059,7 @@ bool HttpMethod::Configuration(string Message) if (pkgAcqMethod::Configuration(Message) == false) return false; + AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut); PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", PipelineDepth); @@ -1041,6 +1073,10 @@ bool HttpMethod::Configuration(string Message) /* */ int HttpMethod::Loop() { + typedef vector StringVector; + typedef vector::iterator StringVectorIterator; + map Redirected; + signal(SIGTERM,SigTerm); signal(SIGINT,SigTerm); @@ -1227,6 +1263,46 @@ int HttpMethod::Loop() break; } + // Try again with a new URL + case 6: + { + // Clear rest of response if there is content + if (Server->HaveContent) + { + File = new FileFd("/dev/null",FileFd::WriteExists); + Server->RunData(); + delete File; + File = 0; + } + + /* Detect redirect loops. No more redirects are allowed + after the same URI is seen twice in a queue item. */ + StringVector &R = Redirected[Queue->DestFile]; + bool StopRedirects = false; + if (R.size() == 0) + R.push_back(Queue->Uri); + else if (R[0] == "STOP" || R.size() > 10) + StopRedirects = true; + else + { + for (StringVectorIterator I = R.begin(); I != R.end(); I++) + if (Queue->Uri == *I) + { + R[0] = "STOP"; + break; + } + + R.push_back(Queue->Uri); + } + + if (StopRedirects == false) + Redirect(NextURI); + else + Fail(); + + break; + } + default: Fail(_("Internal error")); break; diff --git a/methods/http.h b/methods/http.h index dec5cd80f..bc076e1f8 100644 --- a/methods/http.h +++ b/methods/http.h @@ -99,6 +99,7 @@ struct ServerState enum {Chunked,Stream,Closes} Encoding; enum {Header, Data} State; bool Persistent; + string Location; // This is a Persistent attribute of the server itself. bool Pipeline; @@ -145,6 +146,8 @@ class HttpMethod : public pkgAcqMethod protected: virtual bool Fetch(FetchItem *); + string NextURI; + public: friend class ServerState; diff --git a/methods/makefile b/methods/makefile index 5c4fa82bf..1d022be90 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.6 +LIB_APT_PKG_MAJOR = 4.7 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3-70-g09d2 From 8872c430df84660fa994db0ebfa67861239fe739 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Feb 2009 14:36:36 +0100 Subject: * methods/https.cc: - add Acquire::https::AllowRedirect support --- debian/changelog | 2 ++ doc/examples/configure-index | 6 ++++-- methods/https.cc | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 89848f31c..23a0514f6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -32,6 +32,8 @@ apt (0.7.20.2ubuntu1) jaunty; urgency=low * cmdline/apt-get.cc: - default to "false" for the "APT::Get::Build-Dep-Automatic" option (follow debian here) + * methods/https.cc: + - add Acquire::https::AllowRedirect support [ Dereck Wonnacott ] * Clarify the --help for 'purge' (LP: #243948) diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 5cc6187a8..a3ccdded8 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -136,7 +136,8 @@ Acquire Proxy::http.us.debian.org "DIRECT"; // Specific per-host setting Timeout "120"; Pipeline-Depth "5"; - + AllowRedirect "true"; + // Cache Control. Note these do not work with Squid 2.0.2 No-Cache "false"; Max-Age "86400"; // 1 Day age on index files @@ -153,7 +154,8 @@ Acquire Verify-Peer "false"; SslCert "/etc/apt/some.pem"; CaPath "/etc/ssl/certs"; - Verify-Host" "2"; + Verify-Host" "true"; + AllowRedirect "true"; }; ftp diff --git a/methods/https.cc b/methods/https.cc index 98dfeefa1..87de54589 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -208,6 +208,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout); + // set redirect options and default to 10 redirects + bool AllowRedirect = _config->FindI("Acquire::https::AllowRedirect", true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10); + // debug if(_config->FindB("Debug::Acquire::https", false)) curl_easy_setopt(curl, CURLOPT_VERBOSE, true); -- cgit v1.2.3-70-g09d2 From 7e522e9e0e4808202061e0a73345bd8b18eccde3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Feb 2009 14:38:11 +0100 Subject: * apt-pkg/pkgcache.cc: - do not run "dpkg --configure pkg" if pkg is in trigger-awaited state (LP: #322955) - do not unlink files in partial/ (thanks to robbiew) --- apt-pkg/pkgcache.cc | 8 ++++++-- debian/changelog | 4 ++++ methods/https.cc | 1 - 3 files changed, 10 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 8eb62089a..4fbf42c4b 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -275,8 +275,12 @@ pkgCache::PkgIterator::OkState pkgCache::PkgIterator::State() const if (Pkg->CurrentState == pkgCache::State::UnPacked || Pkg->CurrentState == pkgCache::State::HalfConfigured || - Pkg->CurrentState == pkgCache::State::TriggersPending || - Pkg->CurrentState == pkgCache::State::TriggersAwaited) + //we don't need to care for triggers awaiting packages + //dpkg will deal with them automatically when the + //trigger pending action is run (those packages are usually + //in half-configured or triggers-pending state) + //Pkg->CurrentState == pkgCache::State::TriggersAwaited + Pkg->CurrentState == pkgCache::State::TriggersPending) return NeedsConfigure; if (Pkg->CurrentState == pkgCache::State::HalfInstalled || diff --git a/debian/changelog b/debian/changelog index 23a0514f6..0e2d3afe6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -32,8 +32,12 @@ apt (0.7.20.2ubuntu1) jaunty; urgency=low * cmdline/apt-get.cc: - default to "false" for the "APT::Get::Build-Dep-Automatic" option (follow debian here) + * apt-pkg/pkgcache.cc: + - do not run "dpkg --configure pkg" if pkg is in trigger-awaited + state (LP: #322955) * methods/https.cc: - add Acquire::https::AllowRedirect support + - do not unlink files in partial/ (thanks to robbiew) [ Dereck Wonnacott ] * Clarify the --help for 'purge' (LP: #243948) diff --git a/methods/https.cc b/methods/https.cc index 87de54589..8bf44b52a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -253,7 +253,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // cleanup if(success != 0) { - unlink(File->Name().c_str()); _error->Error("%s", curl_errorstr); Fail(); return true; -- cgit v1.2.3-70-g09d2 From 5dad4134da37576de12721f34165c1fe4d138b3c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 19 Mar 2009 15:42:25 +0100 Subject: * methods/mirror.cc: - when download the mirror file and the server is down, return a propper error message (LP: #278635) --- cmdline/apt-report-mirror-failure | 2 +- debian/changelog | 8 ++++++++ methods/mirror.cc | 18 ++++++++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure index 1567e78e3..ef77d4954 100755 --- a/cmdline/apt-report-mirror-failure +++ b/cmdline/apt-report-mirror-failure @@ -5,7 +5,7 @@ import urllib import apt_pkg apt_pkg.init() -url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", None) +url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", "") #"http://people.ubuntu.com:9000/mirror-failure") #"http://localhost:9000/mirror-failure") if not url: diff --git a/debian/changelog b/debian/changelog index bef5c3cad..5a764383e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +apt (0.7.20.2ubuntu3) jaunty; urgency=low + + * methods/mirror.cc: + - when download the mirror file and the server is down, + return a propper error message (LP: #278635) + + -- Michael Vogt Thu, 19 Mar 2009 15:42:15 +0100 + apt (0.7.20.2ubuntu2) jaunty; urgency=low * apt-pkg/deb/dpkgpm.cc: diff --git a/methods/mirror.cc b/methods/mirror.cc index bdd783cc7..b3a956b95 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -123,6 +123,8 @@ bool MirrorMethod::Clean(string Dir) bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) { + if(Debug) + clog << "MirrorMethod::DownloadMirrorFile(): " << endl; // check the file, if it is not older than RefreshInterval just use it // otherwise try to get a new one @@ -251,6 +253,9 @@ string MirrorMethod::GetMirrorFileName(string mirror_uri_str) depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { + if(Debug) + clog << "MirrorMethod::Fetch()" << endl; + // the http method uses Fetch(0) as a way to update the pipeline, // just let it do its work in this case - Fetch() with a valid // Itm will always run before the first Fetch(0) @@ -269,8 +274,17 @@ bool MirrorMethod::Fetch(FetchItem *Itm) DownloadMirrorFile(Itm->Uri); } - if(Mirror.empty()) - SelectMirror(); + if(Mirror.empty()) { + if(!SelectMirror()) { + // no valid mirror selected, something went wrong downloading + // from the master mirror site most likely and there is + // no old mirror file availalbe + return false; + } + } + if(Debug) + clog << "selected mirror: " << Mirror << endl; + for (FetchItem *I = Queue; I != 0; I = I->Next) { -- cgit v1.2.3-70-g09d2 From 0b77f4775db7bc45964e0337b8978a170b3f0483 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Apr 2009 22:52:11 +0200 Subject: * methods/gpgv.cc: - properly check for expired and revoked keys (closes: #433091) LP: #356012 --- debian/changelog | 5 +++++ methods/gpgv.cc | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 43827dada..9618cc1c6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,11 @@ apt (0.7.20.2ubuntu6) jaunty; urgency=low * apt.cron.daily: catch invalid dates due to DST time changes in the stamp files + [ Michael Vogt ] + * methods/gpgv.cc: + - properly check for expired and revoked keys (closes: #433091) + LP: #356012 + -- Michael Vogt Wed, 08 Apr 2009 22:39:50 +0200 apt (0.7.20.2ubuntu5) jaunty; urgency=low diff --git a/methods/gpgv.cc b/methods/gpgv.cc index f3277b300..150c1d315 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -17,13 +17,18 @@ #define GNUPGBADSIG "[GNUPG:] BADSIG" #define GNUPGNOPUBKEY "[GNUPG:] NO_PUBKEY" #define GNUPGVALIDSIG "[GNUPG:] VALIDSIG" +#define GNUPGGOODSIG "[GNUPG:] GOODSIG" +#define GNUPGKEYEXPIRED "[GNUPG:] KEYEXPIRED" +#define GNUPGREVKEYSIG "[GNUPG:] REVKEYSIG" #define GNUPGNODATA "[GNUPG:] NODATA" class GPGVMethod : public pkgAcqMethod { private: string VerifyGetSigners(const char *file, const char *outfile, - vector &GoodSigners, vector &BadSigners, + vector &GoodSigners, + vector &BadSigners, + vector &WorthlessSigners, vector &NoPubKeySigners); protected: @@ -37,6 +42,7 @@ class GPGVMethod : public pkgAcqMethod string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, vector &GoodSigners, vector &BadSigners, + vector &WorthlessSigners, vector &NoPubKeySigners) { // setup a (empty) stringstream for formating the return value @@ -179,15 +185,27 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, std::cerr << "Got NODATA! " << std::endl; BadSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); } - if (strncmp(buffer, GNUPGVALIDSIG, sizeof(GNUPGVALIDSIG)-1) == 0) + if (strncmp(buffer, GNUPGKEYEXPIRED, sizeof(GNUPGKEYEXPIRED)-1) == 0) + { + if (_config->FindB("Debug::Acquire::gpgv", false)) + std::cerr << "Got KEYEXPIRED! " << std::endl; + WorthlessSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); + } + if (strncmp(buffer, GNUPGREVKEYSIG, sizeof(GNUPGREVKEYSIG)-1) == 0) + { + if (_config->FindB("Debug::Acquire::gpgv", false)) + std::cerr << "Got REVKEYSIG! " << std::endl; + WorthlessSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); + } + if (strncmp(buffer, GNUPGGOODSIG, sizeof(GNUPGGOODSIG)-1) == 0) { char *sig = buffer + sizeof(GNUPGPREFIX); - char *p = sig + sizeof("VALIDSIG"); + char *p = sig + sizeof("GOODSIG"); while (*p && isxdigit(*p)) p++; *p = 0; if (_config->FindB("Debug::Acquire::gpgv", false)) - std::cerr << "Got VALIDSIG, key ID:" << sig << std::endl; + std::cerr << "Got GOODSIG, key ID:" << sig << std::endl; GoodSigners.push_back(string(sig)); } } @@ -227,6 +245,8 @@ bool GPGVMethod::Fetch(FetchItem *Itm) string keyID; vector GoodSigners; vector BadSigners; + // a worthless signature is a expired or revoked one + vector WorthlessSigners; vector NoPubKeySigners; FetchResult Res; @@ -235,13 +255,14 @@ bool GPGVMethod::Fetch(FetchItem *Itm) // Run gpgv on file, extract contents and get the key ID of the signer string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), - GoodSigners, BadSigners, NoPubKeySigners); + GoodSigners, BadSigners, WorthlessSigners, + NoPubKeySigners); if (GoodSigners.empty() || !BadSigners.empty() || !NoPubKeySigners.empty()) { string errmsg; // In this case, something bad probably happened, so we just go // with what the other method gave us for an error message. - if (BadSigners.empty() && NoPubKeySigners.empty()) + if (BadSigners.empty() && WorthlessSigners.empty() && NoPubKeySigners.empty()) errmsg = msg; else { @@ -252,6 +273,13 @@ bool GPGVMethod::Fetch(FetchItem *Itm) I != BadSigners.end(); I++) errmsg += (*I + "\n"); } + if (!WorthlessSigners.empty()) + { + errmsg += _("The following signatures were invalid:\n"); + for (vector::iterator I = WorthlessSigners.begin(); + I != WorthlessSigners.end(); I++) + errmsg += (*I + "\n"); + } if (!NoPubKeySigners.empty()) { errmsg += _("The following signatures couldn't be verified because the public key is not available:\n"); -- cgit v1.2.3-70-g09d2 From 217d575b5713f4d9275177b58a36c581a7880c03 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 14 Jul 2009 11:22:41 +0200 Subject: * apt-pkg/acquire-worker.cc: - show error details of failed methods * apt-pkg/contrib/fileutl.cc: - if a process aborts with signal, show signal number * methods/http.cc: - ignore SIGPIPE, we deal with EPIPE from write in HttpMethod::ServerDie() (LP: #385144) --- apt-pkg/acquire-worker.cc | 7 ++----- apt-pkg/contrib/fileutl.cc | 7 +++++-- debian/changelog | 12 ++++++++++++ methods/http_main.cc | 5 +++++ 4 files changed, 24 insertions(+), 7 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 78c68737c..4f0b52af9 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -527,10 +527,6 @@ bool pkgAcquire::Worker::OutFdReady() if (Res <= 0) return MethodFailure(); - - // Hmm.. this should never happen. - if (Res < 0) - return true; OutQueue.erase(0,Res); if (OutQueue.empty() == true) @@ -558,7 +554,8 @@ bool pkgAcquire::Worker::MethodFailure() { _error->Error("Method %s has died unexpectedly!",Access.c_str()); - ExecWait(Process,Access.c_str(),true); + // do not reap the child here to show meaningfull error to the user + ExecWait(Process,Access.c_str(),false); Process = -1; close(InFd); close(OutFd); diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index a5976cf3a..a7de09c44 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -450,8 +450,11 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) { if (Reap == true) return false; - if (WIFSIGNALED(Status) != 0 && WTERMSIG(Status) == SIGSEGV) - return _error->Error(_("Sub-process %s received a segmentation fault."),Name); + if (WIFSIGNALED(Status) != 0) + if( WTERMSIG(Status) == SIGSEGV) + return _error->Error(_("Sub-process %s received a segmentation fault."),Name); + else + return _error->Error(_("Sub-process %s received signal %u."),Name, WTERMSIG(Status)); if (WIFEXITED(Status) != 0) return _error->Error(_("Sub-process %s returned an error code (%u)"),Name,WEXITSTATUS(Status)); diff --git a/debian/changelog b/debian/changelog index 2db8ae677..d6e6a5997 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +apt (0.7.21ubuntu2~ppa1) karmic; urgency=low + + * apt-pkg/acquire-worker.cc: + - show error details of failed methods + * apt-pkg/contrib/fileutl.cc: + - if a process aborts with signal, show signal number + * methods/http.cc: + - ignore SIGPIPE, we deal with EPIPE from write in + HttpMethod::ServerDie() (LP: #385144) + + -- Michael Vogt Thu, 02 Jul 2009 14:11:35 +0200 + apt (0.7.21ubuntu1) karmic; urgency=low * merged from the debian-sid bzr branch diff --git a/methods/http_main.cc b/methods/http_main.cc index 2c46ab19d..7815c2fc1 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -1,5 +1,6 @@ #include #include +#include #include "connect.h" #include "rfc2553emu.h" @@ -10,6 +11,10 @@ int main() { setlocale(LC_ALL, ""); + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + HttpMethod Mth; return Mth.Loop(); } -- cgit v1.2.3-70-g09d2 From 056539eeb8eb84cea65c0c15466111b978ee156d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Jul 2009 18:25:16 +0200 Subject: methods/http.cc: remove merge artifact --- methods/http.cc | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 476fb751f..859dbd39c 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -1316,15 +1316,4 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - - HttpMethod Mth; - return Mth.Loop(); -} - -- cgit v1.2.3-70-g09d2 From 1082d4c7ad98c091688bdeb427e48913d1327279 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 4 May 2010 16:10:47 +0200 Subject: * methods/rred.cc: - use the patchfile modification time instead of the one from the "old" file - thanks to Philipp Weis for noticing! (Closes: #571541) --- debian/changelog | 3 +++ methods/rred.cc | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 0f01d1867..482f0860e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -57,6 +57,9 @@ apt (0.7.26~exp4) UNRELEASED; urgency=low * apt-pkg/pkgcache.h: - enhance the Groups ABI by providing a ID as the other structs does - check also the size of the Group struct then checking for the others + * methods/rred.cc: + - use the patchfile modification time instead of the one from the + "old" file - thanks to Philipp Weis for noticing! (Closes: #571541) [ Jari Aalto ] * cmdline/apt-get.cc: diff --git a/methods/rred.cc b/methods/rred.cc index 262c78cab..f42c7a072 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -477,23 +477,26 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ Patch.Close(); To.Close(); - // Transfer the modification times - struct stat Buf; - if (stat(Path.c_str(),&Buf) != 0) + /* Transfer the modification times from the patch file + to be able to see in which state the file should be + and use the access time from the "old" file */ + struct stat BufBase, BufPatch; + if (stat(Path.c_str(),&BufBase) != 0 || + stat(string(Path+".ed").c_str(),&BufPatch) != 0) return _error->Errno("stat",_("Failed to stat")); struct utimbuf TimeBuf; - TimeBuf.actime = Buf.st_atime; - TimeBuf.modtime = Buf.st_mtime; + TimeBuf.actime = BufBase.st_atime; + TimeBuf.modtime = BufPatch.st_mtime; if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0) return _error->Errno("utime",_("Failed to set modification time")); - if (stat(Itm->DestFile.c_str(),&Buf) != 0) + if (stat(Itm->DestFile.c_str(),&BufBase) != 0) return _error->Errno("stat",_("Failed to stat")); // return done - Res.LastModified = Buf.st_mtime; - Res.Size = Buf.st_size; + Res.LastModified = BufBase.st_mtime; + Res.Size = BufBase.st_size; Res.TakeHashes(Hash); URIDone(Res); -- cgit v1.2.3-70-g09d2 From 7273e49443e480d57bd8455f9cf9a0f39ef181f4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 7 May 2010 11:20:31 +0200 Subject: * methods/http.cc: - code cleanup, add (some) doxygen strings --- debian/changelog | 2 ++ methods/http.cc | 45 +++++++++++++++++++++++---------------------- methods/http.h | 37 ++++++++++++++++++++++++++++++++++--- 3 files changed, 59 insertions(+), 25 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 29c8ea33b..f4cdeeec5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -45,6 +45,8 @@ apt (0.7.25.4) UNRELEASED; urgency=low - fix max tag buffer size (LP: #545336, closes: #578959) * debian/rules: - install html doxygen in libapt-pkg-doc as well + * methods/http.cc: + - code cleanup, add (some) doxygen strings [ Robert Collins ] * Change the package index Info methods to allow apt-cache policy to be diff --git a/methods/http.cc b/methods/http.cc index b05444691..c05abc862 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -67,7 +67,7 @@ unsigned long CircleBuf::BwReadLimit=0; unsigned long CircleBuf::BwTickReadData=0; struct timeval CircleBuf::BwReadTick={0,0}; const unsigned int CircleBuf::BW_HZ=10; - + // CircleBuf::CircleBuf - Circular input buffer /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -378,7 +378,7 @@ bool ServerState::Close() // --------------------------------------------------------------------- /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header parse error occurred */ -int ServerState::RunHeaders() +ServerState::RunHeadersResult ServerState::RunHeaders() { State = Header; @@ -407,7 +407,7 @@ int ServerState::RunHeaders() string::const_iterator J = I; for (; J != Data.end() && *J != '\n' && *J != '\r';J++); if (HeaderLine(string(I,J)) == false) - return 2; + return RUN_HEADERS_PARSE_ERROR; I = J; } @@ -419,11 +419,11 @@ int ServerState::RunHeaders() if (Encoding == Closes && HaveContent == true) Persistent = false; - return 0; + return RUN_HEADERS_OK; } while (Owner->Go(false,this) == true); - return 1; + return RUN_HEADERS_IO_ERROR; } /*}}}*/ // ServerState::RunData - Transfer the data from the socket /*{{{*/ @@ -922,7 +922,8 @@ bool HttpMethod::ServerDie(ServerState *Srv) 5 - Unrecoverable non-server error (close the connection) 6 - Try again with a new or changed URI */ -int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) +HttpMethod::DealWithHeadersResult +HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { // Not Modified if (Srv->Result == 304) @@ -930,7 +931,7 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) unlink(Queue->DestFile.c_str()); Res.IMSHit = true; Res.LastModified = Queue->LastModified; - return 1; + return IMS_HIT; } /* Redirect @@ -949,7 +950,7 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) if (!Srv->Location.empty()) { NextURI = Srv->Location; - return 6; + return TRY_AGAIN_OR_REDIRECT; } /* else pass through for error message */ } @@ -960,8 +961,8 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { _error->Error("%u %s",Srv->Result,Srv->Code); if (Srv->HaveContent == true) - return 4; - return 3; + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; } // This is some sort of 2xx 'data follows' reply @@ -972,7 +973,7 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) delete File; File = new FileFd(Queue->DestFile,FileFd::WriteAny); if (_error->PendingError() == true) - return 5; + return ERROR_NOT_FROM_SERVER; FailFile = Queue->DestFile; FailFile.c_str(); // Make sure we dont do a malloc in the signal handler @@ -1000,13 +1001,13 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false) { _error->Errno("read",_("Problem hashing file")); - return 5; + return ERROR_NOT_FROM_SERVER; } lseek(File->Fd(),0,SEEK_END); } SetNonBlock(File->Fd(),true); - return 0; + return FILE_IS_OPEN; } /*}}}*/ // HttpMethod::SigTerm - Handle a fatal signal /*{{{*/ @@ -1147,11 +1148,11 @@ int HttpMethod::Loop() // Fetch the next URL header data from the server. switch (Server->RunHeaders()) { - case 0: + case ServerState::RUN_HEADERS_OK: break; // The header data is bad - case 2: + case ServerState::RUN_HEADERS_PARSE_ERROR: { _error->Error(_("Bad header data")); Fail(true); @@ -1161,7 +1162,7 @@ int HttpMethod::Loop() // The server closed a connection during the header get.. default: - case 1: + case ServerState::RUN_HEADERS_IO_ERROR: { FailCounter++; _error->Discard(); @@ -1185,7 +1186,7 @@ int HttpMethod::Loop() switch (DealWithHeaders(Res,Server)) { // Ok, the file is Open - case 0: + case FILE_IS_OPEN: { URIStart(Res); @@ -1238,21 +1239,21 @@ int HttpMethod::Loop() } // IMS hit - case 1: + case IMS_HIT: { URIDone(Res); break; } // Hard server error, not found or something - case 3: + case ERROR_UNRECOVERABLE: { Fail(); break; } // Hard internal error, kill the connection and fail - case 5: + case ERROR_NOT_FROM_SERVER: { delete File; File = 0; @@ -1264,7 +1265,7 @@ int HttpMethod::Loop() } // We need to flush the data, the header is like a 404 w/ error text - case 4: + case ERROR_WITH_CONTENT_PAGE: { Fail(); @@ -1277,7 +1278,7 @@ int HttpMethod::Loop() } // Try again with a new URL - case 6: + case TRY_AGAIN_OR_REDIRECT: { // Clear rest of response if there is content if (Server->HaveContent) diff --git a/methods/http.h b/methods/http.h index ceee36cbe..af0f5e033 100644 --- a/methods/http.h +++ b/methods/http.h @@ -117,7 +117,19 @@ struct ServerState void Reset() {Major = 0; Minor = 0; Result = 0; Size = 0; StartPos = 0; Encoding = Closes; time(&Date); ServerFd = -1; Pipeline = true;}; - int RunHeaders(); + + /** \brief Result of the header acquire */ + enum RunHeadersResult { + /** \brief Header ok */ + RUN_HEADERS_OK, + /** \brief IO error while retrieving */ + RUN_HEADERS_IO_ERROR, + /** \brief Parse error after retrieving */ + RUN_HEADERS_PARSE_ERROR, + }; + /** \brief Get the headers before the data */ + RunHeadersResult RunHeaders(); + /** \brief Transfer the data from the socket */ bool RunData(); bool Open(); @@ -133,7 +145,26 @@ class HttpMethod : public pkgAcqMethod bool Go(bool ToFile,ServerState *Srv); bool Flush(ServerState *Srv); bool ServerDie(ServerState *Srv); - int DealWithHeaders(FetchResult &Res,ServerState *Srv); + + /** \brief Result of the header parsing */ + enum DealWithHeadersResult { + /** \brief The file is open and ready */ + FILE_IS_OPEN, + /** \brief We got a IMS hit, the file has not changed */ + IMS_HIT, + /** \brief The server reported a unrecoverable error */ + ERROR_UNRECOVERABLE, + /** \brief The server reported a error with a error content page */ + ERROR_WITH_CONTENT_PAGE, + /** \brief A error on the client side */ + ERROR_NOT_FROM_SERVER, + /** \brief A redirect or retry request */ + TRY_AGAIN_OR_REDIRECT + }; + /** \brief Handle the retrieved header data */ + DealWithHeadersResult DealWithHeaders(FetchResult &Res,ServerState *Srv); + + /** \brief Try to AutoDetect the proxy */ bool AutoDetectProxy(); virtual bool Fetch(FetchItem *); @@ -147,7 +178,7 @@ class HttpMethod : public pkgAcqMethod string NextURI; string AutoDetectProxyCmd; - + public: friend class ServerState; -- cgit v1.2.3-70-g09d2 From f5a3d0095a5937c14c509a588eb6b7e946faf0c1 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 12 May 2010 11:44:50 +0200 Subject: * methods/gpgv.cc: - remove the keyrings count limit by using vector magic --- debian/changelog | 2 ++ methods/gpgv.cc | 42 +++++++++++++++++------------------------- 2 files changed, 19 insertions(+), 25 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 845a37581..a328c1e1e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -13,6 +13,8 @@ apt (0.7.26~exp5) UNRELEASED; urgency=low - remove targets refering to CVS or arch as they are useless * apt-pkg/depcache.cc: - be doublesure that the killer query is empty before starting reinstall + * methods/gpgv.cc: + - remove the keyrings count limit by using vector magic [ Jari Aalto ] * debian/rules: diff --git a/methods/gpgv.cc b/methods/gpgv.cc index c58e6cc45..a149d67dd 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -12,6 +12,8 @@ #include #include +#include + #define GNUPGPREFIX "[GNUPG:]" #define GNUPGBADSIG "[GNUPG:] BADSIG" #define GNUPGNOPUBKEY "[GNUPG:] NO_PUBKEY" @@ -87,23 +89,18 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, return string("Couldn't spawn new process") + strerror(errno); else if (pid == 0) { - const char *Args[400]; - unsigned int i = 0; + std::vector Args; + Args.reserve(30); - Args[i++] = gpgvpath.c_str(); - Args[i++] = "--status-fd"; - Args[i++] = "3"; - Args[i++] = "--ignore-time-conflict"; + Args.push_back(gpgvpath.c_str()); + Args.push_back("--status-fd"); + Args.push_back("3"); + Args.push_back("--ignore-time-conflict"); for (vector::const_iterator K = keyrings.begin(); K != keyrings.end(); ++K) { - Args[i++] = "--keyring"; - Args[i++] = K->c_str(); - // check overflow (minus a bit of extra space at the end) - if(i >= sizeof(Args)/sizeof(char*)-5) { - std::clog << _("E: Too many keyrings should be passed to gpgv. Exiting.") << std::endl; - exit(111); - } + Args.push_back("--keyring"); + Args.push_back(K->c_str()); } Configuration::Item const *Opts; @@ -115,23 +112,18 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, { if (Opts->Value.empty() == true) continue; - Args[i++] = Opts->Value.c_str(); - // check overflow (minus a bit of extra space at the end) - if(i >= sizeof(Args)/sizeof(char*)-5) { - std::clog << _("E: Argument list from Acquire::gpgv::Options too long. Exiting.") << std::endl; - exit(111); - } + Args.push_back(Opts->Value.c_str()); } } - Args[i++] = file; - Args[i++] = outfile; - Args[i++] = NULL; + Args.push_back(file); + Args.push_back(outfile); + Args.push_back(NULL); if (Debug == true) { std::clog << "Preparing to exec: " << gpgvpath; - for(unsigned int j=0;Args[j] != NULL; j++) - std::clog << " " << Args[j]; + for(std::vector::const_iterator a = Args.begin();*a != NULL; ++a) + std::clog << " " << *a; std::clog << std::endl; } int const nullfd = open("/dev/null", O_RDONLY); @@ -145,7 +137,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, putenv((char *)"LANG="); putenv((char *)"LC_ALL="); putenv((char *)"LC_MESSAGES="); - execvp(gpgvpath.c_str(), (char **)Args); + execvp(gpgvpath.c_str(), (char **) &Args[0]); exit(111); } -- cgit v1.2.3-70-g09d2 From f64684a40c97fc52bcd7942bed81bf08b02fdd28 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 25 May 2010 17:45:12 +0200 Subject: methods/http.cc: shorten comment on return values of DealWithHeaders (because we use a enum now) --- debian/changelog | 4 ++-- methods/http.cc | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 037e424d6..b4aa5753b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -apt (0.7.26~exp5) UNRELEASED; urgency=low +apt (0.7.26~exp5) experimental; urgency=low [ David Kalnischkies ] * cmdline/apt-get.cc: @@ -55,7 +55,7 @@ apt (0.7.26~exp5) UNRELEASED; urgency=low * apt-pkg/acquire-item.cc: - Fix pkgAcqFile::Custom600Headers() to always return something. - -- David Kalnischkies Thu, 06 May 2010 16:10:39 +0200 + -- Michael Vogt Tue, 25 May 2010 16:01:42 +0200 apt (0.7.26~exp4) experimental; urgency=low diff --git a/methods/http.cc b/methods/http.cc index c05abc862..d43dd14c8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -914,13 +914,7 @@ bool HttpMethod::ServerDie(ServerState *Srv) // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/ // --------------------------------------------------------------------- /* We look at the header data we got back from the server and decide what - to do. Returns - 0 - File is open, - 1 - IMS hit - 3 - Unrecoverable error - 4 - Error with error content page - 5 - Unrecoverable non-server error (close the connection) - 6 - Try again with a new or changed URI + to do. Returns DealWithHeadersResult (see http.h for details). */ HttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) -- cgit v1.2.3-70-g09d2 From 96db74ce38e9451609fe33f9e25f3f9d42b1fe22 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 16:40:40 +0200 Subject: * apt-pkg/deb/dpkgpm.cc: - add missing include * methods/mirror.{cc,h}: - add SelectNextMirror() and InitMirrors() functions - read all mirrors into the AllMirrors vector --- apt-pkg/deb/dpkgpm.cc | 1 + methods/mirror.cc | 30 +++++++++++++++++++++++------- methods/mirror.h | 4 +++- 3 files changed, 27 insertions(+), 8 deletions(-) (limited to 'methods') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 18c731788..7e5171eda 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/methods/mirror.cc b/methods/mirror.cc index b3a956b95..0a0d07e6b 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -162,7 +162,21 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } -bool MirrorMethod::SelectMirror() +bool MirrorMethod::SelectNextMirror() +{ + if (AllMirrors.size() < 1) + return false; + + Mirror = AllMirrors[0]; + AllMirrors.erase(AllMirrors.begin()); + if(Debug) + cerr << "using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +bool MirrorMethod::InitMirrors() { // if we do not have a MirrorFile, fallback if(!FileExists(MirrorFile)) @@ -179,11 +193,13 @@ bool MirrorMethod::SelectMirror() // get into sync issues (got indexfiles from mirror A, // but packages from mirror B - one might be out of date etc) ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; + string s; + while (!in.eof()) + { + getline(in, s); + AllMirrors.push_back(s); + } + SelectNextMirror(); return true; } @@ -275,7 +291,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) } if(Mirror.empty()) { - if(!SelectMirror()) { + if(!InitMirrors()) { // no valid mirror selected, something went wrong downloading // from the master mirror site most likely and there is // no old mirror file availalbe diff --git a/methods/mirror.h b/methods/mirror.h index ed817806b..1b506dc87 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,6 +26,7 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) + vector AllMirrors; // all available mirrors string MirrorFile; // the file that contains the list of mirrors bool DownloadedMirrorFile; // already downloaded this session @@ -34,7 +35,8 @@ class MirrorMethod : public HttpMethod protected: bool DownloadMirrorFile(string uri); string GetMirrorFileName(string uri); - bool SelectMirror(); + bool InitMirrors(); + bool SelectNextMirror(); bool Clean(string dir); // we need to overwrite those to transform the url back -- cgit v1.2.3-70-g09d2 From 483dfdd8aced593e966d221073c056c2e332584f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 16:51:25 +0200 Subject: methods/mirror.cc: on fail try the next mirror --- methods/mirror.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 0a0d07e6b..b8bd6db73 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -197,7 +197,8 @@ bool MirrorMethod::InitMirrors() while (!in.eof()) { getline(in, s); - AllMirrors.push_back(s); + if (s.size() > 0) + AllMirrors.push_back(s); } SelectNextMirror(); return true; @@ -314,6 +315,15 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // try the next mirror on fail + string old_mirror = Mirror; + if (SelectNextMirror()) + { + Queue->Uri.replace(0, old_mirror.size(), Mirror); + return; + } + + // all mirrors failed, so bail out if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); -- cgit v1.2.3-70-g09d2 From 0ded3ad3438666f833773895ef6318f84d2f849d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 22:44:49 +0200 Subject: improve error message if mirror method fails --- methods/mirror.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b8bd6db73..b9fa55d87 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -25,6 +25,8 @@ using namespace std; +#include + #include "mirror.h" #include "http.h" #include "apti18n.h" @@ -164,16 +166,12 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) bool MirrorMethod::SelectNextMirror() { - if (AllMirrors.size() < 1) - return false; - - Mirror = AllMirrors[0]; - AllMirrors.erase(AllMirrors.begin()); if(Debug) cerr << "using mirror: " << Mirror << endl; + Mirror = AllMirrors[0]; UsedMirror = Mirror; - return true; + return false; } bool MirrorMethod::InitMirrors() @@ -324,6 +322,10 @@ void MirrorMethod::Fail(string Err,bool Transient) } // all mirrors failed, so bail out + string s; + strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); + SetIP(s); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); -- cgit v1.2.3-70-g09d2 From 661f7b1c727aada703ffbb350d44759189a441fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 22:47:49 +0200 Subject: methods/mirror.cc: remove Acquire::Mirror::RefreshInterval (not really useful) --- methods/mirror.cc | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b9fa55d87..4d767c9f3 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -128,28 +128,6 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) if(Debug) clog << "MirrorMethod::DownloadMirrorFile(): " << endl; - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) - { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) - { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - DownloadedMirrorFile = true; - return true; - } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; - } - // not that great to use pkgAcquire here, but we do not have // any other way right now string fetch = BaseUri; -- cgit v1.2.3-70-g09d2 From 0391542729e5e8a2ba9ae9b1d470f252e24eb296 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:13:42 +0200 Subject: methods/mirrors.cc: make cycle through the mirrors work properly --- methods/mirror.cc | 55 ++++++++++++++++++++++++++++++++++++++++--------------- methods/mirror.h | 3 ++- 2 files changed, 42 insertions(+), 16 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 4d767c9f3..567522478 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -142,13 +142,44 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } -bool MirrorMethod::SelectNextMirror() +/* convert a the Queue->Uri back to the mirror base uri and look + * at all mirrors we have for this, this is needed as queue->uri + * may point to different mirrors (if TryNextMirror() was run) + */ +void MirrorMethod::CurrentQueueUriToMirror() +{ + // already in mirror:// style so nothing to do + if(Queue->Uri.find("mirror://") == 0) + return; + + // find current mirror and select next one + for (int i=0; i < AllMirrors.size(); i++) + { + if (Queue->Uri.find(AllMirrors[i]) == 0) + { + Queue->Uri.replace(0, AllMirrors[i].size(), BaseUri); + return; + } + } + _error->Error("Internal error: Failed to convert %s back to %s", + Queue->Uri, BaseUri); +} + +bool MirrorMethod::TryNextMirror() { if(Debug) cerr << "using mirror: " << Mirror << endl; - Mirror = AllMirrors[0]; - UsedMirror = Mirror; + // find current mirror and select next one + for (int i=0; i < AllMirrors.size()-1; i++) + { + if (Queue->Uri.find(AllMirrors[i]) == 0) + { + Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); + return true; + } + } + return false; } @@ -176,7 +207,8 @@ bool MirrorMethod::InitMirrors() if (s.size() > 0) AllMirrors.push_back(s); } - SelectNextMirror(); + Mirror = AllMirrors[0]; + UsedMirror = Mirror; return true; } @@ -292,34 +324,27 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { // try the next mirror on fail - string old_mirror = Mirror; - if (SelectNextMirror()) - { - Queue->Uri.replace(0, old_mirror.size(), Mirror); + if (TryNextMirror()) return; - } // all mirrors failed, so bail out string s; strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); SetIP(s); - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::Fail(Err, Transient); } void MirrorMethod::URIStart(FetchResult &Res) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIStart(Res); } void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIDone(Res, Alt); } diff --git a/methods/mirror.h b/methods/mirror.h index 1b506dc87..0a3ea6e92 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -36,7 +36,8 @@ class MirrorMethod : public HttpMethod bool DownloadMirrorFile(string uri); string GetMirrorFileName(string uri); bool InitMirrors(); - bool SelectNextMirror(); + bool TryNextMirror(); + void CurrentQueueUriToMirror(); bool Clean(string dir); // we need to overwrite those to transform the url back -- cgit v1.2.3-70-g09d2 From b86f642111954754dd9932ed2f28a9ea85035e87 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:15:55 +0200 Subject: methods/mirror.cc: simplify uri.startswith() --- methods/mirror.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 567522478..cfc155f58 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -106,7 +106,7 @@ bool MirrorMethod::Clean(string Dir) for(I=list.begin(); I != list.end(); I++) { string uri = (*I)->GetURI(); - if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + if(uri.find("mirror://") != 0) continue; string BaseUri = uri.substr(0,uri.size()-1); if (URItoFileName(BaseUri) == Dir->d_name) -- cgit v1.2.3-70-g09d2 From 963b16dcebba149ae2c35bd255b34242923fbea0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:30:01 +0200 Subject: implement Fail-Ignore bool in FetchItem that tells the method that a failure of this item is ok and does not need to be tried on all mirrors --- apt-pkg/acquire-item.cc | 7 +++++++ apt-pkg/acquire-item.h | 1 + apt-pkg/acquire-method.cc | 1 + apt-pkg/acquire-method.h | 1 + methods/mirror.cc | 23 +++++++++++------------ 5 files changed, 21 insertions(+), 12 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index a959253bc..3cc2c8717 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -820,6 +820,13 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc) : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "") { +} + /*}}}*/ +// AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +string pkgAcqIndexTrans::Custom600Headers() +{ + return "\nFail-Ignore: true"; } /*}}}*/ // AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 06fcffc73..3ac8a19e2 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -573,6 +573,7 @@ class pkgAcqIndexTrans : public pkgAcqIndex public: virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual string Custom600Headers(); /** \brief Create a pkgAcqIndexTrans. * diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 3008c8d1a..6e021a445 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -381,6 +381,7 @@ int pkgAcqMethod::Run(bool Single) if (StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified) == false) Tmp->LastModified = 0; Tmp->IndexFile = StringToBool(LookupTag(Message,"Index-File"),false); + Tmp->FailIgnore = StringToBool(LookupTag(Message,"Fail-Ignore"),false); Tmp->Next = 0; // Append it to the list diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 99a4605b1..03851e823 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -37,6 +37,7 @@ class pkgAcqMethod string DestFile; time_t LastModified; bool IndexFile; + bool FailIgnore; }; struct FetchResult diff --git a/methods/mirror.cc b/methods/mirror.cc index cfc155f58..ea0fba438 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -162,24 +162,26 @@ void MirrorMethod::CurrentQueueUriToMirror() } } _error->Error("Internal error: Failed to convert %s back to %s", - Queue->Uri, BaseUri); + Queue->Uri.c_str(), BaseUri.c_str()); } bool MirrorMethod::TryNextMirror() { - if(Debug) - cerr << "using mirror: " << Mirror << endl; - // find current mirror and select next one for (int i=0; i < AllMirrors.size()-1; i++) { if (Queue->Uri.find(AllMirrors[i]) == 0) { Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); + if (Debug) + clog << "TryNextMirror: " << Queue->Uri << endl; return true; } } + if (Debug) + clog << "TryNextMirror could not find another mirror to try" << endl; + return false; } @@ -307,15 +309,12 @@ bool MirrorMethod::Fetch(FetchItem *Itm) return false; } } - if(Debug) - clog << "selected mirror: " << Mirror << endl; + if(Itm->Uri.find("mirror://") != string::npos) + Itm->Uri.replace(0,BaseUri.size(), Mirror); - for (FetchItem *I = Queue; I != 0; I = I->Next) - { - if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(), Mirror); - } + if(Debug) + clog << "Fetch: " << Itm->Uri << endl << endl; // now run the real fetcher return HttpMethod::Fetch(Itm); @@ -324,7 +323,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { // try the next mirror on fail - if (TryNextMirror()) + if (!Queue->FailIgnore && TryNextMirror()) return; // all mirrors failed, so bail out -- cgit v1.2.3-70-g09d2 From 2ac9b90b7b3ebeea9a2580b6f317f1dfefc8c8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:57:00 +0200 Subject: methods/mirror.cc: debug improvements --- methods/mirror.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index ea0fba438..b2b6b2ecf 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -301,7 +301,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) DownloadMirrorFile(Itm->Uri); } - if(Mirror.empty()) { + if(AllMirrors.empty()) { if(!InitMirrors()) { // no valid mirror selected, something went wrong downloading // from the master mirror site most likely and there is @@ -322,7 +322,14 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // try the next mirror on fail + // FIXME: TryNextMirror is not ideal for indexfile as we may + // run into auth issues + + if (Debug) + clog << "Failure to get " << Queue->Uri << endl; + + // try the next mirror on fail (if its not a expected failure, + // e.g. translations are ok to ignore) if (!Queue->FailIgnore && TryNextMirror()) return; -- cgit v1.2.3-70-g09d2 From 96cc64a521957d63704de72ed95f1c839698c53c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 00:53:44 +0200 Subject: move the users away from the deprecated StrToTime() method --- apt-pkg/acquire-method.cc | 2 +- apt-pkg/contrib/strutl.h | 4 ++-- methods/ftp.cc | 3 +-- methods/http.cc | 2 +- methods/rsh.cc | 3 +-- 5 files changed, 6 insertions(+), 8 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index fe066741c..b82dceecb 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -373,7 +373,7 @@ int pkgAcqMethod::Run(bool Single) Tmp->Uri = LookupTag(Message,"URI"); Tmp->DestFile = LookupTag(Message,"FileName"); - if (StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified) == false) + if (RFC1123StrToTime(LookupTag(Message,"Last-Modified").c_str(),Tmp->LastModified) == false) Tmp->LastModified = 0; Tmp->IndexFile = StringToBool(LookupTag(Message,"Index-File"),false); Tmp->Next = 0; diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h index b5de0802e..a457ff047 100644 --- a/apt-pkg/contrib/strutl.h +++ b/apt-pkg/contrib/strutl.h @@ -45,8 +45,8 @@ string Base64Encode(const string &Str); string OutputInDepth(const unsigned long Depth, const char* Separator=" "); string URItoFileName(const string &URI); string TimeRFC1123(time_t Date); -bool RFC1123StrToTime(const char* const str,time_t &time) __attrib_const; -bool FTPMDTMStrToTime(const char* const str,time_t &time) __attrib_const; +bool RFC1123StrToTime(const char* const str,time_t &time) __must_check; +bool FTPMDTMStrToTime(const char* const str,time_t &time) __must_check; __deprecated bool StrToTime(const string &Val,time_t &Result); string LookupTag(const string &Message,const char *Tag,const char *Default = 0); int StringToBool(const string &Text,int Default = -1); diff --git a/methods/ftp.cc b/methods/ftp.cc index 3e1725823..97248f900 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -661,8 +661,7 @@ bool FTPConn::ModTime(const char *Path, time_t &Time) return true; // Parse it - StrToTime(Msg,Time); - return true; + return FTPMDTMStrToTime(Msg.c_str(), Time); } /*}}}*/ // FTPConn::CreateDataFd - Get a data connection /*{{{*/ diff --git a/methods/http.cc b/methods/http.cc index d43dd14c8..5fdc62696 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -631,7 +631,7 @@ bool ServerState::HeaderLine(string Line) if (stringcasecmp(Tag,"Last-Modified:") == 0) { - if (StrToTime(Val,Date) == false) + if (RFC1123StrToTime(Val.c_str(), Date) == false) return _error->Error(_("Unknown date format")); return true; } diff --git a/methods/rsh.cc b/methods/rsh.cc index f0ccfc42d..97b4ef151 100644 --- a/methods/rsh.cc +++ b/methods/rsh.cc @@ -278,8 +278,7 @@ bool RSHConn::ModTime(const char *Path, time_t &Time) return false; // Parse it - StrToTime(Msg,Time); - return true; + return FTPMDTMStrToTime(Msg.c_str(), Time); } /*}}}*/ // RSHConn::Get - Get a file /*{{{*/ -- cgit v1.2.3-70-g09d2 From 51561c4de75e84c2b2d037eb57387d3d3c2aa494 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 17:13:40 +0200 Subject: fix compiler warning in the new mirror code --- methods/http.h | 2 +- methods/mirror.cc | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) (limited to 'methods') diff --git a/methods/http.h b/methods/http.h index d0677bdaa..0bc019e77 100644 --- a/methods/http.h +++ b/methods/http.h @@ -13,7 +13,7 @@ #define MAXLEN 360 - +#include using std::cout; using std::endl; diff --git a/methods/mirror.cc b/methods/mirror.cc index b2b6b2ecf..e8873d97b 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -153,11 +153,12 @@ void MirrorMethod::CurrentQueueUriToMirror() return; // find current mirror and select next one - for (int i=0; i < AllMirrors.size(); i++) + for (vector::const_iterator mirror = AllMirrors.begin(); + mirror != AllMirrors.end(); ++mirror) { - if (Queue->Uri.find(AllMirrors[i]) == 0) + if (Queue->Uri.find(*mirror) == 0) { - Queue->Uri.replace(0, AllMirrors[i].size(), BaseUri); + Queue->Uri.replace(0, mirror->length(), BaseUri); return; } } @@ -168,15 +169,19 @@ void MirrorMethod::CurrentQueueUriToMirror() bool MirrorMethod::TryNextMirror() { // find current mirror and select next one - for (int i=0; i < AllMirrors.size()-1; i++) + for (vector::const_iterator mirror = AllMirrors.begin(); + mirror != AllMirrors.end(); ++mirror) { - if (Queue->Uri.find(AllMirrors[i]) == 0) - { - Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); - if (Debug) - clog << "TryNextMirror: " << Queue->Uri << endl; - return true; - } + if (Queue->Uri.find(*mirror) != 0) + continue; + + vector::const_iterator nextmirror = mirror + 1; + if (nextmirror != AllMirrors.end()) + break; + Queue->Uri.replace(0, mirror->length(), *nextmirror); + if (Debug) + clog << "TryNextMirror: " << Queue->Uri << endl; + return true; } if (Debug) -- cgit v1.2.3-70-g09d2 From a319c4eeae62511d1cb58986742491d3e224bf20 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 17:19:40 +0200 Subject: * apt-pkg/indexcopy.cc: - move the gpg codecopy to a new method and use it also in methods/gpgv.cc --- apt-pkg/indexcopy.cc | 109 +++++++++++++++++++++++++++++++++------------------ apt-pkg/indexcopy.h | 3 ++ debian/changelog | 4 +- methods/gpgv.cc | 47 +++------------------- 4 files changed, 82 insertions(+), 81 deletions(-) (limited to 'methods') diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index 53eb11172..47eaefc5c 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -590,66 +590,39 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, indexRecords *MetaIndex = new indexRecords; string prefix = *I; + string const releasegpg = *I+"Release.gpg"; + string const release = *I+"Release"; + // a Release.gpg without a Release should never happen - if(!FileExists(*I+"Release")) + if(FileExists(release) == false) { delete MetaIndex; continue; } - - // verify the gpg signature of "Release" - // gpg --verify "*I+Release.gpg", "*I+Release" - const char *Args[400]; - unsigned int i = 0; - - string gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); - string pubringpath = _config->Find("Apt::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg"); - string releasegpg = *I+"Release.gpg"; - string release = *I+"Release"; - - Args[i++] = gpgvpath.c_str(); - Args[i++] = "--keyring"; - Args[i++] = pubringpath.c_str(); - Configuration::Item const *Opts; - Opts = _config->Tree("Acquire::gpgv::Options"); - if (Opts != 0) - { - Opts = Opts->Child; - for (; Opts != 0; Opts = Opts->Next) - { - if (Opts->Value.empty() == true) - continue; - Args[i++] = Opts->Value.c_str(); - if(i >= 390) { - _error->Error("Argument list from Acquire::gpgv::Options too long. Exiting."); - return false; - } - } - } - - Args[i++] = releasegpg.c_str(); - Args[i++] = release.c_str(); - Args[i++] = NULL; - pid_t pid = ExecFork(); if(pid < 0) { _error->Error("Fork failed"); return false; } if(pid == 0) { - execvp(gpgvpath.c_str(), (char**)Args); + string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); + std::vector Args = GetGPGVCommandLine(); + Args.push_back(releasegpg.c_str()); + Args.push_back(release.c_str()); + Args.push_back(NULL); + execvp(gpgvpath.c_str(), (char**) &Args[0]); } if(!ExecWait(pid, "gpgv")) { _error->Warning("Signature verification failed for: %s", - string(*I+"Release.gpg").c_str()); + releasegpg.c_str()); // something went wrong, don't copy the Release.gpg // FIXME: delete any existing gpg file? continue; } // Open the Release file and add it to the MetaIndex - if(!MetaIndex->Load(*I+"Release")) + if(!MetaIndex->Load(release)) { _error->Error("%s",MetaIndex->ErrorText.c_str()); return false; @@ -679,6 +652,64 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, return true; } /*}}}*/ +// SigVerify::GetGPGVCommandLine - returns the command needed for verify/*{{{*/ +// --------------------------------------------------------------------- +/* Generating the commandline for calling gpgv is somehow complicated as + we need to add multiple keyrings and user supplied options. Also, as + the cdrom code currently can not use the gpgv method we have two places + these need to be done - so the place for this method is wrong but better + than code duplication… */ +std::vector SigVerify::GetGPGVCommandLine() +{ + string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); + // FIXME: remove support for deprecated APT::GPGV setting + string const trustedFile = _config->FindFile("Dir::Etc::Trusted", + _config->Find("APT::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg").c_str()); + string const trustedPath = _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d"); + + if (_config->FindB("Debug::Acquire::gpgv", false) == true) + { + std::clog << "gpgv path: " << gpgvpath << std::endl; + std::clog << "Keyring file: " << trustedFile << std::endl; + std::clog << "Keyring path: " << trustedPath << std::endl; + } + + std::vector keyrings = GetListOfFilesInDir(trustedPath, "gpg", false); + if (FileExists(trustedFile) == true) + keyrings.push_back(trustedFile); + + std::vector Args; + Args.reserve(30); + + if (keyrings.empty() == true) + return Args; + + Args.push_back(gpgvpath.c_str()); + Args.push_back("--ignore-time-conflict"); + + for (vector::const_iterator K = keyrings.begin(); + K != keyrings.end(); ++K) + { + Args.push_back("--keyring"); + Args.push_back(K->c_str()); + } + + Configuration::Item const *Opts; + Opts = _config->Tree("Acquire::gpgv::Options"); + if (Opts != 0) + { + Opts = Opts->Child; + for (; Opts != 0; Opts = Opts->Next) + { + if (Opts->Value.empty() == true) + continue; + Args.push_back(Opts->Value.c_str()); + } + } + + return Args; +} + /*}}}*/ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ vector &List, pkgCdromStatus *log) { diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h index 9e5ad4e43..ee6557a3d 100644 --- a/apt-pkg/indexcopy.h +++ b/apt-pkg/indexcopy.h @@ -89,6 +89,9 @@ class SigVerify /*{{{*/ public: bool CopyAndVerify(string CDROM,string Name,vector &SigList, vector PkgList,vector SrcList); + + /** \brief generates the command to verify a file with gpgv */ + static std::vector GetGPGVCommandLine(); }; /*}}}*/ diff --git a/debian/changelog b/debian/changelog index 8d67582e4..a83d4c52a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -84,8 +84,10 @@ apt (0.7.26~exp6) UNRELEASED; urgency=low * apt-pkg/policy.cc: - get the candidate right for a not-installed pseudo package if his non-pseudo friend is installed + * apt-pkg/indexcopy.cc: + - move the gpg codecopy to a new method and use it also in methods/gpgv.cc - -- David Kalnischkies Wed, 09 Jun 2010 14:20:19 +0200 + -- David Kalnischkies Wed, 09 Jun 2010 17:18:26 +0200 apt (0.7.26~exp5) experimental; urgency=low diff --git a/methods/gpgv.cc b/methods/gpgv.cc index a149d67dd..5f5f23f7d 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -58,26 +59,14 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, int fd[2]; FILE *pipein; int status; - string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); - // FIXME: remove support for deprecated APT::GPGV setting - string const trustedFile = _config->FindFile("Dir::Etc::Trusted", - _config->Find("APT::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg").c_str()); - string const trustedPath = _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d"); - if (Debug == true) - { - std::clog << "gpgv path: " << gpgvpath << std::endl; - std::clog << "Keyring file: " << trustedFile << std::endl; - std::clog << "Keyring path: " << trustedPath << std::endl; - } - vector keyrings = GetListOfFilesInDir(trustedPath, "gpg", false); - if (FileExists(trustedFile) == true) - keyrings.push_back(trustedFile); - - if (keyrings.empty() == true) + string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); + std::vector Args = SigVerify::GetGPGVCommandLine(); + if (Args.empty() == true) { // TRANSLATOR: %s is the trusted keyring parts directory - ioprintf(ret, _("No keyring installed in %s."), trustedPath.c_str()); + ioprintf(ret, _("No keyring installed in %s."), + _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d").c_str()); return ret.str(); } @@ -89,32 +78,8 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, return string("Couldn't spawn new process") + strerror(errno); else if (pid == 0) { - std::vector Args; - Args.reserve(30); - - Args.push_back(gpgvpath.c_str()); Args.push_back("--status-fd"); Args.push_back("3"); - Args.push_back("--ignore-time-conflict"); - for (vector::const_iterator K = keyrings.begin(); - K != keyrings.end(); ++K) - { - Args.push_back("--keyring"); - Args.push_back(K->c_str()); - } - - Configuration::Item const *Opts; - Opts = _config->Tree("Acquire::gpgv::Options"); - if (Opts != 0) - { - Opts = Opts->Child; - for (; Opts != 0; Opts = Opts->Next) - { - if (Opts->Value.empty() == true) - continue; - Args.push_back(Opts->Value.c_str()); - } - } Args.push_back(file); Args.push_back(outfile); Args.push_back(NULL); -- cgit v1.2.3-70-g09d2 From cf440facb498fa0ec70148723b13d6d019758c0e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Jun 2010 23:20:26 +0200 Subject: enhance the split out of the gpgv commandline mangling by splitting out the call completely --- apt-pkg/indexcopy.cc | 62 +++++++++++++++++++++++++++++++++++++++++----------- apt-pkg/indexcopy.h | 10 +++++++-- methods/gpgv.cc | 51 +++++++++--------------------------------- 3 files changed, 67 insertions(+), 56 deletions(-) (limited to 'methods') diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index 47eaefc5c..621c18716 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include /*}}}*/ @@ -605,14 +607,9 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, _error->Error("Fork failed"); return false; } - if(pid == 0) { - string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); - std::vector Args = GetGPGVCommandLine(); - Args.push_back(releasegpg.c_str()); - Args.push_back(release.c_str()); - Args.push_back(NULL); - execvp(gpgvpath.c_str(), (char**) &Args[0]); - } + if(pid == 0) + RunGPGV(release, releasegpg); + if(!ExecWait(pid, "gpgv")) { _error->Warning("Signature verification failed for: %s", releasegpg.c_str()); @@ -652,14 +649,15 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, return true; } /*}}}*/ -// SigVerify::GetGPGVCommandLine - returns the command needed for verify/*{{{*/ +// SigVerify::RunGPGV - returns the command needed for verify /*{{{*/ // --------------------------------------------------------------------- /* Generating the commandline for calling gpgv is somehow complicated as we need to add multiple keyrings and user supplied options. Also, as the cdrom code currently can not use the gpgv method we have two places these need to be done - so the place for this method is wrong but better than code duplication… */ -std::vector SigVerify::GetGPGVCommandLine() +bool SigVerify::RunGPGV(std::string const &File, std::string const &FileGPG, + int const &statusfd, int fd[2]) { string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); // FIXME: remove support for deprecated APT::GPGV setting @@ -667,7 +665,9 @@ std::vector SigVerify::GetGPGVCommandLine() _config->Find("APT::GPGV::TrustedKeyring", "/etc/apt/trusted.gpg").c_str()); string const trustedPath = _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d"); - if (_config->FindB("Debug::Acquire::gpgv", false) == true) + bool const Debug = _config->FindB("Debug::Acquire::gpgv", false); + + if (Debug == true) { std::clog << "gpgv path: " << gpgvpath << std::endl; std::clog << "Keyring file: " << trustedFile << std::endl; @@ -682,11 +682,19 @@ std::vector SigVerify::GetGPGVCommandLine() Args.reserve(30); if (keyrings.empty() == true) - return Args; + return false; Args.push_back(gpgvpath.c_str()); Args.push_back("--ignore-time-conflict"); + if (statusfd != -1) + { + Args.push_back("--status-fd"); + char fd[10]; + snprintf(fd, sizeof(fd), "%i", statusfd); + Args.push_back(fd); + } + for (vector::const_iterator K = keyrings.begin(); K != keyrings.end(); ++K) { @@ -707,7 +715,35 @@ std::vector SigVerify::GetGPGVCommandLine() } } - return Args; + Args.push_back(FileGPG.c_str()); + Args.push_back(File.c_str()); + Args.push_back(NULL); + + if (Debug == true) + { + std::clog << "Preparing to exec: " << gpgvpath; + for (std::vector::const_iterator a = Args.begin(); *a != NULL; ++a) + std::clog << " " << *a; + std::clog << std::endl; + } + + if (statusfd != -1) + { + int const nullfd = open("/dev/null", O_RDONLY); + close(fd[0]); + // Redirect output to /dev/null; we read from the status fd + dup2(nullfd, STDOUT_FILENO); + dup2(nullfd, STDERR_FILENO); + // Redirect the pipe to the status fd (3) + dup2(fd[1], statusfd); + + putenv((char *)"LANG="); + putenv((char *)"LC_ALL="); + putenv((char *)"LC_MESSAGES="); + } + + execvp(gpgvpath.c_str(), (char **) &Args[0]); + return true; } /*}}}*/ bool TranslationsCopy::CopyTranslations(string CDROM,string Name, /*{{{*/ diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h index ee6557a3d..6fcd3b8ce 100644 --- a/apt-pkg/indexcopy.h +++ b/apt-pkg/indexcopy.h @@ -90,8 +90,14 @@ class SigVerify /*{{{*/ bool CopyAndVerify(string CDROM,string Name,vector &SigList, vector PkgList,vector SrcList); - /** \brief generates the command to verify a file with gpgv */ - static std::vector GetGPGVCommandLine(); + /** \brief generates and run the command to verify a file with gpgv */ + static bool RunGPGV(std::string const &File, std::string const &FileOut, + int const &statusfd, int fd[2]); + inline static bool RunGPGV(std::string const &File, std::string const &FileOut, + int const &statusfd = -1) { + int fd[2]; + return RunGPGV(File, FileOut, statusfd, fd); + }; }; /*}}}*/ diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 5f5f23f7d..018e4f622 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -55,61 +55,29 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, if (Debug == true) std::clog << "inside VerifyGetSigners" << std::endl; - pid_t pid; int fd[2]; - FILE *pipein; - int status; - - string const gpgvpath = _config->Find("Dir::Bin::gpg", "/usr/bin/gpgv"); - std::vector Args = SigVerify::GetGPGVCommandLine(); - if (Args.empty() == true) - { - // TRANSLATOR: %s is the trusted keyring parts directory - ioprintf(ret, _("No keyring installed in %s."), - _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d").c_str()); - return ret.str(); - } if (pipe(fd) < 0) return "Couldn't create pipe"; - pid = fork(); + pid_t pid = fork(); if (pid < 0) return string("Couldn't spawn new process") + strerror(errno); else if (pid == 0) { - Args.push_back("--status-fd"); - Args.push_back("3"); - Args.push_back(file); - Args.push_back(outfile); - Args.push_back(NULL); - - if (Debug == true) + if (SigVerify::RunGPGV(outfile, file, 3, fd) == false) { - std::clog << "Preparing to exec: " << gpgvpath; - for(std::vector::const_iterator a = Args.begin();*a != NULL; ++a) - std::clog << " " << *a; - std::clog << std::endl; + // TRANSLATOR: %s is the trusted keyring parts directory + ioprintf(ret, _("No keyring installed in %s."), + _config->FindDir("Dir::Etc::TrustedParts", "/etc/apt/trusted.gpg.d").c_str()); + return ret.str(); } - int const nullfd = open("/dev/null", O_RDONLY); - close(fd[0]); - // Redirect output to /dev/null; we read from the status fd - dup2(nullfd, STDOUT_FILENO); - dup2(nullfd, STDERR_FILENO); - // Redirect the pipe to the status fd (3) - dup2(fd[1], 3); - - putenv((char *)"LANG="); - putenv((char *)"LC_ALL="); - putenv((char *)"LC_MESSAGES="); - execvp(gpgvpath.c_str(), (char **) &Args[0]); - exit(111); } close(fd[1]); - pipein = fdopen(fd[0], "r"); - + FILE *pipein = fdopen(fd[0], "r"); + // Loop over the output of gpgv, and check the signatures. size_t buffersize = 64; char *buffer = (char *) malloc(buffersize); @@ -182,6 +150,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, } fclose(pipein); + int status; waitpid(pid, &status, 0); if (Debug == true) { @@ -200,7 +169,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, } else if (WEXITSTATUS(status) == 111) { - ioprintf(ret, _("Could not execute '%s' to verify signature (is gpgv installed?)"), gpgvpath.c_str()); + ioprintf(ret, _("Could not execute 'gpgv' to verify signature (is gpgv installed?)")); return ret.str(); } else -- cgit v1.2.3-70-g09d2 From 330463dd2374bd11757c6f2662f279fc31f035a0 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 24 Jun 2010 10:57:45 +0200 Subject: methods/ftp.h: Handle different logins are on the same server (Closes: #586904). --- debian/changelog | 7 +++++++ methods/ftp.h | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index d29705d24..037dfa7bd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +apt (0.7.26~exp8) experimental; urgency=low + + * methods/ftp.h: + - Handle different logins are on the same server (Closes: #586904). + + -- Julian Andres Klode Thu, 24 Jun 2010 10:56:39 +0200 + apt (0.7.26~exp7) experimental; urgency=low * apt-pkg/cachefile.h: diff --git a/methods/ftp.h b/methods/ftp.h index 1bcea41b6..d7f1f7fbe 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -40,7 +40,7 @@ class FTPConn public: - bool Comp(URI Other) {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; + bool Comp(URI Other) {return Other.Host == ServerName.Host && Other.Port == ServerName.Port && Other.User == ServerName.User && Other.Password == ServerName.Password; }; // Raw connection IO bool ReadResp(unsigned int &Ret,string &Text); -- cgit v1.2.3-70-g09d2 From 127e6df37213a1fda0dd5b44182acf678ccbbf02 Mon Sep 17 00:00:00 2001 From: "martin@piware.de" <> Date: Tue, 6 Jul 2010 13:14:57 +0200 Subject: methods/gzip.cc: With FileFd now being able to read gzipped files, there is no need for the gzip method any more to spawn an external gzip process. Rewrite it to use FileFd directly, which makes the code a lot simpler, and also using less memory and overhead. --- debian/changelog | 4 ++++ methods/gzip.cc | 63 +++++++------------------------------------------------- 2 files changed, 11 insertions(+), 56 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index f3f2d3df4..6f3d2eb71 100644 --- a/debian/changelog +++ b/debian/changelog @@ -38,6 +38,10 @@ apt (0.7.26~exp5) UNRELEASED; urgency=low - Fix return value of pkgAcqFile::Custom600Headers() in the non-index case, to avoid returning NULL and causing crashers in callers. This also fixes a compiler warning. + * methods/gzip.cc: With FileFd now being able to read gzipped files, there + is no need for the gzip method any more to spawn an external gzip process. + Rewrite it to use FileFd directly, which makes the code a lot simpler, and + also using less memory and overhead. -- Christian Perrier Tue, 11 May 2010 19:52:00 +0200 diff --git a/methods/gzip.cc b/methods/gzip.cc index f732c0b86..72e3ac909 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -23,8 +23,6 @@ #include /*}}}*/ -const char *Prog; - class GzipMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); @@ -43,14 +41,12 @@ bool GzipMethod::Fetch(FetchItem *Itm) URI Get = Itm->Uri; string Path = Get.Host + Get.Path; // To account for relative paths - string GzPathOption = "Dir::bin::"+string(Prog); - FetchResult Res; Res.Filename = Itm->DestFile; URIStart(Res); // Open the source and destination files - FileFd From(Path,FileFd::ReadOnly); + FileFd From(Path,FileFd::ReadOnlyGzip); // if the file is empty, just rename it and return if(From.Size() == 0) @@ -59,40 +55,12 @@ bool GzipMethod::Fetch(FetchItem *Itm) return true; } - int GzOut[2]; - if (pipe(GzOut) < 0) - return _error->Errno("pipe",_("Couldn't open pipe for %s"),Prog); - - // Fork gzip - pid_t Process = ExecFork(); - if (Process == 0) - { - close(GzOut[0]); - dup2(From.Fd(),STDIN_FILENO); - dup2(GzOut[1],STDOUT_FILENO); - From.Close(); - close(GzOut[1]); - SetCloseExec(STDIN_FILENO,false); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[3]; - string Tmp = _config->Find(GzPathOption,Prog); - Args[0] = Tmp.c_str(); - Args[1] = "-d"; - Args[2] = 0; - execvp(Args[0],(char **)Args); - _exit(100); - } - From.Close(); - close(GzOut[1]); - - FileFd FromGz(GzOut[0]); // For autoclose FileFd To(Itm->DestFile,FileFd::WriteEmpty); To.EraseOnFailure(); if (_error->PendingError() == true) return false; - // Read data from gzip, generate checksums and write + // Read data from source, generate checksums and write Hashes Hash; bool Failed = false; while (1) @@ -100,36 +68,23 @@ bool GzipMethod::Fetch(FetchItem *Itm) unsigned char Buffer[4*1024]; unsigned long Count; - Count = read(GzOut[0],Buffer,sizeof(Buffer)); - if (Count < 0 && errno == EINTR) - continue; - - if (Count < 0) + if (!From.Read(Buffer,sizeof(Buffer),&Count)) { - _error->Errno("read", _("Read error from %s process"),Prog); - Failed = true; - break; + To.OpFail(); + return false; } - if (Count == 0) break; - + Hash.Add(Buffer,Count); if (To.Write(Buffer,Count) == false) { Failed = true; - FromGz.Close(); break; } } - // Wait for gzip to finish - if (ExecWait(Process,_config->Find(GzPathOption,Prog).c_str(),false) == false) - { - To.OpFail(); - return false; - } - + From.Close(); To.Close(); if (Failed == true) @@ -165,9 +120,5 @@ int main(int argc, char *argv[]) setlocale(LC_ALL, ""); GzipMethod Mth; - - Prog = strrchr(argv[0],'/'); - Prog++; - return Mth.Run(); } -- cgit v1.2.3-70-g09d2 From b09663668c4d8203543e56b25db822ba55d21529 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Jul 2010 21:57:51 +0200 Subject: * methods/bzip2.cc: - add a copycat of the old gzip.cc as we need it for bzip2 and lzma --- debian/changelog | 4 +- methods/bzip2.cc | 177 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ methods/makefile | 20 ++++--- 3 files changed, 191 insertions(+), 10 deletions(-) create mode 100644 methods/bzip2.cc (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index b0efc72e5..cfac23251 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,8 @@ apt (0.7.26~exp10) UNRELEASED; urgency=low * apt-pkg/deb/debmetaindex.cc: - do not query each architecture for flat file archives - fix typo preventing display of architecture in Info() + * methods/bzip2.cc: + - add a copycat of the old gzip.cc as we need it for bzip2 and lzma [ Martin Pitt ] * debian/rules: @@ -43,7 +45,7 @@ apt (0.7.26~exp10) UNRELEASED; urgency=low Rewrite it to use FileFd directly, which makes the code a lot simpler, and also using less memory and overhead. - -- David Kalnischkies Sat, 10 Jul 2010 13:44:32 +0200 + -- David Kalnischkies Sun, 11 Jul 2010 21:56:36 +0200 apt (0.7.26~exp9) experimental; urgency=low diff --git a/methods/bzip2.cc b/methods/bzip2.cc new file mode 100644 index 000000000..5da214bfc --- /dev/null +++ b/methods/bzip2.cc @@ -0,0 +1,177 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Bzip2 method - Take a file URI in and decompress it into the target + file. + + While the method is named "bzip2" it handles also other compression + types as it calls binaries based on the name of the method, + so it can also be used to handle gzip, lzma and others if named + correctly. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + /*}}}*/ + +const char *Prog; + +class Bzip2Method : public pkgAcqMethod +{ + virtual bool Fetch(FetchItem *Itm); + + public: + + Bzip2Method() : pkgAcqMethod("1.1",SingleInstance | SendConfig) {}; +}; + + +// Bzip2Method::Fetch - Decompress the passed URI /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool Bzip2Method::Fetch(FetchItem *Itm) +{ + URI Get = Itm->Uri; + string Path = Get.Host + Get.Path; // To account for relative paths + + string GzPathOption = "Dir::bin::"+string(Prog); + + FetchResult Res; + Res.Filename = Itm->DestFile; + URIStart(Res); + + // Open the source and destination files + FileFd From(Path,FileFd::ReadOnly); + + // if the file is empty, just rename it and return + if(From.Size() == 0) + { + rename(Path.c_str(), Itm->DestFile.c_str()); + return true; + } + + int GzOut[2]; + if (pipe(GzOut) < 0) + return _error->Errno("pipe",_("Couldn't open pipe for %s"),Prog); + + // Fork bzip2 + pid_t Process = ExecFork(); + if (Process == 0) + { + close(GzOut[0]); + dup2(From.Fd(),STDIN_FILENO); + dup2(GzOut[1],STDOUT_FILENO); + From.Close(); + close(GzOut[1]); + SetCloseExec(STDIN_FILENO,false); + SetCloseExec(STDOUT_FILENO,false); + + const char *Args[3]; + string Tmp = _config->Find(GzPathOption,Prog); + Args[0] = Tmp.c_str(); + Args[1] = "-d"; + Args[2] = 0; + execvp(Args[0],(char **)Args); + _exit(100); + } + From.Close(); + close(GzOut[1]); + + FileFd FromGz(GzOut[0]); // For autoclose + FileFd To(Itm->DestFile,FileFd::WriteEmpty); + To.EraseOnFailure(); + if (_error->PendingError() == true) + return false; + + // Read data from bzip2, generate checksums and write + Hashes Hash; + bool Failed = false; + while (1) + { + unsigned char Buffer[4*1024]; + unsigned long Count; + + Count = read(GzOut[0],Buffer,sizeof(Buffer)); + if (Count < 0 && errno == EINTR) + continue; + + if (Count < 0) + { + _error->Errno("read", _("Read error from %s process"),Prog); + Failed = true; + break; + } + + if (Count == 0) + break; + + Hash.Add(Buffer,Count); + if (To.Write(Buffer,Count) == false) + { + Failed = true; + FromGz.Close(); + break; + } + } + + // Wait for bzip2 to finish + if (ExecWait(Process,_config->Find(GzPathOption,Prog).c_str(),false) == false) + { + To.OpFail(); + return false; + } + + To.Close(); + + if (Failed == true) + return false; + + // Transfer the modification times + struct stat Buf; + if (stat(Path.c_str(),&Buf) != 0) + return _error->Errno("stat",_("Failed to stat")); + + struct utimbuf TimeBuf; + TimeBuf.actime = Buf.st_atime; + TimeBuf.modtime = Buf.st_mtime; + if (utime(Itm->DestFile.c_str(),&TimeBuf) != 0) + return _error->Errno("utime",_("Failed to set modification time")); + + if (stat(Itm->DestFile.c_str(),&Buf) != 0) + return _error->Errno("stat",_("Failed to stat")); + + // Return a Done response + Res.LastModified = Buf.st_mtime; + Res.Size = Buf.st_size; + Res.TakeHashes(Hash); + + URIDone(Res); + + return true; +} + /*}}}*/ + +int main(int argc, char *argv[]) +{ + setlocale(LC_ALL, ""); + + Bzip2Method Mth; + + Prog = strrchr(argv[0],'/'); + Prog++; + + return Mth.Run(); +} diff --git a/methods/makefile b/methods/makefile index eabe85cfd..d94a85340 100644 --- a/methods/makefile +++ b/methods/makefile @@ -86,9 +86,16 @@ LIB_MAKES = apt-pkg/makefile SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc include $(PROGRAM_H) -# SSH and bzip2 method symlink -binary: $(BIN)/ssh $(BIN)/bzip2 $(BIN)/lzma -veryclean: clean-$(BIN)/ssh clean-$(BIN)/bzip2 clean-$(BIN)/lzma +# The gzip method +PROGRAM=bzip2 +SLIBS = -lapt-pkg $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile +SOURCE = bzip2.cc +include $(PROGRAM_H) + +# SSH and lzma method symlink +binary: $(BIN)/ssh $(BIN)/lzma +veryclean: clean-$(BIN)/ssh clean-$(BIN)/lzma $(BIN)/ssh: echo "Installing ssh method link" @@ -96,13 +103,8 @@ $(BIN)/ssh: clean-$(BIN)/ssh: -rm $(BIN)/ssh -$(BIN)/bzip2: - echo "Installing bzip2 method link" - ln -fs gzip $(BIN)/bzip2 $(BIN)/lzma: echo "Installing lzma method link" - ln -fs gzip $(BIN)/lzma -clean-$(BIN)/bzip2: - -rm $(BIN)/bzip2 + ln -fs bzip2 $(BIN)/lzma clean-$(BIN)/lzma: -rm $(BIN)/lzma -- cgit v1.2.3-70-g09d2