From 5f6b130d6342965bfa49beb9413bdf742440b8ab Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 21:24:29 +0100 Subject: * prototype of mirror method added --- methods/http.cc | 13 ++----- methods/http.h | 7 ++-- methods/http_main.cc | 15 ++++++++ methods/makefile | 9 ++++- methods/mirror.cc | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ methods/mirror.h | 40 ++++++++++++++++++++++ 6 files changed, 166 insertions(+), 15 deletions(-) create mode 100644 methods/http_main.cc create mode 100644 methods/mirror.cc create mode 100644 methods/mirror.h (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index c6623c46f..deaa8d0c8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,7 +38,6 @@ #include #include #include -#include #include // Internet stuff @@ -57,7 +56,7 @@ time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; bool Debug = false; - +URI Proxy; unsigned long CircleBuf::BwReadLimit=0; unsigned long CircleBuf::BwTickReadData=0; @@ -990,7 +989,7 @@ void HttpMethod::SigTerm(int) depth. */ bool HttpMethod::Fetch(FetchItem *) { - if (Server == 0) + if (Server == 0) return true; // Queue the requests @@ -1223,13 +1222,5 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - - HttpMethod Mth; - - return Mth.Loop(); -} diff --git a/methods/http.h b/methods/http.h index 541e2952c..5eac11401 100644 --- a/methods/http.h +++ b/methods/http.h @@ -13,7 +13,7 @@ #define MAXLEN 360 -#include + using std::cout; using std::endl; @@ -134,7 +134,6 @@ class HttpMethod : public pkgAcqMethod bool ServerDie(ServerState *Srv); int DealWithHeaders(FetchResult &Res,ServerState *Srv); - virtual bool Fetch(FetchItem *); virtual bool Configuration(string Message); // In the event of a fatal signal this file will be closed and timestamped. @@ -142,6 +141,9 @@ class HttpMethod : public pkgAcqMethod static int FailFd; static time_t FailTime; static void SigTerm(int); + + protected: + virtual bool Fetch(FetchItem *); public: friend class ServerState; @@ -158,6 +160,5 @@ class HttpMethod : public pkgAcqMethod }; }; -URI Proxy; #endif diff --git a/methods/http_main.cc b/methods/http_main.cc new file mode 100644 index 000000000..2c46ab19d --- /dev/null +++ b/methods/http_main.cc @@ -0,0 +1,15 @@ +#include +#include + +#include "connect.h" +#include "rfc2553emu.h" +#include "http.h" + + +int main() +{ + setlocale(LC_ALL, ""); + + HttpMethod Mth; + return Mth.Loop(); +} diff --git a/methods/makefile b/methods/makefile index 1e3b1ef85..12e446a55 100644 --- a/methods/makefile +++ b/methods/makefile @@ -49,7 +49,7 @@ include $(PROGRAM_H) PROGRAM=http SLIBS = -lapt-pkg $(SOCKETLIBS) LIB_MAKES = apt-pkg/makefile -SOURCE = http.cc rfc2553emu.cc connect.cc +SOURCE = http.cc http_main.cc rfc2553emu.cc connect.cc include $(PROGRAM_H) # The ftp method @@ -66,6 +66,13 @@ LIB_MAKES = apt-pkg/makefile SOURCE = rsh.cc include $(PROGRAM_H) +# The mirror method +PROGRAM=mirror +SLIBS = -lapt-pkg $(SOCKETLIBS) +LIB_MAKES = apt-pkg/makefile +SOURCE = mirror.cc http.cc rfc2553emu.cc connect.cc +include $(PROGRAM_H) + # SSH and vzip2 method symlink binary: $(BIN)/ssh $(BIN)/bzip2 veryclean: clean-$(BIN)/ssh clean-$(BIN)/bzip2 diff --git a/methods/mirror.cc b/methods/mirror.cc new file mode 100644 index 000000000..fad076e91 --- /dev/null +++ b/methods/mirror.cc @@ -0,0 +1,97 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ +/* ###################################################################### + + Mirror Aquire Method - This is the Mirror aquire method for APT. + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include +#include +#include + +#include +#include +using namespace std; + +#include "mirror.h" +#include "http.h" + + /*}}}*/ + +MirrorMethod::MirrorMethod() + : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) +{ +#if 0 + HasMirrorFile=true; + BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + Mirror="http://de.archive.ubuntu.com/ubuntu/"; +#endif +}; + +bool MirrorMethod::GetMirrorFile(string uri) +{ + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + BaseUri = uri.substr(0,uri.find(Marker)); + BaseUri.replace(0,strlen("mirror://"),"http://"); + + MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + + // FIXME: fetch it with curl + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + cerr << "fetch-result: " << res << endl; + + if(res) + HasMirrorFile = true; + Fetcher.Shutdown(); + return true; +} + +bool MirrorMethod::SelectMirror() +{ + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + cerr << "Mirror: " << Mirror << endl; +} + +// MirrorMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool MirrorMethod::Fetch(FetchItem *Itm) +{ + // get mirror information + if(!HasMirrorFile) + { + GetMirrorFile(Itm->Uri); + SelectMirror(); + } + + // change the items in the queue + Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); + cerr << "new Fetch-uri: " << Itm->Uri << endl; + + // FIXME: fetch it with! + +}; + +int main() +{ + setlocale(LC_ALL, ""); + + MirrorMethod Mth; + + return Mth.Run(); +} + + diff --git a/methods/mirror.h b/methods/mirror.h new file mode 100644 index 000000000..ca3428830 --- /dev/null +++ b/methods/mirror.h @@ -0,0 +1,40 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +/* ###################################################################### + + MIRROR Aquire Method - This is the MIRROR aquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_MIRROR_H +#define APT_MIRROR_H + + +#include + +using std::cout; +using std::cerr; +using std::endl; + +#include "http.h" + +class MirrorMethod : public pkgAcqMethod +{ + FetchResult Res; + string Mirror; + string BaseUri; + string MirrorFile; + bool HasMirrorFile; + + protected: + bool GetMirrorFile(string uri); + bool SelectMirror(); + public: + MirrorMethod(); + virtual bool Fetch(FetchItem *Itm); +}; + + +#endif -- cgit v1.2.3-70-g09d2 From 14e097c105c49a102a642c8108979356c5cc3152 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 22:59:35 +0100 Subject: * working mirror implementation based on http method --- apt-pkg/acquire-method.h | 7 +++-- methods/mirror.cc | 72 ++++++++++++++++++++++++++++++++++++++---------- methods/mirror.h | 15 +++++++--- 3 files changed, 73 insertions(+), 21 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index f46209d12..e0e7c1d09 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -63,9 +63,10 @@ class pkgAcqMethod // Outgoing messages void Fail(bool Transient = false); inline void Fail(const char *Why, bool Transient = false) {Fail(string(Why),Transient);}; - void Fail(string Why, bool Transient = false); - void URIStart(FetchResult &Res); - void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + bool MediaFail(string Required,string Drive); virtual void Exit() {}; diff --git a/methods/mirror.cc b/methods/mirror.cc index fad076e91..c4d911a3c 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -17,6 +17,8 @@ #include #include +#include + using namespace std; #include "mirror.h" @@ -25,31 +27,50 @@ using namespace std; /*}}}*/ MirrorMethod::MirrorMethod() - : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) + : HttpMethod(), HasMirrorFile(false) { #if 0 HasMirrorFile=true; - BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; + MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; Mirror="http://de.archive.ubuntu.com/ubuntu/"; #endif }; +// HttpMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* We stash the desired pipeline depth */ +bool MirrorMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + Debug = _config->FindB("Debug::Acquire::mirror",false); + + return true; +} + /*}}}*/ + + bool MirrorMethod::GetMirrorFile(string uri) { string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); BaseUri = uri.substr(0,uri.find(Marker)); - BaseUri.replace(0,strlen("mirror://"),"http://"); + + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; + if(Debug) + { + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + } // FIXME: fetch it with curl pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - cerr << "fetch-result: " << res << endl; if(res) HasMirrorFile = true; @@ -61,7 +82,9 @@ bool MirrorMethod::SelectMirror() { ifstream in(MirrorFile.c_str()); getline(in, Mirror); - cerr << "Mirror: " << Mirror << endl; + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + return true; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -77,21 +100,42 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - // change the items in the queue - Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); - cerr << "new Fetch-uri: " << Itm->Uri << endl; + if(Queue->Uri.find("mirror://") != string::npos) + Queue->Uri.replace(0,BaseUri.size(),Mirror); - // FIXME: fetch it with! - + // now run the real fetcher + return HttpMethod::Fetch(Itm); }; +void MirrorMethod::Fail(string Err,bool Transient) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::Fail(Err, Transient); +} + +void MirrorMethod::URIStart(FetchResult &Res) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIStart(Res); +} + +void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIDone(Res, Alt); +} + + int main() { setlocale(LC_ALL, ""); MirrorMethod Mth; - return Mth.Run(); + return Mth.Loop(); } diff --git a/methods/mirror.h b/methods/mirror.h index ca3428830..91cf8c64f 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -20,17 +20,24 @@ using std::endl; #include "http.h" -class MirrorMethod : public pkgAcqMethod +class MirrorMethod : public HttpMethod { FetchResult Res; - string Mirror; - string BaseUri; - string MirrorFile; + string BaseUri; // the original mirror://... url + string Mirror; // the selected mirror uri (http://...) + string MirrorFile; // bool HasMirrorFile; + bool Debug; + protected: bool GetMirrorFile(string uri); bool SelectMirror(); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual bool Configuration(string Message); + public: MirrorMethod(); virtual bool Fetch(FetchItem *Itm); -- cgit v1.2.3-70-g09d2 From 86c17f0a4bd3c0e3dde5bb58b3df9c41e5a88763 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:09:26 +0100 Subject: * todo added --- methods/mirror.cc | 8 ++++++++ methods/mirror.h | 3 +++ 2 files changed, 11 insertions(+) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c4d911a3c..c49764044 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -26,6 +26,14 @@ using namespace std; /*}}}*/ +/* + * TODO: + * - support keeping the mirror file around (evil listclearer strikes again) + * - better method to download than having a pkgAcquire interface here + * - testing :) + * + */ + MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { diff --git a/methods/mirror.h b/methods/mirror.h index 91cf8c64f..a47b8fbf1 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -23,6 +23,7 @@ using std::endl; class MirrorMethod : public HttpMethod { FetchResult Res; + // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) string MirrorFile; // @@ -33,6 +34,8 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + + // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); virtual void URIStart(FetchResult &Res); virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); -- cgit v1.2.3-70-g09d2 From 3ed91a17d21d0d456a12accc507b82e1c0f1a697 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:21:47 +0100 Subject: * more todo items --- methods/mirror.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index c49764044..1911d6115 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -28,10 +28,11 @@ using namespace std; /* * TODO: - * - support keeping the mirror file around (evil listclearer strikes again) * - better method to download than having a pkgAcquire interface here + * - support keeping the mirror file around (evil listclearer strikes again) + * -> /var/lib/apt/mirrors dir? how to cleanup? by time? + * - provide some TTL time until the mirror file is get again (1h? 6h?) * - testing :) - * */ MirrorMethod::MirrorMethod() -- cgit v1.2.3-70-g09d2 From ef1e6d8822bb34527bba5d2318b5768013efd010 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Nov 2006 11:06:09 +0100 Subject: * methods/mirror.cc: - check the complette queue on Fetch() for urls to transform --- methods/mirror.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 1911d6115..4de981522 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,6 +32,7 @@ using namespace std; * - support keeping the mirror file around (evil listclearer strikes again) * -> /var/lib/apt/mirrors dir? how to cleanup? by time? * - provide some TTL time until the mirror file is get again (1h? 6h?) + * - deal with runing as non-root (we can't write to the lists dir then) * - testing :) */ @@ -109,8 +110,11 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - if(Queue->Uri.find("mirror://") != string::npos) - Queue->Uri.replace(0,BaseUri.size(),Mirror); + for (FetchItem *I = Queue; I != 0; I = I->Next) + { + if(I->Uri.find("mirror://") != string::npos) + I->Uri.replace(0,BaseUri.size(),Mirror); + } // now run the real fetcher return HttpMethod::Fetch(Itm); -- cgit v1.2.3-70-g09d2 From d731f9c574138a9cb31c586a8f1e7d7181a44456 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:19:32 +0100 Subject: * apt-pkg/init.cc: - added Dir::State::Mirrors * doc/examples/configure-index: - added Acquire::mirror::RefreshInterval * methods/mirror.{cc,h}: - download the mirror file into Dir::State::Mirrors - added RefreshInterval option to not ask for the mirror file too often --- apt-pkg/init.cc | 1 + debian/apt.dirs | 1 + doc/examples/configure-index | 7 +++++- methods/mirror.cc | 56 ++++++++++++++++++++++++++++++++++++-------- methods/mirror.h | 1 + po/apt-all.pot | 4 ++-- 6 files changed, 57 insertions(+), 13 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index b47378d4a..fe085a75e 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -54,6 +54,7 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.Set("Dir::State::lists","lists/"); Cnf.Set("Dir::State::cdroms","cdroms.list"); + Cnf.Set("Dir::State::mirrors","mirrors/"); // Cache Cnf.Set("Dir::Cache","var/cache/apt/"); diff --git a/debian/apt.dirs b/debian/apt.dirs index e1cb738fa..b81878baa 100644 --- a/debian/apt.dirs +++ b/debian/apt.dirs @@ -5,5 +5,6 @@ etc/apt etc/apt/sources.list.d var/cache/apt/archives/partial var/lib/apt/lists/partial +var/lib/apt/mirrors/partial var/lib/apt/periodic usr/share/bug/apt diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 0f0abc30c..cb20d767b 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -160,6 +160,11 @@ Acquire { Options {"--ignore-time-conflict";} // not very usefull on a normal system }; + + mirror + { + RefreshInterval "360"; // refresh interval in minutes + }; }; // Directory layout @@ -258,9 +263,9 @@ Debug Acquire::Ftp "false"; // Show ftp command traffic Acquire::Http "false"; // Show http command traffic Acquire::gpgv "false"; // Show the gpgv traffic + Acquire::Mirror "false"; // Show debugging of the mirror method aptcdrom "false"; // Show found package files IdentCdrom "false"; - } /* Whatever you do, do not use this configuration file!! Take out ONLY diff --git a/methods/mirror.cc b/methods/mirror.cc index 4de981522..e70a40f55 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -18,6 +18,7 @@ #include #include #include +#include using namespace std; @@ -28,11 +29,16 @@ using namespace std; /* * TODO: + * - send expected checksum to the mirror method so that + some checking/falling back can be done here already + * - keep the mirror file around in /var/lib/apt/mirrors + * can't be put into lists/ because of the listclearer + * cleanup by time (mtime relative to the other mtimes) + * - use a TTL time the mirror file is fetched again (6h?) + * - deal with runing as non-root because we can't write to the lists + dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - support keeping the mirror file around (evil listclearer strikes again) - * -> /var/lib/apt/mirrors dir? how to cleanup? by time? - * - provide some TTL time until the mirror file is get again (1h? 6h?) - * - deal with runing as non-root (we can't write to the lists dir then) + * - magicmarker is (a bit) evil * - testing :) */ @@ -60,6 +66,12 @@ bool MirrorMethod::Configuration(string Message) } /*}}}*/ +// clean the mirrors dir based on ttl information +bool MirrorMethod::Clean(string dir) +{ + +} + bool MirrorMethod::GetMirrorFile(string uri) { @@ -69,7 +81,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); - MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { @@ -77,19 +89,43 @@ bool MirrorMethod::GetMirrorFile(string uri) cerr << "mirror-file: " << MirrorFile << endl; } - // FIXME: fetch it with curl + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + HasMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now pkgAcquire Fetcher; new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - - if(res) + if(res) HasMirrorFile = true; Fetcher.Shutdown(); - return true; + return res; } bool MirrorMethod::SelectMirror() { + // FIXME: make the mirror selection more clever, do not + // just use the first one! ifstream in(MirrorFile.c_str()); getline(in, Mirror); if(Debug) @@ -103,7 +139,7 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // get mirror information + // select mirror only once per session if(!HasMirrorFile) { GetMirrorFile(Itm->Uri); diff --git a/methods/mirror.h b/methods/mirror.h index a47b8fbf1..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -34,6 +34,7 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + bool Clean(string dir); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index b3da2ba9a..42c43fe8d 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -2238,12 +2238,12 @@ msgstr "" msgid "Please insert the disc labeled: '%s' in the drive '%s' and press enter." msgstr "" -#: apt-pkg/init.cc:120 +#: apt-pkg/init.cc:121 #, c-format msgid "Packaging system '%s' is not supported" msgstr "" -#: apt-pkg/init.cc:136 +#: apt-pkg/init.cc:137 msgid "Unable to determine a suitable packaging system type" msgstr "" -- cgit v1.2.3-70-g09d2 From 70288656715d2fe4c2c33598124ae48f7bca9cdf Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:38:55 +0100 Subject: * methods/mirror.cc: - implemented simple Clean() implementation based on the time of the last access for a mirror file --- doc/examples/configure-index | 2 ++ methods/mirror.cc | 55 +++++++++++++++++++++++++++++++++++++++++--- po/apt-all.pot | 3 ++- 3 files changed, 56 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/doc/examples/configure-index b/doc/examples/configure-index index cb20d767b..81bb6b3b0 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -164,6 +164,8 @@ Acquire mirror { RefreshInterval "360"; // refresh interval in minutes + MaxAge "90"; // max age for a mirror file in days before + // it gets deleted }; }; diff --git a/methods/mirror.cc b/methods/mirror.cc index e70a40f55..425a2f7f5 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -19,12 +19,14 @@ #include #include #include +#include +#include using namespace std; #include "mirror.h" #include "http.h" - +#include "apti18n.h" /*}}}*/ /* @@ -67,9 +69,54 @@ bool MirrorMethod::Configuration(string Message) /*}}}*/ // clean the mirrors dir based on ttl information -bool MirrorMethod::Clean(string dir) +bool MirrorMethod::Clean(string Dir) { + // FIXME: it would better to have a global idea of the mirrors + // in the sources.list and use this instead of this time + // based approach. currently apt does not support this :/ + + DIR *D = opendir(Dir.c_str()); + if (D == 0) + return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str()); + + string StartDir = SafeGetCWD(); + if (chdir(Dir.c_str()) != 0) + { + closedir(D); + return _error->Errno("chdir",_("Unable to change to %s"),Dir.c_str()); + } + + for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + { + // Skip some files.. + if (strcmp(Dir->d_name,"lock") == 0 || + strcmp(Dir->d_name,"partial") == 0 || + strcmp(Dir->d_name,".") == 0 || + strcmp(Dir->d_name,"..") == 0) + continue; + + // Del everything not touched for MaxAge days + time_t t,now,max; + struct stat buf; + if(stat(Dir->d_name, &buf) != 0) + { + cerr << "Can't stat '" << Dir->d_name << "'" << endl; + continue; + } + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); + if(t + max < now) + { + if(Debug) + clog << "Mirror file is older than MaxAge days, deleting" << endl; + unlink(Dir->d_name); + } + }; + chdir(StartDir.c_str()); + closedir(D); + return true; } @@ -81,6 +128,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); + // get new file MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) @@ -108,7 +156,7 @@ bool MirrorMethod::GetMirrorFile(string uri) return true; } if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; } // not that great to use pkgAcquire here, but we do not have @@ -142,6 +190,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) // select mirror only once per session if(!HasMirrorFile) { + Clean(_config->FindDir("Dir::State::mirrors")); GetMirrorFile(Itm->Uri); SelectMirror(); } diff --git a/po/apt-all.pot b/po/apt-all.pot index 42c43fe8d..d1ce72503 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:76 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,6 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 +#: methods/mirror.cc:82 #, c-format msgid "Unable to change to %s" msgstr "" -- cgit v1.2.3-70-g09d2 From 0c312e0ed24fed69412c181634dd4515e9b1d46f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 12:10:05 +0100 Subject: * implemented proper mirror list cleanup --- methods/mirror.cc | 39 +++++++++++++++++++++------------------ po/apt-all.pot | 4 ++-- 2 files changed, 23 insertions(+), 20 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 425a2f7f5..00f7b7807 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -71,9 +72,14 @@ bool MirrorMethod::Configuration(string Message) // clean the mirrors dir based on ttl information bool MirrorMethod::Clean(string Dir) { - // FIXME: it would better to have a global idea of the mirrors - // in the sources.list and use this instead of this time - // based approach. currently apt does not support this :/ + vector::const_iterator I; + + if(Debug) + clog << "MirrorMethod::Clean(): " << Dir << endl; + + // read sources.list + pkgSourceList list; + list.ReadMainList(); DIR *D = opendir(Dir.c_str()); if (D == 0) @@ -94,24 +100,21 @@ bool MirrorMethod::Clean(string Dir) strcmp(Dir->d_name,".") == 0 || strcmp(Dir->d_name,"..") == 0) continue; - - // Del everything not touched for MaxAge days - time_t t,now,max; - struct stat buf; - if(stat(Dir->d_name, &buf) != 0) + + // see if we have that uri + for(I=list.begin(); I != list.end(); I++) { - cerr << "Can't stat '" << Dir->d_name << "'" << endl; - continue; + string uri = (*I)->GetURI(); + if(uri.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); + string BaseUri = uri.substr(0,uri.find(Marker)); + if (URItoFileName(BaseUri) == Dir->d_name) + break; } - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - max = 24*60*60*_config->FindI("Acquire::Mirror::MaxAge",90); - if(t + max < now) - { - if(Debug) - clog << "Mirror file is older than MaxAge days, deleting" << endl; + // nothing found, nuke it + if (I == list.end()) unlink(Dir->d_name); - } }; chdir(StartDir.c_str()); diff --git a/po/apt-all.pot b/po/apt-all.pot index d1ce72503..52fadb81c 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:76 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:86 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:82 +#: methods/mirror.cc:92 #, c-format msgid "Unable to change to %s" msgstr "" -- cgit v1.2.3-70-g09d2 From 933833c51fbfe3dca3f3a3073d441e5856462605 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 13:07:22 +0100 Subject: * methods/mirror.cc: - updated the TODO --- methods/mirror.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 00f7b7807..f08b324ec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -34,14 +34,18 @@ using namespace std; * TODO: * - send expected checksum to the mirror method so that some checking/falling back can be done here already - * - keep the mirror file around in /var/lib/apt/mirrors - * can't be put into lists/ because of the listclearer - * cleanup by time (mtime relative to the other mtimes) - * - use a TTL time the mirror file is fetched again (6h?) + use pkgAcquire::Custom600Header() for this? what about gpgv + failures? + #OR# + * - implement it at the pkgAcquire::Item::Failed() level but then + we need to send back what uri exactly was failing + * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil + * - magicmarker is (a bit) evil, maybe just use a similar approach as in + clean and read the sources.list and use the GetURI() method to find + the prefix? * - testing :) */ -- cgit v1.2.3-70-g09d2 From cae9cdcefc34eba6d023cb30cbbdb9bbf909b8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 16:35:56 +0100 Subject: * basic error reporting from apt in place now (ReportMirrorFailures()) --- apt-pkg/acquire-item.cc | 10 +++++++++- apt-pkg/acquire-item.h | 1 + apt-pkg/acquire-method.cc | 1 + apt-pkg/acquire-method.h | 1 + methods/mirror.cc | 27 +++++++++++++++++++++------ methods/mirror.h | 1 + po/apt-all.pot | 16 ++++++++-------- 7 files changed, 42 insertions(+), 15 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 8ec4ba2c0..7b2a89763 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -176,7 +176,8 @@ string pkgAcqIndex::Custom600Headers() struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true"; - + if(ExpectedMD5 != "") + return "\nExpectedMD5: " + ExpectedMD5; return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ @@ -1015,6 +1016,13 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } } /*}}}*/ +// --------------------------------------------------------------------- +string pkgAcqArchive::Custom600Headers() +{ + if(MD5 != "") + return "\nExpectedMD5: " + MD5; + return ""; +} // AcqArchive::IsTrusted - Determine whether this archive comes from a // trusted source /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index da1bea801..c9cd75321 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -205,6 +205,7 @@ class pkgAcqArchive : public pkgAcquire::Item virtual string ShortDesc() {return Desc.ShortDesc;}; virtual void Finished(); virtual bool IsTrusted(); + virtual string Custom600Headers(); pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 41b832f3b..638797657 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -368,6 +368,7 @@ int pkgAcqMethod::Run(bool Single) Tmp->Uri = LookupTag(Message,"URI"); Tmp->DestFile = LookupTag(Message,"FileName"); + Tmp->ExpectedMD5 = LookupTag(Message,"ExpectedMD5"); if (StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified) == false) Tmp->LastModified = 0; Tmp->IndexFile = StringToBool(LookupTag(Message,"Index-File"),false); diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index e0e7c1d09..9e908d32c 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -33,6 +33,7 @@ class pkgAcqMethod string DestFile; time_t LastModified; bool IndexFile; + string ExpectedMD5; }; struct FetchResult diff --git a/methods/mirror.cc b/methods/mirror.cc index f08b324ec..428726a3d 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,14 +32,14 @@ using namespace std; /* * TODO: - * - send expected checksum to the mirror method so that - some checking/falling back can be done here already - use pkgAcquire::Custom600Header() for this? what about gpgv - failures? + * - what about gpgv failures? better standard format for errors + to send back to LP #OR# * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing - + we need to send back what uri exactly was failing + [mvo: the problem with this approach is ::Failed() is not really + called for all failures :/ e.g. md5sum mismatch in a archive + is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here @@ -214,6 +214,9 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // FIXME: queue next mirror? + ReportMirrorFailure(Err); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -228,11 +231,23 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { + // FIXME: queue next mirror? + if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) + ReportMirrorFailure("499 Hash mismatch"); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } +void MirrorMethod::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed + std::cerr << "\nReportMirrorFailure: " + << Queue->Uri + << " FailCode: " + << FailCode << std::endl; +} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..3ff9e1a96 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,6 +35,7 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); + void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index 52fadb81c..aee5c09c8 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:86 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:92 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:92 +#: methods/mirror.cc:98 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2362,35 +2362,35 @@ msgstr "" msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:236 apt-pkg/acquire-item.cc:945 +#: apt-pkg/acquire-item.cc:237 apt-pkg/acquire-item.cc:946 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:640 +#: apt-pkg/acquire-item.cc:641 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:753 +#: apt-pkg/acquire-item.cc:754 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:812 +#: apt-pkg/acquire-item.cc:813 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:848 +#: apt-pkg/acquire-item.cc:849 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:935 +#: apt-pkg/acquire-item.cc:936 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 2769f0bcf6c1ac73e883f47857fa227de92f4525 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 17:16:42 +0100 Subject: * mirror-failure.py: example mirror failure cgi * methods/mirror.cc: prepare for the failure submit --- methods/mirror.cc | 6 ++++++ mirror-failure.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 mirror-failure.py (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 428726a3d..6621d47e2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -247,6 +247,12 @@ void MirrorMethod::ReportMirrorFailure(string FailCode) << Queue->Uri << " FailCode: " << FailCode << std::endl; +#if 0 // FIXME: do not use system, make sure to properly encode + // URI/FailCode, do not hardcode the submit url + system("curl -d url=" + Queue->Uri + + " -d FailureCode=" + FailCode + + " http://localhost:8000/ &"); +#endif } int main() diff --git a/mirror-failure.py b/mirror-failure.py new file mode 100644 index 000000000..e7d2bbf54 --- /dev/null +++ b/mirror-failure.py @@ -0,0 +1,23 @@ +# File: cgihttpserver-example-1.py + +import CGIHTTPServer +import BaseHTTPServer + +class Handler(CGIHTTPServer.CGIHTTPRequestHandler): + #cgi_directories = ["/cgi"] + def do_POST(self): + print "do_POST" + #print self.command + #print self.path + #print self.headers + print self.client_address + data = self.rfile.read(int(self.headers["content-length"])) + print data + self.wfile.write("200 Ok\n"); + +PORT = 8000 + +httpd = BaseHTTPServer.HTTPServer(("", PORT), Handler) +print "serving at port", PORT +httpd.serve_forever() + -- cgit v1.2.3-70-g09d2 From 362d29343e5d25248bcd84300aa1b18effe891e7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Jan 2007 14:42:17 +0100 Subject: make the mirror failures actually produce a error message --- cmdline/makefile | 6 ++++++ cmdline/report-mirror-failure | 21 -------------------- configure.in | 2 +- debian/changelog | 2 +- methods/mirror.cc | 46 ++++++++++++++++++++++++++++++++----------- po/apt-all.pot | 6 +++--- 6 files changed, 46 insertions(+), 37 deletions(-) delete mode 100755 cmdline/report-mirror-failure (limited to 'methods') diff --git a/cmdline/makefile b/cmdline/makefile index 882a0e1b5..f07c7da3b 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -52,3 +52,9 @@ SOURCE=apt-key TO=$(BIN) TARGET=program include $(COPY_H) + +# The apt-key program +SOURCE=apt-report-mirror-failure +TO=$(LIB) +TARGET=program +include $(COPY_H) diff --git a/cmdline/report-mirror-failure b/cmdline/report-mirror-failure deleted file mode 100755 index 8301ed079..000000000 --- a/cmdline/report-mirror-failure +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/python - -import sys -import urllib -import apt_pkg - -apt_pkg.init() -url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", - "http://people.ubuntu.com:9000/mirror-failure") - #"http://localhost:9000/mirror-failure") -if not url: - sys.exit(0) - -data = {} -data['url'] = sys.argv[1] -data['error'] = sys.argv[2] -f = urllib.urlopen(url, urllib.urlencode(data)) -f.read() -f.close() - - diff --git a/configure.in b/configure.in index 81f327c37..df452ba34 100644 --- a/configure.in +++ b/configure.in @@ -18,7 +18,7 @@ AC_CONFIG_AUX_DIR(buildlib) AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in) dnl -- SET THIS TO THE RELEASE VERSION -- -AC_DEFINE_UNQUOTED(VERSION,"0.6.46.3") +AC_DEFINE_UNQUOTED(VERSION,"0.6.46.4") PACKAGE="apt" AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE") AC_SUBST(PACKAGE) diff --git a/debian/changelog b/debian/changelog index 24e10dcd5..6c9ada3a5 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,7 +2,7 @@ apt (0.6.46.4) unstable; urgency=low * add apt-secure.8 to "See also" section - -- + -- Michael Vogt Wed, 24 Jan 2007 14:34:15 +0100 apt (0.6.46.3) unstable; urgency=low diff --git a/methods/mirror.cc b/methods/mirror.cc index 6621d47e2..8f9b8ed34 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -30,13 +30,18 @@ using namespace std; #include "apti18n.h" /*}}}*/ -/* +/* Done: + * - works with http only + * - always picks the first mirror from the list + * - call out to problem reporting script + * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * * TODO: - * - what about gpgv failures? better standard format for errors - to send back to LP - #OR# - * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing + * what about gpgv failures? this should call-out to the problem reporting + script, but we need to know what mirror was used + * better standard format for errors to send back + * - implement failure reporting at the pkgAcquire::Item::Failed() level + but then we need to send back what uri exactly was failing [mvo: the problem with this approach is ::Failed() is not really called for all failures :/ e.g. md5sum mismatch in a archive is not] @@ -46,6 +51,7 @@ using namespace std; * - magicmarker is (a bit) evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? + * support more than http * - testing :) */ @@ -243,16 +249,34 @@ void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) void MirrorMethod::ReportMirrorFailure(string FailCode) { // report that Queue->Uri failed +#if 0 std::cerr << "\nReportMirrorFailure: " << Queue->Uri << " FailCode: " << FailCode << std::endl; -#if 0 // FIXME: do not use system, make sure to properly encode - // URI/FailCode, do not hardcode the submit url - system("curl -d url=" + Queue->Uri + - " -d FailureCode=" + FailCode + - " http://localhost:8000/ &"); #endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/lib/apt/report-mirror-failure"); + Args[i++] = report.c_str(); + Args[i++] = Queue->Uri.c_str(); + Args[i++] = FailCode.c_str(); + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(report.c_str(), (char**)Args); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Acquire::Mirror::ReportFailures").c_str()); + } } int main() diff --git a/po/apt-all.pot b/po/apt-all.pot index aee5c09c8..dd140bd63 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2006-11-24 16:37+0100\n" +"POT-Creation-Date: 2007-01-24 12:51+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:92 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:96 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:98 +#: methods/mirror.cc:102 #, c-format msgid "Unable to change to %s" msgstr "" -- cgit v1.2.3-70-g09d2 From 36280399db0ae203d3f1ae4d44b946f31e9a38ce Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 1 Feb 2007 12:32:50 +0100 Subject: * commited the latest mirror failure detection code --- apt-pkg/acquire-item.cc | 51 +++++++++++++++++++++++++++++++-------- apt-pkg/acquire-item.h | 5 +++- apt-pkg/acquire-method.cc | 21 ++++++++++------ apt-pkg/acquire-method.h | 8 +++--- cmdline/apt-report-mirror-failure | 23 ++++++++++++++++++ methods/connect.cc | 17 ++++++------- methods/mirror.cc | 45 +++------------------------------- methods/mirror.h | 1 - po/apt-all.pot | 44 ++++++++++++++++----------------- 9 files changed, 120 insertions(+), 95 deletions(-) create mode 100755 cmdline/apt-report-mirror-failure (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7b2a89763..61564c7aa 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -79,6 +79,10 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Status = StatError; Dequeue(); } + + // report mirror failure back to LP if we actually use a mirror + if(!UsedMirror.empty()) + ReportMirrorFailure(ErrorText); } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -100,6 +104,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string, { // We just downloaded something.. string FileName = LookupTag(Message,"Filename"); + UsedMirror = LookupTag(Message,"UsedMirror"); if (Complete == false && FileName == DestFile) { if (Owner->Log != 0) @@ -108,7 +113,6 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string, if (FileSize == 0) FileSize= Size; - Status = StatDone; ErrorText = string(); Owner->Dequeue(this); @@ -131,6 +135,42 @@ void pkgAcquire::Item::Rename(string From,string To) } /*}}}*/ +void pkgAcquire::Item::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed +#if 0 + std::cerr << "\nReportMirrorFailure: " + << UsedMirror + << " FailCode: " + << FailCode << std::endl; +#endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/bin/apt-report-mirror-failure"); + if(!FileExists(report)) + return; + Args[i++] = report.c_str(); + Args[i++] = UsedMirror.c_str(); + Args[i++] = FailCode.c_str(); + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(report.c_str(), (char**)Args); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Acquire::Mirror::ReportFailures").c_str()); + } +} + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is @@ -176,8 +216,6 @@ string pkgAcqIndex::Custom600Headers() struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true"; - if(ExpectedMD5 != "") - return "\nExpectedMD5: " + ExpectedMD5; return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ @@ -1016,13 +1054,6 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } } /*}}}*/ -// --------------------------------------------------------------------- -string pkgAcqArchive::Custom600Headers() -{ - if(MD5 != "") - return "\nExpectedMD5: " + MD5; - return ""; -} // AcqArchive::IsTrusted - Determine whether this archive comes from a // trusted source /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index c9cd75321..9949d0044 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -56,6 +56,7 @@ class pkgAcquire::Item unsigned long ID; bool Complete; bool Local; + string UsedMirror; // Number of queues we are inserted into unsigned int QueueCounter; @@ -78,6 +79,9 @@ class pkgAcquire::Item pkgAcquire *GetOwner() {return Owner;}; virtual bool IsTrusted() {return false;}; + // report mirror problems + void ReportMirrorFailure(string FailCode); + Item(pkgAcquire *Owner); virtual ~Item(); }; @@ -205,7 +209,6 @@ class pkgAcqArchive : public pkgAcquire::Item virtual string ShortDesc() {return Desc.ShortDesc;}; virtual void Finished(); virtual bool IsTrusted(); - virtual string Custom600Headers(); pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 638797657..13201d310 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -99,12 +99,11 @@ void pkgAcqMethod::Fail(string Err,bool Transient) } char S[1024]; + char *End = S; if (Queue != 0) { - snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" - "Message: %s %s\n",Queue->Uri.c_str(),Err.c_str(), - FailExtra.c_str()); - + End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" + "Message: %s %s\n",Queue->Uri.c_str(), Err.c_str(), IP.c_str()); // Dequeue FetchItem *Tmp = Queue; Queue = Queue->Next; @@ -113,10 +112,14 @@ void pkgAcqMethod::Fail(string Err,bool Transient) QueueBack = Queue; } else - snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: \n" - "Message: %s %s\n",Err.c_str(), - FailExtra.c_str()); - + { + End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: \n" + "Message: %s\n",Err.c_str()); + } + if(FailReason.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"FailReason: %s\n",FailReason.c_str()); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); // Set the transient flag if (Transient == true) strcat(S,"Transient-Failure: true\n\n"); @@ -182,6 +185,8 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) End += snprintf(End,sizeof(S)-50 - (End - S),"MD5-Hash: %s\n",Res.MD5Sum.c_str()); if (Res.SHA1Sum.empty() == false) End += snprintf(End,sizeof(S)-50 - (End - S),"SHA1-Hash: %s\n",Res.SHA1Sum.c_str()); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); if (Res.GPGVOutput.size() > 0) End += snprintf(End,sizeof(S)-50 - (End - S),"GPGVOutput:\n"); for (vector::iterator I = Res.GPGVOutput.begin(); diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 9e908d32c..ee68c75f7 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -55,7 +55,9 @@ class pkgAcqMethod vector Messages; FetchItem *Queue; FetchItem *QueueBack; - string FailExtra; + string FailReason; + string UsedMirror; + string IP; // Handlers for messages virtual bool Configuration(string Message); @@ -72,7 +74,6 @@ class pkgAcqMethod virtual void Exit() {}; public: - enum CnfFlags {SingleInstance = (1<<0), Pipeline = (1<<1), SendConfig = (1<<2), LocalOnly = (1<<3), NeedsCleanup = (1<<4), @@ -82,7 +83,8 @@ class pkgAcqMethod void Status(const char *Format,...); int Run(bool Single = false); - inline void SetFailExtraMsg(string Msg) {FailExtra = Msg;}; + inline void SetFailReason(string Msg) {FailReason = Msg;}; + inline void SetIP(string aIP) {IP = aIP;}; pkgAcqMethod(const char *Ver,unsigned long Flags = 0); virtual ~pkgAcqMethod() {}; diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure new file mode 100755 index 000000000..70b16cf4f --- /dev/null +++ b/cmdline/apt-report-mirror-failure @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import sys +import urllib +import apt_pkg + +print "apt-report-mirror-failure" + +apt_pkg.init() +url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", + "http://people.ubuntu.com:9000/mirror-failure") + #"http://localhost:9000/mirror-failure") +if not url: + sys.exit(0) + +data = {} +data['url'] = sys.argv[1] +data['error'] = sys.argv[2] +f = urllib.urlopen(url, urllib.urlencode(data)) +f.read() +f.close() + + diff --git a/methods/connect.cc b/methods/connect.cc index 8c2ac6d56..4e227c3fd 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -18,6 +18,7 @@ #include #include #include +#include // Internet stuff #include @@ -67,12 +68,10 @@ static bool DoConnect(struct addrinfo *Addr,string Host, wrong this will get tacked onto the end of the error message */ if (LastHostAddr->ai_next != 0) { - char Name2[NI_MAXHOST + NI_MAXSERV + 10]; - snprintf(Name2,sizeof(Name2),_("[IP: %s %s]"),Name,Service); - Owner->SetFailExtraMsg(string(Name2)); - } - else - Owner->SetFailExtraMsg(""); + std::stringstream ss; + ioprintf(ss, _("[IP: %s %s]"),Name,Service); + Owner->SetIP(ss.str()); + } // Get a socket if ((Fd = socket(Addr->ai_family,Addr->ai_socktype, @@ -89,7 +88,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, /* This implements a timeout for connect by opening the connection nonblocking */ if (WaitFd(Fd,true,TimeOut) == false) { - Owner->SetFailExtraMsg("\nFailReason: Timeout"); + Owner->SetFailReason("Timeout"); return _error->Error(_("Could not connect to %s:%s (%s), " "connection timed out"),Host.c_str(),Service,Name); } @@ -104,7 +103,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, { errno = Err; if(errno == ECONNREFUSED) - Owner->SetFailExtraMsg("\nFailReason: ConnectionRefused"); + Owner->SetFailReason("ConnectionRefused"); return _error->Errno("connect",_("Could not connect to %s:%s (%s)."),Host.c_str(), Service,Name); } @@ -169,7 +168,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, if (Res == EAI_AGAIN) { - Owner->SetFailExtraMsg("\nFailReason: TmpResolveFailure"); + Owner->SetFailReason("TmpResolveFailure"); return _error->Error(_("Temporary failure resolving '%s'"), Host.c_str()); } diff --git a/methods/mirror.cc b/methods/mirror.cc index 8f9b8ed34..8ccfb8559 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -38,7 +38,7 @@ using namespace std; * * TODO: * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used + script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() * better standard format for errors to send back * - implement failure reporting at the pkgAcquire::Item::Failed() level but then we need to send back what uri exactly was failing @@ -48,7 +48,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil, maybe just use a similar approach as in + * - magicmarker is evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? * support more than http @@ -191,6 +191,8 @@ bool MirrorMethod::SelectMirror() getline(in, Mirror); if(Debug) cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; return true; } @@ -220,9 +222,6 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // FIXME: queue next mirror? - ReportMirrorFailure(Err); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -237,47 +236,11 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - // FIXME: queue next mirror? - if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) - ReportMirrorFailure("499 Hash mismatch"); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } -void MirrorMethod::ReportMirrorFailure(string FailCode) -{ - // report that Queue->Uri failed -#if 0 - std::cerr << "\nReportMirrorFailure: " - << Queue->Uri - << " FailCode: " - << FailCode << std::endl; -#endif - const char *Args[40]; - unsigned int i = 0; - string report = _config->Find("Methods::Mirror::ProblemReporting", - "/usr/lib/apt/report-mirror-failure"); - Args[i++] = report.c_str(); - Args[i++] = Queue->Uri.c_str(); - Args[i++] = FailCode.c_str(); - pid_t pid = ExecFork(); - if(pid < 0) - { - _error->Error("ReportMirrorFailure Fork failed"); - return; - } - else if(pid == 0) - { - execvp(report.c_str(), (char**)Args); - } - if(!ExecWait(pid, "report-mirror-failure")) - { - _error->Warning("Couldn't report problem to '%s'", - _config->Find("Acquire::Mirror::ReportFailures").c_str()); - } -} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 3ff9e1a96..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,7 +35,6 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); - void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index dd140bd63..15b650fda 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2007-01-24 12:51+0100\n" +"POT-Creation-Date: 2007-01-24 17:20+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -950,7 +950,7 @@ msgstr "" msgid "Calculating upgrade... " msgstr "" -#: cmdline/apt-get.cc:1716 methods/ftp.cc:702 methods/connect.cc:101 +#: cmdline/apt-get.cc:1716 methods/ftp.cc:702 methods/connect.cc:100 msgid "Failed" msgstr "" @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:96 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:99 #, c-format msgid "Unable to read %s" msgstr "" @@ -1648,59 +1648,59 @@ msgstr "" msgid "Unable to invoke " msgstr "" -#: methods/connect.cc:64 +#: methods/connect.cc:65 #, c-format msgid "Connecting to %s (%s)" msgstr "" -#: methods/connect.cc:71 +#: methods/connect.cc:72 #, c-format msgid "[IP: %s %s]" msgstr "" -#: methods/connect.cc:80 +#: methods/connect.cc:79 #, c-format msgid "Could not create a socket for %s (f=%u t=%u p=%u)" msgstr "" -#: methods/connect.cc:86 +#: methods/connect.cc:85 #, c-format msgid "Cannot initiate the connection to %s:%s (%s)." msgstr "" -#: methods/connect.cc:93 +#: methods/connect.cc:92 #, c-format msgid "Could not connect to %s:%s (%s), connection timed out" msgstr "" -#: methods/connect.cc:108 +#: methods/connect.cc:107 #, c-format msgid "Could not connect to %s:%s (%s)." msgstr "" #. We say this mainly because the pause here is for the #. ssh connection that is still going -#: methods/connect.cc:136 methods/rsh.cc:425 +#: methods/connect.cc:135 methods/rsh.cc:425 #, c-format msgid "Connecting to %s" msgstr "" -#: methods/connect.cc:167 +#: methods/connect.cc:166 #, c-format msgid "Could not resolve '%s'" msgstr "" -#: methods/connect.cc:173 +#: methods/connect.cc:172 #, c-format msgid "Temporary failure resolving '%s'" msgstr "" -#: methods/connect.cc:176 +#: methods/connect.cc:175 #, c-format msgid "Something wicked happened resolving '%s:%s' (%i)" msgstr "" -#: methods/connect.cc:223 +#: methods/connect.cc:222 #, c-format msgid "Unable to connect to %s %s:" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:102 +#: methods/mirror.cc:105 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2357,40 +2357,40 @@ msgstr "" msgid "IO Error saving source cache" msgstr "" -#: apt-pkg/acquire-item.cc:126 +#: apt-pkg/acquire-item.cc:128 #, c-format msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:237 apt-pkg/acquire-item.cc:946 +#: apt-pkg/acquire-item.cc:275 apt-pkg/acquire-item.cc:984 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:641 +#: apt-pkg/acquire-item.cc:679 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:754 +#: apt-pkg/acquire-item.cc:792 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:813 +#: apt-pkg/acquire-item.cc:851 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:849 +#: apt-pkg/acquire-item.cc:887 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:936 +#: apt-pkg/acquire-item.cc:974 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 59271f62e4a291c8d96e1f6073203c395734b6ca Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 17:52:28 +0100 Subject: * use pkgAcqMethod::FailReason() for consistent error reporting --- apt-pkg/acquire-item.cc | 10 +++++++--- methods/connect.cc | 1 + methods/http.cc | 3 +++ methods/mirror.cc | 12 +++--------- po/apt-all.pot | 34 +++++++++++++++++----------------- 5 files changed, 31 insertions(+), 29 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 0d3d6a083..08a029ff4 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -82,8 +82,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } // report mirror failure back to LP if we actually use a mirror - if(!UsedMirror.empty()) - ReportMirrorFailure(ErrorText); + ReportMirrorFailure(ErrorText); } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -138,10 +137,13 @@ void pkgAcquire::Item::Rename(string From,string To) void pkgAcquire::Item::ReportMirrorFailure(string FailCode) { - // report that Queue->Uri failed + // we only act if a mirror was used at all + if(UsedMirror.empty()) + return; #if 0 std::cerr << "\nReportMirrorFailure: " << UsedMirror + << " Uri: " << DescURI() << " FailCode: " << FailCode << std::endl; #endif @@ -278,6 +280,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5, Status = StatAuthError; ErrorText = _("MD5Sum mismatch"); Rename(DestFile,DestFile + ".FAILED"); + ReportMirrorFailure("HashChecksumFailure"); return; } // Done, move it into position @@ -765,6 +768,7 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } // gpgv method failed + ReportMirrorFailure("GPGFailure"); _error->Warning("GPG error: %s: %s", Desc.Description.c_str(), LookupTag(Message,"Message").c_str()); diff --git a/methods/connect.cc b/methods/connect.cc index 4e227c3fd..145001fb3 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -163,6 +163,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, DefPort = 0; continue; } + Owner->SetFailReason("ResolveFailure"); return _error->Error(_("Could not resolve '%s'"),Host.c_str()); } diff --git a/methods/http.cc b/methods/http.cc index deaa8d0c8..01ad14655 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -914,6 +914,9 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) failure */ if (Srv->Result < 200 || Srv->Result >= 300) { + char err[255]; + snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result); + SetFailReason(err); _error->Error("%u %s",Srv->Result,Srv->Code); if (Srv->HaveContent == true) return 4; diff --git a/methods/mirror.cc b/methods/mirror.cc index 8ccfb8559..b64879bec 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -31,20 +31,14 @@ using namespace std; /*}}}*/ /* Done: - * - works with http only + * - works with http (only!) * - always picks the first mirror from the list * - call out to problem reporting script * - supports "deb mirror://host/path/to/mirror-list/// dist component" + * - use pkgAcqMethod::FailReason() to have a string representation + * of the failure that is also send to LP * * TODO: - * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() - * better standard format for errors to send back - * - implement failure reporting at the pkgAcquire::Item::Failed() level - but then we need to send back what uri exactly was failing - [mvo: the problem with this approach is ::Failed() is not really - called for all failures :/ e.g. md5sum mismatch in a archive - is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here diff --git a/po/apt-all.pot b/po/apt-all.pot index 39759f265..f24f19ba7 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:96 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:98 #, c-format msgid "Unable to read %s" msgstr "" @@ -1621,7 +1621,7 @@ msgstr "" msgid "Unable to accept connection" msgstr "" -#: methods/ftp.cc:864 methods/http.cc:957 methods/rsh.cc:303 +#: methods/ftp.cc:864 methods/http.cc:960 methods/rsh.cc:303 msgid "Problem hashing file" msgstr "" @@ -1685,22 +1685,22 @@ msgstr "" msgid "Connecting to %s" msgstr "" -#: methods/connect.cc:166 +#: methods/connect.cc:167 #, c-format msgid "Could not resolve '%s'" msgstr "" -#: methods/connect.cc:172 +#: methods/connect.cc:173 #, c-format msgid "Temporary failure resolving '%s'" msgstr "" -#: methods/connect.cc:175 +#: methods/connect.cc:176 #, c-format msgid "Something wicked happened resolving '%s:%s' (%i)" msgstr "" -#: methods/connect.cc:222 +#: methods/connect.cc:223 #, c-format msgid "Unable to connect to %s %s:" msgstr "" @@ -1813,15 +1813,15 @@ msgstr "" msgid "Error reading from server" msgstr "" -#: methods/http.cc:1106 +#: methods/http.cc:1109 msgid "Bad header data" msgstr "" -#: methods/http.cc:1123 +#: methods/http.cc:1126 msgid "Connection failed" msgstr "" -#: methods/http.cc:1214 +#: methods/http.cc:1217 msgid "Internal error" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:102 +#: methods/mirror.cc:104 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2357,40 +2357,40 @@ msgstr "" msgid "IO Error saving source cache" msgstr "" -#: apt-pkg/acquire-item.cc:131 +#: apt-pkg/acquire-item.cc:130 #, c-format msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:279 apt-pkg/acquire-item.cc:988 +#: apt-pkg/acquire-item.cc:281 apt-pkg/acquire-item.cc:992 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:683 +#: apt-pkg/acquire-item.cc:686 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:796 +#: apt-pkg/acquire-item.cc:800 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:855 +#: apt-pkg/acquire-item.cc:859 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:891 +#: apt-pkg/acquire-item.cc:895 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:978 +#: apt-pkg/acquire-item.cc:982 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 066b53e95ba224f81d078e7d8df3002275f8a092 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 18:46:45 +0100 Subject: * get rid of magic mirror --- methods/mirror.cc | 62 +++++++++++++++++++++++++++++++++++++++++-------------- po/apt-all.pot | 4 ++-- 2 files changed, 48 insertions(+), 18 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index b64879bec..ff91130b8 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,22 +42,13 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is evil, maybe just use a similar approach as in - clean and read the sources.list and use the GetURI() method to find - the prefix? - * support more than http + * - support more than http * - testing :) */ MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { -#if 0 - HasMirrorFile=true; - BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; - MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; - Mirror="http://de.archive.ubuntu.com/ubuntu/"; -#endif }; // HttpMethod::Configuration - Handle a configuration message /*{{{*/ @@ -111,8 +102,7 @@ bool MirrorMethod::Clean(string Dir) string uri = (*I)->GetURI(); if(uri.substr(0,strlen("mirror://")) != string("mirror://")) continue; - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - string BaseUri = uri.substr(0,uri.find(Marker)); + string BaseUri = uri.substr(0,uri.size()-1); if (URItoFileName(BaseUri) == Dir->d_name) break; } @@ -127,11 +117,51 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string uri) +bool MirrorMethod::GetMirrorFile(string mirror_uri_str) { - string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); - BaseUri = uri.substr(0,uri.find(Marker)); - + /* + - a mirror_uri_str looks like this: + mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg + + - the matching source.list entry + deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main + + - we actually want to go after: + http://people.ubuntu.com/~mvo/apt/mirror/mirrors + + And we need to save the BaseUri for later: + - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors + + FIXME: what if we have two similar prefixes? + mirror://people.ubuntu.com/~mvo/mirror + mirror://people.ubuntu.com/~mvo/mirror2 + then mirror_uri_str looks like: + mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg + mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg + we search sources.list and find: + mirror://people.ubuntu.com/~mvo/apt/mirror + in both cases! So we need to apply some domain knowledge here :( and + check for /dists/ or /Release.gpg as suffixes + */ + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + + // read sources.list and find match + vector::const_iterator I; + pkgSourceList list; + list.ReadMainList(); + for(I=list.begin(); I != list.end(); I++) + { + string uristr = (*I)->GetURI(); + std::cerr << "Checking: " << uristr << std::endl; + if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) + continue; + // find matching uri in sources.list + if(mirror_uri_str.substr(0,uristr.size()) == uristr) + { + std::cerr << "found BaseURI: " << uristr << std::endl; + BaseUri = uristr.substr(0,uristr.size()-1); + } + } string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); diff --git a/po/apt-all.pot b/po/apt-all.pot index f24f19ba7..8fa68d296 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:98 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:90 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:104 +#: methods/mirror.cc:96 #, c-format msgid "Unable to change to %s" msgstr "" -- cgit v1.2.3-70-g09d2 From f0b509cdb44cb5e79e9c5ddd7ebec46965138534 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 5 Feb 2007 19:06:42 +0100 Subject: * apt-pkg/acquire-item.cc: - use FailReason in pkgAcquire::Item::Failed when available * methods/mirror.cc: - move some debug output into if(Debug) --- apt-pkg/acquire-item.cc | 7 ++++++- cmdline/apt-report-mirror-failure | 5 +++-- methods/mirror.cc | 9 ++++++--- po/apt-all.pot | 18 +++++++++--------- 4 files changed, 24 insertions(+), 15 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 08a029ff4..14610d6e9 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -82,7 +82,11 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } // report mirror failure back to LP if we actually use a mirror - ReportMirrorFailure(ErrorText); + string FailReason = LookupTag(Message, "FailReason"); + if(FailReason.size() != 0) + ReportMirrorFailure(FailReason); + else + ReportMirrorFailure(ErrorText); } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -155,6 +159,7 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) return; Args[i++] = report.c_str(); Args[i++] = UsedMirror.c_str(); + Args[i++] = DescURI().c_str(); Args[i++] = FailCode.c_str(); Args[i++] = NULL; pid_t pid = ExecFork(); diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure index 277b23e9a..fb43f0e74 100755 --- a/cmdline/apt-report-mirror-failure +++ b/cmdline/apt-report-mirror-failure @@ -14,8 +14,9 @@ if not url: print "Reporting mirror failure to '%s'" % url data = {} -data['url'] = sys.argv[1] -data['error'] = sys.argv[2] +data['mirror'] = sys.argv[1] +data['failurl'] = sys.argv[2] +data['error'] = sys.argv[3] f = urllib.urlopen(url, urllib.urlencode(data)) f.read() f.close() diff --git a/methods/mirror.cc b/methods/mirror.cc index ff91130b8..c5c0c7461 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -143,7 +143,8 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + if(Debug) + std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -152,13 +153,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) for(I=list.begin(); I != list.end(); I++) { string uristr = (*I)->GetURI(); - std::cerr << "Checking: " << uristr << std::endl; + if(Debug) + std::cerr << "Checking: " << uristr << std::endl; if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) continue; // find matching uri in sources.list if(mirror_uri_str.substr(0,uristr.size()) == uristr) { - std::cerr << "found BaseURI: " << uristr << std::endl; + if(Debug) + std::cerr << "found BaseURI: " << uristr << std::endl; BaseUri = uristr.substr(0,uristr.size()-1); } } diff --git a/po/apt-all.pot b/po/apt-all.pot index 8fa68d296..2355eee20 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:90 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:81 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:96 +#: methods/mirror.cc:87 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2357,40 +2357,40 @@ msgstr "" msgid "IO Error saving source cache" msgstr "" -#: apt-pkg/acquire-item.cc:130 +#: apt-pkg/acquire-item.cc:134 #, c-format msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:281 apt-pkg/acquire-item.cc:992 +#: apt-pkg/acquire-item.cc:286 apt-pkg/acquire-item.cc:997 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:686 +#: apt-pkg/acquire-item.cc:691 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:800 +#: apt-pkg/acquire-item.cc:805 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:859 +#: apt-pkg/acquire-item.cc:864 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:895 +#: apt-pkg/acquire-item.cc:900 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:982 +#: apt-pkg/acquire-item.cc:987 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 3f599bb721c4ac58d8ff18991c9704b5f30eaa2b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 10:49:25 +0100 Subject: * apt-pkg/acquire-item.cc: - default to "/usr/lib/apt/apt-report-mirror-failure" * cmdline/apt-report-mirror-failure: - no default comit url for now * debian/rules: - move apt-report-mirror-failure into /usr/lib/apt * doc/examples/configure-index: - more documentation * methods/mirror.cc: - updated TODO --- apt-pkg/acquire-item.cc | 2 +- cmdline/apt-report-mirror-failure | 4 ++-- cmdline/makefile | 2 +- debian/rules | 4 ++++ doc/examples/configure-index | 4 ++++ methods/mirror.cc | 1 + 6 files changed, 13 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 14610d6e9..38530152f 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -154,7 +154,7 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) const char *Args[40]; unsigned int i = 0; string report = _config->Find("Methods::Mirror::ProblemReporting", - "/usr/bin/apt-report-mirror-failure"); + "/usr/lib/apt/apt-report-mirror-failure"); if(!FileExists(report)) return; Args[i++] = report.c_str(); diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure index fb43f0e74..1567e78e3 100755 --- a/cmdline/apt-report-mirror-failure +++ b/cmdline/apt-report-mirror-failure @@ -5,8 +5,8 @@ import urllib import apt_pkg apt_pkg.init() -url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", - "http://people.ubuntu.com:9000/mirror-failure") +url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", None) + #"http://people.ubuntu.com:9000/mirror-failure") #"http://localhost:9000/mirror-failure") if not url: sys.exit(0) diff --git a/cmdline/makefile b/cmdline/makefile index f07c7da3b..8d284c2f3 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -55,6 +55,6 @@ include $(COPY_H) # The apt-key program SOURCE=apt-report-mirror-failure -TO=$(LIB) +TO=$(BIN) TARGET=program include $(COPY_H) diff --git a/debian/rules b/debian/rules index 02ee734bb..9e87b1723 100755 --- a/debian/rules +++ b/debian/rules @@ -214,6 +214,10 @@ apt: build debian/shlibs.local # head -n 500 ChangeLog > debian/ChangeLog + # move the mirror failure script in place + mv debian/$@/usr/bin/apt-report-mirror-failure \ + debian/$@/usr/lib/apt/apt-report-mirror-failure \ + dh_installexamples -p$@ $(BLD)/docs/examples/* dh_installman -p$@ dh_installcron -p$@ diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 81bb6b3b0..81fa4c52f 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -166,6 +166,10 @@ Acquire RefreshInterval "360"; // refresh interval in minutes MaxAge "90"; // max age for a mirror file in days before // it gets deleted + // mirror failure reporting script + ProblemReporting "/usr/lib/apt/apt-report-mirror-failure"; + // mirror failure reporting url + ReportFailures "http://example.com/mirror-failure"; }; }; diff --git a/methods/mirror.cc b/methods/mirror.cc index c5c0c7461..9a86a10c2 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -42,6 +42,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here + * and better error handling there! * - support more than http * - testing :) */ -- cgit v1.2.3-70-g09d2 From 68acfb437f0c6c2e026be6fd3b7da3fc8d64c523 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 6 Feb 2007 16:37:20 +0100 Subject: * ABI version bumped --- apt-pkg/init.h | 2 +- apt-pkg/makefile | 2 +- debian/changelog | 1 + methods/makefile | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.h b/apt-pkg/init.h index b036b8609..0e91f164d 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -18,7 +18,7 @@ // See the makefile #define APT_PKG_MAJOR 3 -#define APT_PKG_MINOR 51 +#define APT_PKG_MINOR 52 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 6cc1414b2..10b012774 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=3.51 +MAJOR=3.52 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/debian/changelog b/debian/changelog index 6fdf646f0..caed5e531 100644 --- a/debian/changelog +++ b/debian/changelog @@ -7,6 +7,7 @@ apt (0.6.46.4ubuntu7) feisty; urgency=low It also supports error reporting to a configurable url for mirror problems/failures. + * Bump ABI version -- Michael Vogt Tue, 6 Feb 2007 11:38:06 +0100 diff --git a/methods/makefile b/methods/makefile index f971e56e8..d66ba11e3 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 3.51 +LIB_APT_PKG_MAJOR = 3.52 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3-70-g09d2 From d715b9c969264eec96913b577f63a3e9e498b2e6 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 21 Feb 2007 09:59:53 +0100 Subject: * fix in the pkgRecords destructor * Bump ABI version --- apt-pkg/init.h | 2 +- apt-pkg/makefile | 2 +- apt-pkg/pkgrecords.cc | 16 +++++++++------- apt-pkg/pkgrecords.h | 5 +++-- configure.in | 2 +- debian/changelog | 7 +++++++ methods/makefile | 2 +- po/apt-all.pot | 48 ++++++++++++++++++++++++------------------------ 8 files changed, 47 insertions(+), 37 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.h b/apt-pkg/init.h index 0e91f164d..905173a7a 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -18,7 +18,7 @@ // See the makefile #define APT_PKG_MAJOR 3 -#define APT_PKG_MINOR 52 +#define APT_PKG_MINOR 53 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 10b012774..d36747d85 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=3.52 +MAJOR=3.53 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/apt-pkg/pkgrecords.cc b/apt-pkg/pkgrecords.cc index b22f3e73f..456b5aef4 100644 --- a/apt-pkg/pkgrecords.cc +++ b/apt-pkg/pkgrecords.cc @@ -23,11 +23,9 @@ // Records::pkgRecords - Constructor /*{{{*/ // --------------------------------------------------------------------- /* This will create the necessary structures to access the status files */ -pkgRecords::pkgRecords(pkgCache &Cache) : Cache(Cache), Files(0) +pkgRecords::pkgRecords(pkgCache &Cache) : Cache(Cache), + Files(Cache.HeaderP->PackageFileCount) { - Files = new Parser *[Cache.HeaderP->PackageFileCount]; - memset(Files,0,sizeof(*Files)*Cache.HeaderP->PackageFileCount); - for (pkgCache::PkgFileIterator I = Cache.FileBegin(); I.end() == false; I++) { @@ -49,9 +47,13 @@ pkgRecords::pkgRecords(pkgCache &Cache) : Cache(Cache), Files(0) /* */ pkgRecords::~pkgRecords() { - for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++) - delete Files[I]; - delete [] Files; + for ( vector::iterator it = Files.begin(); + it != Files.end(); + ++it) + { + delete *it; + } + } /*}}}*/ // Records::Lookup - Get a parser for the package version file /*{{{*/ diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h index 31c444dbf..ad3946c1b 100644 --- a/apt-pkg/pkgrecords.h +++ b/apt-pkg/pkgrecords.h @@ -23,6 +23,7 @@ #include #include +#include class pkgRecords { @@ -32,8 +33,8 @@ class pkgRecords private: pkgCache &Cache; - Parser **Files; - + std::vectorFiles; + public: // Lookup function diff --git a/configure.in b/configure.in index ce166b31e..7d93de955 100644 --- a/configure.in +++ b/configure.in @@ -18,7 +18,7 @@ AC_CONFIG_AUX_DIR(buildlib) AC_CONFIG_HEADER(include/config.h:buildlib/config.h.in include/apti18n.h:buildlib/apti18n.h.in) dnl -- SET THIS TO THE RELEASE VERSION -- -AC_DEFINE_UNQUOTED(VERSION,"0.6.46.4ubuntu7") +AC_DEFINE_UNQUOTED(VERSION,"0.6.46.4ubuntu8") PACKAGE="apt" AC_DEFINE_UNQUOTED(PACKAGE,"$PACKAGE") AC_SUBST(PACKAGE) diff --git a/debian/changelog b/debian/changelog index caed5e531..b926a495f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +apt (0.6.46.4ubuntu8) feisty; urgency=low + + * fix in the pkgRecords destructor + * Bump ABI version + + -- + apt (0.6.46.4ubuntu7) feisty; urgency=low * Merged the apt--mirror branch. This means that a new 'mirror' diff --git a/methods/makefile b/methods/makefile index d66ba11e3..610bd5b49 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 3.52 +LIB_APT_PKG_MAJOR = 3.53 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method diff --git a/po/apt-all.pot b/po/apt-all.pot index f2f151dd3..7233cd617 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2007-02-06 11:14+0100\n" +"POT-Creation-Date: 2007-02-09 16:54+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -251,7 +251,7 @@ msgid "" " -o=? Set an arbitrary configuration option, eg -o dir::cache=/tmp\n" msgstr "" -#: cmdline/apt-extracttemplates.cc:267 apt-pkg/pkgcachegen.cc:819 +#: cmdline/apt-extracttemplates.cc:267 apt-pkg/pkgcachegen.cc:820 #, c-format msgid "Unable to write to %s" msgstr "" @@ -1397,9 +1397,9 @@ msgid "The info and temp directories need to be on the same filesystem" msgstr "" #. Build the status cache -#: apt-inst/deb/dpkgdb.cc:139 apt-pkg/pkgcachegen.cc:752 -#: apt-pkg/pkgcachegen.cc:821 apt-pkg/pkgcachegen.cc:826 -#: apt-pkg/pkgcachegen.cc:949 +#: apt-inst/deb/dpkgdb.cc:139 apt-pkg/pkgcachegen.cc:753 +#: apt-pkg/pkgcachegen.cc:822 apt-pkg/pkgcachegen.cc:827 +#: apt-pkg/pkgcachegen.cc:950 msgid "Reading package lists" msgstr "" @@ -2337,92 +2337,92 @@ msgstr "" msgid "Cache has an incompatible versioning system" msgstr "" -#: apt-pkg/pkgcachegen.cc:119 +#: apt-pkg/pkgcachegen.cc:120 #, c-format msgid "Error occurred while processing %s (NewPackage)" msgstr "" -#: apt-pkg/pkgcachegen.cc:134 +#: apt-pkg/pkgcachegen.cc:135 #, c-format msgid "Error occurred while processing %s (UsePackage1)" msgstr "" -#: apt-pkg/pkgcachegen.cc:157 +#: apt-pkg/pkgcachegen.cc:158 #, c-format msgid "Error occured while processing %s (NewFileDesc1)" msgstr "" -#: apt-pkg/pkgcachegen.cc:182 +#: apt-pkg/pkgcachegen.cc:183 #, c-format msgid "Error occurred while processing %s (UsePackage2)" msgstr "" -#: apt-pkg/pkgcachegen.cc:186 +#: apt-pkg/pkgcachegen.cc:187 #, c-format msgid "Error occurred while processing %s (NewFileVer1)" msgstr "" -#: apt-pkg/pkgcachegen.cc:217 +#: apt-pkg/pkgcachegen.cc:218 #, c-format msgid "Error occurred while processing %s (NewVersion1)" msgstr "" -#: apt-pkg/pkgcachegen.cc:221 +#: apt-pkg/pkgcachegen.cc:222 #, c-format msgid "Error occurred while processing %s (UsePackage3)" msgstr "" -#: apt-pkg/pkgcachegen.cc:225 +#: apt-pkg/pkgcachegen.cc:226 #, c-format msgid "Error occurred while processing %s (NewVersion2)" msgstr "" -#: apt-pkg/pkgcachegen.cc:249 +#: apt-pkg/pkgcachegen.cc:250 #, c-format msgid "Error occured while processing %s (NewFileDesc2)" msgstr "" -#: apt-pkg/pkgcachegen.cc:255 +#: apt-pkg/pkgcachegen.cc:256 msgid "Wow, you exceeded the number of package names this APT is capable of." msgstr "" -#: apt-pkg/pkgcachegen.cc:258 +#: apt-pkg/pkgcachegen.cc:259 msgid "Wow, you exceeded the number of versions this APT is capable of." msgstr "" -#: apt-pkg/pkgcachegen.cc:261 +#: apt-pkg/pkgcachegen.cc:262 msgid "Wow, you exceeded the number of descriptions this APT is capable of." msgstr "" -#: apt-pkg/pkgcachegen.cc:264 +#: apt-pkg/pkgcachegen.cc:265 msgid "Wow, you exceeded the number of dependencies this APT is capable of." msgstr "" -#: apt-pkg/pkgcachegen.cc:292 +#: apt-pkg/pkgcachegen.cc:293 #, c-format msgid "Error occurred while processing %s (FindPkg)" msgstr "" -#: apt-pkg/pkgcachegen.cc:305 +#: apt-pkg/pkgcachegen.cc:306 #, c-format msgid "Error occurred while processing %s (CollectFileProvides)" msgstr "" -#: apt-pkg/pkgcachegen.cc:311 +#: apt-pkg/pkgcachegen.cc:312 #, c-format msgid "Package %s %s was not found while processing file dependencies" msgstr "" -#: apt-pkg/pkgcachegen.cc:682 +#: apt-pkg/pkgcachegen.cc:683 #, c-format msgid "Couldn't stat source package list %s" msgstr "" -#: apt-pkg/pkgcachegen.cc:767 +#: apt-pkg/pkgcachegen.cc:768 msgid "Collecting File Provides" msgstr "" -#: apt-pkg/pkgcachegen.cc:894 apt-pkg/pkgcachegen.cc:901 +#: apt-pkg/pkgcachegen.cc:895 apt-pkg/pkgcachegen.cc:902 msgid "IO Error saving source cache" msgstr "" -- cgit v1.2.3-70-g09d2 From d530b4c24fc452efc82be4b7e1ebc536a00c09b7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 3 Oct 2007 02:21:52 +0200 Subject: * Reset curl options and timestamp between downloaded files. Thanks to Ryan Murray for the patch --- debian/changelog | 2 ++ methods/https.cc | 13 +++++++++++++ 2 files changed, 15 insertions(+) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 87b9dd024..375a6bb1e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,8 @@ apt (0.7.6ubuntu12) gutsy; urgency=low (thanks to Laurent Bigonville, LP: #140019) * apt-pkg/deb/debmetaindex.cc: comparison with string literal results in unspecified behaviour; + * Reset curl options and timestamp between downloaded files. Thanks to + Ryan Murray for the patch [Paul Sladen] * Have 'cron.daily/apt' send D-Bus doesn't exist error messages diff --git a/methods/https.cc b/methods/https.cc index e6717e63a..3b2b0bb19 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -115,6 +115,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // - error checking/reporting // - more debug options? (CURLOPT_DEBUGFUNCTION?) + curl_easy_reset(curl); SetupProxy(); // callbacks @@ -125,6 +126,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); curl_easy_setopt(curl, CURLOPT_FAILONERROR, true); + curl_easy_setopt(curl, CURLOPT_FILETIME, true); // FIXME: https: offer various options of verification bool peer_verify = _config->FindB("Acquire::https::Verify-Peer", false); @@ -202,6 +204,9 @@ bool HttpsMethod::Fetch(FetchItem *Itm) CURLcode success = curl_easy_perform(curl); curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &curl_responsecode); + long curl_servdate; + curl_easy_getinfo(curl, CURLINFO_FILETIME, &curl_servdate); + // cleanup if(success != 0) { @@ -215,6 +220,14 @@ bool HttpsMethod::Fetch(FetchItem *Itm) if (Res.Size == 0) Res.Size = File->Size(); + // Timestamp + struct utimbuf UBuf; + if (curl_servdate != -1) { + UBuf.actime = curl_servdate; + UBuf.modtime = curl_servdate; + utime(File->Name().c_str(),&UBuf); + } + // check the downloaded result struct stat Buf; if (stat(File->Name().c_str(),&Buf) == 0) -- cgit v1.2.3-70-g09d2 From 3de6d4585be0b8a9bd98b5fb1d62ff922537c435 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 5 Oct 2007 07:50:24 +0200 Subject: * methods/https.cc: - fix off-by-one error I-M-S handling - cleanup after I-M-S hit - only send LastModified if we actually have a file - delete zero size I-M-S hits --- debian/changelog | 6 +++++- methods/https.cc | 26 +++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index b25e75ef5..30ac4e10d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,9 @@ apt (0.7.6ubuntu13) gutsy; urgency=low - fix crash in WriteApportReport (LP: #144537) * apt-pkg/acquire-item.cc - fix disappearing local Packages.gz file (LP: #131166) + * methods/https.cc: + - fix off-by-one error I-M-S handling + - cleanup after I-M-S hit -- @@ -124,10 +127,11 @@ apt (0.7.6ubuntu1) gutsy; urgency=low * apt-pkg/acquire-item.cc: - remove zero size files on I-M-S hit * methods/https.cc: - - only send LastModified if we actually have one + - only send LastModified if we actually have a file - send range request with if-range - delete failed downloads (thanks to Thom May for his help here) + - delete zero size I-M-S hits * apt-pkg/deb/dpkgpm.{cc,h}: - merged dpkg-log branch, this lets you specify a Dir::Log::Terminal file to log dpkg output to diff --git a/methods/https.cc b/methods/https.cc index 3b2b0bb19..b2bbbddb1 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -161,13 +161,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - // set time values - if(Itm->LastModified > 0) - { - curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); - curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); - } - // speed limit int dlLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024; if (dlLimit > 0) @@ -183,7 +176,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // error handling curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); - // In this case we send an if-range query with a range header + // if we have the file send an if-range query with a range header if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { char Buf[1000]; @@ -191,11 +184,17 @@ bool HttpsMethod::Fetch(FetchItem *Itm) (long)SBuf.st_size - 1, TimeRFC1123(SBuf.st_mtime).c_str()); headers = curl_slist_append(headers, Buf); + } + else if(Itm->LastModified > 0) + { + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); } // go for it - if the file exists, append on it File = new FileFd(Itm->DestFile, FileFd::WriteAny); - File->Seek(File->Size()); + if (File->Size() > 0) + File->Seek(File->Size() - 1); // keep apt updated Res.Filename = Itm->DestFile; @@ -217,9 +216,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) } File->Close(); - if (Res.Size == 0) - Res.Size = File->Size(); - // Timestamp struct utimbuf UBuf; if (curl_servdate != -1) { @@ -232,15 +228,19 @@ bool HttpsMethod::Fetch(FetchItem *Itm) struct stat Buf; if (stat(File->Name().c_str(),&Buf) == 0) { - Res.Size = Buf.st_size; Res.Filename = File->Name(); Res.LastModified = Buf.st_mtime; Res.IMSHit = false; if (curl_responsecode == 304) { + unlink(File->Name().c_str()); Res.IMSHit = true; Res.LastModified = Itm->LastModified; + Res.Size = 0; + URIDone(Res); + return true; } + Res.Size = Buf.st_size; } // take hashes -- cgit v1.2.3-70-g09d2 From 38eedeb77790c106eb35940e3d7835a7e0f0297e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Nov 2007 15:54:17 +0100 Subject: methods/mirror.{cc,h}: - improve the mirror method so that it only updates the mirror list from the server if we get new indexfiles - code cleanup --- methods/mirror.cc | 152 +++++++++++++++++++++++++++++++++--------------------- methods/mirror.h | 7 +-- 2 files changed, 97 insertions(+), 62 deletions(-) (limited to 'methods') diff --git a/methods/mirror.cc b/methods/mirror.cc index 9a86a10c2..43e56c71a 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -48,7 +48,7 @@ using namespace std; */ MirrorMethod::MirrorMethod() - : HttpMethod(), HasMirrorFile(false) + : HttpMethod(), DownloadedMirrorFile(false) { }; @@ -73,6 +73,9 @@ bool MirrorMethod::Clean(string Dir) if(Debug) clog << "MirrorMethod::Clean(): " << Dir << endl; + if(Dir == "/") + return _error->Error("will not clean: '/'"); + // read sources.list pkgSourceList list; list.ReadMainList(); @@ -118,7 +121,71 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string mirror_uri_str) +bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) +{ + + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + DownloadedMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); + + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + if(res) + DownloadedMirrorFile = true; + Fetcher.Shutdown(); + return res; +} + +bool MirrorMethod::SelectMirror() +{ + // if we do not have a MirrorFile, fallback + if(!FileExists(MirrorFile)) + { + // FIXME: fallback to a default mirror here instead + // and provide a config option to define that default + return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + } + + // FIXME: make the mirror selection more clever, do not + // just use the first one! + // BUT: we can not make this random, the mirror has to be + // stable accross session, because otherwise we can + // get into sync issues (got indexfiles from mirror A, + // but packages from mirror B - one might be out of date etc) + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +string MirrorMethod::GetMirrorFileName(string mirror_uri_str) { /* - a mirror_uri_str looks like this: @@ -144,8 +211,9 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ + string name; if(Debug) - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -166,62 +234,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) BaseUri = uristr.substr(0,uristr.size()-1); } } - string fetch = BaseUri; - fetch.replace(0,strlen("mirror://"),"http://"); - // get new file - MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); + name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; - } - - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) - { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) - { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - HasMirrorFile = true; - return true; - } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + cerr << "mirror-file: " << name << endl; } - - // not that great to use pkgAcquire here, but we do not have - // any other way right now - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) - HasMirrorFile = true; - Fetcher.Shutdown(); - return res; -} - -bool MirrorMethod::SelectMirror() -{ - // FIXME: make the mirror selection more clever, do not - // just use the first one! - ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; - return true; + return name; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -230,20 +251,33 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // select mirror only once per session - if(!HasMirrorFile) + // the http method uses Fetch(0) as a way to update the pipeline, + // just let it do its work in this case - Fetch() with a valid + // Itm will always run before the first Fetch(0) + if(Itm == NULL) + return HttpMethod::Fetch(Itm); + + // if we don't have the name of the mirror file on disk yet, + // calculate it now (can be derived from the uri) + if(MirrorFile.empty()) + MirrorFile = GetMirrorFileName(Itm->Uri); + + // download mirror file once (if we are after index files) + if(Itm->IndexFile && !DownloadedMirrorFile) { Clean(_config->FindDir("Dir::State::mirrors")); - GetMirrorFile(Itm->Uri); - SelectMirror(); + DownloadMirrorFile(Itm->Uri); } + if(Mirror.empty()) + SelectMirror(); + for (FetchItem *I = Queue; I != 0; I = I->Next) { if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(),Mirror); + I->Uri.replace(0,BaseUri.size(), Mirror); } - + // now run the real fetcher return HttpMethod::Fetch(Itm); }; diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..ed817806b 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,13 +26,14 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) - string MirrorFile; // - bool HasMirrorFile; + string MirrorFile; // the file that contains the list of mirrors + bool DownloadedMirrorFile; // already downloaded this session bool Debug; protected: - bool GetMirrorFile(string uri); + bool DownloadMirrorFile(string uri); + string GetMirrorFileName(string uri); bool SelectMirror(); bool Clean(string dir); -- cgit v1.2.3-70-g09d2 From 0df1d3ac4312a2051b10478cf1c8b242864cb3b1 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 6 Dec 2007 19:47:36 +0100 Subject: we do not break the abi --- apt-pkg/init.h | 2 +- apt-pkg/makefile | 2 +- methods/makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.h b/apt-pkg/init.h index 6d8693be9..23e951eff 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -18,7 +18,7 @@ // See the makefile #define APT_PKG_MAJOR 4 -#define APT_PKG_MINOR 6 +#define APT_PKG_MINOR 5 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 1b78c94f6..b327dbf64 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=4.6 +MAJOR=4.5 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) -lutil APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/methods/makefile b/methods/makefile index 085b357b8..3c12bb974 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.6 +LIB_APT_PKG_MAJOR = 4.5 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3-70-g09d2 From 5148def81fb20b22fc9ec54bc212e34260fbfebb Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 7 Dec 2007 22:54:41 +0100 Subject: update ABI, the homepage change broke it --- apt-pkg/init.h | 2 +- apt-pkg/makefile | 2 +- methods/makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/init.h b/apt-pkg/init.h index 23e951eff..6d8693be9 100644 --- a/apt-pkg/init.h +++ b/apt-pkg/init.h @@ -18,7 +18,7 @@ // See the makefile #define APT_PKG_MAJOR 4 -#define APT_PKG_MINOR 5 +#define APT_PKG_MINOR 6 #define APT_PKG_RELEASE 0 extern const char *pkgVersion; diff --git a/apt-pkg/makefile b/apt-pkg/makefile index b327dbf64..1b78c94f6 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=4.5 +MAJOR=4.6 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) -lutil APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/methods/makefile b/methods/makefile index 3c12bb974..085b357b8 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.5 +LIB_APT_PKG_MAJOR = 4.6 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3-70-g09d2 From d7bab8f1142e474ac4bafd139e93439aeddfd546 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 3 Jan 2008 15:22:44 +0100 Subject: methods/makefile: remove duplicated entry for https --- methods/makefile | 7 ------- 1 file changed, 7 deletions(-) (limited to 'methods') diff --git a/methods/makefile b/methods/makefile index 085b357b8..f765a4d37 100644 --- a/methods/makefile +++ b/methods/makefile @@ -59,13 +59,6 @@ LIB_MAKES = apt-pkg/makefile SOURCE = https.cc include $(PROGRAM_H) -# The https method -PROGRAM=https -SLIBS = -lapt-pkg -lcurl -LIB_MAKES = apt-pkg/makefile -SOURCE = https.cc -include $(PROGRAM_H) - # The ftp method PROGRAM=ftp SLIBS = -lapt-pkg $(SOCKETLIBS) -- cgit v1.2.3-70-g09d2 From 484befd1518245e795ed92e6c97ad50786f3b4a2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 11 Mar 2008 09:59:32 +0100 Subject: * typo fixes (LP: #107960) --- apt-pkg/depcache.h | 4 ++-- debian/changelog | 1 + methods/http.cc | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/apt-pkg/depcache.h b/apt-pkg/depcache.h index 7adf6fe7f..f41ad17e9 100644 --- a/apt-pkg/depcache.h +++ b/apt-pkg/depcache.h @@ -92,7 +92,7 @@ class pkgDepCache : protected pkgCache::Namespace * \param rootFunc A callback that can be used to add extra * packages to the root set. * - * \return \b false if an error occured. + * \return \b false if an error occurred. */ bool MarkRequired(InRootSetFunc &rootFunc); @@ -103,7 +103,7 @@ class pkgDepCache : protected pkgCache::Namespace * are tested to see whether they are actually garbage. If so, * they are marked as such. * - * \return \b false if an error occured. + * \return \b false if an error occurred. */ bool Sweep(); diff --git a/debian/changelog b/debian/changelog index e531be876..6fe4e5ece 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,6 +11,7 @@ apt (0.7.9ubuntu13) hardy; urgency=low - updated for hardy (LP: #195879) * cmdline/apt-get.cc: - fix incorrect help output for -f (LP: #57487) + * typo fixes (LP: #107960) -- diff --git a/methods/http.cc b/methods/http.cc index 6aa4261ff..61321c850 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -367,8 +367,8 @@ bool ServerState::Close() /*}}}*/ // ServerState::RunHeaders - Get the headers before the data /*{{{*/ // --------------------------------------------------------------------- -/* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header - parse error occured */ +/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header + parse error occurred */ int ServerState::RunHeaders() { State = Header; -- cgit v1.2.3-70-g09d2 From 236527f0fffe30fc8988e17d926778c23ac1f902 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Jan 2009 17:39:37 +0100 Subject: LP: #263089 --- apt-pkg/deb/debsystem.cc | 10 +++++----- cmdline/apt-get.cc | 2 +- debian/changelog | 2 ++ methods/gpgv.cc | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) (limited to 'methods') diff --git a/apt-pkg/deb/debsystem.cc b/apt-pkg/deb/debsystem.cc index e1aacdbdb..ee2039c75 100644 --- a/apt-pkg/deb/debsystem.cc +++ b/apt-pkg/deb/debsystem.cc @@ -18,7 +18,7 @@ #include #include #include - + #include #include #include @@ -67,11 +67,11 @@ bool debSystem::Lock() if (LockFD == -1) { if (errno == EACCES || errno == EAGAIN) - return _error->Error("Unable to lock the administration directory (%s), " - "is another process using it?",AdminDir.c_str()); + return _error->Error(_("Unable to lock the administration directory (%s), " + "is another process using it?"),AdminDir.c_str()); else - return _error->Error("Unable to lock the administration directory (%s), " - "are you root?",AdminDir.c_str()); + return _error->Error(_("Unable to lock the administration directory (%s), " + "are you root?"),AdminDir.c_str()); } // See if we need to abort with a dirty journal diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index d5869da43..38775f70d 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1297,7 +1297,7 @@ pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, // show name mismatches if (IsMatch == true && Parse->Package() != Src) - ioprintf(c1out, _("No source package '%s' picking '%s' instead"), Parse->Package(), Src); + ioprintf(c1out, _("No source package '%s' picking '%s' instead"), Parse->Package().c_str(), Src.c_str()); if (VerTag.empty() == false) { diff --git a/debian/changelog b/debian/changelog index 2c4802e3b..478638a63 100644 --- a/debian/changelog +++ b/debian/changelog @@ -9,6 +9,8 @@ apt (0.7.19ubuntu3) jaunty; urgency=low * debian/apt.conf.autoremove: - readd "linux-image" (and friends) to the auto-remove blacklist + * fix some i18n issues (thanks to Gabor Kelemen) + LP: #263089 [ Christian Perrier ] * Translations: diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 9f4683e6e..f3277b300 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -264,7 +264,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) // least one bad signature. good signatures and NoPubKey signatures // happen easily when a file is signed with multiple signatures if(GoodSigners.empty() or !BadSigners.empty()) - return _error->Error(errmsg.c_str()); + return _error->Error("%s", errmsg.c_str()); } // Just pass the raw output up, because passing it as a real data -- cgit v1.2.3-70-g09d2 From ebb461fdf02ee3e038d4b3a4ab1a0a60188edf9a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Feb 2009 12:14:36 +0100 Subject: * [ABI break] merge support for http redirects, thanks to Jeff Licquia and Anthony Towns * [ABI break] use int for the package IDs (thanks to Steve Cotton) - Galician updated. Closes: #509151 - Catalan updated. Closes: #509375 - Vietnamese updated. Closes: #509422 - Traditional Chinese added. Closes: #510664 * COPYING: - Actualized. Removed obsolete Qt section, added GPLv2 clause. (Closes: #440049, #509337) * Clarify the --help for 'purge' (LP: #243948) --- apt-pkg/acquire-method.cc | 32 +++++++++++++++++++ apt-pkg/acquire-method.h | 2 ++ apt-pkg/acquire-worker.cc | 14 +++++++++ apt-pkg/makefile | 2 +- apt-pkg/pkgcache.h | 8 ++--- debian/changelog | 44 ++++++++++++-------------- methods/http.cc | 78 ++++++++++++++++++++++++++++++++++++++++++++++- methods/http.h | 3 ++ methods/makefile | 2 +- 9 files changed, 153 insertions(+), 32 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 26f992bcf..150fbb77b 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -452,6 +452,38 @@ void pkgAcqMethod::Status(const char *Format,...) } /*}}}*/ +// AcqMethod::Redirect - Send a redirect message /*{{{*/ +// --------------------------------------------------------------------- +/* This method sends the redirect message and also manipulates the queue + to keep the pipeline synchronized. */ +void pkgAcqMethod::Redirect(const string &NewURI) +{ + string CurrentURI = ""; + if (Queue != 0) + CurrentURI = Queue->Uri; + + char S[1024]; + snprintf(S, sizeof(S)-50, "103 Redirect\nURI: %s\nNew-URI: %s\n\n", + CurrentURI.c_str(), NewURI.c_str()); + + if (write(STDOUT_FILENO,S,strlen(S)) != (ssize_t)strlen(S)) + exit(100); + + // Change the URI for the request. + Queue->Uri = NewURI; + + /* To keep the pipeline synchronized, move the current request to + the end of the queue, past the end of the current pipeline. */ + FetchItem *I; + for (I = Queue; I->Next != 0; I = I->Next) ; + I->Next = Queue; + Queue = Queue->Next; + I->Next->Next = 0; + if (QueueBack == 0) + QueueBack = I->Next; +} + /*}}}*/ + // AcqMethod::FetchResult::FetchResult - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 18c2cf009..99a4605b1 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -86,6 +86,8 @@ class pkgAcqMethod void Log(const char *Format,...); void Status(const char *Format,...); + void Redirect(const string &NewURI); + int Run(bool Single = false); inline void SetFailReason(string Msg) {FailReason = Msg;}; inline void SetIP(string aIP) {IP = aIP;}; diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 1a754dae9..78c68737c 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -220,6 +220,20 @@ bool pkgAcquire::Worker::RunMessages() Status = LookupTag(Message,"Message"); break; + // 103 Redirect + case 103: + { + if (Itm == 0) + { + _error->Error("Method gave invalid 103 Redirect message"); + break; + } + + string NewURI = LookupTag(Message,"New-URI",URI.c_str()); + Itm->URI = NewURI; + break; + } + // 200 URI Start case 200: { diff --git a/apt-pkg/makefile b/apt-pkg/makefile index 1b78c94f6..087f17740 100644 --- a/apt-pkg/makefile +++ b/apt-pkg/makefile @@ -13,7 +13,7 @@ include ../buildlib/defaults.mak # methods/makefile - FIXME LIBRARY=apt-pkg LIBEXT=$(GLIBC_VER)$(LIBSTDCPP_VER) -MAJOR=4.6 +MAJOR=4.7 MINOR=0 SLIBS=$(PTHREADLIB) $(INTLLIBS) -lutil APT_DOMAIN:=libapt-pkg$(MAJOR) diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 59d5003bb..759e9a225 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -214,7 +214,7 @@ struct pkgCache::Package unsigned char InstState; // Flags unsigned char CurrentState; // State - unsigned short ID; + unsigned int ID; unsigned long Flags; }; @@ -235,7 +235,7 @@ struct pkgCache::PackageFile // Linked list map_ptrloc NextFile; // PackageFile - unsigned short ID; + unsigned int ID; time_t mtime; // Modification time for the file }; @@ -272,7 +272,7 @@ struct pkgCache::Version map_ptrloc Size; // These are the .deb size map_ptrloc InstalledSize; unsigned short Hash; - unsigned short ID; + unsigned int ID; unsigned char Priority; }; @@ -289,7 +289,7 @@ struct pkgCache::Description map_ptrloc NextDesc; // Description map_ptrloc ParentPkg; // Package - unsigned short ID; + unsigned int ID; }; struct pkgCache::Dependency diff --git a/debian/changelog b/debian/changelog index 478638a63..bfbe6be5c 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,29 +6,11 @@ apt (0.7.19ubuntu3) jaunty; urgency=low * apt-inst/contrib/arfile.cc: - support members ending with '/' as well (thanks to Michal Cihr, closes: #500988) - * debian/apt.conf.autoremove: - - readd "linux-image" (and friends) to the auto-remove + * debian/apt.conf.autoremove: + - readd "linux-image" (and friends) to the auto-remove blacklist - * fix some i18n issues (thanks to Gabor Kelemen) - LP: #263089 - - [ Christian Perrier ] - * Translations: - - Galician updated. Closes: #509151 - - Catalan updated. Closes: #509375 - - Vietnamese updated. Closes: #509422 - - Traditional Chinese added. Closes: #510664 - - [ Eugene V. Lyubimkin ] - * COPYING: - - Actualized. Removed obsolete Qt section, added GPLv2 clause. - (Closes: #440049, #509337) - - -- Michael Vogt Mon, 05 Jan 2009 08:59:20 +0100 - -apt (0.7.19ubuntu2) jaunty; urgency=low - - [ Michael Vogt ] + * fix some i18n issues (thanks to Gabor Kelemen) + LP: #263089 * apt-pkg/deb/dpkgpm.cc: - filter "ENOMEM" errors when creating apport reports * cmdline/apt-get.cc: @@ -40,15 +22,27 @@ apt (0.7.19ubuntu2) jaunty; urgency=low - add new strprintf() function to make i18n strings easier * apt-pkg/dev/debsystem.cc: - add missing apti18n.h header - - [ Dereck Wonnacott ] - * Clarify the --help for 'purge' (LP: #243948) + * [ABI break] merge support for http redirects, thanks to + Jeff Licquia and Anthony Towns + * [ABI break] use int for the package IDs (thanks to Steve Cotton) [ Christian Perrier ] * Translations: + - Galician updated. Closes: #509151 + - Catalan updated. Closes: #509375 + - Vietnamese updated. Closes: #509422 + - Traditional Chinese added. Closes: #510664 - French corrected (remove awful use of first person) - Finnish updated. Closes: #508449 + [ Eugene V. Lyubimkin ] + * COPYING: + - Actualized. Removed obsolete Qt section, added GPLv2 clause. + (Closes: #440049, #509337) + + [ Dereck Wonnacott ] + * Clarify the --help for 'purge' (LP: #243948) + [ Eugene V. Lyubimkin ] * doc/examples/sources.list: - Removed obsolete commented non-us deb-src entry, replaced it with diff --git a/methods/http.cc b/methods/http.cc index b0fb89fda..44274bd78 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -38,6 +38,7 @@ #include #include #include +#include #include // Internet stuff @@ -56,6 +57,7 @@ int HttpMethod::FailFd = -1; time_t HttpMethod::FailTime = 0; unsigned long PipelineDepth = 10; unsigned long TimeOut = 120; +bool AllowRedirect = false; bool Debug = false; URI Proxy; @@ -627,6 +629,12 @@ bool ServerState::HeaderLine(string Line) return true; } + if (stringcasecmp(Tag,"Location:") == 0) + { + Location = Val; + return true; + } + return true; } /*}}}*/ @@ -899,7 +907,9 @@ bool HttpMethod::ServerDie(ServerState *Srv) 1 - IMS hit 3 - Unrecoverable error 4 - Error with error content page - 5 - Unrecoverable non-server error (close the connection) */ + 5 - Unrecoverable non-server error (close the connection) + 6 - Try again with a new or changed URI + */ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) { // Not Modified @@ -911,6 +921,27 @@ int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) return 1; } + /* Redirect + * + * Note that it is only OK for us to treat all redirection the same + * because we *always* use GET, not other HTTP methods. There are + * three redirection codes for which it is not appropriate that we + * redirect. Pass on those codes so the error handling kicks in. + */ + if (AllowRedirect + && (Srv->Result > 300 && Srv->Result < 400) + && (Srv->Result != 300 // Multiple Choices + && Srv->Result != 304 // Not Modified + && Srv->Result != 306)) // (Not part of HTTP/1.1, reserved) + { + if (!Srv->Location.empty()) + { + NextURI = Srv->Location; + return 6; + } + /* else pass through for error message */ + } + /* We have a reply we dont handle. This should indicate a perm server failure */ if (Srv->Result < 200 || Srv->Result >= 300) @@ -1028,6 +1059,7 @@ bool HttpMethod::Configuration(string Message) if (pkgAcqMethod::Configuration(Message) == false) return false; + AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut); PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", PipelineDepth); @@ -1041,6 +1073,10 @@ bool HttpMethod::Configuration(string Message) /* */ int HttpMethod::Loop() { + typedef vector StringVector; + typedef vector::iterator StringVectorIterator; + map Redirected; + signal(SIGTERM,SigTerm); signal(SIGINT,SigTerm); @@ -1227,6 +1263,46 @@ int HttpMethod::Loop() break; } + // Try again with a new URL + case 6: + { + // Clear rest of response if there is content + if (Server->HaveContent) + { + File = new FileFd("/dev/null",FileFd::WriteExists); + Server->RunData(); + delete File; + File = 0; + } + + /* Detect redirect loops. No more redirects are allowed + after the same URI is seen twice in a queue item. */ + StringVector &R = Redirected[Queue->DestFile]; + bool StopRedirects = false; + if (R.size() == 0) + R.push_back(Queue->Uri); + else if (R[0] == "STOP" || R.size() > 10) + StopRedirects = true; + else + { + for (StringVectorIterator I = R.begin(); I != R.end(); I++) + if (Queue->Uri == *I) + { + R[0] = "STOP"; + break; + } + + R.push_back(Queue->Uri); + } + + if (StopRedirects == false) + Redirect(NextURI); + else + Fail(); + + break; + } + default: Fail(_("Internal error")); break; diff --git a/methods/http.h b/methods/http.h index dec5cd80f..bc076e1f8 100644 --- a/methods/http.h +++ b/methods/http.h @@ -99,6 +99,7 @@ struct ServerState enum {Chunked,Stream,Closes} Encoding; enum {Header, Data} State; bool Persistent; + string Location; // This is a Persistent attribute of the server itself. bool Pipeline; @@ -145,6 +146,8 @@ class HttpMethod : public pkgAcqMethod protected: virtual bool Fetch(FetchItem *); + string NextURI; + public: friend class ServerState; diff --git a/methods/makefile b/methods/makefile index 5c4fa82bf..1d022be90 100644 --- a/methods/makefile +++ b/methods/makefile @@ -7,7 +7,7 @@ include ../buildlib/defaults.mak BIN := $(BIN)/methods # FIXME.. -LIB_APT_PKG_MAJOR = 4.6 +LIB_APT_PKG_MAJOR = 4.7 APT_DOMAIN := libapt-pkg$(LIB_APT_PKG_MAJOR) # The file method -- cgit v1.2.3-70-g09d2 From 8872c430df84660fa994db0ebfa67861239fe739 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Feb 2009 14:36:36 +0100 Subject: * methods/https.cc: - add Acquire::https::AllowRedirect support --- debian/changelog | 2 ++ doc/examples/configure-index | 6 ++++-- methods/https.cc | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 89848f31c..23a0514f6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -32,6 +32,8 @@ apt (0.7.20.2ubuntu1) jaunty; urgency=low * cmdline/apt-get.cc: - default to "false" for the "APT::Get::Build-Dep-Automatic" option (follow debian here) + * methods/https.cc: + - add Acquire::https::AllowRedirect support [ Dereck Wonnacott ] * Clarify the --help for 'purge' (LP: #243948) diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 5cc6187a8..a3ccdded8 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -136,7 +136,8 @@ Acquire Proxy::http.us.debian.org "DIRECT"; // Specific per-host setting Timeout "120"; Pipeline-Depth "5"; - + AllowRedirect "true"; + // Cache Control. Note these do not work with Squid 2.0.2 No-Cache "false"; Max-Age "86400"; // 1 Day age on index files @@ -153,7 +154,8 @@ Acquire Verify-Peer "false"; SslCert "/etc/apt/some.pem"; CaPath "/etc/ssl/certs"; - Verify-Host" "2"; + Verify-Host" "true"; + AllowRedirect "true"; }; ftp diff --git a/methods/https.cc b/methods/https.cc index 98dfeefa1..87de54589 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -208,6 +208,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout); + // set redirect options and default to 10 redirects + bool AllowRedirect = _config->FindI("Acquire::https::AllowRedirect", true); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10); + // debug if(_config->FindB("Debug::Acquire::https", false)) curl_easy_setopt(curl, CURLOPT_VERBOSE, true); -- cgit v1.2.3-70-g09d2 From 7e522e9e0e4808202061e0a73345bd8b18eccde3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 9 Feb 2009 14:38:11 +0100 Subject: * apt-pkg/pkgcache.cc: - do not run "dpkg --configure pkg" if pkg is in trigger-awaited state (LP: #322955) - do not unlink files in partial/ (thanks to robbiew) --- apt-pkg/pkgcache.cc | 8 ++++++-- debian/changelog | 4 ++++ methods/https.cc | 1 - 3 files changed, 10 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 8eb62089a..4fbf42c4b 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -275,8 +275,12 @@ pkgCache::PkgIterator::OkState pkgCache::PkgIterator::State() const if (Pkg->CurrentState == pkgCache::State::UnPacked || Pkg->CurrentState == pkgCache::State::HalfConfigured || - Pkg->CurrentState == pkgCache::State::TriggersPending || - Pkg->CurrentState == pkgCache::State::TriggersAwaited) + //we don't need to care for triggers awaiting packages + //dpkg will deal with them automatically when the + //trigger pending action is run (those packages are usually + //in half-configured or triggers-pending state) + //Pkg->CurrentState == pkgCache::State::TriggersAwaited + Pkg->CurrentState == pkgCache::State::TriggersPending) return NeedsConfigure; if (Pkg->CurrentState == pkgCache::State::HalfInstalled || diff --git a/debian/changelog b/debian/changelog index 23a0514f6..0e2d3afe6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -32,8 +32,12 @@ apt (0.7.20.2ubuntu1) jaunty; urgency=low * cmdline/apt-get.cc: - default to "false" for the "APT::Get::Build-Dep-Automatic" option (follow debian here) + * apt-pkg/pkgcache.cc: + - do not run "dpkg --configure pkg" if pkg is in trigger-awaited + state (LP: #322955) * methods/https.cc: - add Acquire::https::AllowRedirect support + - do not unlink files in partial/ (thanks to robbiew) [ Dereck Wonnacott ] * Clarify the --help for 'purge' (LP: #243948) diff --git a/methods/https.cc b/methods/https.cc index 87de54589..8bf44b52a 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -253,7 +253,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // cleanup if(success != 0) { - unlink(File->Name().c_str()); _error->Error("%s", curl_errorstr); Fail(); return true; -- cgit v1.2.3-70-g09d2 From 5dad4134da37576de12721f34165c1fe4d138b3c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 19 Mar 2009 15:42:25 +0100 Subject: * methods/mirror.cc: - when download the mirror file and the server is down, return a propper error message (LP: #278635) --- cmdline/apt-report-mirror-failure | 2 +- debian/changelog | 8 ++++++++ methods/mirror.cc | 18 ++++++++++++++++-- 3 files changed, 25 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure index 1567e78e3..ef77d4954 100755 --- a/cmdline/apt-report-mirror-failure +++ b/cmdline/apt-report-mirror-failure @@ -5,7 +5,7 @@ import urllib import apt_pkg apt_pkg.init() -url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", None) +url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", "") #"http://people.ubuntu.com:9000/mirror-failure") #"http://localhost:9000/mirror-failure") if not url: diff --git a/debian/changelog b/debian/changelog index bef5c3cad..5a764383e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +apt (0.7.20.2ubuntu3) jaunty; urgency=low + + * methods/mirror.cc: + - when download the mirror file and the server is down, + return a propper error message (LP: #278635) + + -- Michael Vogt Thu, 19 Mar 2009 15:42:15 +0100 + apt (0.7.20.2ubuntu2) jaunty; urgency=low * apt-pkg/deb/dpkgpm.cc: diff --git a/methods/mirror.cc b/methods/mirror.cc index bdd783cc7..b3a956b95 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -123,6 +123,8 @@ bool MirrorMethod::Clean(string Dir) bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) { + if(Debug) + clog << "MirrorMethod::DownloadMirrorFile(): " << endl; // check the file, if it is not older than RefreshInterval just use it // otherwise try to get a new one @@ -251,6 +253,9 @@ string MirrorMethod::GetMirrorFileName(string mirror_uri_str) depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { + if(Debug) + clog << "MirrorMethod::Fetch()" << endl; + // the http method uses Fetch(0) as a way to update the pipeline, // just let it do its work in this case - Fetch() with a valid // Itm will always run before the first Fetch(0) @@ -269,8 +274,17 @@ bool MirrorMethod::Fetch(FetchItem *Itm) DownloadMirrorFile(Itm->Uri); } - if(Mirror.empty()) - SelectMirror(); + if(Mirror.empty()) { + if(!SelectMirror()) { + // no valid mirror selected, something went wrong downloading + // from the master mirror site most likely and there is + // no old mirror file availalbe + return false; + } + } + if(Debug) + clog << "selected mirror: " << Mirror << endl; + for (FetchItem *I = Queue; I != 0; I = I->Next) { -- cgit v1.2.3-70-g09d2 From 0b77f4775db7bc45964e0337b8978a170b3f0483 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Apr 2009 22:52:11 +0200 Subject: * methods/gpgv.cc: - properly check for expired and revoked keys (closes: #433091) LP: #356012 --- debian/changelog | 5 +++++ methods/gpgv.cc | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 43827dada..9618cc1c6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -4,6 +4,11 @@ apt (0.7.20.2ubuntu6) jaunty; urgency=low * apt.cron.daily: catch invalid dates due to DST time changes in the stamp files + [ Michael Vogt ] + * methods/gpgv.cc: + - properly check for expired and revoked keys (closes: #433091) + LP: #356012 + -- Michael Vogt Wed, 08 Apr 2009 22:39:50 +0200 apt (0.7.20.2ubuntu5) jaunty; urgency=low diff --git a/methods/gpgv.cc b/methods/gpgv.cc index f3277b300..150c1d315 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -17,13 +17,18 @@ #define GNUPGBADSIG "[GNUPG:] BADSIG" #define GNUPGNOPUBKEY "[GNUPG:] NO_PUBKEY" #define GNUPGVALIDSIG "[GNUPG:] VALIDSIG" +#define GNUPGGOODSIG "[GNUPG:] GOODSIG" +#define GNUPGKEYEXPIRED "[GNUPG:] KEYEXPIRED" +#define GNUPGREVKEYSIG "[GNUPG:] REVKEYSIG" #define GNUPGNODATA "[GNUPG:] NODATA" class GPGVMethod : public pkgAcqMethod { private: string VerifyGetSigners(const char *file, const char *outfile, - vector &GoodSigners, vector &BadSigners, + vector &GoodSigners, + vector &BadSigners, + vector &WorthlessSigners, vector &NoPubKeySigners); protected: @@ -37,6 +42,7 @@ class GPGVMethod : public pkgAcqMethod string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, vector &GoodSigners, vector &BadSigners, + vector &WorthlessSigners, vector &NoPubKeySigners) { // setup a (empty) stringstream for formating the return value @@ -179,15 +185,27 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, std::cerr << "Got NODATA! " << std::endl; BadSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); } - if (strncmp(buffer, GNUPGVALIDSIG, sizeof(GNUPGVALIDSIG)-1) == 0) + if (strncmp(buffer, GNUPGKEYEXPIRED, sizeof(GNUPGKEYEXPIRED)-1) == 0) + { + if (_config->FindB("Debug::Acquire::gpgv", false)) + std::cerr << "Got KEYEXPIRED! " << std::endl; + WorthlessSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); + } + if (strncmp(buffer, GNUPGREVKEYSIG, sizeof(GNUPGREVKEYSIG)-1) == 0) + { + if (_config->FindB("Debug::Acquire::gpgv", false)) + std::cerr << "Got REVKEYSIG! " << std::endl; + WorthlessSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); + } + if (strncmp(buffer, GNUPGGOODSIG, sizeof(GNUPGGOODSIG)-1) == 0) { char *sig = buffer + sizeof(GNUPGPREFIX); - char *p = sig + sizeof("VALIDSIG"); + char *p = sig + sizeof("GOODSIG"); while (*p && isxdigit(*p)) p++; *p = 0; if (_config->FindB("Debug::Acquire::gpgv", false)) - std::cerr << "Got VALIDSIG, key ID:" << sig << std::endl; + std::cerr << "Got GOODSIG, key ID:" << sig << std::endl; GoodSigners.push_back(string(sig)); } } @@ -227,6 +245,8 @@ bool GPGVMethod::Fetch(FetchItem *Itm) string keyID; vector GoodSigners; vector BadSigners; + // a worthless signature is a expired or revoked one + vector WorthlessSigners; vector NoPubKeySigners; FetchResult Res; @@ -235,13 +255,14 @@ bool GPGVMethod::Fetch(FetchItem *Itm) // Run gpgv on file, extract contents and get the key ID of the signer string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), - GoodSigners, BadSigners, NoPubKeySigners); + GoodSigners, BadSigners, WorthlessSigners, + NoPubKeySigners); if (GoodSigners.empty() || !BadSigners.empty() || !NoPubKeySigners.empty()) { string errmsg; // In this case, something bad probably happened, so we just go // with what the other method gave us for an error message. - if (BadSigners.empty() && NoPubKeySigners.empty()) + if (BadSigners.empty() && WorthlessSigners.empty() && NoPubKeySigners.empty()) errmsg = msg; else { @@ -252,6 +273,13 @@ bool GPGVMethod::Fetch(FetchItem *Itm) I != BadSigners.end(); I++) errmsg += (*I + "\n"); } + if (!WorthlessSigners.empty()) + { + errmsg += _("The following signatures were invalid:\n"); + for (vector::iterator I = WorthlessSigners.begin(); + I != WorthlessSigners.end(); I++) + errmsg += (*I + "\n"); + } if (!NoPubKeySigners.empty()) { errmsg += _("The following signatures couldn't be verified because the public key is not available:\n"); -- cgit v1.2.3-70-g09d2 From 217d575b5713f4d9275177b58a36c581a7880c03 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 14 Jul 2009 11:22:41 +0200 Subject: * apt-pkg/acquire-worker.cc: - show error details of failed methods * apt-pkg/contrib/fileutl.cc: - if a process aborts with signal, show signal number * methods/http.cc: - ignore SIGPIPE, we deal with EPIPE from write in HttpMethod::ServerDie() (LP: #385144) --- apt-pkg/acquire-worker.cc | 7 ++----- apt-pkg/contrib/fileutl.cc | 7 +++++-- debian/changelog | 12 ++++++++++++ methods/http_main.cc | 5 +++++ 4 files changed, 24 insertions(+), 7 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 78c68737c..4f0b52af9 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -527,10 +527,6 @@ bool pkgAcquire::Worker::OutFdReady() if (Res <= 0) return MethodFailure(); - - // Hmm.. this should never happen. - if (Res < 0) - return true; OutQueue.erase(0,Res); if (OutQueue.empty() == true) @@ -558,7 +554,8 @@ bool pkgAcquire::Worker::MethodFailure() { _error->Error("Method %s has died unexpectedly!",Access.c_str()); - ExecWait(Process,Access.c_str(),true); + // do not reap the child here to show meaningfull error to the user + ExecWait(Process,Access.c_str(),false); Process = -1; close(InFd); close(OutFd); diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index a5976cf3a..a7de09c44 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -450,8 +450,11 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) { if (Reap == true) return false; - if (WIFSIGNALED(Status) != 0 && WTERMSIG(Status) == SIGSEGV) - return _error->Error(_("Sub-process %s received a segmentation fault."),Name); + if (WIFSIGNALED(Status) != 0) + if( WTERMSIG(Status) == SIGSEGV) + return _error->Error(_("Sub-process %s received a segmentation fault."),Name); + else + return _error->Error(_("Sub-process %s received signal %u."),Name, WTERMSIG(Status)); if (WIFEXITED(Status) != 0) return _error->Error(_("Sub-process %s returned an error code (%u)"),Name,WEXITSTATUS(Status)); diff --git a/debian/changelog b/debian/changelog index 2db8ae677..d6e6a5997 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +apt (0.7.21ubuntu2~ppa1) karmic; urgency=low + + * apt-pkg/acquire-worker.cc: + - show error details of failed methods + * apt-pkg/contrib/fileutl.cc: + - if a process aborts with signal, show signal number + * methods/http.cc: + - ignore SIGPIPE, we deal with EPIPE from write in + HttpMethod::ServerDie() (LP: #385144) + + -- Michael Vogt Thu, 02 Jul 2009 14:11:35 +0200 + apt (0.7.21ubuntu1) karmic; urgency=low * merged from the debian-sid bzr branch diff --git a/methods/http_main.cc b/methods/http_main.cc index 2c46ab19d..7815c2fc1 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -1,5 +1,6 @@ #include #include +#include #include "connect.h" #include "rfc2553emu.h" @@ -10,6 +11,10 @@ int main() { setlocale(LC_ALL, ""); + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + HttpMethod Mth; return Mth.Loop(); } -- cgit v1.2.3-70-g09d2 From 056539eeb8eb84cea65c0c15466111b978ee156d Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 30 Jul 2009 18:25:16 +0200 Subject: methods/http.cc: remove merge artifact --- methods/http.cc | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index 476fb751f..859dbd39c 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -1316,15 +1316,4 @@ int HttpMethod::Loop() } /*}}}*/ -int main() -{ - setlocale(LC_ALL, ""); - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - - HttpMethod Mth; - return Mth.Loop(); -} - -- cgit v1.2.3-70-g09d2