From 5f6b130d6342965bfa49beb9413bdf742440b8ab Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 21:24:29 +0100 Subject: * prototype of mirror method added --- methods/mirror.h | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 methods/mirror.h (limited to 'methods/mirror.h') diff --git a/methods/mirror.h b/methods/mirror.h new file mode 100644 index 000000000..ca3428830 --- /dev/null +++ b/methods/mirror.h @@ -0,0 +1,40 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +/* ###################################################################### + + MIRROR Aquire Method - This is the MIRROR aquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_MIRROR_H +#define APT_MIRROR_H + + +#include + +using std::cout; +using std::cerr; +using std::endl; + +#include "http.h" + +class MirrorMethod : public pkgAcqMethod +{ + FetchResult Res; + string Mirror; + string BaseUri; + string MirrorFile; + bool HasMirrorFile; + + protected: + bool GetMirrorFile(string uri); + bool SelectMirror(); + public: + MirrorMethod(); + virtual bool Fetch(FetchItem *Itm); +}; + + +#endif -- cgit v1.2.3-70-g09d2 From 14e097c105c49a102a642c8108979356c5cc3152 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 22:59:35 +0100 Subject: * working mirror implementation based on http method --- apt-pkg/acquire-method.h | 7 +++-- methods/mirror.cc | 72 ++++++++++++++++++++++++++++++++++++++---------- methods/mirror.h | 15 +++++++--- 3 files changed, 73 insertions(+), 21 deletions(-) (limited to 'methods/mirror.h') diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index f46209d12..e0e7c1d09 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -63,9 +63,10 @@ class pkgAcqMethod // Outgoing messages void Fail(bool Transient = false); inline void Fail(const char *Why, bool Transient = false) {Fail(string(Why),Transient);}; - void Fail(string Why, bool Transient = false); - void URIStart(FetchResult &Res); - void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + bool MediaFail(string Required,string Drive); virtual void Exit() {}; diff --git a/methods/mirror.cc b/methods/mirror.cc index fad076e91..c4d911a3c 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -17,6 +17,8 @@ #include #include +#include + using namespace std; #include "mirror.h" @@ -25,31 +27,50 @@ using namespace std; /*}}}*/ MirrorMethod::MirrorMethod() - : pkgAcqMethod("1.2",Pipeline | SendConfig), HasMirrorFile(false) + : HttpMethod(), HasMirrorFile(false) { #if 0 HasMirrorFile=true; - BaseUri="http://people.ubuntu.com/~mvo/mirror/mirrors///"; + BaseUri="mirror://people.ubuntu.com/~mvo/mirror/mirrors"; + MirrorFile="/var/lib/apt/lists/people.ubuntu.com_%7emvo_apt_mirror_mirrors"; Mirror="http://de.archive.ubuntu.com/ubuntu/"; #endif }; +// HttpMethod::Configuration - Handle a configuration message /*{{{*/ +// --------------------------------------------------------------------- +/* We stash the desired pipeline depth */ +bool MirrorMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + Debug = _config->FindB("Debug::Acquire::mirror",false); + + return true; +} + /*}}}*/ + + bool MirrorMethod::GetMirrorFile(string uri) { string Marker = _config->Find("Acquire::Mirror::MagicMarker","///"); BaseUri = uri.substr(0,uri.find(Marker)); - BaseUri.replace(0,strlen("mirror://"),"http://"); + + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; + if(Debug) + { + cerr << "base-uri: " << BaseUri << endl; + cerr << "mirror-file: " << MirrorFile << endl; + } // FIXME: fetch it with curl pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, BaseUri, "", 0, "", "", "", MirrorFile); + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - cerr << "fetch-result: " << res << endl; if(res) HasMirrorFile = true; @@ -61,7 +82,9 @@ bool MirrorMethod::SelectMirror() { ifstream in(MirrorFile.c_str()); getline(in, Mirror); - cerr << "Mirror: " << Mirror << endl; + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + return true; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -77,21 +100,42 @@ bool MirrorMethod::Fetch(FetchItem *Itm) SelectMirror(); } - // change the items in the queue - Itm->Uri.replace(0,BaseUri.size()+_config->Find("Acquire::Mirror::MagicMarker","///").size()+2/*len("mirror")-len("http")*/,Mirror); - cerr << "new Fetch-uri: " << Itm->Uri << endl; + if(Queue->Uri.find("mirror://") != string::npos) + Queue->Uri.replace(0,BaseUri.size(),Mirror); - // FIXME: fetch it with! - + // now run the real fetcher + return HttpMethod::Fetch(Itm); }; +void MirrorMethod::Fail(string Err,bool Transient) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::Fail(Err, Transient); +} + +void MirrorMethod::URIStart(FetchResult &Res) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIStart(Res); +} + +void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) +{ + if(Queue->Uri.find("http://") != string::npos) + Queue->Uri.replace(0,Mirror.size(), BaseUri); + pkgAcqMethod::URIDone(Res, Alt); +} + + int main() { setlocale(LC_ALL, ""); MirrorMethod Mth; - return Mth.Run(); + return Mth.Loop(); } diff --git a/methods/mirror.h b/methods/mirror.h index ca3428830..91cf8c64f 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -20,17 +20,24 @@ using std::endl; #include "http.h" -class MirrorMethod : public pkgAcqMethod +class MirrorMethod : public HttpMethod { FetchResult Res; - string Mirror; - string BaseUri; - string MirrorFile; + string BaseUri; // the original mirror://... url + string Mirror; // the selected mirror uri (http://...) + string MirrorFile; // bool HasMirrorFile; + bool Debug; + protected: bool GetMirrorFile(string uri); bool SelectMirror(); + virtual void Fail(string Why, bool Transient = false); + virtual void URIStart(FetchResult &Res); + virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + virtual bool Configuration(string Message); + public: MirrorMethod(); virtual bool Fetch(FetchItem *Itm); -- cgit v1.2.3-70-g09d2 From 86c17f0a4bd3c0e3dde5bb58b3df9c41e5a88763 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Nov 2006 23:09:26 +0100 Subject: * todo added --- methods/mirror.cc | 8 ++++++++ methods/mirror.h | 3 +++ 2 files changed, 11 insertions(+) (limited to 'methods/mirror.h') diff --git a/methods/mirror.cc b/methods/mirror.cc index c4d911a3c..c49764044 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -26,6 +26,14 @@ using namespace std; /*}}}*/ +/* + * TODO: + * - support keeping the mirror file around (evil listclearer strikes again) + * - better method to download than having a pkgAcquire interface here + * - testing :) + * + */ + MirrorMethod::MirrorMethod() : HttpMethod(), HasMirrorFile(false) { diff --git a/methods/mirror.h b/methods/mirror.h index 91cf8c64f..a47b8fbf1 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -23,6 +23,7 @@ using std::endl; class MirrorMethod : public HttpMethod { FetchResult Res; + // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) string MirrorFile; // @@ -33,6 +34,8 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + + // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); virtual void URIStart(FetchResult &Res); virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); -- cgit v1.2.3-70-g09d2 From d731f9c574138a9cb31c586a8f1e7d7181a44456 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 8 Jan 2007 11:19:32 +0100 Subject: * apt-pkg/init.cc: - added Dir::State::Mirrors * doc/examples/configure-index: - added Acquire::mirror::RefreshInterval * methods/mirror.{cc,h}: - download the mirror file into Dir::State::Mirrors - added RefreshInterval option to not ask for the mirror file too often --- apt-pkg/init.cc | 1 + debian/apt.dirs | 1 + doc/examples/configure-index | 7 +++++- methods/mirror.cc | 56 ++++++++++++++++++++++++++++++++++++-------- methods/mirror.h | 1 + po/apt-all.pot | 4 ++-- 6 files changed, 57 insertions(+), 13 deletions(-) (limited to 'methods/mirror.h') diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index b47378d4a..fe085a75e 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -54,6 +54,7 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.Set("Dir::State::lists","lists/"); Cnf.Set("Dir::State::cdroms","cdroms.list"); + Cnf.Set("Dir::State::mirrors","mirrors/"); // Cache Cnf.Set("Dir::Cache","var/cache/apt/"); diff --git a/debian/apt.dirs b/debian/apt.dirs index e1cb738fa..b81878baa 100644 --- a/debian/apt.dirs +++ b/debian/apt.dirs @@ -5,5 +5,6 @@ etc/apt etc/apt/sources.list.d var/cache/apt/archives/partial var/lib/apt/lists/partial +var/lib/apt/mirrors/partial var/lib/apt/periodic usr/share/bug/apt diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 0f0abc30c..cb20d767b 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -160,6 +160,11 @@ Acquire { Options {"--ignore-time-conflict";} // not very usefull on a normal system }; + + mirror + { + RefreshInterval "360"; // refresh interval in minutes + }; }; // Directory layout @@ -258,9 +263,9 @@ Debug Acquire::Ftp "false"; // Show ftp command traffic Acquire::Http "false"; // Show http command traffic Acquire::gpgv "false"; // Show the gpgv traffic + Acquire::Mirror "false"; // Show debugging of the mirror method aptcdrom "false"; // Show found package files IdentCdrom "false"; - } /* Whatever you do, do not use this configuration file!! Take out ONLY diff --git a/methods/mirror.cc b/methods/mirror.cc index 4de981522..e70a40f55 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -18,6 +18,7 @@ #include #include #include +#include using namespace std; @@ -28,11 +29,16 @@ using namespace std; /* * TODO: + * - send expected checksum to the mirror method so that + some checking/falling back can be done here already + * - keep the mirror file around in /var/lib/apt/mirrors + * can't be put into lists/ because of the listclearer + * cleanup by time (mtime relative to the other mtimes) + * - use a TTL time the mirror file is fetched again (6h?) + * - deal with runing as non-root because we can't write to the lists + dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - support keeping the mirror file around (evil listclearer strikes again) - * -> /var/lib/apt/mirrors dir? how to cleanup? by time? - * - provide some TTL time until the mirror file is get again (1h? 6h?) - * - deal with runing as non-root (we can't write to the lists dir then) + * - magicmarker is (a bit) evil * - testing :) */ @@ -60,6 +66,12 @@ bool MirrorMethod::Configuration(string Message) } /*}}}*/ +// clean the mirrors dir based on ttl information +bool MirrorMethod::Clean(string dir) +{ + +} + bool MirrorMethod::GetMirrorFile(string uri) { @@ -69,7 +81,7 @@ bool MirrorMethod::GetMirrorFile(string uri) string fetch = BaseUri; fetch.replace(0,strlen("mirror://"),"http://"); - MirrorFile = _config->FindDir("Dir::State::lists") + URItoFileName(BaseUri); + MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { @@ -77,19 +89,43 @@ bool MirrorMethod::GetMirrorFile(string uri) cerr << "mirror-file: " << MirrorFile << endl; } - // FIXME: fetch it with curl + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + HasMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << ", re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now pkgAcquire Fetcher; new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); bool res = (Fetcher.Run() == pkgAcquire::Continue); - - if(res) + if(res) HasMirrorFile = true; Fetcher.Shutdown(); - return true; + return res; } bool MirrorMethod::SelectMirror() { + // FIXME: make the mirror selection more clever, do not + // just use the first one! ifstream in(MirrorFile.c_str()); getline(in, Mirror); if(Debug) @@ -103,7 +139,7 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // get mirror information + // select mirror only once per session if(!HasMirrorFile) { GetMirrorFile(Itm->Uri); diff --git a/methods/mirror.h b/methods/mirror.h index a47b8fbf1..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -34,6 +34,7 @@ class MirrorMethod : public HttpMethod protected: bool GetMirrorFile(string uri); bool SelectMirror(); + bool Clean(string dir); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index b3da2ba9a..42c43fe8d 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -2238,12 +2238,12 @@ msgstr "" msgid "Please insert the disc labeled: '%s' in the drive '%s' and press enter." msgstr "" -#: apt-pkg/init.cc:120 +#: apt-pkg/init.cc:121 #, c-format msgid "Packaging system '%s' is not supported" msgstr "" -#: apt-pkg/init.cc:136 +#: apt-pkg/init.cc:137 msgid "Unable to determine a suitable packaging system type" msgstr "" -- cgit v1.2.3-70-g09d2 From cae9cdcefc34eba6d023cb30cbbdb9bbf909b8fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 17 Jan 2007 16:35:56 +0100 Subject: * basic error reporting from apt in place now (ReportMirrorFailures()) --- apt-pkg/acquire-item.cc | 10 +++++++++- apt-pkg/acquire-item.h | 1 + apt-pkg/acquire-method.cc | 1 + apt-pkg/acquire-method.h | 1 + methods/mirror.cc | 27 +++++++++++++++++++++------ methods/mirror.h | 1 + po/apt-all.pot | 16 ++++++++-------- 7 files changed, 42 insertions(+), 15 deletions(-) (limited to 'methods/mirror.h') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 8ec4ba2c0..7b2a89763 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -176,7 +176,8 @@ string pkgAcqIndex::Custom600Headers() struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true"; - + if(ExpectedMD5 != "") + return "\nExpectedMD5: " + ExpectedMD5; return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ @@ -1015,6 +1016,13 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } } /*}}}*/ +// --------------------------------------------------------------------- +string pkgAcqArchive::Custom600Headers() +{ + if(MD5 != "") + return "\nExpectedMD5: " + MD5; + return ""; +} // AcqArchive::IsTrusted - Determine whether this archive comes from a // trusted source /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index da1bea801..c9cd75321 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -205,6 +205,7 @@ class pkgAcqArchive : public pkgAcquire::Item virtual string ShortDesc() {return Desc.ShortDesc;}; virtual void Finished(); virtual bool IsTrusted(); + virtual string Custom600Headers(); pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 41b832f3b..638797657 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -368,6 +368,7 @@ int pkgAcqMethod::Run(bool Single) Tmp->Uri = LookupTag(Message,"URI"); Tmp->DestFile = LookupTag(Message,"FileName"); + Tmp->ExpectedMD5 = LookupTag(Message,"ExpectedMD5"); if (StrToTime(LookupTag(Message,"Last-Modified"),Tmp->LastModified) == false) Tmp->LastModified = 0; Tmp->IndexFile = StringToBool(LookupTag(Message,"Index-File"),false); diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index e0e7c1d09..9e908d32c 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -33,6 +33,7 @@ class pkgAcqMethod string DestFile; time_t LastModified; bool IndexFile; + string ExpectedMD5; }; struct FetchResult diff --git a/methods/mirror.cc b/methods/mirror.cc index f08b324ec..428726a3d 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -32,14 +32,14 @@ using namespace std; /* * TODO: - * - send expected checksum to the mirror method so that - some checking/falling back can be done here already - use pkgAcquire::Custom600Header() for this? what about gpgv - failures? + * - what about gpgv failures? better standard format for errors + to send back to LP #OR# * - implement it at the pkgAcquire::Item::Failed() level but then - we need to send back what uri exactly was failing - + we need to send back what uri exactly was failing + [mvo: the problem with this approach is ::Failed() is not really + called for all failures :/ e.g. md5sum mismatch in a archive + is not] * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here @@ -214,6 +214,9 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { + // FIXME: queue next mirror? + ReportMirrorFailure(Err); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -228,11 +231,23 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { + // FIXME: queue next mirror? + if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) + ReportMirrorFailure("499 Hash mismatch"); + if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } +void MirrorMethod::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed + std::cerr << "\nReportMirrorFailure: " + << Queue->Uri + << " FailCode: " + << FailCode << std::endl; +} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..3ff9e1a96 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,6 +35,7 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); + void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index 52fadb81c..aee5c09c8 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:86 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:92 #, c-format msgid "Unable to read %s" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:92 +#: methods/mirror.cc:98 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2362,35 +2362,35 @@ msgstr "" msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:236 apt-pkg/acquire-item.cc:945 +#: apt-pkg/acquire-item.cc:237 apt-pkg/acquire-item.cc:946 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:640 +#: apt-pkg/acquire-item.cc:641 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:753 +#: apt-pkg/acquire-item.cc:754 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:812 +#: apt-pkg/acquire-item.cc:813 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:848 +#: apt-pkg/acquire-item.cc:849 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:935 +#: apt-pkg/acquire-item.cc:936 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 36280399db0ae203d3f1ae4d44b946f31e9a38ce Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 1 Feb 2007 12:32:50 +0100 Subject: * commited the latest mirror failure detection code --- apt-pkg/acquire-item.cc | 51 +++++++++++++++++++++++++++++++-------- apt-pkg/acquire-item.h | 5 +++- apt-pkg/acquire-method.cc | 21 ++++++++++------ apt-pkg/acquire-method.h | 8 +++--- cmdline/apt-report-mirror-failure | 23 ++++++++++++++++++ methods/connect.cc | 17 ++++++------- methods/mirror.cc | 45 +++------------------------------- methods/mirror.h | 1 - po/apt-all.pot | 44 ++++++++++++++++----------------- 9 files changed, 120 insertions(+), 95 deletions(-) create mode 100755 cmdline/apt-report-mirror-failure (limited to 'methods/mirror.h') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7b2a89763..61564c7aa 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -79,6 +79,10 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Status = StatError; Dequeue(); } + + // report mirror failure back to LP if we actually use a mirror + if(!UsedMirror.empty()) + ReportMirrorFailure(ErrorText); } /*}}}*/ // Acquire::Item::Start - Item has begun to download /*{{{*/ @@ -100,6 +104,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string, { // We just downloaded something.. string FileName = LookupTag(Message,"Filename"); + UsedMirror = LookupTag(Message,"UsedMirror"); if (Complete == false && FileName == DestFile) { if (Owner->Log != 0) @@ -108,7 +113,6 @@ void pkgAcquire::Item::Done(string Message,unsigned long Size,string, if (FileSize == 0) FileSize= Size; - Status = StatDone; ErrorText = string(); Owner->Dequeue(this); @@ -131,6 +135,42 @@ void pkgAcquire::Item::Rename(string From,string To) } /*}}}*/ +void pkgAcquire::Item::ReportMirrorFailure(string FailCode) +{ + // report that Queue->Uri failed +#if 0 + std::cerr << "\nReportMirrorFailure: " + << UsedMirror + << " FailCode: " + << FailCode << std::endl; +#endif + const char *Args[40]; + unsigned int i = 0; + string report = _config->Find("Methods::Mirror::ProblemReporting", + "/usr/bin/apt-report-mirror-failure"); + if(!FileExists(report)) + return; + Args[i++] = report.c_str(); + Args[i++] = UsedMirror.c_str(); + Args[i++] = FailCode.c_str(); + pid_t pid = ExecFork(); + if(pid < 0) + { + _error->Error("ReportMirrorFailure Fork failed"); + return; + } + else if(pid == 0) + { + execvp(report.c_str(), (char**)Args); + } + if(!ExecWait(pid, "report-mirror-failure")) + { + _error->Warning("Couldn't report problem to '%s'", + _config->Find("Acquire::Mirror::ReportFailures").c_str()); + } +} + + // AcqIndex::AcqIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The package file is added to the queue and a second class is @@ -176,8 +216,6 @@ string pkgAcqIndex::Custom600Headers() struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nIndex-File: true"; - if(ExpectedMD5 != "") - return "\nExpectedMD5: " + ExpectedMD5; return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ @@ -1016,13 +1054,6 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) } } /*}}}*/ -// --------------------------------------------------------------------- -string pkgAcqArchive::Custom600Headers() -{ - if(MD5 != "") - return "\nExpectedMD5: " + MD5; - return ""; -} // AcqArchive::IsTrusted - Determine whether this archive comes from a // trusted source /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index c9cd75321..9949d0044 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -56,6 +56,7 @@ class pkgAcquire::Item unsigned long ID; bool Complete; bool Local; + string UsedMirror; // Number of queues we are inserted into unsigned int QueueCounter; @@ -78,6 +79,9 @@ class pkgAcquire::Item pkgAcquire *GetOwner() {return Owner;}; virtual bool IsTrusted() {return false;}; + // report mirror problems + void ReportMirrorFailure(string FailCode); + Item(pkgAcquire *Owner); virtual ~Item(); }; @@ -205,7 +209,6 @@ class pkgAcqArchive : public pkgAcquire::Item virtual string ShortDesc() {return Desc.ShortDesc;}; virtual void Finished(); virtual bool IsTrusted(); - virtual string Custom600Headers(); pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, pkgRecords *Recs,pkgCache::VerIterator const &Version, diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 638797657..13201d310 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -99,12 +99,11 @@ void pkgAcqMethod::Fail(string Err,bool Transient) } char S[1024]; + char *End = S; if (Queue != 0) { - snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" - "Message: %s %s\n",Queue->Uri.c_str(),Err.c_str(), - FailExtra.c_str()); - + End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: %s\n" + "Message: %s %s\n",Queue->Uri.c_str(), Err.c_str(), IP.c_str()); // Dequeue FetchItem *Tmp = Queue; Queue = Queue->Next; @@ -113,10 +112,14 @@ void pkgAcqMethod::Fail(string Err,bool Transient) QueueBack = Queue; } else - snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: \n" - "Message: %s %s\n",Err.c_str(), - FailExtra.c_str()); - + { + End += snprintf(S,sizeof(S)-50,"400 URI Failure\nURI: \n" + "Message: %s\n",Err.c_str()); + } + if(FailReason.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"FailReason: %s\n",FailReason.c_str()); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); // Set the transient flag if (Transient == true) strcat(S,"Transient-Failure: true\n\n"); @@ -182,6 +185,8 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) End += snprintf(End,sizeof(S)-50 - (End - S),"MD5-Hash: %s\n",Res.MD5Sum.c_str()); if (Res.SHA1Sum.empty() == false) End += snprintf(End,sizeof(S)-50 - (End - S),"SHA1-Hash: %s\n",Res.SHA1Sum.c_str()); + if (UsedMirror.empty() == false) + End += snprintf(End,sizeof(S)-50 - (End - S),"UsedMirror: %s\n",UsedMirror.c_str()); if (Res.GPGVOutput.size() > 0) End += snprintf(End,sizeof(S)-50 - (End - S),"GPGVOutput:\n"); for (vector::iterator I = Res.GPGVOutput.begin(); diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 9e908d32c..ee68c75f7 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -55,7 +55,9 @@ class pkgAcqMethod vector Messages; FetchItem *Queue; FetchItem *QueueBack; - string FailExtra; + string FailReason; + string UsedMirror; + string IP; // Handlers for messages virtual bool Configuration(string Message); @@ -72,7 +74,6 @@ class pkgAcqMethod virtual void Exit() {}; public: - enum CnfFlags {SingleInstance = (1<<0), Pipeline = (1<<1), SendConfig = (1<<2), LocalOnly = (1<<3), NeedsCleanup = (1<<4), @@ -82,7 +83,8 @@ class pkgAcqMethod void Status(const char *Format,...); int Run(bool Single = false); - inline void SetFailExtraMsg(string Msg) {FailExtra = Msg;}; + inline void SetFailReason(string Msg) {FailReason = Msg;}; + inline void SetIP(string aIP) {IP = aIP;}; pkgAcqMethod(const char *Ver,unsigned long Flags = 0); virtual ~pkgAcqMethod() {}; diff --git a/cmdline/apt-report-mirror-failure b/cmdline/apt-report-mirror-failure new file mode 100755 index 000000000..70b16cf4f --- /dev/null +++ b/cmdline/apt-report-mirror-failure @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import sys +import urllib +import apt_pkg + +print "apt-report-mirror-failure" + +apt_pkg.init() +url = apt_pkg.Config.Find("Acquire::Mirror::ReportFailures", + "http://people.ubuntu.com:9000/mirror-failure") + #"http://localhost:9000/mirror-failure") +if not url: + sys.exit(0) + +data = {} +data['url'] = sys.argv[1] +data['error'] = sys.argv[2] +f = urllib.urlopen(url, urllib.urlencode(data)) +f.read() +f.close() + + diff --git a/methods/connect.cc b/methods/connect.cc index 8c2ac6d56..4e227c3fd 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -18,6 +18,7 @@ #include #include #include +#include // Internet stuff #include @@ -67,12 +68,10 @@ static bool DoConnect(struct addrinfo *Addr,string Host, wrong this will get tacked onto the end of the error message */ if (LastHostAddr->ai_next != 0) { - char Name2[NI_MAXHOST + NI_MAXSERV + 10]; - snprintf(Name2,sizeof(Name2),_("[IP: %s %s]"),Name,Service); - Owner->SetFailExtraMsg(string(Name2)); - } - else - Owner->SetFailExtraMsg(""); + std::stringstream ss; + ioprintf(ss, _("[IP: %s %s]"),Name,Service); + Owner->SetIP(ss.str()); + } // Get a socket if ((Fd = socket(Addr->ai_family,Addr->ai_socktype, @@ -89,7 +88,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, /* This implements a timeout for connect by opening the connection nonblocking */ if (WaitFd(Fd,true,TimeOut) == false) { - Owner->SetFailExtraMsg("\nFailReason: Timeout"); + Owner->SetFailReason("Timeout"); return _error->Error(_("Could not connect to %s:%s (%s), " "connection timed out"),Host.c_str(),Service,Name); } @@ -104,7 +103,7 @@ static bool DoConnect(struct addrinfo *Addr,string Host, { errno = Err; if(errno == ECONNREFUSED) - Owner->SetFailExtraMsg("\nFailReason: ConnectionRefused"); + Owner->SetFailReason("ConnectionRefused"); return _error->Errno("connect",_("Could not connect to %s:%s (%s)."),Host.c_str(), Service,Name); } @@ -169,7 +168,7 @@ bool Connect(string Host,int Port,const char *Service,int DefPort,int &Fd, if (Res == EAI_AGAIN) { - Owner->SetFailExtraMsg("\nFailReason: TmpResolveFailure"); + Owner->SetFailReason("TmpResolveFailure"); return _error->Error(_("Temporary failure resolving '%s'"), Host.c_str()); } diff --git a/methods/mirror.cc b/methods/mirror.cc index 8f9b8ed34..8ccfb8559 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -38,7 +38,7 @@ using namespace std; * * TODO: * what about gpgv failures? this should call-out to the problem reporting - script, but we need to know what mirror was used + script, but we need to know what mirror was used -> just run pkgAcquire::Item::ReportMirrorFailure() * better standard format for errors to send back * - implement failure reporting at the pkgAcquire::Item::Failed() level but then we need to send back what uri exactly was failing @@ -48,7 +48,7 @@ using namespace std; * - deal with runing as non-root because we can't write to the lists dir then -> use the cached mirror file * - better method to download than having a pkgAcquire interface here - * - magicmarker is (a bit) evil, maybe just use a similar approach as in + * - magicmarker is evil, maybe just use a similar approach as in clean and read the sources.list and use the GetURI() method to find the prefix? * support more than http @@ -191,6 +191,8 @@ bool MirrorMethod::SelectMirror() getline(in, Mirror); if(Debug) cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; return true; } @@ -220,9 +222,6 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { - // FIXME: queue next mirror? - ReportMirrorFailure(Err); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::Fail(Err, Transient); @@ -237,47 +236,11 @@ void MirrorMethod::URIStart(FetchResult &Res) void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - // FIXME: queue next mirror? - if(Queue->ExpectedMD5 != "" && Res.MD5Sum != Queue->ExpectedMD5) - ReportMirrorFailure("499 Hash mismatch"); - if(Queue->Uri.find("http://") != string::npos) Queue->Uri.replace(0,Mirror.size(), BaseUri); pkgAcqMethod::URIDone(Res, Alt); } -void MirrorMethod::ReportMirrorFailure(string FailCode) -{ - // report that Queue->Uri failed -#if 0 - std::cerr << "\nReportMirrorFailure: " - << Queue->Uri - << " FailCode: " - << FailCode << std::endl; -#endif - const char *Args[40]; - unsigned int i = 0; - string report = _config->Find("Methods::Mirror::ProblemReporting", - "/usr/lib/apt/report-mirror-failure"); - Args[i++] = report.c_str(); - Args[i++] = Queue->Uri.c_str(); - Args[i++] = FailCode.c_str(); - pid_t pid = ExecFork(); - if(pid < 0) - { - _error->Error("ReportMirrorFailure Fork failed"); - return; - } - else if(pid == 0) - { - execvp(report.c_str(), (char**)Args); - } - if(!ExecWait(pid, "report-mirror-failure")) - { - _error->Warning("Couldn't report problem to '%s'", - _config->Find("Acquire::Mirror::ReportFailures").c_str()); - } -} int main() { diff --git a/methods/mirror.h b/methods/mirror.h index 3ff9e1a96..798f5a9b5 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -35,7 +35,6 @@ class MirrorMethod : public HttpMethod bool GetMirrorFile(string uri); bool SelectMirror(); bool Clean(string dir); - void ReportMirrorFailure(string FailCode); // we need to overwrite those to transform the url back virtual void Fail(string Why, bool Transient = false); diff --git a/po/apt-all.pot b/po/apt-all.pot index dd140bd63..15b650fda 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2007-01-24 12:51+0100\n" +"POT-Creation-Date: 2007-01-24 17:20+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -950,7 +950,7 @@ msgstr "" msgid "Calculating upgrade... " msgstr "" -#: cmdline/apt-get.cc:1716 methods/ftp.cc:702 methods/connect.cc:101 +#: cmdline/apt-get.cc:1716 methods/ftp.cc:702 methods/connect.cc:100 msgid "Failed" msgstr "" @@ -1330,7 +1330,7 @@ msgstr "" #: apt-inst/extract.cc:467 apt-pkg/contrib/configuration.cc:750 #: apt-pkg/contrib/cdromutl.cc:153 apt-pkg/sourcelist.cc:324 -#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:96 +#: apt-pkg/acquire.cc:421 apt-pkg/clean.cc:38 methods/mirror.cc:99 #, c-format msgid "Unable to read %s" msgstr "" @@ -1648,59 +1648,59 @@ msgstr "" msgid "Unable to invoke " msgstr "" -#: methods/connect.cc:64 +#: methods/connect.cc:65 #, c-format msgid "Connecting to %s (%s)" msgstr "" -#: methods/connect.cc:71 +#: methods/connect.cc:72 #, c-format msgid "[IP: %s %s]" msgstr "" -#: methods/connect.cc:80 +#: methods/connect.cc:79 #, c-format msgid "Could not create a socket for %s (f=%u t=%u p=%u)" msgstr "" -#: methods/connect.cc:86 +#: methods/connect.cc:85 #, c-format msgid "Cannot initiate the connection to %s:%s (%s)." msgstr "" -#: methods/connect.cc:93 +#: methods/connect.cc:92 #, c-format msgid "Could not connect to %s:%s (%s), connection timed out" msgstr "" -#: methods/connect.cc:108 +#: methods/connect.cc:107 #, c-format msgid "Could not connect to %s:%s (%s)." msgstr "" #. We say this mainly because the pause here is for the #. ssh connection that is still going -#: methods/connect.cc:136 methods/rsh.cc:425 +#: methods/connect.cc:135 methods/rsh.cc:425 #, c-format msgid "Connecting to %s" msgstr "" -#: methods/connect.cc:167 +#: methods/connect.cc:166 #, c-format msgid "Could not resolve '%s'" msgstr "" -#: methods/connect.cc:173 +#: methods/connect.cc:172 #, c-format msgid "Temporary failure resolving '%s'" msgstr "" -#: methods/connect.cc:176 +#: methods/connect.cc:175 #, c-format msgid "Something wicked happened resolving '%s:%s' (%i)" msgstr "" -#: methods/connect.cc:223 +#: methods/connect.cc:222 #, c-format msgid "Unable to connect to %s %s:" msgstr "" @@ -1956,7 +1956,7 @@ msgid "Unable to stat the mount point %s" msgstr "" #: apt-pkg/contrib/cdromutl.cc:149 apt-pkg/acquire.cc:427 apt-pkg/clean.cc:44 -#: methods/mirror.cc:102 +#: methods/mirror.cc:105 #, c-format msgid "Unable to change to %s" msgstr "" @@ -2357,40 +2357,40 @@ msgstr "" msgid "IO Error saving source cache" msgstr "" -#: apt-pkg/acquire-item.cc:126 +#: apt-pkg/acquire-item.cc:128 #, c-format msgid "rename failed, %s (%s -> %s)." msgstr "" -#: apt-pkg/acquire-item.cc:237 apt-pkg/acquire-item.cc:946 +#: apt-pkg/acquire-item.cc:275 apt-pkg/acquire-item.cc:984 msgid "MD5Sum mismatch" msgstr "" -#: apt-pkg/acquire-item.cc:641 +#: apt-pkg/acquire-item.cc:679 msgid "There is no public key available for the following key IDs:\n" msgstr "" -#: apt-pkg/acquire-item.cc:754 +#: apt-pkg/acquire-item.cc:792 #, c-format msgid "" "I wasn't able to locate a file for the %s package. This might mean you need " "to manually fix this package. (due to missing arch)" msgstr "" -#: apt-pkg/acquire-item.cc:813 +#: apt-pkg/acquire-item.cc:851 #, c-format msgid "" "I wasn't able to locate file for the %s package. This might mean you need to " "manually fix this package." msgstr "" -#: apt-pkg/acquire-item.cc:849 +#: apt-pkg/acquire-item.cc:887 #, c-format msgid "" "The package index files are corrupted. No Filename: field for package %s." msgstr "" -#: apt-pkg/acquire-item.cc:936 +#: apt-pkg/acquire-item.cc:974 msgid "Size mismatch" msgstr "" -- cgit v1.2.3-70-g09d2 From 38eedeb77790c106eb35940e3d7835a7e0f0297e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 Nov 2007 15:54:17 +0100 Subject: methods/mirror.{cc,h}: - improve the mirror method so that it only updates the mirror list from the server if we get new indexfiles - code cleanup --- methods/mirror.cc | 152 +++++++++++++++++++++++++++++++++--------------------- methods/mirror.h | 7 +-- 2 files changed, 97 insertions(+), 62 deletions(-) (limited to 'methods/mirror.h') diff --git a/methods/mirror.cc b/methods/mirror.cc index 9a86a10c2..43e56c71a 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -48,7 +48,7 @@ using namespace std; */ MirrorMethod::MirrorMethod() - : HttpMethod(), HasMirrorFile(false) + : HttpMethod(), DownloadedMirrorFile(false) { }; @@ -73,6 +73,9 @@ bool MirrorMethod::Clean(string Dir) if(Debug) clog << "MirrorMethod::Clean(): " << Dir << endl; + if(Dir == "/") + return _error->Error("will not clean: '/'"); + // read sources.list pkgSourceList list; list.ReadMainList(); @@ -118,7 +121,71 @@ bool MirrorMethod::Clean(string Dir) } -bool MirrorMethod::GetMirrorFile(string mirror_uri_str) +bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) +{ + + // check the file, if it is not older than RefreshInterval just use it + // otherwise try to get a new one + if(FileExists(MirrorFile)) + { + struct stat buf; + time_t t,now,refresh; + if(stat(MirrorFile.c_str(), &buf) != 0) + return false; + t = std::max(buf.st_mtime, buf.st_ctime); + now = time(NULL); + refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); + if(t + refresh > now) + { + if(Debug) + clog << "Mirror file is in RefreshInterval" << endl; + DownloadedMirrorFile = true; + return true; + } + if(Debug) + clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + } + + // not that great to use pkgAcquire here, but we do not have + // any other way right now + string fetch = BaseUri; + fetch.replace(0,strlen("mirror://"),"http://"); + + pkgAcquire Fetcher; + new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); + bool res = (Fetcher.Run() == pkgAcquire::Continue); + if(res) + DownloadedMirrorFile = true; + Fetcher.Shutdown(); + return res; +} + +bool MirrorMethod::SelectMirror() +{ + // if we do not have a MirrorFile, fallback + if(!FileExists(MirrorFile)) + { + // FIXME: fallback to a default mirror here instead + // and provide a config option to define that default + return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + } + + // FIXME: make the mirror selection more clever, do not + // just use the first one! + // BUT: we can not make this random, the mirror has to be + // stable accross session, because otherwise we can + // get into sync issues (got indexfiles from mirror A, + // but packages from mirror B - one might be out of date etc) + ifstream in(MirrorFile.c_str()); + getline(in, Mirror); + if(Debug) + cerr << "Using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +string MirrorMethod::GetMirrorFileName(string mirror_uri_str) { /* - a mirror_uri_str looks like this: @@ -144,8 +211,9 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) in both cases! So we need to apply some domain knowledge here :( and check for /dists/ or /Release.gpg as suffixes */ + string name; if(Debug) - std::cerr << "GetMirrorFile: " << mirror_uri_str << std::endl; + std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; // read sources.list and find match vector::const_iterator I; @@ -166,62 +234,15 @@ bool MirrorMethod::GetMirrorFile(string mirror_uri_str) BaseUri = uristr.substr(0,uristr.size()-1); } } - string fetch = BaseUri; - fetch.replace(0,strlen("mirror://"),"http://"); - // get new file - MirrorFile = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); + name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); if(Debug) { cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << MirrorFile << endl; - } - - // check the file, if it is not older than RefreshInterval just use it - // otherwise try to get a new one - if(FileExists(MirrorFile)) - { - struct stat buf; - time_t t,now,refresh; - if(stat(MirrorFile.c_str(), &buf) != 0) - return false; - t = std::max(buf.st_mtime, buf.st_ctime); - now = time(NULL); - refresh = 60*_config->FindI("Acquire::Mirror::RefreshInterval",360); - if(t + refresh > now) - { - if(Debug) - clog << "Mirror file is in RefreshInterval" << endl; - HasMirrorFile = true; - return true; - } - if(Debug) - clog << "Mirror file " << MirrorFile << " older than " << refresh << "min, re-download it" << endl; + cerr << "mirror-file: " << name << endl; } - - // not that great to use pkgAcquire here, but we do not have - // any other way right now - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) - HasMirrorFile = true; - Fetcher.Shutdown(); - return res; -} - -bool MirrorMethod::SelectMirror() -{ - // FIXME: make the mirror selection more clever, do not - // just use the first one! - ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; - return true; + return name; } // MirrorMethod::Fetch - Fetch an item /*{{{*/ @@ -230,20 +251,33 @@ bool MirrorMethod::SelectMirror() depth. */ bool MirrorMethod::Fetch(FetchItem *Itm) { - // select mirror only once per session - if(!HasMirrorFile) + // the http method uses Fetch(0) as a way to update the pipeline, + // just let it do its work in this case - Fetch() with a valid + // Itm will always run before the first Fetch(0) + if(Itm == NULL) + return HttpMethod::Fetch(Itm); + + // if we don't have the name of the mirror file on disk yet, + // calculate it now (can be derived from the uri) + if(MirrorFile.empty()) + MirrorFile = GetMirrorFileName(Itm->Uri); + + // download mirror file once (if we are after index files) + if(Itm->IndexFile && !DownloadedMirrorFile) { Clean(_config->FindDir("Dir::State::mirrors")); - GetMirrorFile(Itm->Uri); - SelectMirror(); + DownloadMirrorFile(Itm->Uri); } + if(Mirror.empty()) + SelectMirror(); + for (FetchItem *I = Queue; I != 0; I = I->Next) { if(I->Uri.find("mirror://") != string::npos) - I->Uri.replace(0,BaseUri.size(),Mirror); + I->Uri.replace(0,BaseUri.size(), Mirror); } - + // now run the real fetcher return HttpMethod::Fetch(Itm); }; diff --git a/methods/mirror.h b/methods/mirror.h index 798f5a9b5..ed817806b 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,13 +26,14 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) - string MirrorFile; // - bool HasMirrorFile; + string MirrorFile; // the file that contains the list of mirrors + bool DownloadedMirrorFile; // already downloaded this session bool Debug; protected: - bool GetMirrorFile(string uri); + bool DownloadMirrorFile(string uri); + string GetMirrorFileName(string uri); bool SelectMirror(); bool Clean(string dir); -- cgit v1.2.3-70-g09d2 From 96db74ce38e9451609fe33f9e25f3f9d42b1fe22 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 16:40:40 +0200 Subject: * apt-pkg/deb/dpkgpm.cc: - add missing include * methods/mirror.{cc,h}: - add SelectNextMirror() and InitMirrors() functions - read all mirrors into the AllMirrors vector --- apt-pkg/deb/dpkgpm.cc | 1 + methods/mirror.cc | 30 +++++++++++++++++++++++------- methods/mirror.h | 4 +++- 3 files changed, 27 insertions(+), 8 deletions(-) (limited to 'methods/mirror.h') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 18c731788..7e5171eda 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include diff --git a/methods/mirror.cc b/methods/mirror.cc index b3a956b95..0a0d07e6b 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -162,7 +162,21 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } -bool MirrorMethod::SelectMirror() +bool MirrorMethod::SelectNextMirror() +{ + if (AllMirrors.size() < 1) + return false; + + Mirror = AllMirrors[0]; + AllMirrors.erase(AllMirrors.begin()); + if(Debug) + cerr << "using mirror: " << Mirror << endl; + + UsedMirror = Mirror; + return true; +} + +bool MirrorMethod::InitMirrors() { // if we do not have a MirrorFile, fallback if(!FileExists(MirrorFile)) @@ -179,11 +193,13 @@ bool MirrorMethod::SelectMirror() // get into sync issues (got indexfiles from mirror A, // but packages from mirror B - one might be out of date etc) ifstream in(MirrorFile.c_str()); - getline(in, Mirror); - if(Debug) - cerr << "Using mirror: " << Mirror << endl; - - UsedMirror = Mirror; + string s; + while (!in.eof()) + { + getline(in, s); + AllMirrors.push_back(s); + } + SelectNextMirror(); return true; } @@ -275,7 +291,7 @@ bool MirrorMethod::Fetch(FetchItem *Itm) } if(Mirror.empty()) { - if(!SelectMirror()) { + if(!InitMirrors()) { // no valid mirror selected, something went wrong downloading // from the master mirror site most likely and there is // no old mirror file availalbe diff --git a/methods/mirror.h b/methods/mirror.h index ed817806b..1b506dc87 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -26,6 +26,7 @@ class MirrorMethod : public HttpMethod // we simply transform between BaseUri and Mirror string BaseUri; // the original mirror://... url string Mirror; // the selected mirror uri (http://...) + vector AllMirrors; // all available mirrors string MirrorFile; // the file that contains the list of mirrors bool DownloadedMirrorFile; // already downloaded this session @@ -34,7 +35,8 @@ class MirrorMethod : public HttpMethod protected: bool DownloadMirrorFile(string uri); string GetMirrorFileName(string uri); - bool SelectMirror(); + bool InitMirrors(); + bool SelectNextMirror(); bool Clean(string dir); // we need to overwrite those to transform the url back -- cgit v1.2.3-70-g09d2 From 0391542729e5e8a2ba9ae9b1d470f252e24eb296 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 2 Jun 2010 23:13:42 +0200 Subject: methods/mirrors.cc: make cycle through the mirrors work properly --- methods/mirror.cc | 55 ++++++++++++++++++++++++++++++++++++++++--------------- methods/mirror.h | 3 ++- 2 files changed, 42 insertions(+), 16 deletions(-) (limited to 'methods/mirror.h') diff --git a/methods/mirror.cc b/methods/mirror.cc index 4d767c9f3..567522478 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -142,13 +142,44 @@ bool MirrorMethod::DownloadMirrorFile(string mirror_uri_str) return res; } -bool MirrorMethod::SelectNextMirror() +/* convert a the Queue->Uri back to the mirror base uri and look + * at all mirrors we have for this, this is needed as queue->uri + * may point to different mirrors (if TryNextMirror() was run) + */ +void MirrorMethod::CurrentQueueUriToMirror() +{ + // already in mirror:// style so nothing to do + if(Queue->Uri.find("mirror://") == 0) + return; + + // find current mirror and select next one + for (int i=0; i < AllMirrors.size(); i++) + { + if (Queue->Uri.find(AllMirrors[i]) == 0) + { + Queue->Uri.replace(0, AllMirrors[i].size(), BaseUri); + return; + } + } + _error->Error("Internal error: Failed to convert %s back to %s", + Queue->Uri, BaseUri); +} + +bool MirrorMethod::TryNextMirror() { if(Debug) cerr << "using mirror: " << Mirror << endl; - Mirror = AllMirrors[0]; - UsedMirror = Mirror; + // find current mirror and select next one + for (int i=0; i < AllMirrors.size()-1; i++) + { + if (Queue->Uri.find(AllMirrors[i]) == 0) + { + Queue->Uri.replace(0, AllMirrors[i].size(), AllMirrors[i+1]); + return true; + } + } + return false; } @@ -176,7 +207,8 @@ bool MirrorMethod::InitMirrors() if (s.size() > 0) AllMirrors.push_back(s); } - SelectNextMirror(); + Mirror = AllMirrors[0]; + UsedMirror = Mirror; return true; } @@ -292,34 +324,27 @@ bool MirrorMethod::Fetch(FetchItem *Itm) void MirrorMethod::Fail(string Err,bool Transient) { // try the next mirror on fail - string old_mirror = Mirror; - if (SelectNextMirror()) - { - Queue->Uri.replace(0, old_mirror.size(), Mirror); + if (TryNextMirror()) return; - } // all mirrors failed, so bail out string s; strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); SetIP(s); - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::Fail(Err, Transient); } void MirrorMethod::URIStart(FetchResult &Res) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIStart(Res); } void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) { - if(Queue->Uri.find("http://") != string::npos) - Queue->Uri.replace(0,Mirror.size(), BaseUri); + CurrentQueueUriToMirror(); pkgAcqMethod::URIDone(Res, Alt); } diff --git a/methods/mirror.h b/methods/mirror.h index 1b506dc87..0a3ea6e92 100644 --- a/methods/mirror.h +++ b/methods/mirror.h @@ -36,7 +36,8 @@ class MirrorMethod : public HttpMethod bool DownloadMirrorFile(string uri); string GetMirrorFileName(string uri); bool InitMirrors(); - bool SelectNextMirror(); + bool TryNextMirror(); + void CurrentQueueUriToMirror(); bool Clean(string dir); // we need to overwrite those to transform the url back -- cgit v1.2.3-70-g09d2