diff options
Diffstat (limited to 'methods')
| -rw-r--r-- | methods/CMakeLists.txt | 8 | ||||
| -rw-r--r-- | methods/aptmethod.h | 10 | ||||
| -rw-r--r-- | methods/http.cc | 14 | ||||
| -rw-r--r-- | methods/http_main.cc | 17 | ||||
| -rw-r--r-- | methods/mirror.cc | 728 | ||||
| -rw-r--r-- | methods/mirror.h | 57 |
6 files changed, 360 insertions, 474 deletions
diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index a25d4b525..c4a32b4f5 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -2,7 +2,6 @@ include_directories($<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_INCLUDE_DIR}>) link_libraries(apt-pkg $<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_LIBRARIES}>) -add_library(httplib OBJECT http.cc basehttp.cc) add_library(connectlib OBJECT connect.cc rfc2553emu.cc) add_executable(file file.cc) @@ -10,8 +9,8 @@ add_executable(copy copy.cc) add_executable(store store.cc) add_executable(gpgv gpgv.cc) add_executable(cdrom cdrom.cc) -add_executable(http http_main.cc $<TARGET_OBJECTS:httplib> $<TARGET_OBJECTS:connectlib>) -add_executable(mirror mirror.cc $<TARGET_OBJECTS:httplib> $<TARGET_OBJECTS:connectlib>) +add_executable(http http.cc basehttp.cc $<TARGET_OBJECTS:connectlib>) +add_executable(mirror mirror.cc) add_executable(ftp ftp.cc $<TARGET_OBJECTS:connectlib>) add_executable(rred rred.cc) add_executable(rsh rsh.cc) @@ -21,14 +20,13 @@ target_include_directories(connectlib PRIVATE ${GNUTLS_INCLUDE_DIR}) # Additional libraries to link against for networked stuff target_link_libraries(http ${GNUTLS_LIBRARIES}) -target_link_libraries(mirror ${RESOLV_LIBRARIES} ${GNUTLS_LIBRARIES}) target_link_libraries(ftp ${GNUTLS_LIBRARIES}) # Install the library install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods) -add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods store) +add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+ftp mirror+http mirror+https mirror+file mirror+copy) add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh) diff --git a/methods/aptmethod.h b/methods/aptmethod.h index 88d325cba..331411571 100644 --- a/methods/aptmethod.h +++ b/methods/aptmethod.h @@ -448,6 +448,16 @@ protected: return true; } + // This is a copy of #pkgAcqMethod::Dequeue which is private & hidden + void Dequeue() + { + FetchItem const *const Tmp = Queue; + Queue = Queue->Next; + if (Tmp == QueueBack) + QueueBack = Queue; + delete Tmp; + } + aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3) : pkgAcqMethod(Ver, Flags), Binary(Binary), SeccompFlags(0), methodNames({Binary}) { diff --git a/methods/http.cc b/methods/http.cc index 2d23b1646..5d286bcb4 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -31,6 +31,7 @@ #include <sstream> #include <arpa/inet.h> #include <errno.h> +#include <signal.h> #include <stddef.h> #include <stdio.h> #include <stdlib.h> @@ -1034,7 +1035,7 @@ BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &R return FILE_IS_OPEN; } /*}}}*/ -HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/ +HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg), "1.2", Pipeline | SendConfig) /*{{{*/ { SeccompFlags = aptMethod::BASE | aptMethod::NETWORK; @@ -1051,3 +1052,14 @@ HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2 } } /*}}}*/ + +int main(int, const char *argv[]) +{ + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + std::string Binary = flNotDir(argv[0]); + if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http") + Binary.append("+http"); + return HttpMethod(std::move(Binary)).Loop(); +} diff --git a/methods/http_main.cc b/methods/http_main.cc deleted file mode 100644 index 792b5e22f..000000000 --- a/methods/http_main.cc +++ /dev/null @@ -1,17 +0,0 @@ -#include <config.h> -#include <apt-pkg/error.h> -#include <apt-pkg/fileutl.h> -#include <signal.h> - -#include "http.h" - -int main(int, const char *argv[]) -{ - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - std::string Binary = flNotDir(argv[0]); - if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http") - Binary.append("+http"); - return HttpMethod(std::move(Binary)).Loop(); -} diff --git a/methods/mirror.cc b/methods/mirror.cc index b551802e4..add9f0875 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -1,18 +1,17 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ /* ###################################################################### - Mirror Acquire Method - This is the Mirror acquire method for APT. - + Mirror URI – This method helps avoiding hardcoding of mirrors in the + sources.lists by looking up a list of mirrors first to which the + following requests are redirected. + ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ #include <config.h> -#include <apt-pkg/acquire-item.h> -#include <apt-pkg/acquire.h> -#include <apt-pkg/aptconfiguration.h> +#include "aptmethod.h" #include <apt-pkg/configuration.h> #include <apt-pkg/error.h> #include <apt-pkg/fileutl.h> @@ -20,447 +19,388 @@ #include <apt-pkg/sourcelist.h> #include <apt-pkg/strutl.h> -#include <algorithm> -#include <fstream> -#include <iostream> +#include <functional> +#include <random> +#include <string> +#include <unordered_map> -#include <dirent.h> -#include <fcntl.h> -#include <stdlib.h> -#include <string.h> -#include <sys/stat.h> -#include <sys/types.h> #include <sys/utsname.h> -#include <unistd.h> - -using namespace std; - -#include <sstream> -#include "http.h" -#include "mirror.h" #include <apti18n.h> /*}}}*/ +constexpr char const *const disallowLocal[] = {"ftp", "http", "https"}; -/* Done: - * - works with http (only!) - * - always picks the first mirror from the list - * - call out to problem reporting script - * - supports "deb mirror://host/path/to/mirror-list/// dist component" - * - uses pkgAcqMethod::FailReason() to have a string representation - * of the failure that is also send to LP - * - * TODO: - * - deal with running as non-root because we can't write to the lists - dir then -> use the cached mirror file - * - better method to download than having a pkgAcquire interface here - * and better error handling there! - * - support more than http - * - testing :) - */ - -MirrorMethod::MirrorMethod() - : HttpMethod("mirror"), DownloadedMirrorFile(false), Debug(false) +static void sortByLength(std::vector<std::string> &vec) /*{{{*/ { -} - -// HttpMethod::Configuration - Handle a configuration message /*{{{*/ -// --------------------------------------------------------------------- -/* We stash the desired pipeline depth */ -bool MirrorMethod::Configuration(string Message) -{ - if (HttpMethod::Configuration(Message) == false) - return false; - Debug = DebugEnabled(); - - return true; + // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected + // by checking for the longest matches first + std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) { + return a.length() > b.length(); + }); } /*}}}*/ - -// clean the mirrors dir based on ttl information -bool MirrorMethod::Clean(string Dir) +class MirrorMethod : public aptMethod /*{{{*/ { - vector<metaIndex *>::const_iterator I; - - if(Debug) - clog << "MirrorMethod::Clean(): " << Dir << endl; - - if(Dir == "/") - return _error->Error("will not clean: '/'"); - - // read sources.list - pkgSourceList list; - list.ReadMainList(); - - int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); - if (dirfd == -1) - return _error->Errno("open",_("Unable to read %s"), Dir.c_str()); - DIR * const D = fdopendir(dirfd); - if (D == nullptr) - return _error->Errno("fdopendir",_("Unable to read %s"),Dir.c_str()); - - for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + std::mt19937 genrng; + std::vector<std::string> sourceslist; + std::unordered_map<std::string, std::string> msgCache; + enum MirrorFileState { - // Skip some files.. - if (strcmp(Dir->d_name,"lock") == 0 || - strcmp(Dir->d_name,"partial") == 0 || - strcmp(Dir->d_name,"lost+found") == 0 || - strcmp(Dir->d_name,".") == 0 || - strcmp(Dir->d_name,"..") == 0) - continue; - - // see if we have that uri - for(I=list.begin(); I != list.end(); ++I) + REQUESTED, + FAILED, + AVAILABLE + }; + struct MirrorInfo + { + std::string uri; + unsigned long priority = std::numeric_limits<decltype(priority)>::max(); + decltype(genrng)::result_type seed = 0; + std::unordered_map<std::string, std::vector<std::string>> tags; + MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u) { - string uri = (*I)->GetURI(); - if(uri.compare(0, strlen("mirror://"), "mirror://") != 0) - continue; - string BaseUri = uri.substr(0,uri.size()-1); - if (URItoFileName(BaseUri) == Dir->d_name) - break; + for (auto &&tag : ptags) + { + auto const colonfound = tag.find(':'); + if (unlikely(colonfound == std::string::npos)) + continue; + auto name = tag.substr(0, colonfound); + auto value = tag.substr(colonfound + 1); + if (name == "arch") + tags["Architecture"].emplace_back(std::move(value)); + else if (name == "lang") + tags["Language"].emplace_back(std::move(value)); + else if (name == "priority") + priority = std::strtoul(value.c_str(), nullptr, 10); + else if (likely(name.empty() == false)) + { + if (name == "codename" || name == "suite") + tags["Release"].push_back(value); + name[0] = std::toupper(name[0]); + tags[std::move(name)].emplace_back(std::move(value)); + } + } } - // nothing found, nuke it - if (I == list.end()) - RemoveFileAt("mirror", dirfd, Dir->d_name); - } - closedir(D); - return true; -} - - -bool MirrorMethod::DownloadMirrorFile(string /*mirror_uri_str*/) -{ - // not that great to use pkgAcquire here, but we do not have - // any other way right now - string fetch = BaseUri; - fetch.replace(0,strlen("mirror://"),"http://"); - -#if 0 // no need for this, the getArchitectures() will also include the main - // arch - // append main architecture - fetch += "?arch=" + _config->Find("Apt::Architecture"); -#endif - - // append all architectures - std::vector<std::string> vec = APT::Configuration::getArchitectures(); - for (std::vector<std::string>::const_iterator I = vec.begin(); - I != vec.end(); ++I) - if (I == vec.begin()) - fetch += "?arch=" + (*I); - else - fetch += "&arch=" + (*I); + }; + struct MirrorListInfo + { + MirrorFileState state; + std::string baseuri; + std::vector<MirrorInfo> list; + }; + std::unordered_map<std::string, MirrorListInfo> mirrorfilestate; - // append the dist as a query string - if (Dist != "") - fetch += "&dist=" + Dist; + virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; - if(Debug) - clog << "MirrorMethod::DownloadMirrorFile(): '" << fetch << "'" - << " to " << MirrorFile << endl; + void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message); + bool MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm); + std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm); + void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler); - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) { - DownloadedMirrorFile = true; - chmod(MirrorFile.c_str(), 0644); + public: + MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests), genrng(clock()) + { + SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY; } - Fetcher.Shutdown(); - - if(Debug) - clog << "MirrorMethod::DownloadMirrorFile() success: " << res << endl; - - return res; -} - -// Randomizes the lines in the mirror file, this is used so that -// we spread the load on the mirrors evenly -bool MirrorMethod::RandomizeMirrorFile(string mirror_file) +}; + /*}}}*/ +void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/ { - vector<string> content; - string line; - - if (!FileExists(mirror_file)) - return false; - - // read - ifstream in(mirror_file.c_str()); - while ( !in.eof() ) { - getline(in, line); - content.push_back(line); - } - - // we want the file to be random for each different machine, but also - // "stable" on the same machine. this is to avoid running into out-of-sync - // issues (i.e. Release/Release.gpg different on each mirror) - struct utsname buf; - int seed=1; - if(uname(&buf) == 0) { - for(int i=0,seed=1; buf.nodename[i] != 0; ++i) { - seed = seed * 31 + buf.nodename[i]; + std::unordered_map<std::string, std::string> matchers; + matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture")); + matchers.emplace("Codename", LookupTag(Message, "Target-Codename")); + matchers.emplace("Component", LookupTag(Message, "Target-Component")); + matchers.emplace("Language", LookupTag(Message, "Target-Language")); + matchers.emplace("Release", LookupTag(Message, "Target-Release")); + matchers.emplace("Suite", LookupTag(Message, "Target-Suite")); + matchers.emplace("Type", LookupTag(Message, "Target-Type")); + decltype(info.list) possMirrors; + for (auto const &mirror : info.list) + { + bool failedMatch = false; + for (auto const &m : matchers) + { + if (m.second.empty()) + continue; + auto const tagsetiter = mirror.tags.find(m.first); + if (tagsetiter == mirror.tags.end()) + continue; + auto const tagset = tagsetiter->second; + if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end()) + { + failedMatch = true; + break; + } } + if (failedMatch) + continue; + possMirrors.push_back(mirror); } - srand( seed ); - random_shuffle(content.begin(), content.end()); - - // write - ofstream out(mirror_file.c_str()); - while ( !content.empty()) { - line = content.back(); - content.pop_back(); - out << line << "\n"; + for (auto &&mirror : possMirrors) + mirror.seed = genrng(); + std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) { + if (a.priority != b.priority) + return a.priority < b.priority; + return a.seed < b.seed; + }); + std::string const path = Itm->Uri.substr(info.baseuri.length()); + std::string altMirrors; + std::unordered_map<std::string, std::string> fields; + fields.emplace("URI", Queue->Uri); + for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror) + { + std::string mirror = curMirror->uri; + if (APT::String::Endswith(mirror, "/") == false) + mirror.append("/"); + mirror.append(path); + if (curMirror == possMirrors.cbegin()) + fields.emplace("New-URI", mirror); + else if (altMirrors.empty()) + altMirrors.append(mirror); + else + altMirrors.append("\n").append(mirror); } - - return true; + fields.emplace("Alternate-URIs", altMirrors); + SendMessage("103 Redirect", std::move(fields)); + Dequeue(); } - -/* convert a the Queue->Uri back to the mirror base uri and look - * at all mirrors we have for this, this is needed as queue->uri - * may point to different mirrors (if TryNextMirror() was run) - */ -void MirrorMethod::CurrentQueueUriToMirror() + /*}}}*/ +void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/ + MirrorListInfo const &info, FetchItem *const Itm, + std::function<void()> handler) { - // already in mirror:// style so nothing to do - if(Queue->Uri.find("mirror://") == 0) - return; - - // find current mirror and select next one - for (vector<string>::const_iterator mirror = AllMirrors.begin(); - mirror != AllMirrors.end(); ++mirror) + FetchItem **LastItm = &Itm->Next; + while (*LastItm != nullptr) + LastItm = &((*LastItm)->Next); + while (Queue != Itm) { - if (Queue->Uri.find(*mirror) == 0) + if (APT::String::Startswith(Queue->Uri, info.baseuri) == false || + std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); })) { - Queue->Uri.replace(0, mirror->length(), BaseUri); - return; + // move the item behind the aux file not related to it + *LastItm = Queue; + Queue = QueueBack = Queue->Next; + (*LastItm)->Next = nullptr; + LastItm = &((*LastItm)->Next); + } + else + { + handler(); } } - _error->Error("Internal error: Failed to convert %s back to %s", - Queue->Uri.c_str(), BaseUri.c_str()); + // now remove out trigger + QueueBack = Queue = Queue->Next; + delete Itm; } - -bool MirrorMethod::TryNextMirror() + /*}}}*/ +bool MirrorMethod::MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/ { - // find current mirror and select next one - for (vector<string>::const_iterator mirror = AllMirrors.begin(); - mirror != AllMirrors.end(); ++mirror) + std::vector<std::string> baseuris; + for (auto const &i : mirrorfilestate) + if (info.baseuri.length() < i.second.baseuri.length() && + i.second.state == REQUESTED && + APT::String::Startswith(i.second.baseuri, info.baseuri)) + baseuris.push_back(i.second.baseuri); + sortByLength(baseuris); + + FileFd mirrorlist; + if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension)) { - if (Queue->Uri.find(*mirror) != 0) - continue; - - vector<string>::const_iterator nextmirror = mirror + 1; - if (nextmirror == AllMirrors.end()) - break; - Queue->Uri.replace(0, mirror->length(), *nextmirror); - if (Debug) - clog << "TryNextMirror: " << Queue->Uri << endl; - - // inform parent - UsedMirror = *nextmirror; - Log("Switching mirror"); - return true; - } - - if (Debug) - clog << "TryNextMirror could not find another mirror to try" << endl; - - return false; -} + auto const accessColon = info.baseuri.find(':'); + auto access = info.baseuri.substr(0, accessColon); + std::string prefixAccess; + if (APT::String::Startswith(access, "mirror") == false) + { + auto const plus = info.baseuri.find('+'); + prefixAccess = info.baseuri.substr(0, plus); + access.erase(0, plus + 1); + } + std::vector<std::string> limitAccess; + // If the mirror file comes from an online source, allow only other online + // sources, not e.g. file:///. If the mirrorlist comes from there we can assume + // the admin knows what (s)he is doing through and not limit the options. + if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal), + [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) || + access == "mirror") + { + for (auto const &a : disallowLocal) + limitAccess.emplace_back(a); + } + std::string line; + while (mirrorlist.ReadLine(line)) + { + if (line.empty() || line[0] == '#') + continue; + auto const access = line.substr(0, line.find(':')); + if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end()) + continue; + auto const tab = line.find('\t'); + if (tab == std::string::npos) + { + if (prefixAccess.empty()) + info.list.emplace_back(std::move(line)); + else + info.list.emplace_back(prefixAccess + '+' + line); + } + else + { + auto uri = line.substr(0, tab); + if (prefixAccess.empty() == false) + uri = prefixAccess + '+' + uri; + auto tagline = line.substr(tab + 1); + std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' '); + auto tags = VectorizeString(tagline, ' '); + tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end()); + info.list.emplace_back(std::move(uri), std::move(tags)); + } + } + mirrorlist.Close(); -bool MirrorMethod::InitMirrors() -{ - // if we do not have a MirrorFile, fallback - if(!FileExists(MirrorFile)) - { - // FIXME: fallback to a default mirror here instead - // and provide a config option to define that default - return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + if (info.list.empty()) + { + info.state = FAILED; + DealWithPendingItems(baseuris, info, Itm, [&]() { + std::string msg; + strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str()); + Fail(msg, false); + }); + } + else + { + info.state = AVAILABLE; + DealWithPendingItems(baseuris, info, Itm, [&]() { + RedirectItem(info, Queue, msgCache[Queue->Uri]); + }); + msgCache.clear(); + } } - - if (access(MirrorFile.c_str(), R_OK) != 0) - { - // FIXME: fallback to a default mirror here instead - // and provide a config option to define that default - return _error->Error(_("Can not read mirror file '%s'"), MirrorFile.c_str()); - } - - // FIXME: make the mirror selection more clever, do not - // just use the first one! - // BUT: we can not make this random, the mirror has to be - // stable across session, because otherwise we can - // get into sync issues (got indexfiles from mirror A, - // but packages from mirror B - one might be out of date etc) - ifstream in(MirrorFile.c_str()); - string s; - while (!in.eof()) + else { - getline(in, s); - - // ignore lines that start with # - if (s.find("#") == 0) - continue; - // ignore empty lines - if (s.size() == 0) - continue; - // ignore non http lines - if (s.compare(0, strlen("http://"), "http://") != 0) - continue; - - AllMirrors.push_back(s); + info.state = FAILED; + DealWithPendingItems(baseuris, info, Itm, [&]() { + std::string msg; + strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str()); + Fail(msg, false); + }); } - if (AllMirrors.empty()) { - return _error->Error(_("No entry found in mirror file '%s'"), MirrorFile.c_str()); - } - Mirror = AllMirrors[0]; - UsedMirror = Mirror; return true; } - -string MirrorMethod::GetMirrorFileName(string mirror_uri_str) + /*}}}*/ +std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/ { - /* - - a mirror_uri_str looks like this: - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg - - - the matching source.list entry - deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main - - - we actually want to go after: - http://people.ubuntu.com/~mvo/apt/mirror/mirrors - - And we need to save the BaseUri for later: - - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors - - FIXME: what if we have two similar prefixes? - mirror://people.ubuntu.com/~mvo/mirror - mirror://people.ubuntu.com/~mvo/mirror2 - then mirror_uri_str looks like: - mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg - mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg - we search sources.list and find: - mirror://people.ubuntu.com/~mvo/apt/mirror - in both cases! So we need to apply some domain knowledge here :( and - check for /dists/ or /Release.gpg as suffixes - */ - string name; - if(Debug) - std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; - - // read sources.list and find match - vector<metaIndex *>::const_iterator I; - pkgSourceList list; - list.ReadMainList(); - for(I=list.begin(); I != list.end(); ++I) + if (APT::String::Startswith(Itm->Uri, Binary)) { - string uristr = (*I)->GetURI(); - if(Debug) - std::cerr << "Checking: " << uristr << std::endl; - if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) - continue; - // find matching uri in sources.list - if(mirror_uri_str.substr(0,uristr.size()) == uristr) + std::string const repouri = LookupTag(Message, "Target-Repo-Uri"); + if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend()) + sourceslist.push_back(repouri); + } + if (sourceslist.empty()) + { + // read sources.list and find the matching base uri + pkgSourceList sl; + if (sl.ReadMainList() == false) + { + _error->Error(_("The list of sources could not be read.")); + return ""; + } + std::string const needle = Binary + ":"; + for (auto const &SL : sl) { - if(Debug) - std::cerr << "found BaseURI: " << uristr << std::endl; - BaseUri = uristr.substr(0,uristr.size()-1); - Dist = (*I)->GetDist(); + std::string uristr = SL->GetURI(); + if (APT::String::Startswith(uristr, needle)) + sourceslist.push_back(uristr); } + sortByLength(sourceslist); } - // get new file - name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); - - if(Debug) + for (auto uristr : sourceslist) { - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << name << endl; + if (APT::String::Startswith(Itm->Uri, uristr)) + { + uristr.erase(uristr.length() - 1); // remove the ending '/' + auto const colon = uristr.find(':'); + if (unlikely(colon == std::string::npos)) + continue; + auto const plus = uristr.find("+"); + if (plus < colon) + { + // started as tor+mirror+http we want to get the file via tor+http + auto access = uristr.substr(0, colon); + std::string prefixAccess; + if (APT::String::Startswith(access, "mirror") == false) + { + prefixAccess = uristr.substr(0, plus); + access.erase(0, plus + 1); + uristr.erase(plus, strlen("mirror") + 1); + return uristr; + } + else + return uristr.substr(plus + 1); + } + else + { + uristr.replace(0, strlen("mirror"), "http"); + return uristr; + } + } } - return name; + return ""; } - -// MirrorMethod::Fetch - Fetch an item /*{{{*/ -// --------------------------------------------------------------------- -/* This adds an item to the pipeline. We keep the pipeline at a fixed - depth. */ -bool MirrorMethod::Fetch(FetchItem *Itm) + /*}}}*/ +bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/ { - if(Debug) - clog << "MirrorMethod::Fetch()" << endl; - - // the http method uses Fetch(0) as a way to update the pipeline, - // just let it do its work in this case - Fetch() with a valid - // Itm will always run before the first Fetch(0) - if(Itm == NULL) - return HttpMethod::Fetch(Itm); - - // if we don't have the name of the mirror file on disk yet, - // calculate it now (can be derived from the uri) - if(MirrorFile.empty()) - MirrorFile = GetMirrorFileName(Itm->Uri); + auto mirrorinfo = mirrorfilestate.find(Itm->Uri); + if (mirrorinfo != mirrorfilestate.end()) + return MirrorListFileRecieved(mirrorinfo->second, Itm); - // download mirror file once (if we are after index files) - if(Itm->IndexFile && !DownloadedMirrorFile) + std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm); + if (mirrorfileuri.empty()) { - Clean(_config->FindDir("Dir::State::mirrors")); - if (DownloadMirrorFile(Itm->Uri)) - RandomizeMirrorFile(MirrorFile); + _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str()); + return false; } + if (DebugEnabled()) + std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl; - if(AllMirrors.empty()) { - if(!InitMirrors()) { - // no valid mirror selected, something went wrong downloading - // from the master mirror site most likely and there is - // no old mirror file availalbe + // have we requested this mirror file already? + auto const state = mirrorfilestate.find(mirrorfileuri); + if (state == mirrorfilestate.end()) + { + msgCache[Itm->Uri] = Message; + MirrorListInfo info; + info.state = REQUESTED; + info.baseuri = mirrorfileuri + '/'; + auto const colon = info.baseuri.find(':'); + if (unlikely(colon == std::string::npos)) return false; - } + info.baseuri.replace(0, colon, Binary); + mirrorfilestate[mirrorfileuri] = info; + std::unordered_map<std::string, std::string> fields; + fields.emplace("URI", Itm->Uri); + fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone + fields.emplace("Aux-ShortDesc", "Mirrorlist"); + fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist"); + fields.emplace("Aux-Uri", mirrorfileuri); + SendMessage("351 Aux Request", std::move(fields)); + return true; } - if(Itm->Uri.find("mirror://") != string::npos) - Itm->Uri.replace(0,BaseUri.size(), Mirror); - - if(Debug) - clog << "Fetch: " << Itm->Uri << endl << endl; - - // now run the real fetcher - return HttpMethod::Fetch(Itm); -} - -void MirrorMethod::Fail(string Err,bool Transient) -{ - // FIXME: TryNextMirror is not ideal for indexfile as we may - // run into auth issues - - if (Debug) - clog << "Failure to get " << Queue->Uri << endl; - - // try the next mirror on fail (if its not a expected failure, - // e.g. translations are ok to ignore) - if (!Queue->FailIgnore && TryNextMirror()) - return; - - // all mirrors failed, so bail out - string s; - strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); - SetIP(s); - - CurrentQueueUriToMirror(); - pkgAcqMethod::Fail(Err, Transient); -} - -void MirrorMethod::URIStart(FetchResult &Res) -{ - CurrentQueueUriToMirror(); - pkgAcqMethod::URIStart(Res); -} - -void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) -{ - CurrentQueueUriToMirror(); - pkgAcqMethod::URIDone(Res, Alt); + switch (state->second.state) + { + case REQUESTED: + // lets wait for the requested mirror file + msgCache[Itm->Uri] = Message; + return true; + case FAILED: + Fail("Downloading mirror file failed", false); + return true; + case AVAILABLE: + RedirectItem(state->second, Itm, Message); + return true; + } + return false; } + /*}}}*/ - -int main() +int main(int, const char *argv[]) { - return MirrorMethod().Loop(); + return MirrorMethod(flNotDir(argv[0])).Run(); } - - diff --git a/methods/mirror.h b/methods/mirror.h deleted file mode 100644 index 6ebe08e6b..000000000 --- a/methods/mirror.h +++ /dev/null @@ -1,57 +0,0 @@ -// -*- mode: cpp; mode: fold -*- -// Description /*{{{*/ -/* ###################################################################### - - MIRROR Acquire Method - This is the MIRROR acquire method for APT. - - ##################################################################### */ - /*}}}*/ - -#ifndef APT_MIRROR_H -#define APT_MIRROR_H - -#include <iostream> -#include <string> -#include <vector> - -using std::cout; -using std::cerr; -using std::endl; - -#include "http.h" - -class MirrorMethod : public HttpMethod -{ - FetchResult Res; - // we simply transform between BaseUri and Mirror - std::string BaseUri; // the original mirror://... url - std::string Mirror; // the selected mirror uri (http://...) - std::vector<std::string> AllMirrors; // all available mirrors - std::string MirrorFile; // the file that contains the list of mirrors - bool DownloadedMirrorFile; // already downloaded this session - std::string Dist; // the target distrubtion (e.g. sid, oneiric) - - bool Debug; - - protected: - bool DownloadMirrorFile(std::string uri); - bool RandomizeMirrorFile(std::string file); - std::string GetMirrorFileName(std::string uri); - bool InitMirrors(); - bool TryNextMirror(); - void CurrentQueueUriToMirror(); - bool Clean(std::string dir); - - // we need to overwrite those to transform the url back - virtual void Fail(std::string Why, bool Transient = false) APT_OVERRIDE; - virtual void URIStart(FetchResult &Res) APT_OVERRIDE; - virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0) APT_OVERRIDE; - virtual bool Configuration(std::string Message) APT_OVERRIDE; - - public: - MirrorMethod(); - virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; -}; - - -#endif |