diff options
36 files changed, 1484 insertions, 959 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 9b163081b..611876dd2 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -16,6 +16,7 @@ #include <config.h> #include <apt-pkg/acquire-item.h> +#include <apt-pkg/acquire-worker.h> #include <apt-pkg/acquire.h> #include <apt-pkg/aptconfiguration.h> #include <apt-pkg/configuration.h> @@ -34,6 +35,7 @@ #include <algorithm> #include <ctime> #include <iostream> +#include <memory> #include <numeric> #include <random> #include <sstream> @@ -1070,6 +1072,12 @@ bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/ /*}}}*/ int pkgAcquire::Item::Priority() /*{{{*/ { + // Stage 0: Files requested by methods + // - they will usually not end up here, but if they do we make sure + // to get them as soon as possible as they are probably blocking + // the processing of files by the requesting method + if (dynamic_cast<pkgAcqAuxFile *>(this) != nullptr) + return 5000; // Stage 1: Meta indices and diff indices // - those need to be fetched first to have progress reporting working // for the rest @@ -3892,3 +3900,114 @@ string pkgAcqFile::Custom600Headers() const /*{{{*/ } /*}}}*/ pkgAcqFile::~pkgAcqFile() {} + +void pkgAcqAuxFile::Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/ +{ + pkgAcqFile::Failed(Message, Cnf); + if (Status == StatIdle) + return; + if (RealFileExists(DestFile)) + Rename(DestFile, DestFile + ".FAILED"); + Worker->ReplyAux(Desc); +} + /*}}}*/ +void pkgAcqAuxFile::Done(std::string const &Message, HashStringList const &CalcHashes, /*{{{*/ + pkgAcquire::MethodConfig const *const Cnf) +{ + pkgAcqFile::Done(Message, CalcHashes, Cnf); + if (Status == StatDone) + Worker->ReplyAux(Desc); + else if (Status == StatAuthError || Status == StatError) + Worker->ReplyAux(Desc); +} + /*}}}*/ +std::string pkgAcqAuxFile::Custom600Headers() const /*{{{*/ +{ + if (MaximumSize == 0) + return pkgAcqFile::Custom600Headers(); + std::string maxsize; + strprintf(maxsize, "\nMaximum-Size: %llu", MaximumSize); + return pkgAcqFile::Custom600Headers().append(maxsize); +} + /*}}}*/ +void pkgAcqAuxFile::Finished() /*{{{*/ +{ + auto dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"); + if (APT::String::Startswith(DestFile, dirname)) + { + // the file is never returned by method requesting it, so fix up the permission now + if (FileExists(DestFile)) + { + ChangeOwnerAndPermissionOfFile("pkgAcqAuxFile", DestFile.c_str(), "root", ROOT_GROUP, 0644); + if (Status == StatDone) + return; + } + } + else + { + dirname = flNotFile(DestFile); + RemoveFile("pkgAcqAuxFile::Finished", DestFile); + RemoveFile("pkgAcqAuxFile::Finished", DestFile + ".FAILED"); + rmdir(dirname.c_str()); + } + DestFile.clear(); +} + /*}}}*/ +// GetAuxFileNameFromURI /*{{{*/ +static std::string GetAuxFileNameFromURIInLists(std::string const &uri) +{ + // check if we have write permission for our usual location. + auto const dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"); + char const * const filetag = ".apt-acquire-privs-test.XXXXXX"; + std::string const tmpfile_tpl = flCombine(dirname, filetag); + std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free }; + int const fd = mkstemp(tmpfile.get()); + if (fd == -1 && errno == EACCES) + return ""; + RemoveFile("GetAuxFileNameFromURI", tmpfile.get()); + close(fd); + return flCombine(dirname, URItoFileName(uri)); +} +static std::string GetAuxFileNameFromURI(std::string const &uri) +{ + auto const lists = GetAuxFileNameFromURIInLists(uri); + if (lists.empty() == false) + return lists; + + std::string tmpdir_tpl; + strprintf(tmpdir_tpl, "%s/apt-auxfiles-XXXXXX", GetTempDir().c_str()); + std::unique_ptr<char, decltype(std::free) *> tmpdir { strndup(tmpdir_tpl.data(), tmpdir_tpl.length()), std::free }; + if (mkdtemp(tmpdir.get()) == nullptr) + { + _error->Errno("GetAuxFileNameFromURI", "mkdtemp of %s failed", tmpdir.get()); + return flCombine("/nonexistent/auxfiles/", URItoFileName(uri)); + } + chmod(tmpdir.get(), 0755); + auto const filename = flCombine(tmpdir.get(), URItoFileName(uri)); + _error->PushToStack(); + FileFd in(flCombine(flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"), URItoFileName(uri)), FileFd::ReadOnly); + if (in.IsOpen()) + { + FileFd out(filename, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive); + CopyFile(in, out); + } + _error->RevertToStack(); + return filename; +} + /*}}}*/ +pkgAcqAuxFile::pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, + std::string const &ShortDesc, std::string const &Desc, std::string const &URI, + HashStringList const &Hashes, unsigned long long const MaximumSize) : pkgAcqFile(Owner->GetOwner(), URI, Hashes, Hashes.FileSize(), Desc, ShortDesc, "", GetAuxFileNameFromURI(URI), false), + Owner(Owner), Worker(Worker), MaximumSize(MaximumSize) +{ + /* very bad failures can happen while constructing which causes + us to hang as the aux request is never answered (e.g. method not available) + Ideally we catch failures earlier, but a safe guard can't hurt. */ + if (Status == pkgAcquire::Item::StatIdle || Status == pkgAcquire::Item::StatFetching) + return; + Failed(std::string("400 URI Failure\n") + + "URI: " + URI + "\n" + + "Filename: " + DestFile, + nullptr); +} +pkgAcqAuxFile::~pkgAcqAuxFile() {} diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index cf227d1b5..46d79df92 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -1238,6 +1238,25 @@ class pkgAcqFile : public pkgAcquire::Item virtual ~pkgAcqFile(); }; /*}}}*/ +class APT_HIDDEN pkgAcqAuxFile : public pkgAcqFile /*{{{*/ +{ + pkgAcquire::Item *const Owner; + pkgAcquire::Worker *const Worker; + unsigned long long MaximumSize; + + public: + virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE; + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE; + virtual std::string Custom600Headers() const APT_OVERRIDE; + virtual void Finished() APT_OVERRIDE; + + pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker, + std::string const &ShortDesc, std::string const &Desc, std::string const &URI, + HashStringList const &Hashes, unsigned long long const MaximumSize); + virtual ~pkgAcqAuxFile(); +}; + /*}}}*/ /** @} */ #endif diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 309b5dcf9..8934d87a0 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -27,7 +27,10 @@ #include <apt-pkg/sha2.h> #include <apt-pkg/strutl.h> +#include <algorithm> #include <iostream> +#include <iterator> +#include <sstream> #include <string> #include <vector> #include <stdarg.h> @@ -39,33 +42,44 @@ using namespace std; +// poor mans unordered_map::try_emplace for C++11 as it is a C++17 feature /*{{{*/ +template <typename Arg> +static void try_emplace(std::unordered_map<std::string, std::string> &fields, std::string &&name, Arg &&value) +{ + if (fields.find(name) == fields.end()) + fields.emplace(std::move(name), std::forward<Arg>(value)); +} + /*}}}*/ + // AcqMethod::pkgAcqMethod - Constructor /*{{{*/ // --------------------------------------------------------------------- /* This constructs the initialization text */ pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags) { - std::cout << "100 Capabilities\n" - << "Version: " << Ver << "\n"; - + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "Version", Ver); if ((Flags & SingleInstance) == SingleInstance) - std::cout << "Single-Instance: true\n"; + try_emplace(fields, "Single-Instance", "true"); if ((Flags & Pipeline) == Pipeline) - std::cout << "Pipeline: true\n"; + try_emplace(fields, "Pipeline", "true"); if ((Flags & SendConfig) == SendConfig) - std::cout << "Send-Config: true\n"; + try_emplace(fields, "Send-Config", "true"); if ((Flags & LocalOnly) == LocalOnly) - std::cout <<"Local-Only: true\n"; + try_emplace(fields, "Local-Only", "true"); if ((Flags & NeedsCleanup) == NeedsCleanup) - std::cout << "Needs-Cleanup: true\n"; + try_emplace(fields, "Needs-Cleanup", "true"); if ((Flags & Removable) == Removable) - std::cout << "Removable: true\n"; + try_emplace(fields, "Removable", "true"); - std::cout << "\n" << std::flush; + if ((Flags & AuxRequests) == AuxRequests) + try_emplace(fields, "AuxRequests", "true"); + + SendMessage("100 Capabilities", std::move(fields)); SetNonBlock(STDIN_FILENO,true); @@ -73,6 +87,26 @@ pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags) QueueBack = 0; } /*}}}*/ +void pkgAcqMethod::SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields) /*{{{*/ +{ + std::cout << header << '\n'; + for (auto const &f : fields) + { + if (f.second.empty()) + continue; + std::cout << f.first << ": "; + auto const lines = VectorizeString(f.second, '\n'); + if (likely(lines.empty() == false)) + { + std::copy(lines.begin(), std::prev(lines.end()), std::ostream_iterator<std::string>(std::cout, "\n ")); + std::cout << *lines.rbegin(); + } + std::cout << '\n'; + } + std::cout << '\n' + << std::flush; +} + /*}}}*/ // AcqMethod::Fail - A fetch has failed /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -97,40 +131,35 @@ void pkgAcqMethod::Fail(bool Transient) } /*}}}*/ // AcqMethod::Fail - A fetch has failed /*{{{*/ -// --------------------------------------------------------------------- -/* */ void pkgAcqMethod::Fail(string Err,bool Transient) { // Strip out junk from the error messages - for (string::iterator I = Err.begin(); I != Err.end(); ++I) - { - if (*I == '\r') - *I = ' '; - if (*I == '\n') - *I = ' '; - } - - if (Queue != 0) - { - std::cout << "400 URI Failure\nURI: " << Queue->Uri << "\n" - << "Message: " << Err; - if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true) - std::cout << " " << IP; - std::cout << "\n"; - Dequeue(); - } + std::transform(Err.begin(), Err.end(), Err.begin(), [](char const c) { + if (c == '\r' || c == '\n') + return ' '; + return c; + }); + if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true) + Err.append(" ").append(IP); + + std::unordered_map<std::string, std::string> fields; + if (Queue != nullptr) + try_emplace(fields, "URI", Queue->Uri); else - std::cout << "400 URI Failure\nURI: <UNKNOWN>\nMessage: " << Err << "\n"; + try_emplace(fields, "URI", "<UNKNOWN>"); + try_emplace(fields, "Message", Err); if(FailReason.empty() == false) - std::cout << "FailReason: " << FailReason << "\n"; + try_emplace(fields, "FailReason", FailReason); if (UsedMirror.empty() == false) - std::cout << "UsedMirror: " << UsedMirror << "\n"; - // Set the transient flag + try_emplace(fields, "UsedMirror", UsedMirror); if (Transient == true) - std::cout << "Transient-Failure: true\n"; + try_emplace(fields, "Transient-Failure", "true"); - std::cout << "\n" << std::flush; + SendMessage("400 URI Failure", std::move(fields)); + + if (Queue != nullptr) + Dequeue(); } /*}}}*/ // AcqMethod::DropPrivsOrDie - Drop privileges or die /*{{{*/ @@ -146,96 +175,79 @@ void pkgAcqMethod::DropPrivsOrDie() /*}}}*/ // AcqMethod::URIStart - Indicate a download is starting /*{{{*/ -// --------------------------------------------------------------------- -/* */ void pkgAcqMethod::URIStart(FetchResult &Res) { if (Queue == 0) abort(); - std::cout << "200 URI Start\n" - << "URI: " << Queue->Uri << "\n"; + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); if (Res.Size != 0) - std::cout << "Size: " << std::to_string(Res.Size) << "\n"; - + try_emplace(fields, "Size", std::to_string(Res.Size)); if (Res.LastModified != 0) - std::cout << "Last-Modified: " << TimeRFC1123(Res.LastModified, true) << "\n"; - + try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true)); if (Res.ResumePoint != 0) - std::cout << "Resume-Point: " << std::to_string(Res.ResumePoint) << "\n"; - + try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint)); if (UsedMirror.empty() == false) - std::cout << "UsedMirror: " << UsedMirror << "\n"; + try_emplace(fields, "UsedMirror", UsedMirror); - std::cout << "\n" << std::flush; + SendMessage("200 URI Start", std::move(fields)); } /*}}}*/ // AcqMethod::URIDone - A URI is finished /*{{{*/ -// --------------------------------------------------------------------- -/* */ -static void printHashStringList(char const *const Prefix, HashStringList const *const list) +static void printHashStringList(std::unordered_map<std::string, std::string> &fields, std::string const &Prefix, HashStringList const &list) { - for (HashStringList::const_iterator hash = list->begin(); hash != list->end(); ++hash) - { - // very old compatibility name for MD5Sum - if (hash->HashType() == "MD5Sum") - std::cout << Prefix << "MD5-Hash: " << hash->HashValue() << "\n"; - std::cout << hash->HashType() << "-Hash: " << hash->HashValue() << "\n"; - } + for (auto const &hash : list) + { + // very old compatibility name for MD5Sum + if (hash.HashType() == "MD5Sum") + try_emplace(fields, Prefix + "MD5-Hash", hash.HashValue()); + try_emplace(fields, Prefix + hash.HashType() + "-Hash", hash.HashValue()); + } } void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) { if (Queue == 0) abort(); - std::cout << "201 URI Done\n" - << "URI: " << Queue->Uri << "\n"; - + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); if (Res.Filename.empty() == false) - std::cout << "Filename: " << Res.Filename << "\n"; - + try_emplace(fields, "Filename", Res.Filename); if (Res.Size != 0) - std::cout << "Size: " << std::to_string(Res.Size) << "\n"; - + try_emplace(fields, "Size", std::to_string(Res.Size)); if (Res.LastModified != 0) - std::cout << "Last-Modified: " << TimeRFC1123(Res.LastModified, true) << "\n"; - - printHashStringList("", &Res.Hashes); + try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true)); + printHashStringList(fields, "", Res.Hashes); if (UsedMirror.empty() == false) - std::cout << "UsedMirror: " << UsedMirror << "\n"; + try_emplace(fields, "UsedMirror", UsedMirror); if (Res.GPGVOutput.empty() == false) { - std::cout << "GPGVOutput:\n"; - for (vector<string>::const_iterator I = Res.GPGVOutput.begin(); - I != Res.GPGVOutput.end(); ++I) - std::cout << " " << *I << "\n"; + std::ostringstream os; + std::copy(Res.GPGVOutput.begin(), Res.GPGVOutput.end() - 1, std::ostream_iterator<std::string>(os, "\n")); + os << *Res.GPGVOutput.rbegin(); + try_emplace(fields, "GPGVOutput", os.str()); } - if (Res.ResumePoint != 0) - std::cout << "Resume-Point: " << std::to_string(Res.ResumePoint) << "\n"; - + try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint)); if (Res.IMSHit == true) - std::cout << "IMS-Hit: true\n"; + try_emplace(fields, "IMS-Hit", "true"); - if (Alt != 0) + if (Alt != nullptr) { if (Alt->Filename.empty() == false) - std::cout << "Alt-Filename: " << Alt->Filename << "\n"; - + try_emplace(fields, "Alt-Filename", Alt->Filename); if (Alt->Size != 0) - std::cout << "Alt-Size: " << std::to_string(Alt->Size) << "\n"; - + try_emplace(fields, "Alt-Size", std::to_string(Alt->Size)); if (Alt->LastModified != 0) - std::cout << "Alt-Last-Modified: " << TimeRFC1123(Alt->LastModified, true) << "\n"; - - printHashStringList("Alt-", &Alt->Hashes); - + try_emplace(fields, "Alt-Last-Modified", TimeRFC1123(Alt->LastModified, true)); if (Alt->IMSHit == true) - std::cout << "Alt-IMS-Hit: true\n"; + try_emplace(fields, "Alt-IMS-Hit", "true"); + printHashStringList(fields, "Alt-", Alt->Hashes); } - std::cout << "\n" << std::flush; + SendMessage("201 URI Done", std::move(fields)); Dequeue(); } /*}}}*/ @@ -459,9 +471,10 @@ void pkgAcqMethod::Status(const char *Format,...) * the worker will enqueue again later on to the right queue */ void pkgAcqMethod::Redirect(const string &NewURI) { - std::cout << "103 Redirect\nURI: " << Queue->Uri << "\n" - << "New-URI: " << NewURI << "\n" - << "\n" << std::flush; + std::unordered_map<std::string, std::string> fields; + try_emplace(fields, "URI", Queue->Uri); + try_emplace(fields, "New-URI", NewURI); + SendMessage("103 Redirect", std::move(fields)); Dequeue(); } /*}}}*/ diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 2de9cf5c2..664b95c18 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -26,6 +26,7 @@ #include <time.h> #include <string> +#include <unordered_map> #include <vector> #ifndef APT_8_CLEANER_HEADERS @@ -99,6 +100,7 @@ class pkgAcqMethod virtual void Fail(std::string Why, bool Transient = false); virtual void URIStart(FetchResult &Res); virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0); + void SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields); bool MediaFail(std::string Required,std::string Drive); virtual void Exit() {}; @@ -106,10 +108,16 @@ class pkgAcqMethod void PrintStatus(char const * const header, const char* Format, va_list &args) const; public: - enum CnfFlags {SingleInstance = (1<<0), - Pipeline = (1<<1), SendConfig = (1<<2), - LocalOnly = (1<<3), NeedsCleanup = (1<<4), - Removable = (1<<5)}; + enum CnfFlags + { + SingleInstance = (1 << 0), + Pipeline = (1 << 1), + SendConfig = (1 << 2), + LocalOnly = (1 << 3), + NeedsCleanup = (1 << 4), + Removable = (1 << 5), + AuxRequests = (1 << 6) + }; void Log(const char *Format,...); void Status(const char *Format,...); diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 995750dea..6cbf8b7aa 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -192,7 +192,8 @@ bool pkgAcquire::Worker::ReadMessages() // --------------------------------------------------------------------- /* This takes the messages from the message queue and runs them through the parsers in order. */ -enum class APT_HIDDEN MessageType { +enum class APT_HIDDEN MessageType +{ CAPABILITIES = 100, LOG = 101, STATUS = 102, @@ -200,6 +201,7 @@ enum class APT_HIDDEN MessageType { WARNING = 104, URI_START = 200, URI_DONE = 201, + AUX_REQUEST = 351, URI_FAILURE = 400, GENERAL_FAILURE = 401, MEDIA_CHANGE = 403 @@ -306,6 +308,7 @@ bool pkgAcquire::Worker::RunMessages() std::string const NewURI = LookupTag(Message,"New-URI",URI.c_str()); Itm->URI = NewURI; + auto const AltUris = VectorizeString(LookupTag(Message, "Alternate-URIs"), '\n'); ItemDone(); @@ -333,28 +336,14 @@ bool pkgAcquire::Worker::RunMessages() if (Log != nullptr) Log->Done(desc); - // if we change site, treat it as a mirror change - if (URI::SiteOnly(NewURI) != URI::SiteOnly(desc.URI)) - { - auto const firstSpace = desc.Description.find(" "); - if (firstSpace != std::string::npos) - { - std::string const OldSite = desc.Description.substr(0, firstSpace); - if (likely(APT::String::Startswith(desc.URI, OldSite))) - { - std::string const OldExtra = desc.URI.substr(OldSite.length() + 1); - if (likely(APT::String::Endswith(NewURI, OldExtra))) - { - std::string const NewSite = NewURI.substr(0, NewURI.length() - OldExtra.length()); - Owner->UsedMirror = URI::ArchiveOnly(NewSite); - desc.Description.replace(0, firstSpace, Owner->UsedMirror); - } - } - } - } + ChangeSiteIsMirrorChange(NewURI, desc, Owner); desc.URI = NewURI; if (isDoomedItem(Owner) == false) + { + for (auto alt = AltUris.crbegin(); alt != AltUris.crend(); ++alt) + Owner->PushAlternativeURI(std::string(*alt), {}, false); OwnerQ->Owner->Enqueue(desc); + } } break; } @@ -512,6 +501,42 @@ bool pkgAcquire::Worker::RunMessages() break; } + case MessageType::AUX_REQUEST: + { + if (Itm == nullptr) + { + _error->Error("Method gave invalid Aux Request message"); + break; + } + else if (Config->GetAuxRequests() == false) + { + std::vector<Item *> const ItmOwners = Itm->Owners; + Message.append("\nMessage: Method tried to make an Aux Request while not being allowed to do them"); + OwnerQ->ItemDone(Itm); + Itm = nullptr; + HandleFailure(ItmOwners, Config, Log, Message, false, false); + ItemDone(); + + std::string Msg = "600 URI Acquire\n"; + Msg.reserve(200); + Msg += "URI: " + LookupTag(Message, "Aux-URI", ""); + Msg += "\nFilename: /nonexistent/auxrequest.blocked"; + Msg += "\n\n"; + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(Msg, "\n") << endl; + OutQueue += Msg; + OutReady = true; + break; + } + + auto maxsizestr = LookupTag(Message, "MaximumSize", ""); + unsigned long long const MaxSize = maxsizestr.empty() ? 0 : strtoull(maxsizestr.c_str(), nullptr, 10); + new pkgAcqAuxFile(Itm->Owner, this, LookupTag(Message, "Aux-ShortDesc", ""), + LookupTag(Message, "Aux-Description", ""), LookupTag(Message, "Aux-URI", ""), + GetHashesFromMessage("Aux-", Message), MaxSize); + break; + } + case MessageType::URI_FAILURE: { if (Itm == nullptr) @@ -549,44 +574,7 @@ bool pkgAcquire::Worker::RunMessages() errAuthErr = std::find(std::begin(reasons), std::end(reasons), failReason) != std::end(reasons); } } - - for (auto const Owner: ItmOwners) - { - std::string NewURI; - if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0) - { - --Owner->ModifyRetries(); - Owner->FailMessage(Message); - auto SavedDesc = Owner->GetItemDesc(); - if (Log != nullptr) - Log->Fail(SavedDesc); - if (isDoomedItem(Owner) == false) - OwnerQ->Owner->Enqueue(SavedDesc); - } - else if (Owner->PopAlternativeURI(NewURI)) - { - Owner->FailMessage(Message); - auto &desc = Owner->GetItemDesc(); - if (Log != nullptr) - Log->Fail(desc); - ChangeSiteIsMirrorChange(NewURI, desc, Owner); - desc.URI = NewURI; - if (isDoomedItem(Owner) == false) - OwnerQ->Owner->Enqueue(desc); - } - else - { - if (errAuthErr && Owner->GetExpectedHashes().empty() == false) - Owner->Status = pkgAcquire::Item::StatAuthError; - else if (errTransient) - Owner->Status = pkgAcquire::Item::StatTransientNetworkError; - auto SavedDesc = Owner->GetItemDesc(); - if (isDoomedItem(Owner) == false) - Owner->Failed(Message, Config); - if (Log != nullptr) - Log->Fail(SavedDesc); - } - } + HandleFailure(ItmOwners, Config, Log, Message, errTransient, errAuthErr); ItemDone(); break; @@ -604,6 +592,49 @@ bool pkgAcquire::Worker::RunMessages() return true; } /*}}}*/ +void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &ItmOwners, /*{{{*/ + pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log, + std::string const &Message, bool const errTransient, bool const errAuthErr) +{ + for (auto const Owner : ItmOwners) + { + std::string NewURI; + if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0) + { + --Owner->ModifyRetries(); + Owner->FailMessage(Message); + auto SavedDesc = Owner->GetItemDesc(); + if (Log != nullptr) + Log->Fail(SavedDesc); + if (isDoomedItem(Owner) == false) + OwnerQ->Owner->Enqueue(SavedDesc); + } + else if (Owner->PopAlternativeURI(NewURI)) + { + Owner->FailMessage(Message); + auto &desc = Owner->GetItemDesc(); + if (Log != nullptr) + Log->Fail(desc); + ChangeSiteIsMirrorChange(NewURI, desc, Owner); + desc.URI = NewURI; + if (isDoomedItem(Owner) == false) + OwnerQ->Owner->Enqueue(desc); + } + else + { + if (errAuthErr && Owner->GetExpectedHashes().empty() == false) + Owner->Status = pkgAcquire::Item::StatAuthError; + else if (errTransient) + Owner->Status = pkgAcquire::Item::StatTransientNetworkError; + auto SavedDesc = Owner->GetItemDesc(); + if (isDoomedItem(Owner) == false) + Owner->Failed(Message, Config); + if (Log != nullptr) + Log->Fail(SavedDesc); + } + } +} + /*}}}*/ // Worker::Capabilities - 100 Capabilities handler /*{{{*/ // --------------------------------------------------------------------- /* This parses the capabilities message and dumps it into the configuration @@ -620,18 +651,13 @@ bool pkgAcquire::Worker::Capabilities(string Message) Config->LocalOnly = StringToBool(LookupTag(Message,"Local-Only"),false); Config->NeedsCleanup = StringToBool(LookupTag(Message,"Needs-Cleanup"),false); Config->Removable = StringToBool(LookupTag(Message,"Removable"),false); + Config->SetAuxRequests(StringToBool(LookupTag(Message, "AuxRequests"), false)); // Some debug text if (Debug == true) { clog << "Configured access method " << Config->Access << endl; - clog << "Version:" << Config->Version << - " SingleInstance:" << Config->SingleInstance << - " Pipeline:" << Config->Pipeline << - " SendConfig:" << Config->SendConfig << - " LocalOnly: " << Config->LocalOnly << - " NeedsCleanup: " << Config->NeedsCleanup << - " Removable: " << Config->Removable << endl; + clog << "Version:" << Config->Version << " SingleInstance:" << Config->SingleInstance << " Pipeline:" << Config->Pipeline << " SendConfig:" << Config->SendConfig << " LocalOnly: " << Config->LocalOnly << " NeedsCleanup: " << Config->NeedsCleanup << " Removable: " << Config->Removable << " AuxRequests: " << Config->GetAuxRequests() << endl; } return true; @@ -772,6 +798,46 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) return true; } /*}}}*/ +// Worker::ReplyAux - reply to an aux request from this worker /*{{{*/ +bool pkgAcquire::Worker::ReplyAux(pkgAcquire::ItemDesc const &Item) +{ + if (OutFd == -1) + return false; + + if (isDoomedItem(Item.Owner)) + return true; + + string Message = "600 URI Acquire\n"; + Message.reserve(200); + Message += "URI: " + Item.URI; + if (RealFileExists(Item.Owner->DestFile)) + { + if (Item.Owner->Status == pkgAcquire::Item::StatDone) + { + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + ChangeOwnerAndPermissionOfFile("Worker::ReplyAux", Item.Owner->DestFile.c_str(), + SandboxUser.c_str(), ROOT_GROUP, 0600); + Message += "\nFilename: " + Item.Owner->DestFile; + } + else + { + // we end up here in case we would need root-rights to delete a file, + // but we run the command as non-root… (yes, it is unlikely) + Message += "\nFilename: " + flCombine("/nonexistent", Item.Owner->DestFile); + } + } + else + Message += "\nFilename: " + Item.Owner->DestFile; + Message += "\n\n"; + + if (Debug == true) + clog << " -> " << Access << ':' << QuoteString(Message, "\n") << endl; + OutQueue += Message; + OutReady = true; + + return true; +} + /*}}}*/ // Worker::OutFdRead - Out bound FD is ready /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h index 8fc686880..f04db2b3c 100644 --- a/apt-pkg/acquire-worker.h +++ b/apt-pkg/acquire-worker.h @@ -276,6 +276,7 @@ class pkgAcquire::Worker : public WeakPointable * queue. */ bool QueueItem(pkgAcquire::Queue::QItem *Item); + APT_HIDDEN bool ReplyAux(pkgAcquire::ItemDesc const &Item); /** \brief Start up the worker and fill in #Config. * @@ -328,6 +329,9 @@ class pkgAcquire::Worker : public WeakPointable private: APT_HIDDEN void PrepareFiles(char const * const caller, pkgAcquire::Queue::QItem const * const Itm); + APT_HIDDEN void HandleFailure(std::vector<pkgAcquire::Item *> const &ItmOwners, + pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log, + std::string const &Message, bool const errTransient, bool const errAuthErr); }; /** @} */ diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index aabcb0aba..b25a4434a 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -78,13 +78,13 @@ void pkgAcquire::Initialize() } /*}}}*/ // Acquire::GetLock - lock directory and prepare for action /*{{{*/ -static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent) +static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode) { - std::string const partial = parent + "partial"; - mode_t const mode = umask(S_IWGRP | S_IWOTH); + std::string const partial = parent + postfix; + mode_t const old_umask = umask(S_IWGRP | S_IWOTH); bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false && CreateAPTDirectoryIfNeeded(parent, partial) == false); - umask(mode); + umask(old_umask); if (creation_fail == true) return false; @@ -100,7 +100,7 @@ static bool SetupAPTPartialDirectory(std::string const &grand, std::string const _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str()); } } - if (chmod(partial.c_str(), 0700) != 0) + if (chmod(partial.c_str(), mode) != 0) _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str()); _error->PushToStack(); @@ -117,10 +117,12 @@ bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) if (Lock.empty()) { string const listDir = _config->FindDir("Dir::State::lists"); - if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false) return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false) + return _error->Errno("Acquire", _("List directory %sauxfiles is missing."), listDir.c_str()); string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false) return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); return true; } @@ -137,14 +139,19 @@ bool pkgAcquire::GetLock(std::string const &Lock) if (Lock == listDir) { - if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false) return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); } if (Lock == archivesDir) { - if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false) return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); } + if (Lock == listDir || Lock == archivesDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false) + return _error->Errno("Acquire", _("List directory %sauxfiles is missing."), listDir.c_str()); + } if (_config->FindB("Debug::NoLocking", false) == true) return true; @@ -288,7 +295,6 @@ static bool CheckForBadItemAndFailIt(pkgAcquire::Item * const Item, "\nFilename: " + Item->DestFile + "\nFailReason: WeakHashSums"; - auto SavedDesc = Item->GetItemDesc(); Item->Status = pkgAcquire::Item::StatAuthError; Item->Failed(Message, Config); if (Log != nullptr) @@ -303,7 +309,10 @@ void pkgAcquire::Enqueue(ItemDesc &Item) const MethodConfig *Config; string Name = QueueName(Item.URI,Config); if (Name.empty() == true) + { + Item.Owner->Status = pkgAcquire::Item::StatError; return; + } /* the check for running avoids that we produce errors in logging before we actually have started, which would @@ -769,11 +778,12 @@ bool pkgAcquire::Clean(string Dir) for (struct dirent *E = readdir(D); E != nullptr; E = readdir(D)) { // Skip some entries - if (strcmp(E->d_name,"lock") == 0 || - strcmp(E->d_name,"partial") == 0 || - strcmp(E->d_name,"lost+found") == 0 || - strcmp(E->d_name,".") == 0 || - strcmp(E->d_name,"..") == 0) + if (strcmp(E->d_name, "lock") == 0 || + strcmp(E->d_name, "partial") == 0 || + strcmp(E->d_name, "auxfiles") == 0 || + strcmp(E->d_name, "lost+found") == 0 || + strcmp(E->d_name, ".") == 0 || + strcmp(E->d_name, "..") == 0) continue; // Look in the get list and if not found nuke @@ -845,12 +855,25 @@ pkgAcquire::UriIterator pkgAcquire::UriEnd() } /*}}}*/ // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ -pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false), - Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false), - Removable(false) +class pkgAcquire::MethodConfig::Private +{ + public: + bool AuxRequests = false; +}; +pkgAcquire::MethodConfig::MethodConfig() : d(new Private()), Next(0), SingleInstance(false), + Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false), + Removable(false) +{ +} + /*}}}*/ +bool pkgAcquire::MethodConfig::GetAuxRequests() const /*{{{*/ +{ + return d->AuxRequests; +} + /*}}}*/ +void pkgAcquire::MethodConfig::SetAuxRequests(bool const value) /*{{{*/ { + d->AuxRequests = value; } /*}}}*/ // Queue::Queue - Constructor /*{{{*/ diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index e58aeef65..1cf4da5bf 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -636,9 +636,10 @@ class pkgAcquire::UriIterator /** \brief Information about the properties of a single acquire method. {{{*/ struct pkgAcquire::MethodConfig { + class Private; /** \brief dpointer placeholder (for later in case we need it) */ - void * const d; - + Private *const d; + /** \brief The next link on the acquire method list. * * \todo Why not an STL container? @@ -688,6 +689,9 @@ struct pkgAcquire::MethodConfig */ MethodConfig(); + APT_HIDDEN bool GetAuxRequests() const; + APT_HIDDEN void SetAuxRequests(bool const value); + virtual ~MethodConfig(); }; /*}}}*/ diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc index 04a7c4910..1abc638ee 100644 --- a/apt-pkg/clean.cc +++ b/apt-pkg/clean.cc @@ -62,11 +62,12 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache) for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) { // Skip some files.. - if (strcmp(Dir->d_name,"lock") == 0 || - strcmp(Dir->d_name,"partial") == 0 || - strcmp(Dir->d_name,"lost+found") == 0 || - strcmp(Dir->d_name,".") == 0 || - strcmp(Dir->d_name,"..") == 0) + if (strcmp(Dir->d_name, "lock") == 0 || + strcmp(Dir->d_name, "partial") == 0 || + strcmp(Dir->d_name, "auxfiles") == 0 || + strcmp(Dir->d_name, "lost+found") == 0 || + strcmp(Dir->d_name, ".") == 0 || + strcmp(Dir->d_name, "..") == 0) continue; struct stat St; diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index 207f0d902..5c34113e7 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -138,7 +138,6 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.CndSet("Dir::State", STATE_DIR + 1); Cnf.CndSet("Dir::State::lists","lists/"); Cnf.CndSet("Dir::State::cdroms","cdroms.list"); - Cnf.CndSet("Dir::State::mirrors","mirrors/"); // Cache Cnf.CndSet("Dir::Cache", CACHE_DIR + 1); diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc index 77c35f4de..8bd7e33e0 100644 --- a/apt-private/private-download.cc +++ b/apt-private/private-download.cc @@ -225,7 +225,8 @@ bool DoDownload(CommandLine &CmdL) std::string const filename = cwd + flNotDir((*I)->DestFile); if ((*I)->Local == true && filename != (*I)->DestFile && - (*I)->Status == pkgAcquire::Item::StatDone) + (*I)->Status == pkgAcquire::Item::StatDone && + dynamic_cast<pkgAcqArchive*>(*I) != nullptr) { std::ifstream src((*I)->DestFile.c_str(), std::ios::binary); std::ofstream dst(filename.c_str(), std::ios::binary); diff --git a/debian/apt.install b/debian/apt.install index d30f429b0..705ea12d2 100644 --- a/debian/apt.install +++ b/debian/apt.install @@ -35,6 +35,9 @@ usr/share/man/*/apt-get.* usr/share/man/*/apt-key.* usr/share/man/*/apt-mark.* usr/share/man/*/apt-secure.* +usr/share/man/*/apt-transport-http.* +usr/share/man/*/apt-transport-https.* +usr/share/man/*/apt-transport-mirror.* usr/share/man/*/apt.* usr/share/man/*/apt_auth.* usr/share/man/*/apt_preferences.* diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 73d808c64..7cca4cf81 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -84,6 +84,9 @@ add_docbook(apt-man MANPAGE ALL apt_preferences.5.xml apt-secure.8.xml apt-sortpkgs.1.xml + apt-transport-http.1.xml + apt-transport-https.1.xml + apt-transport-mirror.1.xml sources.list.5.xml DEPENDS ${ENTITIES} TRANSLATED_ENTITIES ${TRANSLATED_ENTITIES} diff --git a/doc/apt-transport-http.1.xml b/doc/apt-transport-http.1.xml new file mode 100644 index 000000000..546e47761 --- /dev/null +++ b/doc/apt-transport-http.1.xml @@ -0,0 +1,138 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" + "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ +<!ENTITY % aptent SYSTEM "apt.ent"> %aptent; +<!ENTITY % aptverbatiment SYSTEM "apt-verbatim.ent"> %aptverbatiment; +<!ENTITY % aptvendor SYSTEM "apt-vendor.ent"> %aptvendor; +]> + +<refentry> + + <refentryinfo> + &apt-author.team; + &apt-email; + &apt-product; + <!-- The last update date --> + <date>2017-11-22T00:00:00Z</date> + </refentryinfo> + + <refmeta> + <refentrytitle>apt-transport-http</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="manual">APT</refmiscinfo> + </refmeta> + + <!-- Man page title --> + <refnamediv> + <refname>apt-transport-http</refname> + <refpurpose>APT transport for downloading via the Hypertext Transfer Protocol (HTTP)</refpurpose> + </refnamediv> + +<refsect1><title>Description</title> +<para>This APT transport allows the use of repositories accessed via the +Hypertext Transfer Protocol (HTTP). It is available by default and probably the +most used of all transports. Note that a transport is never called directly by +a user but used by APT tools based on user configuration.</para> +<para>HTTP is an unencrypted transport protocol meaning that the +whole communication with the remote server (or proxy) can be observed by a +sufficiently capable attacker referred to commonly as man in the middle (MITM). +Such an attacker can <emphasis>not</emphasis> modify the communication to compromise +the security of your system through as APTs data security model is independent of the +chosen transport method. This is explained in detail in &apt-secure;. An overview over +available transport methods is given in &sources-list;.</para> +</refsect1> + +<refsect1><title>Options</title> +<para>Various options are available to modify its behaviour which can be set in +an &apt-conf; file ranging from proxy configuration to workaround for specific +server insufficiencies.</para> + +<refsect2><title>Proxy Configuration</title> +<para>The environment variable <envar>http_proxy</envar> is supported for system wide configuration. +Proxies specific to apt can be configured via the option <literal>Acquire::http::Proxy</literal>. +Proxies which should be used only for certain hosts can be specified via +<literal>Acquire::http::Proxy::<replaceable>host</replaceable></literal>. Even more finegrained control +can be achieved via proxy autodetection detailed further below. +All these options use the URI format <literal><replaceable>scheme</replaceable>://[[<replaceable>user</replaceable>][:<replaceable>pass</replaceable>]@]<replaceable>host</replaceable>[:<replaceable>port</replaceable>]/</literal>. +Supported URI schemes are <literal>socks5h</literal> (SOCKS5 with remote DNS resolution), <literal>http</literal> and <literal>https</literal>. +Authentification details can be supplied via &apt-authconf; instead of including it in the URI directly.</para> +<para>The various APT configuration options support the special value <literal>DIRECT</literal> meaning that +no proxy should be used. The environment variable <envar>no_proxy</envar> with the same propose is also supported.</para> +<para>Further more there are three settings provided for cache control with HTTP/1.1 compliant proxy caches: +<literal>Acquire::http::No-Cache</literal> tells the proxy not to use its +cached response under any circumstances. +<literal>Acquire::http::Max-Age</literal> sets the allowed maximum age (in +seconds) of an index file in the cache of the proxy. +<literal>Acquire::http::No-Store</literal> specifies that the proxy should not +store the requested archive files in its cache, which can be used to prevent +the proxy from polluting its cache with (big) .deb files.</para> +</refsect2> + +<refsect2><title>Automatic Proxy Configuration</title> +<para><literal>Acquire::http::Proxy-Auto-Detect</literal> can be used to +specify an external command to discover the http proxy to use. The first +and only parameter is an URI denoting the host to be contacted to allow +for host-specific configuration. APT expects the command to output the +proxy on stdout as a single line in the previously specified URI format +or the word <literal>DIRECT</literal> if no proxy should be used. No output +indicates that the generic proxy settings should be used.</para> +<para>Note that auto-detection will not be used for a host if a host-specific proxy +configuration is already set via <literal>Acquire::http::Proxy::<replaceable>host</replaceable></literal>.</para> +<para>See the &squid-deb-proxy-client; and &auto-apt-proxy; packages for example implementations.</para> +<para>This option takes precedence over the legacy option name <literal>Acquire::http::ProxyAutoDetect</literal>.</para> +</refsect2> + +<refsect2><title>Connection Configuration</title> +<para>The option <literal>Acquire::http::Timeout</literal> sets the timeout timer used by the method; +this value applies to the connection as well as the data timeout.</para> +<para>The used bandwidth can be limited with +<literal>Acquire::http::Dl-Limit</literal> which accepts integer values in +kilobytes per second. The default value is 0 which deactivates the limit and +tries to use all available bandwidth. Note that this option implicitly +disables downloading from multiple servers at the same time.</para> +<para>The setting <literal>Acquire::http::Pipeline-Depth</literal> can be used to +enable HTTP pipelining (RFC 2616 section 8.1.2.2) which can be beneficial e.g. on +high-latency connections. It specifies how many requests are sent in a pipeline. +APT tries to detect and workaround misbehaving webservers and proxies at runtime, but +if you know that yours does not conform to the HTTP/1.1 specification pipelining can +be disabled by setting the value to 0. It is enabled by default with the value 10.</para> +<para><literal>Acquire::http::AllowRedirect</literal> controls whether APT will follow +redirects, which is enabled by default.</para> +<para><literal>Acquire::http::User-Agent</literal> can be used to set a different +User-Agent for the http download method as some proxies allow access for clients +only if the client uses a known identifier.</para> +<para><literal>Acquire::http::SendAccept</literal> is enabled by default and +sends a <literal>Accept: text/*</literal> header field to the server for +requests without file extensions to prevent the server from attempting content +negotiation.</para> +</refsect2> +</refsect1> + +<refsect1><title>Examples</title> +<literallayout> +Acquire::http { + Proxy::example.org "DIRECT"; + Proxy "socks5h://apt:pass@localhost:9050"; + Proxy-Auto-Detect "/usr/local/bin/apt-http-proxy-auto-detect"; + No-Cache "true"; + Max-Age "3600"; + No-Store "true"; + Timeout "10"; + Dl-Limit "42"; + Pipeline-Depth "0"; + AllowRedirect "false"; + User-Agent "My APT-HTTP"; + SendAccept "false"; +}; +</literallayout> +</refsect1> + +<refsect1> +<title>See Also</title> +<para>&apt-conf; &apt-authconf; &sources-list; +</para> +</refsect1> + + &manbugs; + +</refentry> diff --git a/doc/apt-transport-https.1.xml b/doc/apt-transport-https.1.xml new file mode 100644 index 000000000..97137fc2c --- /dev/null +++ b/doc/apt-transport-https.1.xml @@ -0,0 +1,133 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" + "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ +<!ENTITY % aptent SYSTEM "apt.ent"> %aptent; +<!ENTITY % aptverbatiment SYSTEM "apt-verbatim.ent"> %aptverbatiment; +<!ENTITY % aptvendor SYSTEM "apt-vendor.ent"> %aptvendor; +]> + +<refentry> + + <refentryinfo> + &apt-author.team; + &apt-email; + &apt-product; + <!-- The last update date --> + <date>2017-11-22T00:00:00Z</date> + </refentryinfo> + + <refmeta> + <refentrytitle>apt-transport-https</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="manual">APT</refmiscinfo> + </refmeta> + + <!-- Man page title --> + <refnamediv> + <refname>apt-transport-https</refname> + <refpurpose>APT transport for downloading via the HTTP Secure protocol (HTTPS)</refpurpose> + </refnamediv> + +<refsect1><title>Description</title> +<para>This APT transport allows the use of repositories accessed via the +HTTP Secure protocol (HTTPS) also referred to as HTTP over TLS. It is available +by default since apt 1.5 and was before that available in a <package>apt-transport-https</package> +package. Note that a transport is never called directly by +a user but used by APT tools based on user configuration.</para> +<para>HTTP is by itself an unencrypted transport protocol (compare &apt-transport-http;), +which, as indicated by the appended S is wrapped in an encrypted layer known as +Transport Layer Security (TLS) which provides end-to-end encryption. +A sufficiently capable attacker can still observe the communication partners +and deeper analyse of the encrypted communcation might still reveal important details. +An overview over available alternative transport methods is given in &sources-list;.</para> +</refsect1> + +<refsect1><title>Options</title> +<para>The HTTPS protocol is based on the HTTP protocol and as such this implementation +has the same relation meaning that all options supported by &apt-transport-http; are also +available via <literal>Acquire::https</literal> and will default to the same values specified +for <literal>Acquire::http</literal>. This manpage will only document the options +<emphasis>unique to https</emphasis>.</para> + +<refsect2><title>Server credentials</title> +<para>By default all certificates trusted by the system (see +<package>ca-certificates</package> package) are used for the verification of +the server certificate. An alternative certificate authority (CA) can be +configured with the <literal>Acquire::https::CAInfo</literal> option and its +host-specific option <literal>Acquire::https::CAInfo::<replaceable>host</replaceable></literal>. +The option specifies a file is made of the concatenation of the CA certificates +(in PEM format) creating the chain used for the verification of the path +from the root (self signed one). If the remote server provides the +whole chain during the exchange, the file need only contain the root +certificate. Otherwise, the whole chain is required. If you need to support +multiple authorities, the only way is to concatenate everything.</para> +<para>A custom certificate revocation list (CRL) can be configured with the options +<literal>Acquire::https::CRLFile</literal> and +<literal>Acquire::https::CRLFile::<replaceable>host</replaceable></literal> respectively. +Like the previous option a file in PEM format needs to be specified.</para> +</refsect2> + +<refsect2><title>Disabling security</title> +<para>When authenticating the server, if the certificate verification fails +for some reason (expired, revoked, man in the middle, …), the connection fails. +This is obviously what you want in all cases and what the default value (true) +of the option <literal>Acquire::https::Verify-Peer</literal> and its host-specific +variant provides. If you know <emphasis>exactly</emphasis> what you are doing, +setting this option to "false" allows you to skip peer certificate verification and +make the exchange succeed. Again, this option is for debugging or testing purpose +only as it removes all security provided by the use of HTTPS.</para> +<para>Similarly the option <literal>Acquire::https::Verify-Host</literal> and its +host-specific variant can be used to deactivate a security feature: The certificate +provided by the server includes the identity of the server which should match the +DNS name used to access it. By default, as requested by RFC 2818, the name of the +mirror is checked against the identity found in the certificate. This default behavior +is safe and should not be changed, but if you know that the server you are using has a +DNS name which does not match the identity in its certificate, you can set the option to +"false", which will prevent the comparison to be done.</para> +</refsect2> + +<refsect2><title>Client authentication</title> +<para>Beside the password based authentication available (see &apt-authconf;) HTTPS supports +authentication based on client certificates as well via <literal>Acquire::https::SSLCert</literal> +and <literal>Acquire::https::SSLKey</literal>. They respectively should be set to the filename of +the X.509 client certificate and the associated (unencrypted) private key, both in PEM format. +In practice the use of the host-specific variants of both options is highly recommended.</para> +</refsect2> + +</refsect1> + +<refsect1><title>Examples</title> +<literallayout> +Acquire::https { + Proxy::example.org "DIRECT"; + Proxy "socks5h://apt:pass@localhost:9050"; + Proxy-Auto-Detect "/usr/local/bin/apt-https-proxy-auto-detect"; + No-Cache "true"; + Max-Age "3600"; + No-Store "true"; + Timeout "10"; + Dl-Limit "42"; + Pipeline-Depth "0"; + AllowRedirect "false"; + User-Agent "My APT-HTTPS"; + SendAccept "false"; + + CAInfo "/path/to/ca/certs.pem"; + CRLFile "/path/to/all/crl.pem"; + Verify-Peer "true"; + Verify-Host::broken.example.org "false"; + SSLCert::example.org "/path/to/client/cert.pem"; + SSLKey::example.org "/path/to/client/key.pem" +}; +</literallayout> +</refsect1> + +<refsect1> +<title>See Also</title> +<para>&apt-transport-http; &apt-conf; &apt-authconf; &sources-list; +</para> +</refsect1> + + &manbugs; + +</refentry> diff --git a/doc/apt-transport-mirror.1.xml b/doc/apt-transport-mirror.1.xml new file mode 100644 index 000000000..72001fad5 --- /dev/null +++ b/doc/apt-transport-mirror.1.xml @@ -0,0 +1,150 @@ +<?xml version="1.0" encoding="utf-8" standalone="no"?> +<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" + "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ +<!ENTITY % aptent SYSTEM "apt.ent"> %aptent; +<!ENTITY % aptverbatiment SYSTEM "apt-verbatim.ent"> %aptverbatiment; +<!ENTITY % aptvendor SYSTEM "apt-vendor.ent"> %aptvendor; +]> + +<refentry> + + <refentryinfo> + &apt-author.team; + &apt-email; + &apt-product; + <!-- The last update date --> + <date>2017-12-09T00:00:00Z</date> + </refentryinfo> + + <refmeta> + <refentrytitle>apt-transport-mirror</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="manual">APT</refmiscinfo> + </refmeta> + + <!-- Man page title --> + <refnamediv> + <refname>apt-transport-mirror</refname> + <refpurpose>APT transport for more automated mirror selection</refpurpose> + </refnamediv> + +<refsect1><title>Description</title> +<para>This APT transport isn't implementing a protocol to access local or remote repositories +on its own, but acquires a mirrorlist and redirects all requests to the mirror(s) picked from +this list access via other transports like &apt-transport-http;. The basic functionality is +available since apt 0.7.24, but was undocumented until apt 1.6 which contained a complete +rework of the transport and its supported features. Note that a transport is never called directly +by a user but used by APT tools based on user configuration.</para> +<para>If the acquisition of a file via a mirror fails the method ensures that automatically +another possible mirror of the list is tried until either the file is retrieved or no mirror is +left in the list handling transparently server downtimes and similar problems.</para> +<para>The security implications of the transport are based on the security considerations +associated with the transport used to acquire the mirrorlist and the transports involved in +accessing the chosen mirror(s) by the transport.</para> +</refsect1> + +<refsect1><title>Options</title> +<para>This transport has no configuration options at present. The mirror selection is +based entirely on the mirrors offered in the mirrorlist and the files apt needs to +acquire.</para> + +<refsect2><title>Mirrorlist format</title> +<para>A mirrorlist contains at least one line each specifying an URI for a mirror. +Empty lines and those starting with a hash key (<literal>#</literal>) are ignored. +An URI always starts with a URI scheme which defines the transport used for this +mirror. If the URI e.g. starts with <literal>http:</literal> the responsible transport +is &apt-transport-http; which might have specific requirements for the format of +the remaining part of the URI.</para> +<para>An URI can optionally be separated from metadata about the mirror by a tab. +Multiple datapoints in the provided metadata can itself be separated by spaces for tabs. +(This is an advanced feature only available with apt >= 1.6. Earlier apt versions will +fail to parse mirrorlists using this feature)</para> +<para>Since apt 1.6 the usage of compressed mirrorlists is also supported. +Note that the filename of the mirrorlist must specify the compression algorithm used, +there is no auto-detection based on file content performed.</para> +</refsect2> + +<refsect2><title>Mirror selection by metadata</title> +<para>As specified in the format a mirror can have additional metadata attached to +prevent a mirror from being selected for acquiring a file not matching this metadata. +This way the mirrorlist can e.g. contain partial mirrors serving only certain +architectures and apt will automatically choose a different mirror for files requiring +an unlisted architecture. Supported are limits for the architecture (<literal>arch</literal>), +codename of the release (<literal>codename</literal>), component of the repository +the file is in (<literal>component</literal>), language the file applies to (<literal>lang</literal>), +suite name of the release (<literal>suite</literal>) and type of the file (<literal>type</literal>).</para> +</refsect2> + +<refsect2><title>Fallback order for mirrors</title> +<para>If no priority is given via the metadata key <literal>priority</literal> for a +mirror the order in which mirrors are contacted is random. If a certain set of mirrors +should be tried first before any of another set is tried a priority can be explicitly +set. The mirrors with the lowest number are tried first. Mirrors which have no explicit +priority set default to the highest possible number and are therefore tried last. The +choice between mirrors with the same priority is again random.</para> +</refsect2> + +<refsect2><title>Allowed transports in a mirrorlist</title> +<para>The availability and choice of transports in a mirrorlist is limited by how the apt +client is accessing the mirrorlist. If a local transport like <literal>file</literal> +or <literal>copy</literal> is used the mirrorlist can also include local sources while a +mirrorlist accessed via <literal>http</literal> can not. Additionally, a mirrorlist can +not contain a mirrorlist or other wrapping transports (like <package>apt-transport-tor</package>). +See the documentation of these transports on how to use them with the mirror method.</para> +<para>Note that apt versions before 1.6 do not support any other transport than <literal>http</literal>.</para> +</refsect2> +</refsect1> + +<refsect1><title>Examples</title> +<refsect2><title>Basic example</title> +<para>A basic mirrorlist example supported by all apt versions with a mirror method +(>= 0.7.24) in which the client will pick any of the three mirrors:</para> +<literallayout> +http://ftp.de.debian.org/debian/ +http://ftp.us.debian.org/debian/ +http://deb.debian.org/debian/ +</literallayout> +<para>Assuming a file with this content is stored as <filename>/etc/apt/mirrorlist.txt</filename> +on your machine it can be used like this in &sources-list; (since apt 1.6):</para> +<literallayout> +deb mirror+file:/etc/apt/mirrorlist.txt &debian-stable-codename; main +</literallayout> +<para>All versions of the mirror method support a mirrorlist accessible via http, so assuming +it is available at <literal>http://apt.example.org/mirror.lst</literal> the sources.list entry +from above could be written instead as:</para> +<literallayout> +deb mirror://apt.example.org/mirror.lst &debian-stable-codename; main +</literallayout> +<para>Note that since apt 1.6 the use of <literal>mirror+http</literal> should +be preferred over <literal>mirror</literal> for uniformity. The functionality is the same.</para> +</refsect2> +<refsect2><title>Example with metadata-enhanced mirror selection</title> +<para>As explained in the format definition apt versions before 1.6 do not support this and +will fail parsing the mirrorlist. The example mirrorlist is proposefully complicated to show some +aspects of the selection. The following setup is assumed: The first mirror is local mirror +accessible via the file method, but potentially incomplete. The second mirror has a great +connection, but is a partial mirror in sofar as it only contains files related +to the architectures <literal>amd64</literal> and <literal>all</literal>. The remaining mirrors +are average mirrors which should be contacted only if the earlier ones didn't work. +</para> +<literallayout> +file:/srv/local/debian/mirror/ priority:1 type:index +http://partial.example.org/mirror/ priority:2 arch:amd64 arch:all type:deb +http://ftp.us.debian.org/debian/ type:deb +http://ftp.de.debian.org/debian/ type:deb +https://deb.debian.org/debian/ +</literallayout> +<para>In this setup with this mirrorlist the first mirror will be used to download all +index files assuming the mirrorlist itself is accessed via a local transport like +<literal>file</literal>. If it isn't, the mirror is otherwise inaccessible or does not +contain the requested file another mirror will be used to acquire the file, which one +depending on the type of the file: An index file will be served by the last +mirror in the list, while a package of architecture <literal>amd64</literal> is served by +the second and those of e.g. architecture <literal>i386</literal> by one of the last three.</para> + +</refsect2> +</refsect1> + + &manbugs; + +</refentry> diff --git a/doc/apt-verbatim.ent b/doc/apt-verbatim.ent index 946c7cc7d..8d0a56b6c 100644 --- a/doc/apt-verbatim.ent +++ b/doc/apt-verbatim.ent @@ -81,6 +81,17 @@ </citerefentry>" > +<!ENTITY apt-transport-http "<citerefentry> + <refentrytitle><command>apt-transport-http</command></refentrytitle> + <manvolnum>1</manvolnum> + </citerefentry>" +> + +<!ENTITY apt-transport-https "<citerefentry> + <refentrytitle><command>apt-transport-https</command></refentrytitle> + <manvolnum>1</manvolnum> + </citerefentry>" +> <!ENTITY sources-list "<citerefentry> <refentrytitle><filename>sources.list</filename></refentrytitle> @@ -148,6 +159,12 @@ </citerefentry>" > +<!ENTITY auto-apt-proxy "<citerefentry> + <refentrytitle><command>auto-apt-proxy</command></refentrytitle> + <manvolnum>1</manvolnum> + </citerefentry>" +> + <!ENTITY debsign "<citerefentry> <refentrytitle><command>debsign</command></refentrytitle> <manvolnum>1</manvolnum> diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index 6f47bb029..e0be6a37d 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -409,99 +409,12 @@ APT::Compressor::rev { be symlinked when possible instead of copying. True is the default.</para></listitem> </varlistentry> - <varlistentry><term><option>http</option></term> - <listitem><para><literal>http::Proxy</literal> sets the default proxy to use for HTTP - URIs. It is in the standard form of <literal>http://[[user][:pass]@]host[:port]/</literal>. - Per host proxies can also be specified by using the form - <literal>http::Proxy::<host></literal> with the special keyword <literal>DIRECT</literal> - meaning to use no proxies. If no one of the above settings is specified, - <envar>http_proxy</envar> environment variable - will be used.</para> - - <para>Three settings are provided for cache control with HTTP/1.1 compliant - proxy caches. - <literal>No-Cache</literal> tells the proxy not to use its cached - response under any circumstances. - <literal>Max-Age</literal> sets the allowed maximum age (in seconds) of - an index file in the cache of the proxy. - <literal>No-Store</literal> specifies that the proxy should not store - the requested archive files in its cache, which can be used to prevent - the proxy from polluting its cache with (big) .deb files.</para> - - <para>The option <literal>timeout</literal> sets the timeout timer used by the method; - this value applies to the connection as well as the data timeout.</para> - - <para>The setting <literal>Acquire::http::Pipeline-Depth</literal> can be used to - enable HTTP pipelining (RFC 2616 section 8.1.2.2) which can be beneficial e.g. on - high-latency connections. It specifies how many requests are sent in a pipeline. - APT tries to detect and workaround misbehaving webservers and proxies at runtime, but - if you know that yours does not conform to the HTTP/1.1 specification pipelining can - be disabled by setting the value to 0. It is enabled by default with the value 10.</para> - - <para><literal>Acquire::http::AllowRedirect</literal> controls whether APT will follow - redirects, which is enabled by default.</para> - - <para>The used bandwidth can be limited with - <literal>Acquire::http::Dl-Limit</literal> which accepts integer - values in kilobytes per second. The default value is 0 which - deactivates the limit and tries to use all available bandwidth. - Note that this option implicitly disables downloading from - multiple servers at the same time.</para> - - <para><literal>Acquire::http::User-Agent</literal> can be used to set a different - User-Agent for the http download method as some proxies allow access for clients - only if the client uses a known identifier.</para> - - <para><literal>Acquire::http::Proxy-Auto-Detect</literal> can be used to - specify an external command to discover the http proxy to use. The first - and only parameter is an URI denoting the host to be contacted to allow - for host-specific configuration. APT expects the command to output the - proxy on stdout as a single line in the style <literal>http://proxy:port/</literal> - or the word <literal>DIRECT</literal> if no proxy should be used. No output - indicates that the generic proxy settings should be used. - - Note that auto-detection will not be used for a host if a host-specific proxy - configuration is already set via <literal>Acquire::http::Proxy::<replaceable>HOST</replaceable></literal>. - - See the &squid-deb-proxy-client; package for an example implementation that - uses avahi. - - This option takes precedence over the legacy option name - <literal>ProxyAutoDetect</literal>. - </para> - - </listitem> + <varlistentry><term><option>http</option> <option>https</option></term> + <listitem><para>The options in these scopes configure APTs acquire transports for the protocols + HTTP and HTTPS and are documented in the &apt-transport-http; and &apt-transport-https; + manpages respectively.</para></listitem> </varlistentry> - <varlistentry><term><option>https</option></term> - <listitem><para> - The <literal>Cache-control</literal>, <literal>Timeout</literal>, - <literal>AllowRedirect</literal>, <literal>Dl-Limit</literal> and - <literal>proxy</literal> options work for HTTPS URIs in the same way - as for the <literal>http</literal> method, and default to the same - values if they are not explicitly set. The - <literal>Pipeline-Depth</literal> option is not yet supported. - </para> - - <para><literal>CaInfo</literal> suboption specifies place of file that - holds info about trusted certificates. - <literal><host>::CaInfo</literal> is the corresponding per-host option. - <literal>Verify-Peer</literal> boolean suboption determines whether or not the - server's host certificate should be verified against trusted certificates. - <literal><host>::Verify-Peer</literal> is the corresponding per-host option. - <literal>Verify-Host</literal> boolean suboption determines whether or not the - server's hostname should be verified. - <literal><host>::Verify-Host</literal> is the corresponding per-host option. - <literal>SslCert</literal> determines what certificate to use for client - authentication. <literal><host>::SslCert</literal> is the corresponding per-host option. - <literal>SslKey</literal> determines what private key to use for client - authentication. <literal><host>::SslKey</literal> is the corresponding per-host option. - <literal>SslForceVersion</literal> overrides default SSL version to use. - It can contain either of the strings '<literal>TLSv1</literal>' or - '<literal>SSLv3</literal>'. - <literal><host>::SslForceVersion</literal> is the corresponding per-host option. - </para></listitem></varlistentry> - <varlistentry><term><option>ftp</option></term> <listitem><para> <literal>ftp::Proxy</literal> sets the default proxy to use for FTP URIs. diff --git a/doc/examples/CMakeLists.txt b/doc/examples/CMakeLists.txt index 1998867db..8d9ea068f 100644 --- a/doc/examples/CMakeLists.txt +++ b/doc/examples/CMakeLists.txt @@ -1,4 +1,4 @@ -install(FILES apt.conf apt-https-method-example.conf configure-index preferences +install(FILES apt.conf configure-index preferences DESTINATION ${CMAKE_INSTALL_DOCDIR}/examples) install(FILES apt-ftparchive.conf ftp-archive.conf DESTINATION ${CMAKE_INSTALL_DOCDIR}/../apt-utils/examples) diff --git a/doc/examples/apt-https-method-example.conf b/doc/examples/apt-https-method-example.conf deleted file mode 100644 index 3152ef1e2..000000000 --- a/doc/examples/apt-https-method-example.conf +++ /dev/null @@ -1,186 +0,0 @@ -/* This file is a sample configuration for apt https method. Configuration - parameters found in this example file are expected to be used in main - apt.conf file, just like other configuration parameters for different - methods (ftp, file, ...). - - This example file starts with a common setup that voluntarily exhibits - all available configurations knobs with simple comments. Extended - comments on the behavior of the option is provided at the end for - better readability. As a matter of fact, a common configuration file - will certainly contain far less elements and benefit of default values - for many parameters. - - Because some configuration parameters for apt https method in following - examples apply to specific (fictional) repositories, the associated - sources.list file is provided here: - - ... - - deb https://secure.dom1.tld/debian unstable main contrib non-free - deb-src https://secure.dom1.tld/debian unstable main contrib non-free - - deb https://secure.dom2.tld/debian unstable main contrib non-free - deb-src https://secure.dom2.tld/debian unstable main contrib non-free - - ... - - - Some notes on the servers: - - - secure.dom1.tld is freely accessible using https (no client - authentication is required). - - secure.dom1.tld certificate is part of a multi level PKI, and we - want to specifically check the issuer of its certificate. We do - not have the constraint for secure.dom2.tld - - secure.dom2.tld requires client authentication by certificate - to access its content. - - The certificate presented by both server have (as expected) a CN that - matches their respective DNS names. - - We have CRL available for both dom1.tld and dom2.tld PKI, and intend - to use them. - - It sometimes happens that we had other more generic https available - repository to our list. We want the checks to be performed against - a common list of anchors (like the one provided by ca-certificates - package for instance) - - The sample configuration below basically covers those simple needs. -*/ - - -// Verify peer certificate and also matching between certificate name -// and server name as provided in sources.list (default values) -Acquire::https::Verify-Peer "true"; -Acquire::https::Verify-Host "true"; - -// Except otherwise specified, use that list of anchors -Acquire::https::CaInfo "/etc/ssl/certs/ca-certificates.pem"; - -// Use a specific anchor and associated CRL. Enforce issuer of -// server certificate using its cert. -Acquire::https::secure.dom1.tld::CaInfo "/etc/apt/certs/ca-dom1-crt.pem"; -Acquire::https::secure.dom1.tld::CrlFile "/etc/apt/certs/ca-dom1-crl.pem"; -Acquire::https::secure.dom1.tld::IssuerCert "/etc/apt/certs/secure.dom1-issuer-crt.pem"; - -// Like previous for anchor and CRL, but also provide our -// certificate and keys for client authentication. -Acquire::https::secure.dom2.tld::CaInfo "/etc/apt/certs/ca-dom2-crt.pem"; -Acquire::https::secure.dom2.tld::CrlFile "/etc/apt/certs/ca-dom2-crl.pem"; -Acquire::https::secure.dom2.tld::SslCert "/etc/apt/certs/my-crt.pem"; -Acquire::https::secure.dom2.tld::SslKey "/etc/apt/certs/my-key.pem"; - -// No need to downgrade, TLS will be proposed by default. Uncomment -// to have SSLv3 proposed. -// Acquire::https::mirror.ipv6.ssi.corp::SslForceVersion "SSLv3"; - -// No need for more debug if every is fine (default). Uncomment -// me to get additional information. -// Debug::Acquire::https "true"; - - -/* - Options with extended comments: - - Acquire::https[::repo.domain.tld]::CaInfo "/path/to/ca/certs.pem"; - - A string providing the path of a file containing the list of trusted - CA certificates used to verify the server certificate. The pointed - file is made of the concatenation of the CA certificates (in - PEM format) creating the chain used for the verification of the path - from the root (self signed one). If the remote server provides the - whole chain during the exchange, the file need only contain the root - certificate. Otherwise, the whole chain is required. - - If you need to support multiple authorities, the only way is to - concatenate everything. - - If None is provided, the default CA bundle used by GnuTLS (apt https - method is linked against libcurl-gnutls) is used. At the time of - writing, /etc/ssl/certs/ca-certificates.crt. - - If no specific hostname is provided, the file is used by default - for all https targets. If a specific mirror is provided, it is - used for the https entries in the sources.list file that use that - repository (with the same name). - - Acquire::https[::repo.domain.tld]::CrlFile "/path/to/all/crl.pem"; - - Like previous knob but for passing the list of CRL files (in PEM - format) to be used to verify revocation status. Again, if the - option is defined with no specific mirror (probably makes little - sense), this CRL information is used for all defined https entries - in sources.list file. In a mirror specific context, it only applies - to that mirror. - - Acquire::https[::repo.domain.tld]::IssuerCert "/path/to/issuer/cert.pem"; - - Allows one to constrain the issuer of the server certificate (for all - https mirrors or a specific one) to a specific issuer. If the - server certificate has not been issued by this certificate, - connection fails. - - Acquire::https[::repo.domain.tld]::Verify-Peer "true"; - - When authenticating the server, if the certificate verification fails - for some reason (expired, revoked, man in the middle, lack of anchor, - ...), the connection fails. This is obviously what you want in all - cases and what the default value (true) of this option provides. - - If you know EXACTLY what you are doing, setting this option to "false" - allow you to skip peer certificate verification and make the exchange - succeed. Again, this option is for debugging or testing purpose only. - It removes ALL the security provided by the use of SSL.TLS to secure - the HTTP exchanges. - - Acquire::https[::repo.domain.tld]::Verify-Host "true"; - - The certificate provided by the server during the TLS/SSL exchange - provides the identity of the server which should match the DNS name - used to access it. By default, as requested by RFC 2818, the name - of the mirror is checked against the identity found in the - certificate. This default behavior is safe and should not be - changed. If you know that the server you are using has a DNS name - which does not match the identity in its certificate, you can - [report that issue to its administrator or] set the option to - "false", which will prevent the comparison to be done. - - The options can be set globally or on a per-mirror basis. If set - globally, the DNS name used is the one found in the sources.list - file in the https URI. - - Acquire::https[::repo.domain.tld]::SslCert "/path/to/client/cert.pem"; - Acquire::https[::repo.domain.tld]::SslKey "/path/to/client/key.pem"; - - These two options provides support for client authentication using - certificates. They respectively accept the X.509 client certificate - in PEM format and the associated client key in PEM format (non - encrypted form). - - The options can be set globally (which rarely makes sense) or on a - per-mirror basis. - - Acquire::https[::repo.domain.tld]::SslForceVersion "TLSv1"; - - This option can be use to select the version which will be proposed - to the server. "SSLv3" and "TLSv1" are supported. SSLv2, which is - considered insecure anyway is not supported (by gnutls, which is - used by libcurl against which apt https method is linked). - - When the option is set to "SSLv3" to have apt propose SSLv3 (and - associated sets of ciphersuites) instead of TLSv1 (the default) - when performing the exchange. This prevents the server to select - TLSv1 and use associated ciphersuites. You should probably not use - this option except if you know exactly what you are doing. - - Note that the default setting does not guarantee that the server - will not select SSLv3 (for ciphersuites and SSL/TLS version as - selection is always done by the server, in the end). It only means - that apt will not advertise TLS support. - - Debug::Acquire::https "true"; - - This option can be used to show debug information. Because it is - quite verbose, it is mainly useful to debug problems in case of - failure to connect to a server for some reason. The default value - is "false". - -*/ diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 153637ebc..9088bd844 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -274,9 +274,6 @@ Acquire // - cache-control values // - Dl-Limit, Timeout, ... values // if not set explicit for https - // - // see /usr/share/doc/apt/examples/apt-https-method-example.conf.gz - // for more examples https { Verify-Peer "false"; diff --git a/doc/method.dbk b/doc/method.dbk index e5e035a2b..410d6898c 100644 --- a/doc/method.dbk +++ b/doc/method.dbk @@ -267,6 +267,11 @@ an informational string provided for visual debugging. </listitem> <listitem> <para> +351 Aux Request - Method requests an auxiliary file +</para> +</listitem> +<listitem> +<para> 400 URI Failure - URI has failed to acquire </para> </listitem> @@ -309,9 +314,9 @@ Authorization is User/Pass </itemizedlist> <para> Only the 6xx series of status codes is sent TO the method. Furthermore the -method may not emit status codes in the 6xx range. The Codes 402 and 403 +method may not emit status codes in the 6xx range. The Codes 351, 402 and 403 require that the method continue reading all other 6xx codes until the proper -602/603 code is received. This means the method must be capable of handling an +600/602/603 code is received. This means the method must be capable of handling an unlimited number of 600 messages. </para> <para> @@ -567,6 +572,20 @@ Size, Last-Modified, Filename, MD5-Hash </listitem> </varlistentry> <varlistentry> +<term>351 Aux Request</term> +<listitem> +<para> +Indicates a request for an auxiliary file to be downloaded by the acquire system +(via another method) and made available for the requesting method. The requestor +will get a <emphasis>600 URI Acquire</emphasis> with the URI it requested and the +filename will either be an existing file if the request was success or if the +acquire failed for some reason the file will not exist. +Fields: URI (of the file causing the need for the auxiliary file), MaximumSize, +Aux-ShortDesc, Aux-Description, Aux-URI +</para> +</listitem> +</varlistentry> +<varlistentry> <term>400 URI Failure</term> <listitem> <para> diff --git a/doc/po4a.conf b/doc/po4a.conf index 955ebbc76..587215abc 100644 --- a/doc/po4a.conf +++ b/doc/po4a.conf @@ -27,6 +27,9 @@ [type: manpage] apt-sortpkgs.1.xml $lang:$lang/apt-sortpkgs.$lang.1.xml add_$lang:xml.add [type: manpage] apt-ftparchive.1.xml $lang:$lang/apt-ftparchive.$lang.1.xml add_$lang:xml.add [type: manpage] apt_auth.conf.5.xml $lang:$lang/apt_auth.conf.$lang.5.xml add_$lang:xml.add +[type: manpage] apt-transport-http.1.xml $lang:$lang/apt-transport-http.$lang.1.xml add_$lang:xml.add +[type: manpage] apt-transport-https.1.xml $lang:$lang/apt-transport-https.$lang.1.xml add_$lang:xml.add +[type: manpage] apt-transport-mirror.1.xml $lang:$lang/apt-transport-mirror.$lang.1.xml add_$lang:xml.add [type: docbook] guide.dbk $lang:$lang/guide.$lang.dbk # add_$lang::$lang/addendum/docbook_$lang.add diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml index 694082bea..5572b8da3 100644 --- a/doc/sources.list.5.xml +++ b/doc/sources.list.5.xml @@ -352,17 +352,15 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [. <para>The currently recognized URI types are: <variablelist> - <varlistentry><term><command>http</command></term> + <varlistentry><term><command>http</command> (&apt-transport-http;)</term> <listitem><para> The http scheme specifies an HTTP server for an archive and is the most - commonly used method, with many options in the - <literal>Acquire::http</literal> scope detailed in &apt-conf;. The URI can - directly include login information if the archive requires it, but the use - of &apt-authconf; should be preferred. The method also supports SOCKS5 and - HTTP(S) proxies either configured via apt-specific configuration or - specified by the environment variable <envar>http_proxy</envar> in the - format (assuming an HTTP proxy requiring authentication) - <replaceable>http://user:pass@server:port/</replaceable>. + commonly used method. The URI can directly include login information if the + archive requires it, but the use of &apt-authconf; should be preferred. + The method also supports SOCKS5 and HTTP(S) proxies either configured via + apt-specific configuration or specified by the environment variable + <envar>http_proxy</envar> in the format (assuming an HTTP proxy requiring + authentication) <replaceable>http://user:pass@server:port/</replaceable>. The authentication details for proxies can also be supplied via &apt-authconf;.</para> <para>Note that these forms of authentication are insecure as the whole @@ -373,7 +371,7 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [. chosen transport method. See &apt-secure; for details.</para></listitem> </varlistentry> - <varlistentry><term><command>https</command></term> + <varlistentry><term><command>https</command> (&apt-transport-https;)</term> <listitem><para> The https scheme specifies an HTTPS server for an archive and is very similar in use and available options to the http scheme. The main diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index a25d4b525..c4a32b4f5 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -2,7 +2,6 @@ include_directories($<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_INCLUDE_DIR}>) link_libraries(apt-pkg $<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_LIBRARIES}>) -add_library(httplib OBJECT http.cc basehttp.cc) add_library(connectlib OBJECT connect.cc rfc2553emu.cc) add_executable(file file.cc) @@ -10,8 +9,8 @@ add_executable(copy copy.cc) add_executable(store store.cc) add_executable(gpgv gpgv.cc) add_executable(cdrom cdrom.cc) -add_executable(http http_main.cc $<TARGET_OBJECTS:httplib> $<TARGET_OBJECTS:connectlib>) -add_executable(mirror mirror.cc $<TARGET_OBJECTS:httplib> $<TARGET_OBJECTS:connectlib>) +add_executable(http http.cc basehttp.cc $<TARGET_OBJECTS:connectlib>) +add_executable(mirror mirror.cc) add_executable(ftp ftp.cc $<TARGET_OBJECTS:connectlib>) add_executable(rred rred.cc) add_executable(rsh rsh.cc) @@ -21,14 +20,13 @@ target_include_directories(connectlib PRIVATE ${GNUTLS_INCLUDE_DIR}) # Additional libraries to link against for networked stuff target_link_libraries(http ${GNUTLS_LIBRARIES}) -target_link_libraries(mirror ${RESOLV_LIBRARIES} ${GNUTLS_LIBRARIES}) target_link_libraries(ftp ${GNUTLS_LIBRARIES}) # Install the library install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods) -add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods store) +add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+ftp mirror+http mirror+https mirror+file mirror+copy) add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh) diff --git a/methods/aptmethod.h b/methods/aptmethod.h index 88d325cba..331411571 100644 --- a/methods/aptmethod.h +++ b/methods/aptmethod.h @@ -448,6 +448,16 @@ protected: return true; } + // This is a copy of #pkgAcqMethod::Dequeue which is private & hidden + void Dequeue() + { + FetchItem const *const Tmp = Queue; + Queue = Queue->Next; + if (Tmp == QueueBack) + QueueBack = Queue; + delete Tmp; + } + aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3) : pkgAcqMethod(Ver, Flags), Binary(Binary), SeccompFlags(0), methodNames({Binary}) { diff --git a/methods/http.cc b/methods/http.cc index 2d23b1646..5d286bcb4 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -31,6 +31,7 @@ #include <sstream> #include <arpa/inet.h> #include <errno.h> +#include <signal.h> #include <stddef.h> #include <stdio.h> #include <stdlib.h> @@ -1034,7 +1035,7 @@ BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &R return FILE_IS_OPEN; } /*}}}*/ -HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/ +HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg), "1.2", Pipeline | SendConfig) /*{{{*/ { SeccompFlags = aptMethod::BASE | aptMethod::NETWORK; @@ -1051,3 +1052,14 @@ HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2 } } /*}}}*/ + +int main(int, const char *argv[]) +{ + // ignore SIGPIPE, this can happen on write() if the socket + // closes the connection (this is dealt with via ServerDie()) + signal(SIGPIPE, SIG_IGN); + std::string Binary = flNotDir(argv[0]); + if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http") + Binary.append("+http"); + return HttpMethod(std::move(Binary)).Loop(); +} diff --git a/methods/http_main.cc b/methods/http_main.cc deleted file mode 100644 index 792b5e22f..000000000 --- a/methods/http_main.cc +++ /dev/null @@ -1,17 +0,0 @@ -#include <config.h> -#include <apt-pkg/error.h> -#include <apt-pkg/fileutl.h> -#include <signal.h> - -#include "http.h" - -int main(int, const char *argv[]) -{ - // ignore SIGPIPE, this can happen on write() if the socket - // closes the connection (this is dealt with via ServerDie()) - signal(SIGPIPE, SIG_IGN); - std::string Binary = flNotDir(argv[0]); - if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http") - Binary.append("+http"); - return HttpMethod(std::move(Binary)).Loop(); -} diff --git a/methods/mirror.cc b/methods/mirror.cc index b551802e4..add9f0875 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -1,18 +1,17 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ /* ###################################################################### - Mirror Acquire Method - This is the Mirror acquire method for APT. - + Mirror URI – This method helps avoiding hardcoding of mirrors in the + sources.lists by looking up a list of mirrors first to which the + following requests are redirected. + ##################################################################### */ /*}}}*/ // Include Files /*{{{*/ #include <config.h> -#include <apt-pkg/acquire-item.h> -#include <apt-pkg/acquire.h> -#include <apt-pkg/aptconfiguration.h> +#include "aptmethod.h" #include <apt-pkg/configuration.h> #include <apt-pkg/error.h> #include <apt-pkg/fileutl.h> @@ -20,447 +19,388 @@ #include <apt-pkg/sourcelist.h> #include <apt-pkg/strutl.h> -#include <algorithm> -#include <fstream> -#include <iostream> +#include <functional> +#include <random> +#include <string> +#include <unordered_map> -#include <dirent.h> -#include <fcntl.h> -#include <stdlib.h> -#include <string.h> -#include <sys/stat.h> -#include <sys/types.h> #include <sys/utsname.h> -#include <unistd.h> - -using namespace std; - -#include <sstream> -#include "http.h" -#include "mirror.h" #include <apti18n.h> /*}}}*/ +constexpr char const *const disallowLocal[] = {"ftp", "http", "https"}; -/* Done: - * - works with http (only!) - * - always picks the first mirror from the list - * - call out to problem reporting script - * - supports "deb mirror://host/path/to/mirror-list/// dist component" - * - uses pkgAcqMethod::FailReason() to have a string representation - * of the failure that is also send to LP - * - * TODO: - * - deal with running as non-root because we can't write to the lists - dir then -> use the cached mirror file - * - better method to download than having a pkgAcquire interface here - * and better error handling there! - * - support more than http - * - testing :) - */ - -MirrorMethod::MirrorMethod() - : HttpMethod("mirror"), DownloadedMirrorFile(false), Debug(false) +static void sortByLength(std::vector<std::string> &vec) /*{{{*/ { -} - -// HttpMethod::Configuration - Handle a configuration message /*{{{*/ -// --------------------------------------------------------------------- -/* We stash the desired pipeline depth */ -bool MirrorMethod::Configuration(string Message) -{ - if (HttpMethod::Configuration(Message) == false) - return false; - Debug = DebugEnabled(); - - return true; + // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected + // by checking for the longest matches first + std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) { + return a.length() > b.length(); + }); } /*}}}*/ - -// clean the mirrors dir based on ttl information -bool MirrorMethod::Clean(string Dir) +class MirrorMethod : public aptMethod /*{{{*/ { - vector<metaIndex *>::const_iterator I; - - if(Debug) - clog << "MirrorMethod::Clean(): " << Dir << endl; - - if(Dir == "/") - return _error->Error("will not clean: '/'"); - - // read sources.list - pkgSourceList list; - list.ReadMainList(); - - int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC); - if (dirfd == -1) - return _error->Errno("open",_("Unable to read %s"), Dir.c_str()); - DIR * const D = fdopendir(dirfd); - if (D == nullptr) - return _error->Errno("fdopendir",_("Unable to read %s"),Dir.c_str()); - - for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D)) + std::mt19937 genrng; + std::vector<std::string> sourceslist; + std::unordered_map<std::string, std::string> msgCache; + enum MirrorFileState { - // Skip some files.. - if (strcmp(Dir->d_name,"lock") == 0 || - strcmp(Dir->d_name,"partial") == 0 || - strcmp(Dir->d_name,"lost+found") == 0 || - strcmp(Dir->d_name,".") == 0 || - strcmp(Dir->d_name,"..") == 0) - continue; - - // see if we have that uri - for(I=list.begin(); I != list.end(); ++I) + REQUESTED, + FAILED, + AVAILABLE + }; + struct MirrorInfo + { + std::string uri; + unsigned long priority = std::numeric_limits<decltype(priority)>::max(); + decltype(genrng)::result_type seed = 0; + std::unordered_map<std::string, std::vector<std::string>> tags; + MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u) { - string uri = (*I)->GetURI(); - if(uri.compare(0, strlen("mirror://"), "mirror://") != 0) - continue; - string BaseUri = uri.substr(0,uri.size()-1); - if (URItoFileName(BaseUri) == Dir->d_name) - break; + for (auto &&tag : ptags) + { + auto const colonfound = tag.find(':'); + if (unlikely(colonfound == std::string::npos)) + continue; + auto name = tag.substr(0, colonfound); + auto value = tag.substr(colonfound + 1); + if (name == "arch") + tags["Architecture"].emplace_back(std::move(value)); + else if (name == "lang") + tags["Language"].emplace_back(std::move(value)); + else if (name == "priority") + priority = std::strtoul(value.c_str(), nullptr, 10); + else if (likely(name.empty() == false)) + { + if (name == "codename" || name == "suite") + tags["Release"].push_back(value); + name[0] = std::toupper(name[0]); + tags[std::move(name)].emplace_back(std::move(value)); + } + } } - // nothing found, nuke it - if (I == list.end()) - RemoveFileAt("mirror", dirfd, Dir->d_name); - } - closedir(D); - return true; -} - - -bool MirrorMethod::DownloadMirrorFile(string /*mirror_uri_str*/) -{ - // not that great to use pkgAcquire here, but we do not have - // any other way right now - string fetch = BaseUri; - fetch.replace(0,strlen("mirror://"),"http://"); - -#if 0 // no need for this, the getArchitectures() will also include the main - // arch - // append main architecture - fetch += "?arch=" + _config->Find("Apt::Architecture"); -#endif - - // append all architectures - std::vector<std::string> vec = APT::Configuration::getArchitectures(); - for (std::vector<std::string>::const_iterator I = vec.begin(); - I != vec.end(); ++I) - if (I == vec.begin()) - fetch += "?arch=" + (*I); - else - fetch += "&arch=" + (*I); + }; + struct MirrorListInfo + { + MirrorFileState state; + std::string baseuri; + std::vector<MirrorInfo> list; + }; + std::unordered_map<std::string, MirrorListInfo> mirrorfilestate; - // append the dist as a query string - if (Dist != "") - fetch += "&dist=" + Dist; + virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; - if(Debug) - clog << "MirrorMethod::DownloadMirrorFile(): '" << fetch << "'" - << " to " << MirrorFile << endl; + void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message); + bool MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm); + std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm); + void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler); - pkgAcquire Fetcher; - new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile); - bool res = (Fetcher.Run() == pkgAcquire::Continue); - if(res) { - DownloadedMirrorFile = true; - chmod(MirrorFile.c_str(), 0644); + public: + MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests), genrng(clock()) + { + SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY; } - Fetcher.Shutdown(); - - if(Debug) - clog << "MirrorMethod::DownloadMirrorFile() success: " << res << endl; - - return res; -} - -// Randomizes the lines in the mirror file, this is used so that -// we spread the load on the mirrors evenly -bool MirrorMethod::RandomizeMirrorFile(string mirror_file) +}; + /*}}}*/ +void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/ { - vector<string> content; - string line; - - if (!FileExists(mirror_file)) - return false; - - // read - ifstream in(mirror_file.c_str()); - while ( !in.eof() ) { - getline(in, line); - content.push_back(line); - } - - // we want the file to be random for each different machine, but also - // "stable" on the same machine. this is to avoid running into out-of-sync - // issues (i.e. Release/Release.gpg different on each mirror) - struct utsname buf; - int seed=1; - if(uname(&buf) == 0) { - for(int i=0,seed=1; buf.nodename[i] != 0; ++i) { - seed = seed * 31 + buf.nodename[i]; + std::unordered_map<std::string, std::string> matchers; + matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture")); + matchers.emplace("Codename", LookupTag(Message, "Target-Codename")); + matchers.emplace("Component", LookupTag(Message, "Target-Component")); + matchers.emplace("Language", LookupTag(Message, "Target-Language")); + matchers.emplace("Release", LookupTag(Message, "Target-Release")); + matchers.emplace("Suite", LookupTag(Message, "Target-Suite")); + matchers.emplace("Type", LookupTag(Message, "Target-Type")); + decltype(info.list) possMirrors; + for (auto const &mirror : info.list) + { + bool failedMatch = false; + for (auto const &m : matchers) + { + if (m.second.empty()) + continue; + auto const tagsetiter = mirror.tags.find(m.first); + if (tagsetiter == mirror.tags.end()) + continue; + auto const tagset = tagsetiter->second; + if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end()) + { + failedMatch = true; + break; + } } + if (failedMatch) + continue; + possMirrors.push_back(mirror); } - srand( seed ); - random_shuffle(content.begin(), content.end()); - - // write - ofstream out(mirror_file.c_str()); - while ( !content.empty()) { - line = content.back(); - content.pop_back(); - out << line << "\n"; + for (auto &&mirror : possMirrors) + mirror.seed = genrng(); + std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) { + if (a.priority != b.priority) + return a.priority < b.priority; + return a.seed < b.seed; + }); + std::string const path = Itm->Uri.substr(info.baseuri.length()); + std::string altMirrors; + std::unordered_map<std::string, std::string> fields; + fields.emplace("URI", Queue->Uri); + for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror) + { + std::string mirror = curMirror->uri; + if (APT::String::Endswith(mirror, "/") == false) + mirror.append("/"); + mirror.append(path); + if (curMirror == possMirrors.cbegin()) + fields.emplace("New-URI", mirror); + else if (altMirrors.empty()) + altMirrors.append(mirror); + else + altMirrors.append("\n").append(mirror); } - - return true; + fields.emplace("Alternate-URIs", altMirrors); + SendMessage("103 Redirect", std::move(fields)); + Dequeue(); } - -/* convert a the Queue->Uri back to the mirror base uri and look - * at all mirrors we have for this, this is needed as queue->uri - * may point to different mirrors (if TryNextMirror() was run) - */ -void MirrorMethod::CurrentQueueUriToMirror() + /*}}}*/ +void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/ + MirrorListInfo const &info, FetchItem *const Itm, + std::function<void()> handler) { - // already in mirror:// style so nothing to do - if(Queue->Uri.find("mirror://") == 0) - return; - - // find current mirror and select next one - for (vector<string>::const_iterator mirror = AllMirrors.begin(); - mirror != AllMirrors.end(); ++mirror) + FetchItem **LastItm = &Itm->Next; + while (*LastItm != nullptr) + LastItm = &((*LastItm)->Next); + while (Queue != Itm) { - if (Queue->Uri.find(*mirror) == 0) + if (APT::String::Startswith(Queue->Uri, info.baseuri) == false || + std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); })) { - Queue->Uri.replace(0, mirror->length(), BaseUri); - return; + // move the item behind the aux file not related to it + *LastItm = Queue; + Queue = QueueBack = Queue->Next; + (*LastItm)->Next = nullptr; + LastItm = &((*LastItm)->Next); + } + else + { + handler(); } } - _error->Error("Internal error: Failed to convert %s back to %s", - Queue->Uri.c_str(), BaseUri.c_str()); + // now remove out trigger + QueueBack = Queue = Queue->Next; + delete Itm; } - -bool MirrorMethod::TryNextMirror() + /*}}}*/ +bool MirrorMethod::MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/ { - // find current mirror and select next one - for (vector<string>::const_iterator mirror = AllMirrors.begin(); - mirror != AllMirrors.end(); ++mirror) + std::vector<std::string> baseuris; + for (auto const &i : mirrorfilestate) + if (info.baseuri.length() < i.second.baseuri.length() && + i.second.state == REQUESTED && + APT::String::Startswith(i.second.baseuri, info.baseuri)) + baseuris.push_back(i.second.baseuri); + sortByLength(baseuris); + + FileFd mirrorlist; + if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension)) { - if (Queue->Uri.find(*mirror) != 0) - continue; - - vector<string>::const_iterator nextmirror = mirror + 1; - if (nextmirror == AllMirrors.end()) - break; - Queue->Uri.replace(0, mirror->length(), *nextmirror); - if (Debug) - clog << "TryNextMirror: " << Queue->Uri << endl; - - // inform parent - UsedMirror = *nextmirror; - Log("Switching mirror"); - return true; - } - - if (Debug) - clog << "TryNextMirror could not find another mirror to try" << endl; - - return false; -} + auto const accessColon = info.baseuri.find(':'); + auto access = info.baseuri.substr(0, accessColon); + std::string prefixAccess; + if (APT::String::Startswith(access, "mirror") == false) + { + auto const plus = info.baseuri.find('+'); + prefixAccess = info.baseuri.substr(0, plus); + access.erase(0, plus + 1); + } + std::vector<std::string> limitAccess; + // If the mirror file comes from an online source, allow only other online + // sources, not e.g. file:///. If the mirrorlist comes from there we can assume + // the admin knows what (s)he is doing through and not limit the options. + if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal), + [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) || + access == "mirror") + { + for (auto const &a : disallowLocal) + limitAccess.emplace_back(a); + } + std::string line; + while (mirrorlist.ReadLine(line)) + { + if (line.empty() || line[0] == '#') + continue; + auto const access = line.substr(0, line.find(':')); + if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end()) + continue; + auto const tab = line.find('\t'); + if (tab == std::string::npos) + { + if (prefixAccess.empty()) + info.list.emplace_back(std::move(line)); + else + info.list.emplace_back(prefixAccess + '+' + line); + } + else + { + auto uri = line.substr(0, tab); + if (prefixAccess.empty() == false) + uri = prefixAccess + '+' + uri; + auto tagline = line.substr(tab + 1); + std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' '); + auto tags = VectorizeString(tagline, ' '); + tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end()); + info.list.emplace_back(std::move(uri), std::move(tags)); + } + } + mirrorlist.Close(); -bool MirrorMethod::InitMirrors() -{ - // if we do not have a MirrorFile, fallback - if(!FileExists(MirrorFile)) - { - // FIXME: fallback to a default mirror here instead - // and provide a config option to define that default - return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str()); + if (info.list.empty()) + { + info.state = FAILED; + DealWithPendingItems(baseuris, info, Itm, [&]() { + std::string msg; + strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str()); + Fail(msg, false); + }); + } + else + { + info.state = AVAILABLE; + DealWithPendingItems(baseuris, info, Itm, [&]() { + RedirectItem(info, Queue, msgCache[Queue->Uri]); + }); + msgCache.clear(); + } } - - if (access(MirrorFile.c_str(), R_OK) != 0) - { - // FIXME: fallback to a default mirror here instead - // and provide a config option to define that default - return _error->Error(_("Can not read mirror file '%s'"), MirrorFile.c_str()); - } - - // FIXME: make the mirror selection more clever, do not - // just use the first one! - // BUT: we can not make this random, the mirror has to be - // stable across session, because otherwise we can - // get into sync issues (got indexfiles from mirror A, - // but packages from mirror B - one might be out of date etc) - ifstream in(MirrorFile.c_str()); - string s; - while (!in.eof()) + else { - getline(in, s); - - // ignore lines that start with # - if (s.find("#") == 0) - continue; - // ignore empty lines - if (s.size() == 0) - continue; - // ignore non http lines - if (s.compare(0, strlen("http://"), "http://") != 0) - continue; - - AllMirrors.push_back(s); + info.state = FAILED; + DealWithPendingItems(baseuris, info, Itm, [&]() { + std::string msg; + strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str()); + Fail(msg, false); + }); } - if (AllMirrors.empty()) { - return _error->Error(_("No entry found in mirror file '%s'"), MirrorFile.c_str()); - } - Mirror = AllMirrors[0]; - UsedMirror = Mirror; return true; } - -string MirrorMethod::GetMirrorFileName(string mirror_uri_str) + /*}}}*/ +std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/ { - /* - - a mirror_uri_str looks like this: - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg - - - the matching source.list entry - deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main - - - we actually want to go after: - http://people.ubuntu.com/~mvo/apt/mirror/mirrors - - And we need to save the BaseUri for later: - - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors - - FIXME: what if we have two similar prefixes? - mirror://people.ubuntu.com/~mvo/mirror - mirror://people.ubuntu.com/~mvo/mirror2 - then mirror_uri_str looks like: - mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg - mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg - we search sources.list and find: - mirror://people.ubuntu.com/~mvo/apt/mirror - in both cases! So we need to apply some domain knowledge here :( and - check for /dists/ or /Release.gpg as suffixes - */ - string name; - if(Debug) - std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl; - - // read sources.list and find match - vector<metaIndex *>::const_iterator I; - pkgSourceList list; - list.ReadMainList(); - for(I=list.begin(); I != list.end(); ++I) + if (APT::String::Startswith(Itm->Uri, Binary)) { - string uristr = (*I)->GetURI(); - if(Debug) - std::cerr << "Checking: " << uristr << std::endl; - if(uristr.substr(0,strlen("mirror://")) != string("mirror://")) - continue; - // find matching uri in sources.list - if(mirror_uri_str.substr(0,uristr.size()) == uristr) + std::string const repouri = LookupTag(Message, "Target-Repo-Uri"); + if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend()) + sourceslist.push_back(repouri); + } + if (sourceslist.empty()) + { + // read sources.list and find the matching base uri + pkgSourceList sl; + if (sl.ReadMainList() == false) + { + _error->Error(_("The list of sources could not be read.")); + return ""; + } + std::string const needle = Binary + ":"; + for (auto const &SL : sl) { - if(Debug) - std::cerr << "found BaseURI: " << uristr << std::endl; - BaseUri = uristr.substr(0,uristr.size()-1); - Dist = (*I)->GetDist(); + std::string uristr = SL->GetURI(); + if (APT::String::Startswith(uristr, needle)) + sourceslist.push_back(uristr); } + sortByLength(sourceslist); } - // get new file - name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri); - - if(Debug) + for (auto uristr : sourceslist) { - cerr << "base-uri: " << BaseUri << endl; - cerr << "mirror-file: " << name << endl; + if (APT::String::Startswith(Itm->Uri, uristr)) + { + uristr.erase(uristr.length() - 1); // remove the ending '/' + auto const colon = uristr.find(':'); + if (unlikely(colon == std::string::npos)) + continue; + auto const plus = uristr.find("+"); + if (plus < colon) + { + // started as tor+mirror+http we want to get the file via tor+http + auto access = uristr.substr(0, colon); + std::string prefixAccess; + if (APT::String::Startswith(access, "mirror") == false) + { + prefixAccess = uristr.substr(0, plus); + access.erase(0, plus + 1); + uristr.erase(plus, strlen("mirror") + 1); + return uristr; + } + else + return uristr.substr(plus + 1); + } + else + { + uristr.replace(0, strlen("mirror"), "http"); + return uristr; + } + } } - return name; + return ""; } - -// MirrorMethod::Fetch - Fetch an item /*{{{*/ -// --------------------------------------------------------------------- -/* This adds an item to the pipeline. We keep the pipeline at a fixed - depth. */ -bool MirrorMethod::Fetch(FetchItem *Itm) + /*}}}*/ +bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/ { - if(Debug) - clog << "MirrorMethod::Fetch()" << endl; - - // the http method uses Fetch(0) as a way to update the pipeline, - // just let it do its work in this case - Fetch() with a valid - // Itm will always run before the first Fetch(0) - if(Itm == NULL) - return HttpMethod::Fetch(Itm); - - // if we don't have the name of the mirror file on disk yet, - // calculate it now (can be derived from the uri) - if(MirrorFile.empty()) - MirrorFile = GetMirrorFileName(Itm->Uri); + auto mirrorinfo = mirrorfilestate.find(Itm->Uri); + if (mirrorinfo != mirrorfilestate.end()) + return MirrorListFileRecieved(mirrorinfo->second, Itm); - // download mirror file once (if we are after index files) - if(Itm->IndexFile && !DownloadedMirrorFile) + std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm); + if (mirrorfileuri.empty()) { - Clean(_config->FindDir("Dir::State::mirrors")); - if (DownloadMirrorFile(Itm->Uri)) - RandomizeMirrorFile(MirrorFile); + _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str()); + return false; } + if (DebugEnabled()) + std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl; - if(AllMirrors.empty()) { - if(!InitMirrors()) { - // no valid mirror selected, something went wrong downloading - // from the master mirror site most likely and there is - // no old mirror file availalbe + // have we requested this mirror file already? + auto const state = mirrorfilestate.find(mirrorfileuri); + if (state == mirrorfilestate.end()) + { + msgCache[Itm->Uri] = Message; + MirrorListInfo info; + info.state = REQUESTED; + info.baseuri = mirrorfileuri + '/'; + auto const colon = info.baseuri.find(':'); + if (unlikely(colon == std::string::npos)) return false; - } + info.baseuri.replace(0, colon, Binary); + mirrorfilestate[mirrorfileuri] = info; + std::unordered_map<std::string, std::string> fields; + fields.emplace("URI", Itm->Uri); + fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone + fields.emplace("Aux-ShortDesc", "Mirrorlist"); + fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist"); + fields.emplace("Aux-Uri", mirrorfileuri); + SendMessage("351 Aux Request", std::move(fields)); + return true; } - if(Itm->Uri.find("mirror://") != string::npos) - Itm->Uri.replace(0,BaseUri.size(), Mirror); - - if(Debug) - clog << "Fetch: " << Itm->Uri << endl << endl; - - // now run the real fetcher - return HttpMethod::Fetch(Itm); -} - -void MirrorMethod::Fail(string Err,bool Transient) -{ - // FIXME: TryNextMirror is not ideal for indexfile as we may - // run into auth issues - - if (Debug) - clog << "Failure to get " << Queue->Uri << endl; - - // try the next mirror on fail (if its not a expected failure, - // e.g. translations are ok to ignore) - if (!Queue->FailIgnore && TryNextMirror()) - return; - - // all mirrors failed, so bail out - string s; - strprintf(s, _("[Mirror: %s]"), Mirror.c_str()); - SetIP(s); - - CurrentQueueUriToMirror(); - pkgAcqMethod::Fail(Err, Transient); -} - -void MirrorMethod::URIStart(FetchResult &Res) -{ - CurrentQueueUriToMirror(); - pkgAcqMethod::URIStart(Res); -} - -void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt) -{ - CurrentQueueUriToMirror(); - pkgAcqMethod::URIDone(Res, Alt); + switch (state->second.state) + { + case REQUESTED: + // lets wait for the requested mirror file + msgCache[Itm->Uri] = Message; + return true; + case FAILED: + Fail("Downloading mirror file failed", false); + return true; + case AVAILABLE: + RedirectItem(state->second, Itm, Message); + return true; + } + return false; } + /*}}}*/ - -int main() +int main(int, const char *argv[]) { - return MirrorMethod().Loop(); + return MirrorMethod(flNotDir(argv[0])).Run(); } - - diff --git a/methods/mirror.h b/methods/mirror.h deleted file mode 100644 index 6ebe08e6b..000000000 --- a/methods/mirror.h +++ /dev/null @@ -1,57 +0,0 @@ -// -*- mode: cpp; mode: fold -*- -// Description /*{{{*/ -/* ###################################################################### - - MIRROR Acquire Method - This is the MIRROR acquire method for APT. - - ##################################################################### */ - /*}}}*/ - -#ifndef APT_MIRROR_H -#define APT_MIRROR_H - -#include <iostream> -#include <string> -#include <vector> - -using std::cout; -using std::cerr; -using std::endl; - -#include "http.h" - -class MirrorMethod : public HttpMethod -{ - FetchResult Res; - // we simply transform between BaseUri and Mirror - std::string BaseUri; // the original mirror://... url - std::string Mirror; // the selected mirror uri (http://...) - std::vector<std::string> AllMirrors; // all available mirrors - std::string MirrorFile; // the file that contains the list of mirrors - bool DownloadedMirrorFile; // already downloaded this session - std::string Dist; // the target distrubtion (e.g. sid, oneiric) - - bool Debug; - - protected: - bool DownloadMirrorFile(std::string uri); - bool RandomizeMirrorFile(std::string file); - std::string GetMirrorFileName(std::string uri); - bool InitMirrors(); - bool TryNextMirror(); - void CurrentQueueUriToMirror(); - bool Clean(std::string dir); - - // we need to overwrite those to transform the url back - virtual void Fail(std::string Why, bool Transient = false) APT_OVERRIDE; - virtual void URIStart(FetchResult &Res) APT_OVERRIDE; - virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0) APT_OVERRIDE; - virtual bool Configuration(std::string Message) APT_OVERRIDE; - - public: - MirrorMethod(); - virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; -}; - - -#endif diff --git a/po/CMakeLists.txt b/po/CMakeLists.txt index f69123a26..a8893e8ba 100644 --- a/po/CMakeLists.txt +++ b/po/CMakeLists.txt @@ -15,7 +15,7 @@ apt_add_translation_domain( TARGETS apt apt-cache apt-get apt-config apt-cdrom apt-helper apt-mark apt-private # Methods - connectlib httplib file copy store gpgv cdrom http ftp rred rsh mirror + connectlib file copy store gpgv cdrom http ftp rred rsh mirror SCRIPTS ../dselect/install ../dselect/update EXCLUDE_LANGUAGES ${languages_excluded} ) diff --git a/test/integration/framework b/test/integration/framework index f9d98835c..ff7a7c514 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -2031,9 +2031,11 @@ mkdir() { command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" command mkdir -m 700 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" + command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/auxfiles" touch "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/lock" if [ "$(id -u)" = '0' ]; then chown _apt:$(id -gn) "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" + chown _apt:$(id -gn) "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/auxfiles" fi else command mkdir "$@" diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 616e0234c..a0d7a59d9 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -40,7 +40,8 @@ N: See apt-secure(8) manpage for repository creation and user configuration deta # no package foo testsuccessequal 'Listing...' apt list foo -testequal 'lock +testequal 'auxfiles +lock partial' ls rootdir/var/lib/apt/lists filesize() { diff --git a/test/integration/test-method-mirror b/test/integration/test-method-mirror new file mode 100755 index 000000000..d00118405 --- /dev/null +++ b/test/integration/test-method-mirror @@ -0,0 +1,190 @@ +#!/bin/sh +set -e + +TESTDIR="$(readlink -f "$(dirname "$0")")" +. "$TESTDIR/framework" + +setupenvironment +configarchitecture "i386" + +buildsimplenativepackage 'foo' 'all' '1' 'stable' +buildsimplenativepackage 'foo' 'all' '2' 'unstable' +setupaptarchive --no-update +changetowebserver +webserverconfig 'aptwebserver::redirect::replace::/redirectme/' "http://localhost:${APTHTTPPORT}/" +addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/lib/apt/lists || true;" + +testrundownload_internal() { + cd downloaded + testsuccess apt download "$@" + while [ -n "$1" ]; do + local fndeb="$(echo "$1" | tr '=' '_')_all.deb" + testsuccess test -e "$fndeb" + rm -f "$fndeb" + shift + done + testempty find . -name '*mirror*' + cd .. +} +testrundownload() { + if [ "$(id -u)" = '0' ]; then + testrundownload_internal "$@" + else + chmod -f -R -w rootdir/var/lib/apt/lists + testrundownload_internal "$@" + chmod -f -R +w rootdir/var/lib/apt/lists/auxfiles + rm -f rootdir/var/lib/apt/lists/auxfiles/* + chmod -f -R -w rootdir/var/lib/apt/lists/auxfiles + testrundownload_internal "$@" + chmod -f -R +w rootdir/var/lib/apt/lists + fi +} +testrun() { + rm -rf rootdir/var/lib/apt/lists + testsuccess apt update #-o Debug::Acquire::mirror=1 -o Debug::Acquire::http=1 -o Debug::pkgAcquire::Worker=1 + cp -a rootdir/tmp/testsuccess.output aptupdate.output + test -z "$1" || testempty find rootdir/var/lib/apt/lists -maxdepth 1 -name "$1" -type f + test -z "$2" || testnotempty find rootdir/var/lib/apt/lists -maxdepth 1 -name "$2" -type f + testsuccess apt show foo=1 + testrundownload 'foo=1' 'foo=2' +} + +msgmsg 'basic setup' +testrun '' '' + +msgmsg 'redirect setup' +sed -i -e 's#/ stable#/redirectme stable#' rootdir/etc/apt/sources.list.d/*-stable-* +testrun '' '*_redirectme_*' + +msgmsg 'mirror file does not exist' +sed -i -e 's# http:# mirror:#' -e 's#/redirectme stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/*-stable-* +testfailure apt update + +echo "http://localhost:${APTHTTPPORT}" > aptarchive/mirror.txt + +msgmsg 'stable mirror setup' +testrun '*_redirectme_stable_*' '*_mirror.txt_*' + +msgmsg 'all mirror setup' +sed -i -e 's# http:# mirror:#' -e 's#/ unstable#/mirror.txt unstable#' rootdir/etc/apt/sources.list.d/* +testrun '*_redirectme_stable_*' '*_mirror.txt_*' + +msgmsg 'all mirror+http setup' +sed -i -e 's# mirror:# mirror+http:#' rootdir/etc/apt/sources.list.d/* +testrun '*_redirectme_*' '*_mirror.txt_*' + +msgmsg 'stable gzipped redirect setup' +echo "http://localhost:${APTHTTPPORT}/redirectme" > aptarchive/mirror.txt +compressfile aptarchive/mirror.txt +sed -i -e 's#/mirror\.txt stable#/mirror.txt.gz stable#' rootdir/etc/apt/sources.list.d/*-stable-* +testrun '*_redirectme_*' '*_mirror.txt.gz_*' + +msgmsg 'all mirrored via file' +APTARCHIVE="$(readlink -f ./aptarchive)" +sed -i -e "s#mirror+http://localhost:${APTHTTPPORT}#mirror+file:${APTARCHIVE}#" rootdir/etc/apt/sources.list.d/* +testrun '*_localhost_*' '*_aptarchive_mirror.txt.gz_*' + +msgmsg 'fallback mirrors are used if needed' 'as usual' +sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/* +echo "http://localhost:${APTHTTPPORT}/failure2 priority:3 +http://localhost:${APTHTTPPORT}/redirectme priority:2 +http://localhost:${APTHTTPPORT}/failure priority:1" > aptarchive/mirror.txt +testrun '*_localhost_*' '*_aptarchive_mirror.txt_*' +testsuccessequal "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] +Ign:2 http://localhost:${APTHTTPPORT}/failure stable InRelease + 404 Not Found +Ign:3 http://localhost:${APTHTTPPORT}/failure unstable InRelease + 404 Not Found" head -n 5 aptupdate.output + +msgmsg 'fallback mirrors are used if needed' 'by tags' +echo "http://localhost:${APTHTTPPORT}/failure2 priority:1 release:stable +http://localhost:${APTHTTPPORT}/redirectme priority:2 +http://localhost:${APTHTTPPORT}/failure priority:1 release:unstable" > aptarchive/mirror.txt +testrun '*_localhost_*' '*_aptarchive_mirror.txt_*' +testsuccessequal "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] +Ign:2 http://localhost:${APTHTTPPORT}/failure2 stable InRelease + 404 Not Found +Ign:3 http://localhost:${APTHTTPPORT}/failure unstable InRelease + 404 Not Found" head -n 5 aptupdate.output + +changetohttpswebserver +rm -f rootdir/etc/apt/sources.list.d/*-stable-* +msgmsg 'fallback mirrors are used if needed' 'random' +echo "http://localhost:${APTHTTPPORT}/failure2 priority:1 +http://localhost:${APTHTTPPORT}/redirectme priority:2 +https://localhost:${APTHTTPSPORT}/failure priority:1 +http://localhost:${APTHTTPPORT}/unused-failure1 +http://localhost:${APTHTTPPORT}/unused-failure2 +http://localhost:${APTHTTPPORT}/unused-failure3 +http://localhost:${APTHTTPPORT}/unused-failure4 +http://localhost:${APTHTTPPORT}/unused-failure5 +http://localhost:${APTHTTPPORT}/unused-failure6 +http://localhost:${APTHTTPPORT}/unused-failure7 +http://localhost:${APTHTTPPORT}/unused-failure8 +http://localhost:${APTHTTPPORT}/unused-failure9 +" > aptarchive/mirror.txt +testequalor2 "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] +Ign:2 http://localhost:${APTHTTPPORT}/failure2 unstable InRelease + 404 Not Found +Ign:2 https://localhost:${APTHTTPSPORT}/failure unstable InRelease + 404 Not Found +Hit:2 http://localhost:${APTHTTPPORT}/redirectme unstable InRelease +Reading package lists... +Building dependency tree... +All packages are up to date." "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] +Ign:2 https://localhost:${APTHTTPSPORT}/failure unstable InRelease + 404 Not Found +Ign:2 http://localhost:${APTHTTPPORT}/failure2 unstable InRelease + 404 Not Found +Hit:2 http://localhost:${APTHTTPPORT}/failure2 unstable InRelease +Reading package lists... +Building dependency tree... +All packages are up to date." apt update +testfailure grep '/unused-failure' aptarchive/webserver.log + +msgmsg 'Mirrors can be filtered by' 'type' +echo "http://localhost:${APTHTTPPORT}/failure +http://localhost:${APTHTTPPORT}/redirectme type:deb +" > aptarchive/mirror.txt +testfailure apt update +testrundownload 'foo=2' + +msgmsg 'The prefix for the mirrorlist is' 'passed on' +echo 'Dir::Bin::Methods::foo+mirror+file "mirror"; +Dir::Bin::Methods::foo+mirror+http "mirror"; +Dir::Bin::Methods::foo+http "http"; +' > rootdir/etc/apt/apt.conf.d/99add-foo-method +echo "http://localhost:${APTHTTPPORT}/redirectme +" > aptarchive/mirror.txt +testsuccessequal "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] +Hit:2 http://localhost:${APTHTTPPORT}/redirectme unstable InRelease +Reading package lists... +Building dependency tree... +All packages are up to date." apt update +sed -i -e 's# mirror+# foo+mirror+#' rootdir/etc/apt/sources.list.d/* +testfailure apt update +testsuccess grep 'package apt-transport-foo installed' rootdir/tmp/testfailure.output +echo 'Dir::Bin::Methods::foo+file "file";' >> rootdir/etc/apt/apt.conf.d/99add-foo-method +testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] +Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease +Reading package lists... +Building dependency tree... +All packages are up to date." apt update +echo "file:/nonexistent/apt/archive priority:1 +http://localhost:${APTHTTPPORT}/redirectme +" > aptarchive/mirror.txt +testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] +Get:2 foo+file:/nonexistent/apt/archive unstable InRelease +Ign:2 foo+file:/nonexistent/apt/archive unstable InRelease + File not found - /nonexistent/apt/archive/dists/unstable/InRelease (2: No such file or directory) +Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease +Reading package lists... +Building dependency tree... +All packages are up to date." apt update +sed -i -e "s#+file:${APTARCHIVE}#+http://localhost:${APTHTTPPORT}#" rootdir/etc/apt/sources.list.d/* +testsuccess apt update +testsuccessequal "Get:1 foo+http://localhost:${APTHTTPPORT}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] +Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease +Reading package lists... +Building dependency tree... +All packages are up to date." apt update diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 53586ef32..5a650ad83 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -364,7 +364,7 @@ SHA256-Download: # we let it fail by removing the files so the webserver reports 404 rm -f "$PATCHINDEX" "$PATCHFILE" "${PATCHFILE}.gz" wasmergeused "$@" -o test::cannot-use-pdiff=1 - testsuccess grep '400%20URI%20Failure.*Packages\.diff/Index.*FailReason.*HttpError404' rootdir/tmp/aptupdate.output + testsuccess grep '400%20URI%20Failure.*FailReason.*HttpError404.*Packages\.diff/Index' rootdir/tmp/aptupdate.output testnopackage oldstuff testsuccessequal "$(cat "${PKGFILE}-new") " aptcache show apt newstuff diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall index 3571a9f25..5f2109db9 100755 --- a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall +++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall @@ -41,7 +41,8 @@ runtests() { testsuccess grep "$1" rootdir/tmp/testfailure.output ensure_n_canary_strings_in_dir "$LISTS" 'ni ni ni' 0 - testequal 'lock + testequal 'auxfiles +lock partial' ls "$LISTS" # and again with pre-existing files with "valid data" which should remain |