From 39304fb440f62b3a9f4dd85871b7049fb5b8eb8f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 23 Apr 2022 22:46:33 +0200 Subject: Support uncompressed indexes from partial file:/ mirrors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a file:/ mirror does not contain the requested compressed file, but the uncompressed variant of that file the method will sent us a fitting Done message with data only about the Alt-Filename, but the central hash verification code in the acquire system does not expect that looking for the non-existent compressed file to compare its hashes with the expected result – which fails (if we are verifying at least). That file:/ as well as the item code deals with this suggests that this feature was broken by 448c38bdcd72b52f11ec5f326f822cf57653f81c, but I couldn't (easily) compile that to verify, so this is only a guess. Supporting this allows using index files from /var/lib/apt/lists – which are likely uncompressed or can at least made so easily – to construct a file:/ mirror. --- apt-pkg/acquire-item.cc | 3 +- apt-pkg/acquire-worker.cc | 30 ++++++++-- .../integration/test-apt-get-update-unauth-warning | 66 ++++++++++++---------- .../test-apt-update-incomplete-file-mirror | 31 ++++++++++ 4 files changed, 92 insertions(+), 38 deletions(-) create mode 100755 test/integration/test-apt-update-incomplete-file-mirror diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 12292eb28..1fc07ec05 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1024,8 +1024,7 @@ void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const bool pkgAcquire::Item::VerifyDone(std::string const &Message, pkgAcquire::MethodConfig const * const /*Cnf*/) { - std::string const FileName = LookupTag(Message,"Filename"); - if (FileName.empty() == true) + if (LookupTag(Message,"Filename").empty() && LookupTag(Message, "Alt-Filename").empty()) { Status = StatError; ErrorText = "Method gave a blank filename"; diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 0f22f71e8..035d8ce10 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -434,11 +435,18 @@ bool pkgAcquire::Worker::RunMessages() Log->Pulse((*O)->GetOwner()); HashStringList ReceivedHashes; + bool AltFile = false; { - std::string const givenfilename = LookupTag(Message, "Filename"); - std::string const filename = givenfilename.empty() ? Itm->Owner->DestFile : givenfilename; + std::string filename; + if (filename = LookupTag(Message, "Filename"); filename.empty()) + { + if (filename = LookupTag(Message, "Alt-Filename"); not filename.empty()) + AltFile = true; + else + filename = Itm->Owner->DestFile; + } // see if we got hashes to verify - ReceivedHashes = GetHashesFromMessage("", Message); + ReceivedHashes = GetHashesFromMessage(AltFile ? "Alt-" : "", Message); // not all methods always sent Hashes our way if (ReceivedHashes.usable() == false) { @@ -453,7 +461,7 @@ bool pkgAcquire::Worker::RunMessages() } // only local files can refer other filenames and counting them as fetched would be unfair - if (Log != NULL && Itm->Owner->Complete == false && Itm->Owner->Local == false && givenfilename == filename) + if (Log != nullptr && not Itm->Owner->Complete && not Itm->Owner->Local && not AltFile && Itm->Owner->DestFile == filename) Log->Fetched(ReceivedHashes.FileSize(),atoi(LookupTag(Message,"Resume-Point","0").c_str())); } @@ -466,7 +474,19 @@ bool pkgAcquire::Worker::RunMessages() auto const forcedHash = _config->Find("Acquire::ForceHash"); for (auto const Owner: ItmOwners) { - HashStringList const ExpectedHashes = Owner->GetExpectedHashes(); + HashStringList const ExpectedHashes = [&]() { + if (AltFile) + { + auto const * const transOwner = dynamic_cast(Owner); + if (transOwner != nullptr && transOwner->TransactionManager != nullptr && transOwner->TransactionManager->MetaIndexParser != nullptr) + { + auto const * const hashes = transOwner->TransactionManager->MetaIndexParser->Lookup(transOwner->Target.MetaKey); + if (hashes != nullptr) + return hashes->Hashes; + } + } + return Owner->GetExpectedHashes(); + }(); if(_config->FindB("Debug::pkgAcquire::Auth", false) == true) { std::clog << "201 URI Done: " << Owner->DescURI() << endl diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 42c4e5a9d..46df14c29 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -44,46 +44,15 @@ testequal 'auxfiles lock partial' ls rootdir/var/lib/apt/lists -filesize() { - local CREATEDBY="$1" - shift - stat -c%s "/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 2- ).gz" -} # allow override -#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1 -#exit testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease Ign:1 file:$APTARCHIVE unstable InRelease Get:2 file:$APTARCHIVE unstable Release Ign:2 file:$APTARCHIVE unstable Release Get:3 file:$APTARCHIVE unstable/main Sources -Ign:3 file:$APTARCHIVE unstable/main Sources Get:4 file:$APTARCHIVE unstable/main i386 Packages -Ign:4 file:$APTARCHIVE unstable/main i386 Packages Get:5 file:$APTARCHIVE unstable/main all Packages -Ign:5 file:$APTARCHIVE unstable/main all Packages Get:6 file:$APTARCHIVE unstable/main Translation-en -Ign:6 file:$APTARCHIVE unstable/main Translation-en -Get:3 file:$APTARCHIVE unstable/main Sources -Ign:3 file:$APTARCHIVE unstable/main Sources -Get:4 file:$APTARCHIVE unstable/main i386 Packages -Ign:4 file:$APTARCHIVE unstable/main i386 Packages -Get:5 file:$APTARCHIVE unstable/main all Packages -Ign:5 file:$APTARCHIVE unstable/main all Packages -Get:6 file:$APTARCHIVE unstable/main Translation-en -Ign:6 file:$APTARCHIVE unstable/main Translation-en -Get:3 file:$APTARCHIVE unstable/main Sources -Ign:3 file:$APTARCHIVE unstable/main Sources -Get:4 file:$APTARCHIVE unstable/main i386 Packages -Ign:4 file:$APTARCHIVE unstable/main i386 Packages -Get:5 file:$APTARCHIVE unstable/main all Packages -Ign:5 file:$APTARCHIVE unstable/main all Packages -Get:6 file:$APTARCHIVE unstable/main Translation-en -Ign:6 file:$APTARCHIVE unstable/main Translation-en -Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B] -Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B] -Get:5 file:$APTARCHIVE unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B] -Get:6 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B] Reading package lists... W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use. @@ -92,3 +61,38 @@ N: See apt-secure(8) manpage for repository creation and user configuration deta testfailureequal "WARNING: The following packages cannot be authenticated! foo E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo + +rm -rf rootdir/var/lib/apt/lists +changetowebserver + +filesize() { + local CREATEDBY="$1" + shift + stat -c%s "${APTARCHIVE}/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 4- ).gz" +} +testwarningequal "Ign:1 http://localhost:$APTHTTPPORT unstable InRelease +Ign:2 http://localhost:$APTHTTPPORT unstable Release +Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources +Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages +Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages +Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en +Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources +Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages +Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages +Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en +Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources +Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages +Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages +Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en +Get:3 http://localhost:$APTHTTPPORT unstable/main Sources [$(filesize 'Sources') B] +Get:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B] +Get:5 http://localhost:$APTHTTPPORT unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B] +Get:6 http://localhost:$APTHTTPPORT unstable/main Translation-en [$(filesize 'Translations') B] +Reading package lists... +W: The repository 'http://localhost:$APTHTTPPORT unstable Release' does not have a Release file. +N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use. +N: See apt-secure(8) manpage for repository creation and user configuration details." aptget update --allow-insecure-repositories +# ensure we can not install the package +testfailureequal "WARNING: The following packages cannot be authenticated! + foo +E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror new file mode 100755 index 000000000..d6894d607 --- /dev/null +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -0,0 +1,31 @@ +#!/bin/sh +set -e + +TESTDIR="$(readlink -f "$(dirname "$0")")" +. "$TESTDIR/framework" + +setupenvironment +configarchitecture "amd64" +configcompression 'gz' + +insertpackage 'unstable' 'foo' 'amd64' '1' + +setupaptarchive --no-update + +testfailure aptcache show foo + +# the Release file says gz available, but the mirror has only uncompressed files +find aptarchive/dists -name '*.gz' -delete + +testsuccess apt update +testsuccess aptcache show foo +testfailure aptcache show bar + +testsuccess apt update + +rm -rf rootdir/var/lib/apt/lists + +find aptarchive/dists -name 'Packages' | while read FILE; do + echo 'hacked' > $FILE +done +testfailure apt update -- cgit v1.2.3-70-g09d2 From 496c8b4191790231258e644ac8cd4096d54f445f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 24 Apr 2022 00:03:38 +0200 Subject: Show hashes calculated by main thread in error messages The main thread calculates the hash sums if the method hasn't to be able to verify the files, but while a failure is detected and reported, the error messages did not include the hash sums which looks at least puzzling. So to have a complete error message we ensure that the hashes we calculated are appended to the message from the method so that it looks for the higher level code as if the method had calculated the hashes. --- apt-pkg/acquire-item.cc | 14 +++++++++++++- apt-pkg/acquire-worker.cc | 12 ++++++++++++ test/integration/test-apt-update-incomplete-file-mirror | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 1fc07ec05..7e1c5dad6 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -976,13 +976,25 @@ void pkgAcquire::Item::FailMessage(string const &Message) if (failreason == HASHSUM_MISMATCH) { out << "Hashes of received file:" << std::endl; + size_t hashes = 0; for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) { std::string const tagname = std::string(*type) + "-Hash"; std::string const hashsum = LookupTag(Message, tagname.c_str()); - if (hashsum.empty() == false) + if (not hashsum.empty()) + { formatHashsum(out, HashString(*type, hashsum)); + ++hashes; + } } + if (hashes == 0) + for (char const *const *type = HashString::SupportedHashes(); *type != nullptr; ++type) + { + std::string const tagname = std::string("Alt-") + *type + "-Hash"; + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (not hashsum.empty()) + formatHashsum(out, HashString(*type, hashsum)); + } } auto const lastmod = LookupTag(Message, "Last-Modified", ""); if (lastmod.empty() == false) diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 035d8ce10..4c2acb5cc 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -457,6 +457,18 @@ bool pkgAcquire::Worker::RunMessages() FileFd file(filename, FileFd::ReadOnly, FileFd::None); calc.AddFD(file); ReceivedHashes = calc.GetHashStringList(); + for (auto const &h : ReceivedHashes) + { + std::string tagname; + if (AltFile) + tagname = "Alt-"; + tagname.append(h.HashType()).append("-Hash"); + if (not LookupTag(Message, tagname.c_str()).empty()) + continue; + if (not APT::String::Endswith(Message, "\n")) + Message.append("\n"); + Message.append(tagname).append(": ").append(h.HashValue()); + } } } diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index d6894d607..86eb890a7 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -29,3 +29,4 @@ find aptarchive/dists -name 'Packages' | while read FILE; do echo 'hacked' > $FILE done testfailure apt update +testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output -- cgit v1.2.3-70-g09d2 From 370e024224b8116840173201d51c0c2a170e0e93 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 24 Apr 2022 00:24:40 +0200 Subject: Alternatively calculate alternative file hashes in file method If we do not have the requested file we haven't calculated the hashes for it either and it is likely that the alternative file will be used, so to save the main thread from being busy with calculating hashes we do the calculation here in the method. --- methods/file.cc | 2 ++ test/integration/test-apt-update-incomplete-file-mirror | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/methods/file.cc b/methods/file.cc index b2fe133f2..ff93f83d0 100644 --- a/methods/file.cc +++ b/methods/file.cc @@ -107,6 +107,8 @@ bool FileMethod::Fetch(FetchItem *Itm) AltRes.IMSHit = false; if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) AltRes.IMSHit = true; + if (Res.Filename.empty()) + CalculateHashes(Itm, AltRes); break; } // no break here as we could have situations similar to '.gz' vs '.tar.gz' here diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index 86eb890a7..2d2125582 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -28,5 +28,6 @@ rm -rf rootdir/var/lib/apt/lists find aptarchive/dists -name 'Packages' | while read FILE; do echo 'hacked' > $FILE done -testfailure apt update +testfailure apt update -o Debug::pkgAcquire::Worker=1 testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output +testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output -- cgit v1.2.3-70-g09d2 From eb1e6fc4973a844adbea0297c6425b00f64e8b7b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 26 Apr 2022 21:33:16 +0200 Subject: Rewrite hidden config option checker for more patterns The previous version allowed only foo::*::bar as a pattern, now you can have as many starred parts as you want allowing for a finer control instead of hardcoding specific common star-values or giving up with **. It also adds a new INVALID type for an option to mark options which should not be used, but are covered by a pattern which would normally allow it. Our tests are more or less the only users of this hidden feature making this change not very risky even through it has no dedicated test itself. --- apt-pkg/acquire.cc | 2 +- apt-pkg/contrib/configuration.cc | 115 +++++++++++++++++++++++++-------------- doc/examples/configure-index | 49 ++++++++++++----- 3 files changed, 112 insertions(+), 54 deletions(-) diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index f367e4310..07f64cc69 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -524,7 +524,7 @@ pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access) Configs = Conf; /* if a method uses DownloadLimit, we switch to SingleInstance mode */ - if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0) + if (not Conf->SingleInstance && _config->FindI("Acquire::" + Access + "::Dl-Limit", 0) > 0) Conf->SingleInstance = true; return Conf; diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc index c9b2dc0cc..fff10fb17 100644 --- a/apt-pkg/contrib/configuration.cc +++ b/apt-pkg/contrib/configuration.cc @@ -55,12 +55,14 @@ Configuration *_config = new Configuration; but a Cnf-member – but that would need ABI breaks and stuff and for now that really is an apt-dev-only tool, so it isn't that bad that it is unusable and allaround a bit strange */ -enum class APT_HIDDEN ConfigType { UNDEFINED, INT, BOOL, STRING, STRING_OR_BOOL, STRING_OR_LIST, FILE, DIR, LIST, PROGRAM_PATH = FILE }; +enum class APT_HIDDEN ConfigType { INVALID, UNDEFINED, INT, BOOL, STRING, STRING_OR_BOOL, STRING_OR_LIST, FILE, DIR, LIST, PROGRAM_PATH = FILE }; APT_HIDDEN std::unordered_map apt_known_config {}; +APT_HIDDEN std::vector> apt_known_config_patterns {}; static std::string getConfigTypeString(ConfigType const type) /*{{{*/ { switch (type) { + case ConfigType::INVALID: return "INVALID"; case ConfigType::UNDEFINED: return "UNDEFINED"; case ConfigType::INT: return "INT"; case ConfigType::BOOL: return "BOOL"; @@ -94,6 +96,8 @@ static ConfigType getConfigType(std::string const &type) /*{{{*/ return ConfigType::STRING_OR_LIST; else if (type == "") return ConfigType::PROGRAM_PATH; + else if (type == "") + return ConfigType::INVALID; return ConfigType::UNDEFINED; } /*}}}*/ @@ -101,59 +105,86 @@ static ConfigType getConfigType(std::string const &type) /*{{{*/ static void checkFindConfigOptionTypeInternal(std::string name, ConfigType const type) { std::transform(name.begin(), name.end(), name.begin(), ::tolower); - auto known = apt_known_config.find(name); - if (known == apt_known_config.cend()) + bool found = false; + ConfigType expectedType = ConfigType::INVALID; + if (auto const known = apt_known_config.find(name); known != apt_known_config.cend()) { - auto const rcolon = name.rfind(':'); - if (rcolon != std::string::npos) + found = true; + expectedType = known->second; + } + else + for (auto const &known : apt_known_config_patterns) { - known = apt_known_config.find(name.substr(0, rcolon) + ":*"); - if (known == apt_known_config.cend()) + std::string_view n{name}; + std::string_view k{known.first}; + do { - auto const parts = StringSplit(name, "::"); - size_t psize = parts.size(); - if (psize > 1) + if (auto star = k.find('*'); star != std::string_view::npos) { - for (size_t max = psize; max != 1; --max) - { - std::ostringstream os; - std::copy(parts.begin(), parts.begin() + max, std::ostream_iterator(os, "::")); - os << "**"; - known = apt_known_config.find(os.str()); - if (known != apt_known_config.cend() && known->second == ConfigType::UNDEFINED) - return; - } - for (size_t max = psize - 1; max != 1; --max) - { - std::ostringstream os; - std::copy(parts.begin(), parts.begin() + max - 1, std::ostream_iterator(os, "::")); - os << "*::"; - std::copy(parts.begin() + max + 1, parts.end() - 1, std::ostream_iterator(os, "::")); - os << *(parts.end() - 1); - known = apt_known_config.find(os.str()); - if (known != apt_known_config.cend()) - break; - } + if (auto nextstar = star + 1; k.length() > nextstar && k[nextstar] == '*') + star -= 3; + if (n.compare(0, star, k, 0, star) != 0) + break; + n.remove_prefix(star); + k.remove_prefix(star + 1); } - } + else if (k == n) + { + n = {}; + break; + } + else + break; + + if (k.empty()) + { + if (n.find("::") == std::string_view::npos) + n = {}; + break; + } + if (k == "::**") + { + if (known.second == ConfigType::UNDEFINED) + return; + n = {}; + break; + } + if (k.compare(0, 2, "::") == 0) + { + auto const colons = n.find("::"); + if (colons == std::string_view::npos) + break; + n.remove_prefix(colons); + } + else + break; + } while (not n.empty()); + if (not n.empty()) + continue; + found = true; + expectedType = known.second; + break; } - } - if (known == apt_known_config.cend()) + + if (not found) _error->Warning("Using unknown config option »%s« of type %s", name.c_str(), getConfigTypeString(type).c_str()); - else if (known->second != type) + else if (expectedType != type) { - if (known->second == ConfigType::DIR && type == ConfigType::FILE) + if (expectedType == ConfigType::INVALID) + _error->Warning("Using invalid config option »%s« as a type %s", + name.c_str(), getConfigTypeString(type).c_str()); + if (expectedType == ConfigType::DIR && type == ConfigType::FILE) ; // implementation detail - else if (type == ConfigType::STRING && (known->second == ConfigType::FILE || known->second == ConfigType::DIR)) + else if (type == ConfigType::STRING && (expectedType == ConfigType::FILE || expectedType == ConfigType::DIR)) ; // TODO: that might be an error or not, we will figure this out later - else if (known->second == ConfigType::STRING_OR_BOOL && (type == ConfigType::BOOL || type == ConfigType::STRING)) + else if (expectedType == ConfigType::STRING_OR_BOOL && (type == ConfigType::BOOL || type == ConfigType::STRING)) ; - else if (known->second == ConfigType::STRING_OR_LIST && (type == ConfigType::LIST || type == ConfigType::STRING)) + else if (expectedType == ConfigType::STRING_OR_LIST && (type == ConfigType::LIST || type == ConfigType::STRING)) ; else _error->Warning("Using config option »%s« of type %s as a type %s", - name.c_str(), getConfigTypeString(known->second).c_str(), getConfigTypeString(type).c_str()); + name.c_str(), getConfigTypeString(expectedType).c_str(), getConfigTypeString(type).c_str()); } } static void checkFindConfigOptionType(char const * const name, ConfigType const type) @@ -166,6 +197,7 @@ static void checkFindConfigOptionType(char const * const name, ConfigType const static bool LoadConfigurationIndex(std::string const &filename) /*{{{*/ { apt_known_config.clear(); + apt_known_config_patterns.clear(); if (filename.empty()) return true; Configuration Idx; @@ -181,7 +213,10 @@ static bool LoadConfigurationIndex(std::string const &filename) /*{{{*/ { std::string fulltag = Top->FullTag(); std::transform(fulltag.begin(), fulltag.end(), fulltag.begin(), ::tolower); - apt_known_config.emplace(std::move(fulltag), getConfigType(Top->Value)); + if (fulltag.find("::*") == std::string::npos) + apt_known_config.emplace(std::move(fulltag), getConfigType(Top->Value)); + else + apt_known_config_patterns.emplace_back(std::move(fulltag), getConfigType(Top->Value)); } if (Top->Child != nullptr) diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 6723a48da..1b165702d 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -184,18 +184,16 @@ APT Error-Mode ""; }; - /* define a new supported compressor on the fly - Compressor::rev { - Name "rev"; - Extension ".reversed"; - Binary "rev"; - CompressArg {}; - UncompressArg {}; - Cost "10"; - }; - */ Compressor ""; - Compressor::** ""; + // define a new compressor on the fly, like the super-compressor: /usr/bin/rev + Compressor::* { + Name ""; // rev + Extension ""; // .reversed + Binary ""; // rev + CompressArg ""; // {} + UncompressArg ""; // {} + Cost ""; // 10 + }; Authentication { @@ -584,6 +582,9 @@ Debug Acquire::Ftp ""; // Show ftp command traffic Acquire::Http ""; // Show http command traffic Acquire::Https ""; // Show https debug + Acquire::tor ""; + Acquire::tor+http ""; + Acquire::tor+https ""; Acquire::gpgv ""; // Show the gpgv traffic Acquire::cdrom ""; // Show cdrom debug output Acquire::Transaction ""; @@ -819,10 +820,31 @@ acquire::max-pipeline-depth ""; acquire::progress::diffpercent ""; acquire::gzipindexes ""; acquire::indextargets::randomized ""; -acquire::indextargets::deb::** ""; -acquire::indextargets::deb-src::** ""; +acquire::indextargets::* "" { + * { + MetaKey ""; + ShortDescription ""; + Description ""; + flatMetaKey ""; + flatDescription ""; + Identifier ""; + DefaultEnabled ""; + Optional ""; + KeepCompressed ""; + PDiffs ""; + By-Hash ""; + Fallback-Of ""; + CompressionTypes ""; + KeepCompressedAs ""; + }; +}; acquire::progress::ignore::showerrortext ""; acquire::*::dl-limit ""; // catches file: and co which do not have these +acquire::file::dl-limit ""; +acquire::copy::dl-limit ""; +acquire::gpgv::dl-limit ""; +acquire::store::dl-limit ""; +acquire::mirror::dl-limit ""; methods::mirror::problemreporting ""; acquire::http::proxyautodetect ""; acquire::http::proxy-auto-detect ""; @@ -900,6 +922,7 @@ APT::Internal::OpProgress::EraseLines ""; APT::Color ""; APT::Color::Show::Field ""; APT::Color::Show::Package ""; +APT::Color::* ""; update-manager::always-include-phased-updates ""; update-manager::never-include-phased-updates ""; -- cgit v1.2.3-70-g09d2 From d9601bce6e1fa61cbb3dc4d66e5492f9885eef8f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 26 Apr 2022 22:22:46 +0200 Subject: Allow AutoDetectProxy to work with more than just http(s) It is a bit unfair for third-party methods wrapping e.g. http to not have access to such tools, same for our old ftp and e.g. our tor even if in practice this isn't used much even for http. At least that is one less FIXME in the code. --- apt-pkg/acquire-worker.cc | 10 ++++---- apt-pkg/contrib/proxy.cc | 52 +++++++++++++++++++++++++++++----------- apt-pkg/contrib/proxy.h | 4 +++- doc/examples/configure-index | 18 +++++++++----- test/integration/test-apt-helper | 48 +++++++++++++++++++++++++++---------- 5 files changed, 93 insertions(+), 39 deletions(-) diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 4c2acb5cc..533089ea5 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -877,14 +877,12 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) } Message += "\nFilename: " + Item->Owner->DestFile; - // FIXME: We should not hard code proxy protocols here. - if (URL.Access == "http" || URL.Access == "https") + // AutoDetectProxy() checks this already by itself, but we don't want to access unknown configs + if (CanURIBeAccessedViaProxy(URL)) { AutoDetectProxy(URL); - if (_config->Exists("Acquire::" + URL.Access + "::proxy::" + URL.Host)) - { - Message += "\nProxy: " + _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host); - } + if (auto const proxy = _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host); not proxy.empty()) + Message.append("\nProxy: ").append(proxy); } HashStringList const hsl = Item->GetExpectedHashes(); diff --git a/apt-pkg/contrib/proxy.cc b/apt-pkg/contrib/proxy.cc index a99f44f49..9a9a9f29c 100644 --- a/apt-pkg/contrib/proxy.cc +++ b/apt-pkg/contrib/proxy.cc @@ -12,9 +12,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -22,6 +24,16 @@ #include "proxy.h" /*}}}*/ +bool CanURIBeAccessedViaProxy(URI const &URL) /*{{{*/ +{ + // for some methods a proxy doesn't make sense, so we don't have to fork + if (URL.Host.empty() || + APT::String::Startswith(URL.Access, "mirror+") || URL.Access.find("+mirror+") != std::string::npos || APT::String::Endswith(URL.Access, "+mirror")) + return false; + std::array const noproxy{"file", "copy", "store", "gpgv", "rred", "cdrom", "mirror"}; + return std::find(noproxy.begin(), noproxy.end(), URL.Access) == noproxy.end(); +} + /*}}}*/ // AutoDetectProxy - auto detect proxy /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -29,16 +41,19 @@ static std::vector CompatibleProxies(URI const &URL) { if (URL.Access == "http" || URL.Access == "https") return {"http", "https", "socks5h"}; - return {URL.Access}; + if (URL.Access == "tor" || URL.Access == "tor+http" || URL.Access == "tor+https") + return {"socks5h"}; + if (URL.Access == "ftp") + return {"ftp"}; + return {}; } - bool AutoDetectProxy(URI &URL) { - // we support both http/https debug options - bool Debug = _config->FindB("Debug::Acquire::"+URL.Access,false); + if (not CanURIBeAccessedViaProxy(URL)) + return true; // the user already explicitly set a proxy for this host - if(_config->Find("Acquire::"+URL.Access+"::proxy::"+URL.Host, "") != "") + if (not _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host, "").empty()) return true; // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old @@ -49,6 +64,7 @@ bool AutoDetectProxy(URI &URL) if (AutoDetectProxyCmd.empty()) return true; + bool const Debug = _config->FindB("Debug::Acquire::" + URL.Access, false); if (Debug) std::clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << std::endl; @@ -73,7 +89,7 @@ bool AutoDetectProxy(URI &URL) // and apt will use the generic proxy settings if (goodread == false) return true; - auto const cleanedbuf = _strstrip(buf); + APT::StringView const cleanedbuf = _strstrip(buf); // We warn about this as the implementor probably meant to use DIRECT instead if (cleanedbuf[0] == '\0') { @@ -82,17 +98,25 @@ bool AutoDetectProxy(URI &URL) } if (Debug) - std::clog << "auto detect command returned: '" << cleanedbuf << "'" << std::endl; + std::clog << "auto detect command returned: '" << cleanedbuf.data() << "'" << std::endl; - auto compatibleTypes = CompatibleProxies(URL); - bool compatible = strcmp(cleanedbuf, "DIRECT") == 0 || - compatibleTypes.end() != std::find_if(compatibleTypes.begin(), - compatibleTypes.end(), [cleanedbuf](std::string &compat) { - return strstr(cleanedbuf, compat.c_str()) == cleanedbuf; - }); + bool compatible = true; + if (cleanedbuf != "DIRECT") + { + if (auto const compatibleTypes = CompatibleProxies(URL); not compatibleTypes.empty()) + compatible = std::any_of(compatibleTypes.begin(), compatibleTypes.end(), + [cleanedbuf](std::string const &compat) + { + return cleanedbuf.substr(0, compat.size()) == compat; + }); + } + else if (URL.Access == "tor" || URL.Access == "tor+http" || URL.Access == "tor+https") + compatible = false; // Accepting DIRECT would silently disable tor if (compatible) - _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf); + _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf.data()); + else + _error->Warning("ProxyAutoDetect command returned incompatible proxy '%s' for access type %s", cleanedbuf.data(), URL.Access.c_str()); return true; } diff --git a/apt-pkg/contrib/proxy.h b/apt-pkg/contrib/proxy.h index f6d70ea8b..2a4c19fc8 100644 --- a/apt-pkg/contrib/proxy.h +++ b/apt-pkg/contrib/proxy.h @@ -9,8 +9,10 @@ #ifndef PKGLIB_PROXY_H #define PKGLIB_PROXY_H +#include + class URI; APT_PUBLIC bool AutoDetectProxy(URI &URL); - +APT_HIDDEN bool CanURIBeAccessedViaProxy(URI const &URL); #endif diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 1b165702d..9623514b8 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -846,12 +846,18 @@ acquire::gpgv::dl-limit ""; acquire::store::dl-limit ""; acquire::mirror::dl-limit ""; methods::mirror::problemreporting ""; -acquire::http::proxyautodetect ""; -acquire::http::proxy-auto-detect ""; -acquire::http::proxy::* ""; -acquire::https::proxyautodetect ""; -acquire::https::proxy-auto-detect ""; -acquire::https::proxy::* ""; +acquire::*::proxyautodetect ""; +acquire::file::proxyautodetect ""; +acquire::copy::proxyautodetect ""; +acquire::store::proxyautodetect ""; +acquire::*::proxy-auto-detect ""; +acquire::file::proxy-auto-detect ""; +acquire::copy::proxy-auto-detect ""; +acquire::store::proxy-auto-detect ""; +acquire::file::proxy::* ""; +acquire::copy::proxy::* ""; +acquire::store::proxy::* ""; +acquire::*::proxy::* ""; // Options used by apt-ftparchive dir::archivedir ""; diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index ae1ca7456..4f883b440 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -74,10 +74,11 @@ E: Download Failed" setupproxydetect() { local METH="$1" shift - { - echo '#!/bin/sh -e' - echo "$@" - } > "${TMPWORKINGDIRECTORY}/apt-proxy-detect" + cat >"${TMPWORKINGDIRECTORY}/apt-proxy-detect" < rootdir/etc/apt/apt.conf.d/02proxy-detect } @@ -97,15 +98,38 @@ W: ProxyAutoDetect command returned an empty line" apthelper auto-detect-proxy h chmod -x "${TMPWORKINGDIRECTORY}/apt-proxy-detect" testfailureequal "E: ProxyAutoDetect command '${TMPWORKINGDIRECTORY}/apt-proxy-detect' can not be executed! - access (13: Permission denied)" apthelper auto-detect-proxy http://example.com/ - msgmsg "apt-helper $CONFNAME" 'http proxy' - setupproxydetect 'http' 'echo "http://some-proxy"' - testsuccessequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com - testsuccessequal "Using proxy '' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com + for meth in 'http' 'https'; do + msgmsg "apt-helper $CONFNAME" "${meth} proxy" + for type in 'http' 'https' 'socks5h'; do + setupproxydetect "$meth" "echo '${type}://some-proxy'" + testsuccessequal "Using proxy '${type}://some-proxy' for URL '${meth}://www.example.com/'" apthelper auto-detect-proxy "${meth}://www.example.com" + if [ "$meth" = 'http' ]; then + testsuccessequal "Using proxy '' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy 'https://ssl.example.com' + else + testsuccessequal "Using proxy '' for URL 'http://no-ssl.example.com/'" apthelper auto-detect-proxy 'http://no-ssl.example.com' + fi + done + done + + msgmsg "apt-helper $CONFNAME" 'no strange proxy' + for meth in gpgv copy store file; do + setupproxydetect "$meth" "echo '${meth}://some-proxy'" + testsuccessequal "Using proxy '' for URL '${meth}:/foo/bar'" apthelper auto-detect-proxy "${meth}:/foo/bar" + testsuccessequal "Using proxy '' for URL '${meth}://./foo/bar'" apthelper auto-detect-proxy "${meth}://./foo/bar" + done + for url in 'cdrom://Moo Cow Rom/debian' 'cdrom://[The Debian 1.2 disk, 1/2 R16]/debian/'; do + setupproxydetect "${url%%:*}" "echo 'cdrom://some-proxy'" + testsuccessequal "Using proxy '' for URL '${url}'" apthelper auto-detect-proxy "$url" + done + for meth in tor tor+http tor+https; do + setupproxydetect "$meth" "echo 'socks5h://some-proxy'" + testsuccessequal "Using proxy 'socks5h://some-proxy' for URL '${meth}://example.org/'" apthelper auto-detect-proxy "${meth}://example.org" + + setupproxydetect "$meth" "echo 'DIRECT'" + testwarningequal "Using proxy '' for URL '${meth}://example.org/' +W: ProxyAutoDetect command returned incompatible proxy 'DIRECT' for access type ${meth}" apthelper auto-detect-proxy "${meth}://example.org" + done - msgmsg "apt-helper $CONFNAME" 'https proxy' - setupproxydetect 'https' 'echo "https://https-proxy"' - testsuccessequal "Using proxy '' for URL 'http://no-ssl.example.com/'" apthelper auto-detect-proxy http://no-ssl.example.com - testsuccessequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com rm -f rootdir/etc/apt/apt.conf.d/02proxy-detect "${TMPWORKINGDIRECTORY}/apt-proxy-detect" } -- cgit v1.2.3-70-g09d2 From afc7b0dbf1114f3d723ff0e1ab9d0796cd799970 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 28 Apr 2022 16:44:13 +0200 Subject: Do not assume mirror-URIs end in a filename causing a hang In practice most of them will, but making the acquire process hang in response if they don't seems like an overly excessive response if we can just support it gracefully to reply with an error (or actually succeed in the unlikely event this URI is actually correct). --- methods/mirror.cc | 11 +++++++++-- test/integration/test-method-mirror | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/methods/mirror.cc b/methods/mirror.cc index 787e4c7d5..5781ebcf0 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -326,7 +326,11 @@ std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem { if (APT::String::Startswith(Itm->Uri, uristr)) { - uristr.erase(uristr.length() - 1); // remove the ending '/' + if (::URI uri{uristr}; uri.Path.length() > 1 && APT::String::Endswith(uri.Path, "/")) + { + uri.Path.erase(uri.Path.length() - 1); // remove the ending '/' + uristr = uri; + } auto const colon = uristr.find(':'); if (unlikely(colon == std::string::npos)) continue; @@ -375,7 +379,10 @@ bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{* msgCache[Itm->Uri] = Message; MirrorListInfo info; info.state = REQUESTED; - info.baseuri = mirrorfileuri + '/'; + if (not APT::String::Endswith(mirrorfileuri, "/")) + info.baseuri = mirrorfileuri + '/'; + else + info.baseuri = mirrorfileuri; auto const colon = info.baseuri.find(':'); if (unlikely(colon == std::string::npos)) return false; diff --git a/test/integration/test-method-mirror b/test/integration/test-method-mirror index ce44b86e3..1784e1ecb 100755 --- a/test/integration/test-method-mirror +++ b/test/integration/test-method-mirror @@ -124,9 +124,22 @@ msgmsg 'all mirrored via file' APTARCHIVE="$(readlink -f ./aptarchive)" sed -i -e "s#mirror+http://localhost:${APTHTTPPORT}#mirror+file:${APTARCHIVE}#" rootdir/etc/apt/sources.list.d/* testrun '*_localhost_*' '*_aptarchive_mirror.txt.gz_*' +sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/* + +mv rootdir/etc/apt/sources.list.d rootdir/etc/apt/sources.list.d.bak +mkdir rootdir/etc/apt/sources.list.d +msgmsg 'fail gracefully if mirror uri has no filename' +echo "deb mirror://localhost:${APTHTTPPORT}/ stable main" > rootdir/etc/apt/sources.list.d/mirrordir.list +testfailure apt update + +msgmsg 'but succeed if it is indeed a mirror list' +ln -s mirror.txt aptarchive/index.html +testsuccess apt update +rm aptarchive/index.html rootdir/etc/apt/sources.list.d/mirrordir.list +rmdir rootdir/etc/apt/sources.list.d +mv rootdir/etc/apt/sources.list.d.bak rootdir/etc/apt/sources.list.d msgmsg 'fallback mirrors are used if needed' 'as usual' -sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/* echo "http://localhost:${APTHTTPPORT}/failure2 priority:3 http://localhost:${APTHTTPPORT}/redirectme priority:2 http://localhost:${APTHTTPPORT}/failure priority:1" > aptarchive/mirror.txt -- cgit v1.2.3-70-g09d2 From 83ff257477c9e5d4ef1b74cb42780f22660af109 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 29 Apr 2022 12:46:55 +0200 Subject: Do not change UsedMirror on local copy of InRelease While we pick most files from the file:/ mirror directly, we explicitly bring in the InRelease file with a copy one if-clause later, which causes this code to be run again and ends up considering the copy:/ the mirror the file was picked up from, which works, but isn't what we had signed up for. --- apt-pkg/acquire-item.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7e1c5dad6..c845c9153 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1424,7 +1424,7 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st // verified yet) // Save the final base URI we got this Release file from - if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true)) + if (not I->Local && not I->UsedMirror.empty() && _config->FindB("Acquire::SameMirrorForAllIndexes", true)) { auto InReleasePath = Target.Option(IndexTarget::INRELEASE_PATH); if (InReleasePath.empty()) -- cgit v1.2.3-70-g09d2 From 841243df877b39dfcb907a7fab9c4e8b4f65d715 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 29 Apr 2022 16:30:05 +0200 Subject: Clear previous ignored errors before processing acquire item again If an item failed previously, but that failure is ignored and a retry is triggered we might end up in a situation in which it is failing yet again for perhaps totally different reasons, but the error message displayed is the one we have shown (or, because it was ignored likely hidden) before which can be rather confusing in debugging. --- apt-pkg/acquire-worker.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 533089ea5..ba3d52952 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -478,6 +478,8 @@ bool pkgAcquire::Worker::RunMessages() } std::vector const ItmOwners = Itm->Owners; + for (auto const Owner : ItmOwners) + Owner->ErrorText.clear(); OwnerQ->ItemDone(Itm); Itm = NULL; @@ -625,6 +627,8 @@ bool pkgAcquire::Worker::RunMessages() Log->Pulse((*O)->GetOwner()); std::vector const ItmOwners = Itm->Owners; + for (auto const Owner : ItmOwners) + Owner->ErrorText.clear(); OwnerQ->ItemDone(Itm); Itm = nullptr; -- cgit v1.2.3-70-g09d2 From 041317084a8f74e4a571f308e4a099f9a8090da0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 29 Apr 2022 16:36:20 +0200 Subject: Never consider file/copy/cdrom sources bad sites due to errors As the hostname for them is likely empty (for absolute or . or .. for relative paths) considering one a bad site causes all of the sites from this scheme to be considered bad. In a perfect world, we would figure out a good site-path to use for these sources, but as they are local sources which means we don't have to perform costly requests to the internet we just let them happen instead for now. --- apt-pkg/acquire-item.cc | 13 ++++++++++++- apt-pkg/acquire-item.h | 2 +- apt-pkg/acquire-worker.cc | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c845c9153..c50f46909 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -32,7 +32,9 @@ #include #include +#include #include +#include #include #include #include @@ -843,8 +845,17 @@ void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_m d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields)); } /*}}}*/ -void pkgAcquire::Item::RemoveAlternativeSite(std::string &&OldSite) /*{{{*/ +void pkgAcquire::Item::RemoveAlternativeSite(std::string const &AltUriStr)/*{{{*/ { + ::URI AltUri{AltUriStr}; + // the hostnames for these methods are empty for absolute paths which would result + // in the elimination of all sites accessed via those methods. On the upside, those + // methods are local and fast to reply with failure, so it doesn't hurt that much to + // keep them in the loop – so we just exit here rather than trying to guess the site. + std::array const badhosts{"file", "copy", "cdrom"}; + if (std::find(badhosts.begin(), badhosts.end(), AltUri.Access) != badhosts.end()) + return; + auto const OldSite = URI::SiteOnly(AltUriStr); d->AlternativeURIs.erase(std::remove_if(d->AlternativeURIs.begin(), d->AlternativeURIs.end(), [&](decltype(*d->AlternativeURIs.cbegin()) AltUri) { return URI::SiteOnly(AltUri.URI) == OldSite; diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 22da9815d..b1985ada7 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -240,7 +240,7 @@ class APT_PUBLIC pkgAcquire::Item : public WeakPointable /*{{{*/ APT_HIDDEN bool PopAlternativeURI(std::string &NewURI); APT_HIDDEN bool IsGoodAlternativeURI(std::string const &AltUri) const; APT_HIDDEN void PushAlternativeURI(std::string &&NewURI, std::unordered_map &&fields, bool const at_the_back); - APT_HIDDEN void RemoveAlternativeSite(std::string &&OldSite); + APT_HIDDEN void RemoveAlternativeSite(std::string const &AltUri); /** \brief A "descriptive" URI-like string. * diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index ba3d52952..4b9d33505 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -720,7 +720,7 @@ void pkgAcquire::Worker::HandleFailure(std::vector const &It else { if (errAuthErr) - Owner->RemoveAlternativeSite(URI::SiteOnly(Owner->GetItemDesc().URI)); + Owner->RemoveAlternativeSite(Owner->GetItemDesc().URI); if (Owner->PopAlternativeURI(NewURI)) { Owner->FailMessage(Message); -- cgit v1.2.3-70-g09d2 From 15124923b4cdb19af6bf642f396dab7c3e6bd074 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 29 Apr 2022 16:58:23 +0200 Subject: Allow HashSum mismatches to fallback to other mirrors Perhaps a common error in the past nowadays thanks to by-hash we rarely get mismatches due to our calling behaviour and more because the mirror is genuinely bad especially if this mirror didn't work with by-hash. Using our "normal" error handling allows falling back to other mirrors which hopefully do support by-hash and/or are not broken on a filesystem level making us more resistent to the occasional bad apple in a wild mirror forest. --- apt-pkg/acquire-item.cc | 4 +- apt-pkg/acquire-worker.cc | 108 ++++++++++++--------- .../test-apt-update-incomplete-file-mirror | 30 ++++++ 3 files changed, 94 insertions(+), 48 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c50f46909..2c3f45076 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -950,7 +950,7 @@ void pkgAcquire::Item::FailMessage(string const &Message) failreason = WEAK_HASHSUMS; else if (FailReason == "RedirectionLoop") failreason = REDIRECTION_LOOP; - else if (Status == StatAuthError) + else if (Status == StatAuthError || FailReason == "HashSumMismatch") failreason = HASHSUM_MISMATCH; if(ErrorText.empty()) @@ -975,7 +975,7 @@ void pkgAcquire::Item::FailMessage(string const &Message) break; } - if (Status == StatAuthError) + if (Status == StatAuthError || failreason == HASHSUM_MISMATCH) { auto const ExpectedHashes = GetExpectedHashes(); if (ExpectedHashes.empty() == false) diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 4b9d33505..55897945c 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -486,9 +486,21 @@ bool pkgAcquire::Worker::RunMessages() bool const isIMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) || StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false); auto const forcedHash = _config->Find("Acquire::ForceHash"); + + bool consideredOkay = true; + HashStringList ExpectedHashes; + bool const DebugAuth = _config->FindB("Debug::pkgAcquire::Auth", false); + if (DebugAuth) + { + std::clog << "201 URI Done: " << URI << endl + << "ReceivedHash:" << endl; + for (auto const &hs : ReceivedHashes) + std::clog << "\t- " << hs.toStr() << std::endl; + std::clog << "ExpectedHash:" << endl; + } for (auto const Owner: ItmOwners) { - HashStringList const ExpectedHashes = [&]() { + HashStringList const OwnerExpectedHashes = [&]() { if (AltFile) { auto const * const transOwner = dynamic_cast(Owner); @@ -501,46 +513,51 @@ bool pkgAcquire::Worker::RunMessages() } return Owner->GetExpectedHashes(); }(); - if(_config->FindB("Debug::pkgAcquire::Auth", false) == true) + for (auto const &h : OwnerExpectedHashes) { - std::clog << "201 URI Done: " << Owner->DescURI() << endl - << "ReceivedHash:" << endl; - for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs) - std::clog << "\t- " << hs->toStr() << std::endl; - std::clog << "ExpectedHash:" << endl; - for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs) - std::clog << "\t- " << hs->toStr() << std::endl; - std::clog << endl; - } - - // decide if what we got is what we expected - bool consideredOkay = false; - if ((forcedHash.empty() && ExpectedHashes.empty() == false) || - (forcedHash.empty() == false && ExpectedHashes.usable())) - { - if (ReceivedHashes.empty()) + if (not ExpectedHashes.push_back(h)) { - /* IMS-Hits can't be checked here as we will have uncompressed file, - but the hashes for the compressed file. What we have was good through - so all we have to ensure later is that we are not stalled. */ - consideredOkay = isIMSHit; - } - else if (ReceivedHashes == ExpectedHashes) - consideredOkay = true; - else consideredOkay = false; - + std::clog << "\t- " << h.toStr() << " [conflict]" << std::endl; + } + else if (DebugAuth) + std::clog << "\t- " << h.toStr() << std::endl; + } + } + if (DebugAuth) + std::clog << endl; + + // decide if what we got is what we expected + if (not consideredOkay) + ; + else if ((forcedHash.empty() && not ExpectedHashes.empty()) || + (not forcedHash.empty() && ExpectedHashes.usable())) + { + if (ReceivedHashes.empty()) + { + /* IMS-Hits can't be checked here as we will have uncompressed file, + but the hashes for the compressed file. What we have was good through + so all we have to ensure later is that we are not stalled. */ + consideredOkay = isIMSHit; } + else if (ReceivedHashes == ExpectedHashes) + consideredOkay = true; else - consideredOkay = !Owner->HashesRequired(); - - if (consideredOkay == true) - consideredOkay = Owner->VerifyDone(Message, Config); - else // hashsum mismatch - Owner->Status = pkgAcquire::Item::StatAuthError; + consideredOkay = false; + } + else + consideredOkay = std::none_of(ItmOwners.begin(), ItmOwners.end(), [](auto const * const O) { return O->HashesRequired(); }); + bool otherReasons = false; + if (consideredOkay && not std::all_of(ItmOwners.begin(), ItmOwners.end(), [&](auto * const O) { return O->VerifyDone(Message, Config); })) + { + consideredOkay = false; + otherReasons = true; + } - if (consideredOkay == true) + if (consideredOkay) + { + for (auto const Owner : ItmOwners) { if (isDoomedItem(Owner) == false) Owner->Done(Message, ReceivedHashes, Config); @@ -552,22 +569,21 @@ bool pkgAcquire::Worker::RunMessages() Log->Done(Owner->GetItemDesc()); } } + } + else + { + if (otherReasons) + HandleFailure(ItmOwners, Config, Log, Message, false, false); else { - auto SavedDesc = Owner->GetItemDesc(); - if (isDoomedItem(Owner) == false) + if (Message.find("\nFailReason:") == std::string::npos) { - if (Message.find("\nFailReason:") == std::string::npos) - { - if (ReceivedHashes != ExpectedHashes) - Message.append("\nFailReason: HashSumMismatch"); - else - Message.append("\nFailReason: WeakHashSums"); - } - Owner->Failed(Message,Config); + if (ReceivedHashes != ExpectedHashes) + Message.append("\nFailReason: HashSumMismatch"); + else + Message.append("\nFailReason: WeakHashSums"); } - if (Log != nullptr) - Log->Fail(SavedDesc); + HandleFailure(ItmOwners, Config, Log, Message, false, true); } } ItemDone(); diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index 2d2125582..959ddc1a6 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -25,9 +25,39 @@ testsuccess apt update rm -rf rootdir/var/lib/apt/lists + +msgmsg 'File mirror was hacked' +mkdir aptarchive2 +cp -a aptarchive/dists aptarchive2/ +rm -rf rootdir/var/lib/apt/lists find aptarchive/dists -name 'Packages' | while read FILE; do echo 'hacked' > $FILE done testfailure apt update -o Debug::pkgAcquire::Worker=1 testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output + + +msgmsg 'Fallback over hashsum errors' +rm -f rootdir/etc/apt/sources.list rootdir/etc/apt/sources.list.d/* +echo "deb mirror+file:${TMPWORKINGDIRECTORY}/mirror.list unstable main" > rootdir/etc/apt/sources.list +rm -rf rootdir/var/lib/apt/lists +cat > mirror.list < mirror.list < mirror.list < Date: Mon, 9 May 2022 11:42:48 +0200 Subject: Look at non by-hash paths in copy and file methods Ideally copy and file mirrors would support by-hash as well, but its harder to setup and maintain especially if you want to cache an online mirror who has by-hash enabled. We can avoid unneeded roundtrips (in the best case) and entire cache misses (in the usual worst case) by "just" telling methods that the URI we passed it has the requested file perhaps also in other paths. This is done in pseudo-relative paths as we would otherwise need to teach redirection code to rewrite those URIs as well. A method like http can easily ignore this value and await explicit instructions to look at that file, but inspecting the path in local sources via file or copy is (comparatively) free, so we just do it immediately. If that ends up being the wrong version of the file as by-hash would have protected us from we are in this feature branch now falling back to other mirrors which are like the ones online and in support of by-hash. --- apt-pkg/acquire-item.cc | 11 +- methods/aptmethod.h | 11 ++ methods/copy.cc | 90 ++++++++++------- methods/file.cc | 111 ++++++++++++--------- test/integration/test-apt-by-hash-update | 8 +- .../integration/test-apt-get-update-unauth-warning | 8 +- .../test-apt-update-incomplete-file-mirror | 5 + test/integration/test-method-mirror | 1 - 8 files changed, 143 insertions(+), 102 deletions(-) diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 2c3f45076..3950ae030 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -473,15 +473,16 @@ bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item) // now add the actual by-hash uris auto const Expected = GetExpectedHashes(); auto const TargetHash = Expected.find(nullptr); - auto const PushByHashURI = [&](std::string U) { + auto const PushByHashURI = [&](std::string const &U) { if (unlikely(TargetHash == nullptr)) return false; - auto const trailing_slash = U.find_last_of("/"); + ::URI uri{U}; + auto const trailing_slash = uri.Path.find_last_of("/"); if (unlikely(trailing_slash == std::string::npos)) return false; - auto byhashSuffix = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); - U.replace(trailing_slash, U.length() - trailing_slash, std::move(byhashSuffix)); - PushAlternativeURI(std::move(U), {}, false); + auto altPath = uri.Path.substr(0, trailing_slash) + "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + std::swap(uri.Path, altPath); + PushAlternativeURI(uri, {{"Alternate-Paths", "../../" + flNotDir(altPath)}}, false); return true; }; PushByHashURI(Item.URI); diff --git a/methods/aptmethod.h b/methods/aptmethod.h index 1c24f3a98..1ea173075 100644 --- a/methods/aptmethod.h +++ b/methods/aptmethod.h @@ -514,6 +514,17 @@ protected: return part; } + static std::string CombineWithAlternatePath(std::string Path, std::string Change) + { + while (APT::String::Startswith(Change, "../")) + { + Path.erase(Path.find_last_not_of('/')); + Path = flNotFile(Path); + Change.erase(0, 3); + } + return flCombine(Path, Change); + } + aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3) : pkgAcqMethod(Ver, Flags), aptConfigWrapperForMethods(Binary), Binary(std::move(Binary)), SeccompFlags(0) { diff --git a/methods/copy.cc b/methods/copy.cc index 82eed150c..b32131a21 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -26,7 +26,7 @@ class CopyMethod : public aptMethod { - virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; public: CopyMethod() : aptMethod("copy", "1.0", SingleInstance | SendConfig | SendURIEncoded) @@ -36,55 +36,71 @@ class CopyMethod : public aptMethod }; // CopyMethod::Fetch - Fetch a file /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool CopyMethod::Fetch(FetchItem *Itm) +bool CopyMethod::URIAcquire(std::string const &Message, FetchItem *Itm) { - // this ensures that relative paths work in copy - std::string const File = DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1)); - - // Stat the file and send a start message - struct stat Buf; - if (stat(File.c_str(),&Buf) != 0) - return _error->Errno("stat",_("Failed to stat")); + struct FileCopyType { + std::string name; + struct stat stat{}; + explicit FileCopyType(std::string &&file) : name{std::move(file)} {} + }; + std::vector files; + // this ensures that relative paths work + files.emplace_back(DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1))); + for (auto const &AltPath : VectorizeString(LookupTag(Message, "Alternate-Paths"), '\n')) + files.emplace_back(CombineWithAlternatePath(flNotFile(files[0].name), DecodeSendURI(AltPath))); + files.erase(std::remove_if(files.begin(), files.end(), [](auto &file) + { return stat(file.name.c_str(), &file.stat) != 0; }), + files.end()); + if (files.empty()) + return _error->Errno("copy-stat", _("Failed to stat")); // Forumulate a result and send a start message FetchResult Res; - Res.Size = Buf.st_size; Res.Filename = Itm->DestFile; - Res.LastModified = Buf.st_mtime; Res.IMSHit = false; - URIStart(Res); - // just calc the hashes if the source and destination are identical - if (File == Itm->DestFile || Itm->DestFile == "/dev/null") + for (auto const &File : files) { + Res.Size = File.stat.st_size; + Res.LastModified = File.stat.st_mtime; + + // just calc the hashes if the source and destination are identical + if (Itm->DestFile == "/dev/null" || File.name == Itm->DestFile) + { + URIStart(Res); + CalculateHashes(Itm, Res); + URIDone(Res); + return true; + } + + FileFd From(File.name, FileFd::ReadOnly); + FileFd To(Itm->DestFile, FileFd::WriteAtomic); + To.EraseOnFailure(); + if (not From.IsOpen() || not To.IsOpen()) + continue; + + // Copy the file + URIStart(Res); + if (not CopyFile(From, To)) + { + To.OpFail(); + continue; + } + From.Close(); + To.Close(); + CalculateHashes(Itm, Res); - URIDone(Res); - return true; - } + if (not Itm->ExpectedHashes.empty() && Itm->ExpectedHashes != Res.Hashes) + continue; - // See if the file exists - FileFd From(File,FileFd::ReadOnly); - FileFd To(Itm->DestFile,FileFd::WriteAtomic); - To.EraseOnFailure(); + if (not TransferModificationTimes(File.name.c_str(), Res.Filename.c_str(), Res.LastModified)) + continue; - // Copy the file - if (CopyFile(From,To) == false) - { - To.OpFail(); - return false; + URIDone(Res); + return true; } - From.Close(); - To.Close(); - - if (TransferModificationTimes(File.c_str(), Res.Filename.c_str(), Res.LastModified) == false) - return false; - - CalculateHashes(Itm, Res); - URIDone(Res); - return true; + return false; } /*}}}*/ diff --git a/methods/file.cc b/methods/file.cc index ff93f83d0..279711195 100644 --- a/methods/file.cc +++ b/methods/file.cc @@ -29,7 +29,7 @@ class FileMethod : public aptMethod { - virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; public: FileMethod() : aptMethod("file", "1.0", SingleInstance | SendConfig | LocalOnly | SendURIEncoded) @@ -41,88 +41,101 @@ class FileMethod : public aptMethod // FileMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileMethod::Fetch(FetchItem *Itm) +bool FileMethod::URIAcquire(std::string const &Message, FetchItem *Itm) { URI Get(Itm->Uri); - std::string const File = DecodeSendURI(Get.Path); - FetchResult Res; if (Get.Host.empty() == false) return _error->Error(_("Invalid URI, local URIS must not start with //")); - struct stat Buf; + std::vector Files; + Files.emplace_back(DecodeSendURI(Get.Path)); + for (auto const &AltPath : VectorizeString(LookupTag(Message, "Alternate-Paths"), '\n')) + Files.emplace_back(CombineWithAlternatePath(flNotFile(Files[0]), DecodeSendURI(AltPath))); + + FetchResult Res; // deal with destination files which might linger around - if (lstat(Itm->DestFile.c_str(), &Buf) == 0) + struct stat Buf; + if (lstat(Itm->DestFile.c_str(), &Buf) == 0 && S_ISLNK(Buf.st_mode) && Buf.st_size > 0) + { + char name[Buf.st_size + 1]; + if (ssize_t const sp = readlink(Itm->DestFile.c_str(), name, Buf.st_size); sp == -1) + { + Itm->LastModified = 0; + RemoveFile("file", Itm->DestFile); + } + } + + int olderrno = 0; + // See if the file exists + for (auto const &File : Files) { - if ((Buf.st_mode & S_IFREG) != 0) + if (stat(File.c_str(), &Buf) == 0) { + Res.Size = Buf.st_size; + Res.Filename = File; + Res.LastModified = Buf.st_mtime; + Res.IMSHit = false; if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) { - if (Itm->ExpectedHashes.VerifyFile(File)) - { - Res.Filename = Itm->DestFile; + auto const filesize = Itm->ExpectedHashes.FileSize(); + if (filesize != 0 && filesize == Res.Size) Res.IMSHit = true; - } } + break; } + if (olderrno == 0) + olderrno = errno; } - if (Res.IMSHit != true) - RemoveFile("file", Itm->DestFile); - int olderrno = 0; - // See if the file exists - if (stat(File.c_str(),&Buf) == 0) + if (not Res.IMSHit) { - Res.Size = Buf.st_size; - Res.Filename = File; - Res.LastModified = Buf.st_mtime; - Res.IMSHit = false; - if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) + RemoveFile("file", Itm->DestFile); + if (not Res.Filename.empty()) { - unsigned long long const filesize = Itm->ExpectedHashes.FileSize(); - if (filesize != 0 && filesize == Res.Size) - Res.IMSHit = true; + URIStart(Res); + CalculateHashes(Itm, Res); } - - CalculateHashes(Itm, Res); } - else - olderrno = errno; - if (Res.IMSHit == false) - URIStart(Res); // See if the uncompressed file exists and reuse it FetchResult AltRes; - AltRes.Filename.clear(); - std::vector extensions = APT::Configuration::getCompressorExtensions(); - for (std::vector::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext) + for (auto const &File : Files) { - if (APT::String::Endswith(File, *ext) == true) + for (const auto &ext : APT::Configuration::getCompressorExtensions()) { - std::string const unfile = File.substr(0, File.length() - ext->length()); - if (stat(unfile.c_str(),&Buf) == 0) + if (APT::String::Endswith(File, ext)) { - AltRes.Size = Buf.st_size; - AltRes.Filename = unfile; - AltRes.LastModified = Buf.st_mtime; - AltRes.IMSHit = false; - if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) - AltRes.IMSHit = true; - if (Res.Filename.empty()) - CalculateHashes(Itm, AltRes); - break; + std::string const unfile = File.substr(0, File.length() - ext.length()); + if (stat(unfile.c_str(), &Buf) == 0) + { + AltRes.Size = Buf.st_size; + AltRes.Filename = unfile; + AltRes.LastModified = Buf.st_mtime; + AltRes.IMSHit = false; + if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) + AltRes.IMSHit = true; + if (Res.Filename.empty()) + { + URIStart(Res); + CalculateHashes(Itm, AltRes); + } + break; + } + // no break here as we could have situations similar to '.gz' vs '.tar.gz' here } - // no break here as we could have situations similar to '.gz' vs '.tar.gz' here } + if (not AltRes.Filename.empty()) + break; } - if (AltRes.Filename.empty() == false) + if (not AltRes.Filename.empty()) URIDone(Res,&AltRes); - else if (Res.Filename.empty() == false) + else if (not Res.Filename.empty()) URIDone(Res); else { errno = olderrno; - return _error->Errno(File.c_str(), _("File not found")); + return _error->Errno(Files[0].c_str(), _("File not found")); } return true; diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update index 65c3766d0..a33eb9117 100755 --- a/test/integration/test-apt-by-hash-update +++ b/test/integration/test-apt-by-hash-update @@ -13,6 +13,7 @@ insertpackage 'unstable' 'foo' 'all' '1.0' insertpackage 'unstable' 'bar' 'i386' '1.0' setupaptarchive --no-update +changetowebserver # make Packages *only* accessible by-hash for this test makebyhashonly() { @@ -100,8 +101,7 @@ msgmsg 'Test InRelease by-hash with' 'no fallback' rm -rf aptarchive/dists cp -a aptarchive/dists.bak aptarchive/dists -testfailureequal "Get:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease -Err:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease - File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory) +testfailureequal "Err:1 http://localhost:${APTHTTPPORT} unstable InRelease + 404 Not Found Reading package lists... -E: Failed to fetch file:${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/InRelease File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)" aptget update +E: Failed to fetch http://localhost:${APTHTTPPORT}/dists/unstable/InRelease 404 Not Found" aptget update diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 46df14c29..9f252ad9d 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -28,9 +28,7 @@ testwarning aptget update --allow-insecure-repositories rm -rf rootdir/var/lib/apt/lists find "$APTARCHIVE/dists/unstable" -name '*Release*' -delete # update without authenticated files leads to warning -testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease -Ign:1 file:$APTARCHIVE unstable InRelease -Get:2 file:$APTARCHIVE unstable Release +testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease Err:2 file:$APTARCHIVE unstable Release File not found - ${APTARCHIVE}/dists/unstable/Release (2: No such file or directory) Reading package lists... @@ -45,9 +43,7 @@ lock partial' ls rootdir/var/lib/apt/lists # allow override -testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease -Ign:1 file:$APTARCHIVE unstable InRelease -Get:2 file:$APTARCHIVE unstable Release +testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease Ign:2 file:$APTARCHIVE unstable Release Get:3 file:$APTARCHIVE unstable/main Sources Get:4 file:$APTARCHIVE unstable/main i386 Packages diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index 959ddc1a6..dec9383c9 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -37,6 +37,7 @@ testfailure apt update -o Debug::pkgAcquire::Worker=1 testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output +echo 'Acquire::By-Hash "force";' > rootdir/etc/apt/apt.conf.d/99force-by-hash.conf msgmsg 'Fallback over hashsum errors' rm -f rootdir/etc/apt/sources.list rootdir/etc/apt/sources.list.d/* @@ -47,6 +48,8 @@ copy:${TMPWORKINGDIRECTORY}/aptarchive priority:1 file:${TMPWORKINGDIRECTORY}/aptarchive2 priority:2 EOF testsuccess apt update +cp -a rootdir/tmp/testsuccess.output aptupdate.log +testsuccessequal '2' grep -c -- '^Ign:' aptupdate.log rm -rf rootdir/var/lib/apt/lists cat > mirror.list < aptarchive/mirror.txt testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] -Get:2 foo+file:/nonexistent/apt/archive unstable InRelease Ign:2 foo+file:/nonexistent/apt/archive unstable InRelease File not found - /nonexistent/apt/archive/dists/unstable/InRelease (2: No such file or directory) Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease -- cgit v1.2.3-70-g09d2