From 39304fb440f62b3a9f4dd85871b7049fb5b8eb8f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 23 Apr 2022 22:46:33 +0200 Subject: Support uncompressed indexes from partial file:/ mirrors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a file:/ mirror does not contain the requested compressed file, but the uncompressed variant of that file the method will sent us a fitting Done message with data only about the Alt-Filename, but the central hash verification code in the acquire system does not expect that looking for the non-existent compressed file to compare its hashes with the expected result – which fails (if we are verifying at least). That file:/ as well as the item code deals with this suggests that this feature was broken by 448c38bdcd72b52f11ec5f326f822cf57653f81c, but I couldn't (easily) compile that to verify, so this is only a guess. Supporting this allows using index files from /var/lib/apt/lists – which are likely uncompressed or can at least made so easily – to construct a file:/ mirror. --- .../integration/test-apt-get-update-unauth-warning | 66 ++++++++++++---------- .../test-apt-update-incomplete-file-mirror | 31 ++++++++++ 2 files changed, 66 insertions(+), 31 deletions(-) create mode 100755 test/integration/test-apt-update-incomplete-file-mirror (limited to 'test') diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 42c4e5a9d..46df14c29 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -44,46 +44,15 @@ testequal 'auxfiles lock partial' ls rootdir/var/lib/apt/lists -filesize() { - local CREATEDBY="$1" - shift - stat -c%s "/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 2- ).gz" -} # allow override -#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1 -#exit testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease Ign:1 file:$APTARCHIVE unstable InRelease Get:2 file:$APTARCHIVE unstable Release Ign:2 file:$APTARCHIVE unstable Release Get:3 file:$APTARCHIVE unstable/main Sources -Ign:3 file:$APTARCHIVE unstable/main Sources Get:4 file:$APTARCHIVE unstable/main i386 Packages -Ign:4 file:$APTARCHIVE unstable/main i386 Packages Get:5 file:$APTARCHIVE unstable/main all Packages -Ign:5 file:$APTARCHIVE unstable/main all Packages Get:6 file:$APTARCHIVE unstable/main Translation-en -Ign:6 file:$APTARCHIVE unstable/main Translation-en -Get:3 file:$APTARCHIVE unstable/main Sources -Ign:3 file:$APTARCHIVE unstable/main Sources -Get:4 file:$APTARCHIVE unstable/main i386 Packages -Ign:4 file:$APTARCHIVE unstable/main i386 Packages -Get:5 file:$APTARCHIVE unstable/main all Packages -Ign:5 file:$APTARCHIVE unstable/main all Packages -Get:6 file:$APTARCHIVE unstable/main Translation-en -Ign:6 file:$APTARCHIVE unstable/main Translation-en -Get:3 file:$APTARCHIVE unstable/main Sources -Ign:3 file:$APTARCHIVE unstable/main Sources -Get:4 file:$APTARCHIVE unstable/main i386 Packages -Ign:4 file:$APTARCHIVE unstable/main i386 Packages -Get:5 file:$APTARCHIVE unstable/main all Packages -Ign:5 file:$APTARCHIVE unstable/main all Packages -Get:6 file:$APTARCHIVE unstable/main Translation-en -Ign:6 file:$APTARCHIVE unstable/main Translation-en -Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B] -Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B] -Get:5 file:$APTARCHIVE unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B] -Get:6 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B] Reading package lists... W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use. @@ -92,3 +61,38 @@ N: See apt-secure(8) manpage for repository creation and user configuration deta testfailureequal "WARNING: The following packages cannot be authenticated! foo E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo + +rm -rf rootdir/var/lib/apt/lists +changetowebserver + +filesize() { + local CREATEDBY="$1" + shift + stat -c%s "${APTARCHIVE}/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 4- ).gz" +} +testwarningequal "Ign:1 http://localhost:$APTHTTPPORT unstable InRelease +Ign:2 http://localhost:$APTHTTPPORT unstable Release +Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources +Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages +Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages +Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en +Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources +Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages +Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages +Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en +Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources +Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages +Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages +Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en +Get:3 http://localhost:$APTHTTPPORT unstable/main Sources [$(filesize 'Sources') B] +Get:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B] +Get:5 http://localhost:$APTHTTPPORT unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B] +Get:6 http://localhost:$APTHTTPPORT unstable/main Translation-en [$(filesize 'Translations') B] +Reading package lists... +W: The repository 'http://localhost:$APTHTTPPORT unstable Release' does not have a Release file. +N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use. +N: See apt-secure(8) manpage for repository creation and user configuration details." aptget update --allow-insecure-repositories +# ensure we can not install the package +testfailureequal "WARNING: The following packages cannot be authenticated! + foo +E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror new file mode 100755 index 000000000..d6894d607 --- /dev/null +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -0,0 +1,31 @@ +#!/bin/sh +set -e + +TESTDIR="$(readlink -f "$(dirname "$0")")" +. "$TESTDIR/framework" + +setupenvironment +configarchitecture "amd64" +configcompression 'gz' + +insertpackage 'unstable' 'foo' 'amd64' '1' + +setupaptarchive --no-update + +testfailure aptcache show foo + +# the Release file says gz available, but the mirror has only uncompressed files +find aptarchive/dists -name '*.gz' -delete + +testsuccess apt update +testsuccess aptcache show foo +testfailure aptcache show bar + +testsuccess apt update + +rm -rf rootdir/var/lib/apt/lists + +find aptarchive/dists -name 'Packages' | while read FILE; do + echo 'hacked' > $FILE +done +testfailure apt update -- cgit v1.2.3-70-g09d2 From 496c8b4191790231258e644ac8cd4096d54f445f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 24 Apr 2022 00:03:38 +0200 Subject: Show hashes calculated by main thread in error messages The main thread calculates the hash sums if the method hasn't to be able to verify the files, but while a failure is detected and reported, the error messages did not include the hash sums which looks at least puzzling. So to have a complete error message we ensure that the hashes we calculated are appended to the message from the method so that it looks for the higher level code as if the method had calculated the hashes. --- apt-pkg/acquire-item.cc | 14 +++++++++++++- apt-pkg/acquire-worker.cc | 12 ++++++++++++ test/integration/test-apt-update-incomplete-file-mirror | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) (limited to 'test') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 1fc07ec05..7e1c5dad6 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -976,13 +976,25 @@ void pkgAcquire::Item::FailMessage(string const &Message) if (failreason == HASHSUM_MISMATCH) { out << "Hashes of received file:" << std::endl; + size_t hashes = 0; for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) { std::string const tagname = std::string(*type) + "-Hash"; std::string const hashsum = LookupTag(Message, tagname.c_str()); - if (hashsum.empty() == false) + if (not hashsum.empty()) + { formatHashsum(out, HashString(*type, hashsum)); + ++hashes; + } } + if (hashes == 0) + for (char const *const *type = HashString::SupportedHashes(); *type != nullptr; ++type) + { + std::string const tagname = std::string("Alt-") + *type + "-Hash"; + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (not hashsum.empty()) + formatHashsum(out, HashString(*type, hashsum)); + } } auto const lastmod = LookupTag(Message, "Last-Modified", ""); if (lastmod.empty() == false) diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 035d8ce10..4c2acb5cc 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -457,6 +457,18 @@ bool pkgAcquire::Worker::RunMessages() FileFd file(filename, FileFd::ReadOnly, FileFd::None); calc.AddFD(file); ReceivedHashes = calc.GetHashStringList(); + for (auto const &h : ReceivedHashes) + { + std::string tagname; + if (AltFile) + tagname = "Alt-"; + tagname.append(h.HashType()).append("-Hash"); + if (not LookupTag(Message, tagname.c_str()).empty()) + continue; + if (not APT::String::Endswith(Message, "\n")) + Message.append("\n"); + Message.append(tagname).append(": ").append(h.HashValue()); + } } } diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index d6894d607..86eb890a7 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -29,3 +29,4 @@ find aptarchive/dists -name 'Packages' | while read FILE; do echo 'hacked' > $FILE done testfailure apt update +testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output -- cgit v1.2.3-70-g09d2 From 370e024224b8116840173201d51c0c2a170e0e93 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 24 Apr 2022 00:24:40 +0200 Subject: Alternatively calculate alternative file hashes in file method If we do not have the requested file we haven't calculated the hashes for it either and it is likely that the alternative file will be used, so to save the main thread from being busy with calculating hashes we do the calculation here in the method. --- methods/file.cc | 2 ++ test/integration/test-apt-update-incomplete-file-mirror | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'test') diff --git a/methods/file.cc b/methods/file.cc index b2fe133f2..ff93f83d0 100644 --- a/methods/file.cc +++ b/methods/file.cc @@ -107,6 +107,8 @@ bool FileMethod::Fetch(FetchItem *Itm) AltRes.IMSHit = false; if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) AltRes.IMSHit = true; + if (Res.Filename.empty()) + CalculateHashes(Itm, AltRes); break; } // no break here as we could have situations similar to '.gz' vs '.tar.gz' here diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index 86eb890a7..2d2125582 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -28,5 +28,6 @@ rm -rf rootdir/var/lib/apt/lists find aptarchive/dists -name 'Packages' | while read FILE; do echo 'hacked' > $FILE done -testfailure apt update +testfailure apt update -o Debug::pkgAcquire::Worker=1 testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output +testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output -- cgit v1.2.3-70-g09d2 From d9601bce6e1fa61cbb3dc4d66e5492f9885eef8f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 26 Apr 2022 22:22:46 +0200 Subject: Allow AutoDetectProxy to work with more than just http(s) It is a bit unfair for third-party methods wrapping e.g. http to not have access to such tools, same for our old ftp and e.g. our tor even if in practice this isn't used much even for http. At least that is one less FIXME in the code. --- apt-pkg/acquire-worker.cc | 10 ++++---- apt-pkg/contrib/proxy.cc | 52 +++++++++++++++++++++++++++++----------- apt-pkg/contrib/proxy.h | 4 +++- doc/examples/configure-index | 18 +++++++++----- test/integration/test-apt-helper | 48 +++++++++++++++++++++++++++---------- 5 files changed, 93 insertions(+), 39 deletions(-) (limited to 'test') diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 4c2acb5cc..533089ea5 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -877,14 +877,12 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) } Message += "\nFilename: " + Item->Owner->DestFile; - // FIXME: We should not hard code proxy protocols here. - if (URL.Access == "http" || URL.Access == "https") + // AutoDetectProxy() checks this already by itself, but we don't want to access unknown configs + if (CanURIBeAccessedViaProxy(URL)) { AutoDetectProxy(URL); - if (_config->Exists("Acquire::" + URL.Access + "::proxy::" + URL.Host)) - { - Message += "\nProxy: " + _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host); - } + if (auto const proxy = _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host); not proxy.empty()) + Message.append("\nProxy: ").append(proxy); } HashStringList const hsl = Item->GetExpectedHashes(); diff --git a/apt-pkg/contrib/proxy.cc b/apt-pkg/contrib/proxy.cc index a99f44f49..9a9a9f29c 100644 --- a/apt-pkg/contrib/proxy.cc +++ b/apt-pkg/contrib/proxy.cc @@ -12,9 +12,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -22,6 +24,16 @@ #include "proxy.h" /*}}}*/ +bool CanURIBeAccessedViaProxy(URI const &URL) /*{{{*/ +{ + // for some methods a proxy doesn't make sense, so we don't have to fork + if (URL.Host.empty() || + APT::String::Startswith(URL.Access, "mirror+") || URL.Access.find("+mirror+") != std::string::npos || APT::String::Endswith(URL.Access, "+mirror")) + return false; + std::array const noproxy{"file", "copy", "store", "gpgv", "rred", "cdrom", "mirror"}; + return std::find(noproxy.begin(), noproxy.end(), URL.Access) == noproxy.end(); +} + /*}}}*/ // AutoDetectProxy - auto detect proxy /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -29,16 +41,19 @@ static std::vector CompatibleProxies(URI const &URL) { if (URL.Access == "http" || URL.Access == "https") return {"http", "https", "socks5h"}; - return {URL.Access}; + if (URL.Access == "tor" || URL.Access == "tor+http" || URL.Access == "tor+https") + return {"socks5h"}; + if (URL.Access == "ftp") + return {"ftp"}; + return {}; } - bool AutoDetectProxy(URI &URL) { - // we support both http/https debug options - bool Debug = _config->FindB("Debug::Acquire::"+URL.Access,false); + if (not CanURIBeAccessedViaProxy(URL)) + return true; // the user already explicitly set a proxy for this host - if(_config->Find("Acquire::"+URL.Access+"::proxy::"+URL.Host, "") != "") + if (not _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host, "").empty()) return true; // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old @@ -49,6 +64,7 @@ bool AutoDetectProxy(URI &URL) if (AutoDetectProxyCmd.empty()) return true; + bool const Debug = _config->FindB("Debug::Acquire::" + URL.Access, false); if (Debug) std::clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << std::endl; @@ -73,7 +89,7 @@ bool AutoDetectProxy(URI &URL) // and apt will use the generic proxy settings if (goodread == false) return true; - auto const cleanedbuf = _strstrip(buf); + APT::StringView const cleanedbuf = _strstrip(buf); // We warn about this as the implementor probably meant to use DIRECT instead if (cleanedbuf[0] == '\0') { @@ -82,17 +98,25 @@ bool AutoDetectProxy(URI &URL) } if (Debug) - std::clog << "auto detect command returned: '" << cleanedbuf << "'" << std::endl; + std::clog << "auto detect command returned: '" << cleanedbuf.data() << "'" << std::endl; - auto compatibleTypes = CompatibleProxies(URL); - bool compatible = strcmp(cleanedbuf, "DIRECT") == 0 || - compatibleTypes.end() != std::find_if(compatibleTypes.begin(), - compatibleTypes.end(), [cleanedbuf](std::string &compat) { - return strstr(cleanedbuf, compat.c_str()) == cleanedbuf; - }); + bool compatible = true; + if (cleanedbuf != "DIRECT") + { + if (auto const compatibleTypes = CompatibleProxies(URL); not compatibleTypes.empty()) + compatible = std::any_of(compatibleTypes.begin(), compatibleTypes.end(), + [cleanedbuf](std::string const &compat) + { + return cleanedbuf.substr(0, compat.size()) == compat; + }); + } + else if (URL.Access == "tor" || URL.Access == "tor+http" || URL.Access == "tor+https") + compatible = false; // Accepting DIRECT would silently disable tor if (compatible) - _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf); + _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf.data()); + else + _error->Warning("ProxyAutoDetect command returned incompatible proxy '%s' for access type %s", cleanedbuf.data(), URL.Access.c_str()); return true; } diff --git a/apt-pkg/contrib/proxy.h b/apt-pkg/contrib/proxy.h index f6d70ea8b..2a4c19fc8 100644 --- a/apt-pkg/contrib/proxy.h +++ b/apt-pkg/contrib/proxy.h @@ -9,8 +9,10 @@ #ifndef PKGLIB_PROXY_H #define PKGLIB_PROXY_H +#include + class URI; APT_PUBLIC bool AutoDetectProxy(URI &URL); - +APT_HIDDEN bool CanURIBeAccessedViaProxy(URI const &URL); #endif diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 1b165702d..9623514b8 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -846,12 +846,18 @@ acquire::gpgv::dl-limit ""; acquire::store::dl-limit ""; acquire::mirror::dl-limit ""; methods::mirror::problemreporting ""; -acquire::http::proxyautodetect ""; -acquire::http::proxy-auto-detect ""; -acquire::http::proxy::* ""; -acquire::https::proxyautodetect ""; -acquire::https::proxy-auto-detect ""; -acquire::https::proxy::* ""; +acquire::*::proxyautodetect ""; +acquire::file::proxyautodetect ""; +acquire::copy::proxyautodetect ""; +acquire::store::proxyautodetect ""; +acquire::*::proxy-auto-detect ""; +acquire::file::proxy-auto-detect ""; +acquire::copy::proxy-auto-detect ""; +acquire::store::proxy-auto-detect ""; +acquire::file::proxy::* ""; +acquire::copy::proxy::* ""; +acquire::store::proxy::* ""; +acquire::*::proxy::* ""; // Options used by apt-ftparchive dir::archivedir ""; diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index ae1ca7456..4f883b440 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -74,10 +74,11 @@ E: Download Failed" setupproxydetect() { local METH="$1" shift - { - echo '#!/bin/sh -e' - echo "$@" - } > "${TMPWORKINGDIRECTORY}/apt-proxy-detect" + cat >"${TMPWORKINGDIRECTORY}/apt-proxy-detect" < rootdir/etc/apt/apt.conf.d/02proxy-detect } @@ -97,15 +98,38 @@ W: ProxyAutoDetect command returned an empty line" apthelper auto-detect-proxy h chmod -x "${TMPWORKINGDIRECTORY}/apt-proxy-detect" testfailureequal "E: ProxyAutoDetect command '${TMPWORKINGDIRECTORY}/apt-proxy-detect' can not be executed! - access (13: Permission denied)" apthelper auto-detect-proxy http://example.com/ - msgmsg "apt-helper $CONFNAME" 'http proxy' - setupproxydetect 'http' 'echo "http://some-proxy"' - testsuccessequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com - testsuccessequal "Using proxy '' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com + for meth in 'http' 'https'; do + msgmsg "apt-helper $CONFNAME" "${meth} proxy" + for type in 'http' 'https' 'socks5h'; do + setupproxydetect "$meth" "echo '${type}://some-proxy'" + testsuccessequal "Using proxy '${type}://some-proxy' for URL '${meth}://www.example.com/'" apthelper auto-detect-proxy "${meth}://www.example.com" + if [ "$meth" = 'http' ]; then + testsuccessequal "Using proxy '' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy 'https://ssl.example.com' + else + testsuccessequal "Using proxy '' for URL 'http://no-ssl.example.com/'" apthelper auto-detect-proxy 'http://no-ssl.example.com' + fi + done + done + + msgmsg "apt-helper $CONFNAME" 'no strange proxy' + for meth in gpgv copy store file; do + setupproxydetect "$meth" "echo '${meth}://some-proxy'" + testsuccessequal "Using proxy '' for URL '${meth}:/foo/bar'" apthelper auto-detect-proxy "${meth}:/foo/bar" + testsuccessequal "Using proxy '' for URL '${meth}://./foo/bar'" apthelper auto-detect-proxy "${meth}://./foo/bar" + done + for url in 'cdrom://Moo Cow Rom/debian' 'cdrom://[The Debian 1.2 disk, 1/2 R16]/debian/'; do + setupproxydetect "${url%%:*}" "echo 'cdrom://some-proxy'" + testsuccessequal "Using proxy '' for URL '${url}'" apthelper auto-detect-proxy "$url" + done + for meth in tor tor+http tor+https; do + setupproxydetect "$meth" "echo 'socks5h://some-proxy'" + testsuccessequal "Using proxy 'socks5h://some-proxy' for URL '${meth}://example.org/'" apthelper auto-detect-proxy "${meth}://example.org" + + setupproxydetect "$meth" "echo 'DIRECT'" + testwarningequal "Using proxy '' for URL '${meth}://example.org/' +W: ProxyAutoDetect command returned incompatible proxy 'DIRECT' for access type ${meth}" apthelper auto-detect-proxy "${meth}://example.org" + done - msgmsg "apt-helper $CONFNAME" 'https proxy' - setupproxydetect 'https' 'echo "https://https-proxy"' - testsuccessequal "Using proxy '' for URL 'http://no-ssl.example.com/'" apthelper auto-detect-proxy http://no-ssl.example.com - testsuccessequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com rm -f rootdir/etc/apt/apt.conf.d/02proxy-detect "${TMPWORKINGDIRECTORY}/apt-proxy-detect" } -- cgit v1.2.3-70-g09d2 From afc7b0dbf1114f3d723ff0e1ab9d0796cd799970 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 28 Apr 2022 16:44:13 +0200 Subject: Do not assume mirror-URIs end in a filename causing a hang In practice most of them will, but making the acquire process hang in response if they don't seems like an overly excessive response if we can just support it gracefully to reply with an error (or actually succeed in the unlikely event this URI is actually correct). --- methods/mirror.cc | 11 +++++++++-- test/integration/test-method-mirror | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) (limited to 'test') diff --git a/methods/mirror.cc b/methods/mirror.cc index 787e4c7d5..5781ebcf0 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -326,7 +326,11 @@ std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem { if (APT::String::Startswith(Itm->Uri, uristr)) { - uristr.erase(uristr.length() - 1); // remove the ending '/' + if (::URI uri{uristr}; uri.Path.length() > 1 && APT::String::Endswith(uri.Path, "/")) + { + uri.Path.erase(uri.Path.length() - 1); // remove the ending '/' + uristr = uri; + } auto const colon = uristr.find(':'); if (unlikely(colon == std::string::npos)) continue; @@ -375,7 +379,10 @@ bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{* msgCache[Itm->Uri] = Message; MirrorListInfo info; info.state = REQUESTED; - info.baseuri = mirrorfileuri + '/'; + if (not APT::String::Endswith(mirrorfileuri, "/")) + info.baseuri = mirrorfileuri + '/'; + else + info.baseuri = mirrorfileuri; auto const colon = info.baseuri.find(':'); if (unlikely(colon == std::string::npos)) return false; diff --git a/test/integration/test-method-mirror b/test/integration/test-method-mirror index ce44b86e3..1784e1ecb 100755 --- a/test/integration/test-method-mirror +++ b/test/integration/test-method-mirror @@ -124,9 +124,22 @@ msgmsg 'all mirrored via file' APTARCHIVE="$(readlink -f ./aptarchive)" sed -i -e "s#mirror+http://localhost:${APTHTTPPORT}#mirror+file:${APTARCHIVE}#" rootdir/etc/apt/sources.list.d/* testrun '*_localhost_*' '*_aptarchive_mirror.txt.gz_*' +sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/* + +mv rootdir/etc/apt/sources.list.d rootdir/etc/apt/sources.list.d.bak +mkdir rootdir/etc/apt/sources.list.d +msgmsg 'fail gracefully if mirror uri has no filename' +echo "deb mirror://localhost:${APTHTTPPORT}/ stable main" > rootdir/etc/apt/sources.list.d/mirrordir.list +testfailure apt update + +msgmsg 'but succeed if it is indeed a mirror list' +ln -s mirror.txt aptarchive/index.html +testsuccess apt update +rm aptarchive/index.html rootdir/etc/apt/sources.list.d/mirrordir.list +rmdir rootdir/etc/apt/sources.list.d +mv rootdir/etc/apt/sources.list.d.bak rootdir/etc/apt/sources.list.d msgmsg 'fallback mirrors are used if needed' 'as usual' -sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/* echo "http://localhost:${APTHTTPPORT}/failure2 priority:3 http://localhost:${APTHTTPPORT}/redirectme priority:2 http://localhost:${APTHTTPPORT}/failure priority:1" > aptarchive/mirror.txt -- cgit v1.2.3-70-g09d2 From 15124923b4cdb19af6bf642f396dab7c3e6bd074 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 29 Apr 2022 16:58:23 +0200 Subject: Allow HashSum mismatches to fallback to other mirrors Perhaps a common error in the past nowadays thanks to by-hash we rarely get mismatches due to our calling behaviour and more because the mirror is genuinely bad especially if this mirror didn't work with by-hash. Using our "normal" error handling allows falling back to other mirrors which hopefully do support by-hash and/or are not broken on a filesystem level making us more resistent to the occasional bad apple in a wild mirror forest. --- apt-pkg/acquire-item.cc | 4 +- apt-pkg/acquire-worker.cc | 108 ++++++++++++--------- .../test-apt-update-incomplete-file-mirror | 30 ++++++ 3 files changed, 94 insertions(+), 48 deletions(-) (limited to 'test') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index c50f46909..2c3f45076 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -950,7 +950,7 @@ void pkgAcquire::Item::FailMessage(string const &Message) failreason = WEAK_HASHSUMS; else if (FailReason == "RedirectionLoop") failreason = REDIRECTION_LOOP; - else if (Status == StatAuthError) + else if (Status == StatAuthError || FailReason == "HashSumMismatch") failreason = HASHSUM_MISMATCH; if(ErrorText.empty()) @@ -975,7 +975,7 @@ void pkgAcquire::Item::FailMessage(string const &Message) break; } - if (Status == StatAuthError) + if (Status == StatAuthError || failreason == HASHSUM_MISMATCH) { auto const ExpectedHashes = GetExpectedHashes(); if (ExpectedHashes.empty() == false) diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 4b9d33505..55897945c 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -486,9 +486,21 @@ bool pkgAcquire::Worker::RunMessages() bool const isIMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) || StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false); auto const forcedHash = _config->Find("Acquire::ForceHash"); + + bool consideredOkay = true; + HashStringList ExpectedHashes; + bool const DebugAuth = _config->FindB("Debug::pkgAcquire::Auth", false); + if (DebugAuth) + { + std::clog << "201 URI Done: " << URI << endl + << "ReceivedHash:" << endl; + for (auto const &hs : ReceivedHashes) + std::clog << "\t- " << hs.toStr() << std::endl; + std::clog << "ExpectedHash:" << endl; + } for (auto const Owner: ItmOwners) { - HashStringList const ExpectedHashes = [&]() { + HashStringList const OwnerExpectedHashes = [&]() { if (AltFile) { auto const * const transOwner = dynamic_cast(Owner); @@ -501,46 +513,51 @@ bool pkgAcquire::Worker::RunMessages() } return Owner->GetExpectedHashes(); }(); - if(_config->FindB("Debug::pkgAcquire::Auth", false) == true) + for (auto const &h : OwnerExpectedHashes) { - std::clog << "201 URI Done: " << Owner->DescURI() << endl - << "ReceivedHash:" << endl; - for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs) - std::clog << "\t- " << hs->toStr() << std::endl; - std::clog << "ExpectedHash:" << endl; - for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs) - std::clog << "\t- " << hs->toStr() << std::endl; - std::clog << endl; - } - - // decide if what we got is what we expected - bool consideredOkay = false; - if ((forcedHash.empty() && ExpectedHashes.empty() == false) || - (forcedHash.empty() == false && ExpectedHashes.usable())) - { - if (ReceivedHashes.empty()) + if (not ExpectedHashes.push_back(h)) { - /* IMS-Hits can't be checked here as we will have uncompressed file, - but the hashes for the compressed file. What we have was good through - so all we have to ensure later is that we are not stalled. */ - consideredOkay = isIMSHit; - } - else if (ReceivedHashes == ExpectedHashes) - consideredOkay = true; - else consideredOkay = false; - + std::clog << "\t- " << h.toStr() << " [conflict]" << std::endl; + } + else if (DebugAuth) + std::clog << "\t- " << h.toStr() << std::endl; + } + } + if (DebugAuth) + std::clog << endl; + + // decide if what we got is what we expected + if (not consideredOkay) + ; + else if ((forcedHash.empty() && not ExpectedHashes.empty()) || + (not forcedHash.empty() && ExpectedHashes.usable())) + { + if (ReceivedHashes.empty()) + { + /* IMS-Hits can't be checked here as we will have uncompressed file, + but the hashes for the compressed file. What we have was good through + so all we have to ensure later is that we are not stalled. */ + consideredOkay = isIMSHit; } + else if (ReceivedHashes == ExpectedHashes) + consideredOkay = true; else - consideredOkay = !Owner->HashesRequired(); - - if (consideredOkay == true) - consideredOkay = Owner->VerifyDone(Message, Config); - else // hashsum mismatch - Owner->Status = pkgAcquire::Item::StatAuthError; + consideredOkay = false; + } + else + consideredOkay = std::none_of(ItmOwners.begin(), ItmOwners.end(), [](auto const * const O) { return O->HashesRequired(); }); + bool otherReasons = false; + if (consideredOkay && not std::all_of(ItmOwners.begin(), ItmOwners.end(), [&](auto * const O) { return O->VerifyDone(Message, Config); })) + { + consideredOkay = false; + otherReasons = true; + } - if (consideredOkay == true) + if (consideredOkay) + { + for (auto const Owner : ItmOwners) { if (isDoomedItem(Owner) == false) Owner->Done(Message, ReceivedHashes, Config); @@ -552,22 +569,21 @@ bool pkgAcquire::Worker::RunMessages() Log->Done(Owner->GetItemDesc()); } } + } + else + { + if (otherReasons) + HandleFailure(ItmOwners, Config, Log, Message, false, false); else { - auto SavedDesc = Owner->GetItemDesc(); - if (isDoomedItem(Owner) == false) + if (Message.find("\nFailReason:") == std::string::npos) { - if (Message.find("\nFailReason:") == std::string::npos) - { - if (ReceivedHashes != ExpectedHashes) - Message.append("\nFailReason: HashSumMismatch"); - else - Message.append("\nFailReason: WeakHashSums"); - } - Owner->Failed(Message,Config); + if (ReceivedHashes != ExpectedHashes) + Message.append("\nFailReason: HashSumMismatch"); + else + Message.append("\nFailReason: WeakHashSums"); } - if (Log != nullptr) - Log->Fail(SavedDesc); + HandleFailure(ItmOwners, Config, Log, Message, false, true); } } ItemDone(); diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index 2d2125582..959ddc1a6 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -25,9 +25,39 @@ testsuccess apt update rm -rf rootdir/var/lib/apt/lists + +msgmsg 'File mirror was hacked' +mkdir aptarchive2 +cp -a aptarchive/dists aptarchive2/ +rm -rf rootdir/var/lib/apt/lists find aptarchive/dists -name 'Packages' | while read FILE; do echo 'hacked' > $FILE done testfailure apt update -o Debug::pkgAcquire::Worker=1 testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output + + +msgmsg 'Fallback over hashsum errors' +rm -f rootdir/etc/apt/sources.list rootdir/etc/apt/sources.list.d/* +echo "deb mirror+file:${TMPWORKINGDIRECTORY}/mirror.list unstable main" > rootdir/etc/apt/sources.list +rm -rf rootdir/var/lib/apt/lists +cat > mirror.list < mirror.list < mirror.list < Date: Mon, 9 May 2022 11:42:48 +0200 Subject: Look at non by-hash paths in copy and file methods Ideally copy and file mirrors would support by-hash as well, but its harder to setup and maintain especially if you want to cache an online mirror who has by-hash enabled. We can avoid unneeded roundtrips (in the best case) and entire cache misses (in the usual worst case) by "just" telling methods that the URI we passed it has the requested file perhaps also in other paths. This is done in pseudo-relative paths as we would otherwise need to teach redirection code to rewrite those URIs as well. A method like http can easily ignore this value and await explicit instructions to look at that file, but inspecting the path in local sources via file or copy is (comparatively) free, so we just do it immediately. If that ends up being the wrong version of the file as by-hash would have protected us from we are in this feature branch now falling back to other mirrors which are like the ones online and in support of by-hash. --- apt-pkg/acquire-item.cc | 11 +- methods/aptmethod.h | 11 ++ methods/copy.cc | 90 ++++++++++------- methods/file.cc | 111 ++++++++++++--------- test/integration/test-apt-by-hash-update | 8 +- .../integration/test-apt-get-update-unauth-warning | 8 +- .../test-apt-update-incomplete-file-mirror | 5 + test/integration/test-method-mirror | 1 - 8 files changed, 143 insertions(+), 102 deletions(-) (limited to 'test') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 2c3f45076..3950ae030 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -473,15 +473,16 @@ bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item) // now add the actual by-hash uris auto const Expected = GetExpectedHashes(); auto const TargetHash = Expected.find(nullptr); - auto const PushByHashURI = [&](std::string U) { + auto const PushByHashURI = [&](std::string const &U) { if (unlikely(TargetHash == nullptr)) return false; - auto const trailing_slash = U.find_last_of("/"); + ::URI uri{U}; + auto const trailing_slash = uri.Path.find_last_of("/"); if (unlikely(trailing_slash == std::string::npos)) return false; - auto byhashSuffix = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); - U.replace(trailing_slash, U.length() - trailing_slash, std::move(byhashSuffix)); - PushAlternativeURI(std::move(U), {}, false); + auto altPath = uri.Path.substr(0, trailing_slash) + "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue(); + std::swap(uri.Path, altPath); + PushAlternativeURI(uri, {{"Alternate-Paths", "../../" + flNotDir(altPath)}}, false); return true; }; PushByHashURI(Item.URI); diff --git a/methods/aptmethod.h b/methods/aptmethod.h index 1c24f3a98..1ea173075 100644 --- a/methods/aptmethod.h +++ b/methods/aptmethod.h @@ -514,6 +514,17 @@ protected: return part; } + static std::string CombineWithAlternatePath(std::string Path, std::string Change) + { + while (APT::String::Startswith(Change, "../")) + { + Path.erase(Path.find_last_not_of('/')); + Path = flNotFile(Path); + Change.erase(0, 3); + } + return flCombine(Path, Change); + } + aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3) : pkgAcqMethod(Ver, Flags), aptConfigWrapperForMethods(Binary), Binary(std::move(Binary)), SeccompFlags(0) { diff --git a/methods/copy.cc b/methods/copy.cc index 82eed150c..b32131a21 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -26,7 +26,7 @@ class CopyMethod : public aptMethod { - virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; public: CopyMethod() : aptMethod("copy", "1.0", SingleInstance | SendConfig | SendURIEncoded) @@ -36,55 +36,71 @@ class CopyMethod : public aptMethod }; // CopyMethod::Fetch - Fetch a file /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool CopyMethod::Fetch(FetchItem *Itm) +bool CopyMethod::URIAcquire(std::string const &Message, FetchItem *Itm) { - // this ensures that relative paths work in copy - std::string const File = DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1)); - - // Stat the file and send a start message - struct stat Buf; - if (stat(File.c_str(),&Buf) != 0) - return _error->Errno("stat",_("Failed to stat")); + struct FileCopyType { + std::string name; + struct stat stat{}; + explicit FileCopyType(std::string &&file) : name{std::move(file)} {} + }; + std::vector files; + // this ensures that relative paths work + files.emplace_back(DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1))); + for (auto const &AltPath : VectorizeString(LookupTag(Message, "Alternate-Paths"), '\n')) + files.emplace_back(CombineWithAlternatePath(flNotFile(files[0].name), DecodeSendURI(AltPath))); + files.erase(std::remove_if(files.begin(), files.end(), [](auto &file) + { return stat(file.name.c_str(), &file.stat) != 0; }), + files.end()); + if (files.empty()) + return _error->Errno("copy-stat", _("Failed to stat")); // Forumulate a result and send a start message FetchResult Res; - Res.Size = Buf.st_size; Res.Filename = Itm->DestFile; - Res.LastModified = Buf.st_mtime; Res.IMSHit = false; - URIStart(Res); - // just calc the hashes if the source and destination are identical - if (File == Itm->DestFile || Itm->DestFile == "/dev/null") + for (auto const &File : files) { + Res.Size = File.stat.st_size; + Res.LastModified = File.stat.st_mtime; + + // just calc the hashes if the source and destination are identical + if (Itm->DestFile == "/dev/null" || File.name == Itm->DestFile) + { + URIStart(Res); + CalculateHashes(Itm, Res); + URIDone(Res); + return true; + } + + FileFd From(File.name, FileFd::ReadOnly); + FileFd To(Itm->DestFile, FileFd::WriteAtomic); + To.EraseOnFailure(); + if (not From.IsOpen() || not To.IsOpen()) + continue; + + // Copy the file + URIStart(Res); + if (not CopyFile(From, To)) + { + To.OpFail(); + continue; + } + From.Close(); + To.Close(); + CalculateHashes(Itm, Res); - URIDone(Res); - return true; - } + if (not Itm->ExpectedHashes.empty() && Itm->ExpectedHashes != Res.Hashes) + continue; - // See if the file exists - FileFd From(File,FileFd::ReadOnly); - FileFd To(Itm->DestFile,FileFd::WriteAtomic); - To.EraseOnFailure(); + if (not TransferModificationTimes(File.name.c_str(), Res.Filename.c_str(), Res.LastModified)) + continue; - // Copy the file - if (CopyFile(From,To) == false) - { - To.OpFail(); - return false; + URIDone(Res); + return true; } - From.Close(); - To.Close(); - - if (TransferModificationTimes(File.c_str(), Res.Filename.c_str(), Res.LastModified) == false) - return false; - - CalculateHashes(Itm, Res); - URIDone(Res); - return true; + return false; } /*}}}*/ diff --git a/methods/file.cc b/methods/file.cc index ff93f83d0..279711195 100644 --- a/methods/file.cc +++ b/methods/file.cc @@ -29,7 +29,7 @@ class FileMethod : public aptMethod { - virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; + bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE; public: FileMethod() : aptMethod("file", "1.0", SingleInstance | SendConfig | LocalOnly | SendURIEncoded) @@ -41,88 +41,101 @@ class FileMethod : public aptMethod // FileMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool FileMethod::Fetch(FetchItem *Itm) +bool FileMethod::URIAcquire(std::string const &Message, FetchItem *Itm) { URI Get(Itm->Uri); - std::string const File = DecodeSendURI(Get.Path); - FetchResult Res; if (Get.Host.empty() == false) return _error->Error(_("Invalid URI, local URIS must not start with //")); - struct stat Buf; + std::vector Files; + Files.emplace_back(DecodeSendURI(Get.Path)); + for (auto const &AltPath : VectorizeString(LookupTag(Message, "Alternate-Paths"), '\n')) + Files.emplace_back(CombineWithAlternatePath(flNotFile(Files[0]), DecodeSendURI(AltPath))); + + FetchResult Res; // deal with destination files which might linger around - if (lstat(Itm->DestFile.c_str(), &Buf) == 0) + struct stat Buf; + if (lstat(Itm->DestFile.c_str(), &Buf) == 0 && S_ISLNK(Buf.st_mode) && Buf.st_size > 0) + { + char name[Buf.st_size + 1]; + if (ssize_t const sp = readlink(Itm->DestFile.c_str(), name, Buf.st_size); sp == -1) + { + Itm->LastModified = 0; + RemoveFile("file", Itm->DestFile); + } + } + + int olderrno = 0; + // See if the file exists + for (auto const &File : Files) { - if ((Buf.st_mode & S_IFREG) != 0) + if (stat(File.c_str(), &Buf) == 0) { + Res.Size = Buf.st_size; + Res.Filename = File; + Res.LastModified = Buf.st_mtime; + Res.IMSHit = false; if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) { - if (Itm->ExpectedHashes.VerifyFile(File)) - { - Res.Filename = Itm->DestFile; + auto const filesize = Itm->ExpectedHashes.FileSize(); + if (filesize != 0 && filesize == Res.Size) Res.IMSHit = true; - } } + break; } + if (olderrno == 0) + olderrno = errno; } - if (Res.IMSHit != true) - RemoveFile("file", Itm->DestFile); - int olderrno = 0; - // See if the file exists - if (stat(File.c_str(),&Buf) == 0) + if (not Res.IMSHit) { - Res.Size = Buf.st_size; - Res.Filename = File; - Res.LastModified = Buf.st_mtime; - Res.IMSHit = false; - if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) + RemoveFile("file", Itm->DestFile); + if (not Res.Filename.empty()) { - unsigned long long const filesize = Itm->ExpectedHashes.FileSize(); - if (filesize != 0 && filesize == Res.Size) - Res.IMSHit = true; + URIStart(Res); + CalculateHashes(Itm, Res); } - - CalculateHashes(Itm, Res); } - else - olderrno = errno; - if (Res.IMSHit == false) - URIStart(Res); // See if the uncompressed file exists and reuse it FetchResult AltRes; - AltRes.Filename.clear(); - std::vector extensions = APT::Configuration::getCompressorExtensions(); - for (std::vector::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext) + for (auto const &File : Files) { - if (APT::String::Endswith(File, *ext) == true) + for (const auto &ext : APT::Configuration::getCompressorExtensions()) { - std::string const unfile = File.substr(0, File.length() - ext->length()); - if (stat(unfile.c_str(),&Buf) == 0) + if (APT::String::Endswith(File, ext)) { - AltRes.Size = Buf.st_size; - AltRes.Filename = unfile; - AltRes.LastModified = Buf.st_mtime; - AltRes.IMSHit = false; - if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) - AltRes.IMSHit = true; - if (Res.Filename.empty()) - CalculateHashes(Itm, AltRes); - break; + std::string const unfile = File.substr(0, File.length() - ext.length()); + if (stat(unfile.c_str(), &Buf) == 0) + { + AltRes.Size = Buf.st_size; + AltRes.Filename = unfile; + AltRes.LastModified = Buf.st_mtime; + AltRes.IMSHit = false; + if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0) + AltRes.IMSHit = true; + if (Res.Filename.empty()) + { + URIStart(Res); + CalculateHashes(Itm, AltRes); + } + break; + } + // no break here as we could have situations similar to '.gz' vs '.tar.gz' here } - // no break here as we could have situations similar to '.gz' vs '.tar.gz' here } + if (not AltRes.Filename.empty()) + break; } - if (AltRes.Filename.empty() == false) + if (not AltRes.Filename.empty()) URIDone(Res,&AltRes); - else if (Res.Filename.empty() == false) + else if (not Res.Filename.empty()) URIDone(Res); else { errno = olderrno; - return _error->Errno(File.c_str(), _("File not found")); + return _error->Errno(Files[0].c_str(), _("File not found")); } return true; diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update index 65c3766d0..a33eb9117 100755 --- a/test/integration/test-apt-by-hash-update +++ b/test/integration/test-apt-by-hash-update @@ -13,6 +13,7 @@ insertpackage 'unstable' 'foo' 'all' '1.0' insertpackage 'unstable' 'bar' 'i386' '1.0' setupaptarchive --no-update +changetowebserver # make Packages *only* accessible by-hash for this test makebyhashonly() { @@ -100,8 +101,7 @@ msgmsg 'Test InRelease by-hash with' 'no fallback' rm -rf aptarchive/dists cp -a aptarchive/dists.bak aptarchive/dists -testfailureequal "Get:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease -Err:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease - File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory) +testfailureequal "Err:1 http://localhost:${APTHTTPPORT} unstable InRelease + 404 Not Found Reading package lists... -E: Failed to fetch file:${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/InRelease File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)" aptget update +E: Failed to fetch http://localhost:${APTHTTPPORT}/dists/unstable/InRelease 404 Not Found" aptget update diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 46df14c29..9f252ad9d 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -28,9 +28,7 @@ testwarning aptget update --allow-insecure-repositories rm -rf rootdir/var/lib/apt/lists find "$APTARCHIVE/dists/unstable" -name '*Release*' -delete # update without authenticated files leads to warning -testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease -Ign:1 file:$APTARCHIVE unstable InRelease -Get:2 file:$APTARCHIVE unstable Release +testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease Err:2 file:$APTARCHIVE unstable Release File not found - ${APTARCHIVE}/dists/unstable/Release (2: No such file or directory) Reading package lists... @@ -45,9 +43,7 @@ lock partial' ls rootdir/var/lib/apt/lists # allow override -testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease -Ign:1 file:$APTARCHIVE unstable InRelease -Get:2 file:$APTARCHIVE unstable Release +testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease Ign:2 file:$APTARCHIVE unstable Release Get:3 file:$APTARCHIVE unstable/main Sources Get:4 file:$APTARCHIVE unstable/main i386 Packages diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index 959ddc1a6..dec9383c9 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -37,6 +37,7 @@ testfailure apt update -o Debug::pkgAcquire::Worker=1 testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output +echo 'Acquire::By-Hash "force";' > rootdir/etc/apt/apt.conf.d/99force-by-hash.conf msgmsg 'Fallback over hashsum errors' rm -f rootdir/etc/apt/sources.list rootdir/etc/apt/sources.list.d/* @@ -47,6 +48,8 @@ copy:${TMPWORKINGDIRECTORY}/aptarchive priority:1 file:${TMPWORKINGDIRECTORY}/aptarchive2 priority:2 EOF testsuccess apt update +cp -a rootdir/tmp/testsuccess.output aptupdate.log +testsuccessequal '2' grep -c -- '^Ign:' aptupdate.log rm -rf rootdir/var/lib/apt/lists cat > mirror.list < aptarchive/mirror.txt testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] -Get:2 foo+file:/nonexistent/apt/archive unstable InRelease Ign:2 foo+file:/nonexistent/apt/archive unstable InRelease File not found - /nonexistent/apt/archive/dists/unstable/InRelease (2: No such file or directory) Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease -- cgit v1.2.3-70-g09d2