From 7adf8c1fa8d519b8b57292763eb7703e7d3cc580 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 9 May 2022 11:42:48 +0200 Subject: Look at non by-hash paths in copy and file methods Ideally copy and file mirrors would support by-hash as well, but its harder to setup and maintain especially if you want to cache an online mirror who has by-hash enabled. We can avoid unneeded roundtrips (in the best case) and entire cache misses (in the usual worst case) by "just" telling methods that the URI we passed it has the requested file perhaps also in other paths. This is done in pseudo-relative paths as we would otherwise need to teach redirection code to rewrite those URIs as well. A method like http can easily ignore this value and await explicit instructions to look at that file, but inspecting the path in local sources via file or copy is (comparatively) free, so we just do it immediately. If that ends up being the wrong version of the file as by-hash would have protected us from we are in this feature branch now falling back to other mirrors which are like the ones online and in support of by-hash. --- test/integration/test-apt-by-hash-update | 8 ++++---- test/integration/test-apt-get-update-unauth-warning | 8 ++------ test/integration/test-apt-update-incomplete-file-mirror | 5 +++++ test/integration/test-method-mirror | 1 - 4 files changed, 11 insertions(+), 11 deletions(-) (limited to 'test/integration') diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update index 65c3766d0..a33eb9117 100755 --- a/test/integration/test-apt-by-hash-update +++ b/test/integration/test-apt-by-hash-update @@ -13,6 +13,7 @@ insertpackage 'unstable' 'foo' 'all' '1.0' insertpackage 'unstable' 'bar' 'i386' '1.0' setupaptarchive --no-update +changetowebserver # make Packages *only* accessible by-hash for this test makebyhashonly() { @@ -100,8 +101,7 @@ msgmsg 'Test InRelease by-hash with' 'no fallback' rm -rf aptarchive/dists cp -a aptarchive/dists.bak aptarchive/dists -testfailureequal "Get:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease -Err:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease - File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory) +testfailureequal "Err:1 http://localhost:${APTHTTPPORT} unstable InRelease + 404 Not Found Reading package lists... -E: Failed to fetch file:${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/InRelease File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)" aptget update +E: Failed to fetch http://localhost:${APTHTTPPORT}/dists/unstable/InRelease 404 Not Found" aptget update diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 46df14c29..9f252ad9d 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -28,9 +28,7 @@ testwarning aptget update --allow-insecure-repositories rm -rf rootdir/var/lib/apt/lists find "$APTARCHIVE/dists/unstable" -name '*Release*' -delete # update without authenticated files leads to warning -testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease -Ign:1 file:$APTARCHIVE unstable InRelease -Get:2 file:$APTARCHIVE unstable Release +testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease Err:2 file:$APTARCHIVE unstable Release File not found - ${APTARCHIVE}/dists/unstable/Release (2: No such file or directory) Reading package lists... @@ -45,9 +43,7 @@ lock partial' ls rootdir/var/lib/apt/lists # allow override -testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease -Ign:1 file:$APTARCHIVE unstable InRelease -Get:2 file:$APTARCHIVE unstable Release +testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease Ign:2 file:$APTARCHIVE unstable Release Get:3 file:$APTARCHIVE unstable/main Sources Get:4 file:$APTARCHIVE unstable/main i386 Packages diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror index 959ddc1a6..dec9383c9 100755 --- a/test/integration/test-apt-update-incomplete-file-mirror +++ b/test/integration/test-apt-update-incomplete-file-mirror @@ -37,6 +37,7 @@ testfailure apt update -o Debug::pkgAcquire::Worker=1 testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output +echo 'Acquire::By-Hash "force";' > rootdir/etc/apt/apt.conf.d/99force-by-hash.conf msgmsg 'Fallback over hashsum errors' rm -f rootdir/etc/apt/sources.list rootdir/etc/apt/sources.list.d/* @@ -47,6 +48,8 @@ copy:${TMPWORKINGDIRECTORY}/aptarchive priority:1 file:${TMPWORKINGDIRECTORY}/aptarchive2 priority:2 EOF testsuccess apt update +cp -a rootdir/tmp/testsuccess.output aptupdate.log +testsuccessequal '2' grep -c -- '^Ign:' aptupdate.log rm -rf rootdir/var/lib/apt/lists cat > mirror.list < aptarchive/mirror.txt testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B] -Get:2 foo+file:/nonexistent/apt/archive unstable InRelease Ign:2 foo+file:/nonexistent/apt/archive unstable InRelease File not found - /nonexistent/apt/archive/dists/unstable/InRelease (2: No such file or directory) Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease -- cgit v1.2.3-70-g09d2