do not request files if we expect an IMS hit

If we have a file on disk and the hashes are the same in the new Release
file and the old one we have on disk we know that if we ask the server
for the file, we will at best get an IMS hit – at worse the server
doesn't support this and sends us the (unchanged) file and we have to
run all our checks on it again for nothing. So, we can save ourselves
(and the servers) some unneeded requests if we figure this out on our
own.

7 files changed, 77 insertions, 18 deletions

diff --git a/test/integration/framework b/test/integration/framework
index b253deb91..56c4a1216 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -835,7 +835,9 @@ buildaptarchivefromincoming() {
 
 buildaptarchivefromfiles() {
 	msginfo "Build APT archive for ${CCMD}$(basename $0)${CINFO} based on prebuild files…"
-	find aptarchive -name 'Packages' -o -name 'Sources' -o -name 'Translation-*' | while read line; do
+	local DIR='aptarchive'
+	if [ -d "${DIR}/dists" ]; then DIR="${DIR}/dists"; fi
+	find "$DIR" -name 'Packages' -o -name 'Sources' -o -name 'Translation-*' | while read line; do
 		msgninfo "\t${line} file… "
 		compressfile "$line" "$1"
 		msgdone "info"
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 55bba8188..24ca85133 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -26,7 +26,9 @@ test_inreleasetoobig() {
 }
 
 test_packagestoobig() {
-	redatereleasefiles '+1hour'
+	insertpackage 'unstable' 'foo' 'all' '1.0'
+	buildaptarchivefromfiles '+1 hour'
+	signreleasefiles
 	# append junk at the end of the Packages.gz/Packages
 	SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
 	find aptarchive/dists -name 'Packages*' | while read pkg; do
diff --git a/test/integration/test-apt-update-file b/test/integration/test-apt-update-file
index 665f94fa5..94b604f0e 100755
--- a/test/integration/test-apt-update-file
+++ b/test/integration/test-apt-update-file
@@ -26,14 +26,29 @@ testsuccess aptget update
 # the release files aren't an IMS-hit, but the indexes are
 redatereleasefiles '+1 hour'
 
+# we don't download the index if it isn't updated
 testsuccess aptget update -o Debug::pkgAcquire::Auth=1
+# file:/ isn't shown in the log, so see if it was downloaded anyhow
 cp -a rootdir/tmp/testsuccess.output rootdir/tmp/update.output
+canary="SHA512:$(bzcat aptarchive/dists/unstable/main/binary-amd64/Packages.bz2 | sha512sum |cut -f1 -d' ')"
+testfailure grep -- "$canary" rootdir/tmp/update.output
+
+testfoo() {
+	# foo is still available
+	testsuccess aptget install -s foo
+	testsuccess aptcache showsrc foo
+	testsuccess aptget source foo --print-uris
+}
+testfoo
+
+# the release file is new again, the index still isn't, but it is somehow gone now from disk
+redatereleasefiles '+2 hour'
+find rootdir/var/lib/apt/lists -name '*_Packages*' -delete
 
-# ensure that the hash of the uncompressed file was verified even on a local ims hit
+testsuccess aptget update -o Debug::pkgAcquire::Auth=1
+# file:/ isn't shown in the log, so see if it was downloaded anyhow
+cp -a rootdir/tmp/testsuccess.output rootdir/tmp/update.output
 canary="SHA512:$(bzcat aptarchive/dists/unstable/main/binary-amd64/Packages.bz2 | sha512sum |cut -f1 -d' ')"
 testsuccess grep -- "$canary" rootdir/tmp/update.output
 
-# foo is still available
-testsuccess aptget install -s foo
-testsuccess aptcache showsrc foo
-testsuccess aptget source foo --print-uris
+testfoo
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
index a490f00de..32818658f 100755
--- a/test/integration/test-apt-update-not-modified
+++ b/test/integration/test-apt-update-not-modified
@@ -133,6 +133,25 @@ Reading package lists..." aptget update
 
 	rm -rf aptarchive/dists
 	cp -a aptarchive/dists.good aptarchive/dists
+
+	# new release file, but the indexes are the same
+	redatereleasefiles '+2 hours'
+
+	rm -rf rootdir/var/lib/apt/lists.good
+	cp -a rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
+	testsuccessequal "Get:1 $1 unstable InRelease [$(stat -c '%s' 'aptarchive/dists/unstable/InRelease') B]
+Reading package lists..." aptget update
+
+	rm -rf rootdir/var/lib/apt/lists
+	cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists
+	find rootdir/var/lib/apt/lists -name '*_Packages*' -delete
+	testsuccessequal "Get:1 $1 unstable InRelease [$(stat -c '%s' 'aptarchive/dists/unstable/InRelease') B]
+Get:2 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Get:3 $1 unstable/main i386 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-i386/Packages.gz') B]
+Reading package lists..." aptget update
+
+	rm -rf aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
 }
 
 changetowebserver
diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback
index 6fd901715..6ecf322b2 100755
--- a/test/integration/test-apt-update-rollback
+++ b/test/integration/test-apt-update-rollback
@@ -158,7 +158,10 @@ test_inrelease_to_broken_gzip() {
     msgmsg "Test InRelease to broken gzip"
     start_with_good_inrelease
 
-    redatereleasefiles '+2hours'
+    break_repository_sources_index '+1hour'
+    generatereleasefiles '+2hours'
+    signreleasefiles
+
     # append junk at the end of the compressed file
     echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz
     touch -d '+2min' $APTARCHIVE/dists/unstable/main/source/Sources.gz
@@ -166,6 +169,7 @@ test_inrelease_to_broken_gzip() {
     rm $APTARCHIVE/dists/unstable/main/source/Sources
 
     testfailure aptget update
+    testsuccess grep 'Hash Sum mismatch' rootdir/tmp/testfailure.output
     testfileequal lists.before "$(listcurrentlistsdirectory)"
 }
 
diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions
index 152e1617a..a5dac1737 100755
--- a/test/integration/test-apt-update-transactions
+++ b/test/integration/test-apt-update-transactions
@@ -29,6 +29,12 @@ restorefile() {
 }
 
 testrun() {
+	rm -rf aptarchive/dists.good
+	cp -a aptarchive/dists aptarchive/dists.good
+	insertpackage 'unstable' 'bar' 'all' '1.0'
+	insertsource 'unstable' 'bar' 'all' '1.0'
+	buildaptarchivefromfiles '+1 hour'
+
 	# produce an unsigned repository
 	find aptarchive \( -name 'Release.gpg' -o -name 'InRelease' \) -delete
 	testfailure aptget update --no-allow-insecure-repositories
@@ -37,20 +43,27 @@ testrun() {
 	# signed but broken
 	signreleasefiles
 
+	onehashbroken() {
+		testfailure aptget update
+		# each file generates two messages with this string
+		testequal '2' grep --count 'Hash Sum mismatch' rootdir/tmp/testfailure.output
+		testfileequal "$1" "$(listcurrentlistsdirectory)"
+	}
+
 	breakfile aptarchive/dists/unstable/main/binary-i386/Packages
-	testfailure aptget update
-	testfileequal "$1" "$(listcurrentlistsdirectory)"
+	onehashbroken "$1"
 	restorefile aptarchive/dists/unstable/main/binary-i386/Packages
 
 	breakfile aptarchive/dists/unstable/main/source/Sources
-	testfailure aptget update
-	testfileequal "$1" "$(listcurrentlistsdirectory)"
+	onehashbroken "$1"
 	restorefile aptarchive/dists/unstable/main/source/Sources
+
+	rm -rf aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
 }
 
 testsetup() {
 	msgmsg 'Test with no initial data over' "$1"
-	redatereleasefiles 'now'
 	rm -rf rootdir/var/lib/apt/lists
 	mkdir -p rootdir/var/lib/apt/lists/partial
 	listcurrentlistsdirectory > listsdir.lst
@@ -60,7 +73,6 @@ testsetup() {
 	rm -rf rootdir/var/lib/apt/lists
 	testsuccess aptget update -o Debug::pkgAcquire::Worker=1
 	listcurrentlistsdirectory > listsdir.lst
-	redatereleasefiles '+1hour'
 	testrun 'listsdir.lst'
 }
 
diff --git a/test/integration/test-cve-2013-1051-InRelease-parsing b/test/integration/test-cve-2013-1051-InRelease-parsing
index e38e40cc9..d99174553 100755
--- a/test/integration/test-cve-2013-1051-InRelease-parsing
+++ b/test/integration/test-cve-2013-1051-InRelease-parsing
@@ -39,10 +39,15 @@ sed -i '/^-----BEGIN PGP SIGNATURE-----/,/^-----END PGP SIGNATURE-----/ s/^$/  /
 cat aptarchive/dists/stable/Release >> aptarchive/dists/stable/InRelease
 touch -d '+1hour' aptarchive/dists/stable/InRelease
 
-# ensure the update fails
-# useful for debugging to add "-o Debug::pkgAcquire::auth=true"
-msgtest 'apt-get update for should fail with the modified' 'InRelease'
-aptget update 2>&1 | grep -E -q '(Writing more data than expected|Hash Sum mismatch)' > /dev/null && msgpass || msgfail
+# ensure the update doesn't load bad data as good data
+# Note that we will pick up the InRelease itself as we download no other
+# indexes which would trigger a hashsum mismatch, but we ignore the 'bad'
+# part of the InRelease
+listcurrentlistsdirectory | sed '/_InRelease/ d' > listsdir.lst
+msgtest 'apt-get update should ignore unsigned data in the' 'InRelease'
+testsuccessequal "Get:1 http://localhost:8080 stable InRelease [$(stat -c%s aptarchive/dists/stable/InRelease) B]
+Reading package lists..." --nomsg aptget update
+testfileequal './listsdir.lst' "$(listcurrentlistsdirectory | sed '/_InRelease/ d')"
 
 # ensure there is no package
 testfailureequal 'Reading package lists...