require $(HASH)-Download field in .diff/Index files

Now that we ignore SHA1-only files it makes sense to require also the
provision of hashes for the compressed patches as this was introduced in
the same patchset as support for non-SHA1 hashes in the file itself in
dak and adding support in other archive creators (if they support pdiffs
at all) will likely be in the same batch.

The reason for the change itself is simple: If you are 'scared' enough
about the security of SHA1, you shouldn't uncompress a file you haven't
verified at all – after all, it could be exploiting a bug or a zip bomb.

1 files changed, 41 insertions, 36 deletions

diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index e2330d065..2318448f5 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -30,7 +30,7 @@ wasmergeused() {
 	if echo "$*" | grep -q -- '-o test::cannot-use-pdiff=1'; then
 		msgtest 'Check if pdiff was' 'not used'
 		cp -a rootdir/tmp/testsuccess.output rootdir/tmp/aptupdate.output
-		testsuccess --nomsg grep 'diff_Index: Did not find a good hashsum in the index' rootdir/tmp/aptupdate.output
+		testsuccess --nomsg grep "diff/Index with Message: Couldn't parse pdiff index" rootdir/tmp/aptupdate.output
 		return;
 	fi
 
@@ -51,8 +51,6 @@ wasmergeused() {
 testrun() {
 	configcompression '.' 'xz'
 	msgmsg "Testcase: setup the base with: $*"
-	local DOWNLOADHASH=true
-	if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi
 	find aptarchive -name 'Packages*' -type f -delete
 	cp "${PKGFILE}" aptarchive/Packages
 	compressfile 'aptarchive/Packages'
@@ -84,12 +82,10 @@ SHA256-History:
  $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
- $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
-	if $DOWNLOADHASH; then
-		echo "SHA256-Download:
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
+SHA256-Download:
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
- $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
-	fi
+ $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
 
 	generatereleasefiles '+1hour'
 	signreleasefiles
@@ -99,7 +95,7 @@ SHA256-Patches:
 	testsuccessequal "$(cat "${PKGFILE}-new")
 " aptcache show apt newstuff
 
-	msgmsg "Testcase: apply with one patch and SHA1 only: $*"
+	msgmsg "Testcase: SHA1-only patches are not used: $*"
 	find aptarchive -name 'Packages*' -type f -delete
 	cp "${PKGFILE}-new" aptarchive/Packages
 	compressfile 'aptarchive/Packages'
@@ -114,13 +110,35 @@ SHA1-History:
  $(sha1sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA1-Patches:
  7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28
- $(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
-	if $DOWNLOADHASH; then
-		echo "SHA1-Download:
+ $(sha1sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
+SHA1-Download:
  2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
- $(sha1sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
-	fi
+ $(sha1sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
+	generatereleasefiles '+1hour'
+	signreleasefiles
+	rm -rf rootdir/var/lib/apt/lists
+	cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists
+	wasmergeused "$@" -o test::cannot-use-pdiff=1
+	testnopackage oldstuff
+	testsuccessequal "$(cat "${PKGFILE}-new")
+" aptcache show apt newstuff
 
+	msgmsg "Testcase: no download-hashes patches are not used: $*"
+	find aptarchive -name 'Packages*' -type f -delete
+	cp "${PKGFILE}-new" aptarchive/Packages
+	compressfile 'aptarchive/Packages'
+	mkdir -p aptarchive/Packages.diff
+	PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
+	diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true
+	cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
+	PATCHINDEX='aptarchive/Packages.diff/Index'
+	echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
+SHA256-History:
+ 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28
+ $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
+SHA256-Patches:
+ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > "$PATCHINDEX"
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	rm -rf rootdir/var/lib/apt/lists
@@ -157,13 +175,11 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
- $(sha256sum "${PATCHFILE2}" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}") $(basename "${PATCHFILE2}")" > "$PATCHINDEX"
-	if $DOWNLOADHASH; then
-		echo "SHA256-Download:
+ $(sha256sum "${PATCHFILE2}" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}") $(basename "${PATCHFILE2}")
+SHA256-Download:
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
  $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")
- $(sha256sum "${PATCHFILE2}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}.gz") $(basename "${PATCHFILE2}.gz")" >> "$PATCHINDEX"
-	fi
+ $(sha256sum "${PATCHFILE2}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE2}.gz") $(basename "${PATCHFILE2}.gz")" > "$PATCHINDEX"
 
 	generatereleasefiles '+2hour'
 	signreleasefiles
@@ -205,12 +221,10 @@ SHA256-History:
  $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
- $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")" > $PATCHINDEX
-	if $DOWNLOADHASH; then
-		echo "SHA256-Download:
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
+SHA256-Download:
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
- $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" >> $PATCHINDEX
-	fi
+ $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz") $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
 	# needs to look like a valid command, otherwise the parser will fail before hashes are checked
 	echo '1d' > "$PATCHFILE"
 	cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
@@ -236,22 +250,16 @@ SHA256-Patches:
 	diff -e "${PKGFILE}" "${PKGFILE}-new" > "${PATCHFILE}" || true
 	cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz"
 	PATCHINDEX='aptarchive/Packages.diff/Index'
-	BIGSIZE="$(stat -c%s "$PATCHFILE")"
-	if ! $DOWNLOADHASH; then
-		BIGSIZE="${BIGSIZE}000"
-	fi
 	echo "SHA256-Current: $(sha256sum "${PKGFILE}-new" | cut -d' ' -f 1) $(stat -c%s "${PKGFILE}-new")
 SHA256-History:
  01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28
  $(sha256sum "$PKGFILE" | cut -d' ' -f 1) $(stat -c%s "$PKGFILE") $(basename "$PATCHFILE")
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
- $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $BIGSIZE $(basename "$PATCHFILE")" > "$PATCHINDEX"
-	if $DOWNLOADHASH; then
-		echo "SHA256-Download:
+ $(sha256sum "$PATCHFILE" | cut -d' ' -f 1) $(stat -c%s "$PATCHFILE") $(basename "$PATCHFILE")
+SHA256-Download:
  d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
- $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" >> "$PATCHINDEX"
-	fi
+ $(sha256sum "${PATCHFILE}.gz" | cut -d' ' -f 1) $(stat -c%s "${PATCHFILE}.gz")000 $(basename "${PATCHFILE}.gz")" > "$PATCHINDEX"
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	testsuccess apt update -o Debug::pkgAcquire::Diffs=1 "$@"
@@ -266,9 +274,6 @@ Debug::Acquire::Transaction "true";
 Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
 
 testcase() {
-	testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@"
-	testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 "$@"
-
 	testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1 "$@"
 	testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1 "$@"
 	testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0 "$@"