From f2f8e89f08cdf01c83a0b8ab053c65329d85ca90 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 26 Jul 2017 18:35:42 +0200 Subject: fail early in http if server answer is too small as well Failing on too much data is good, but we can do better by checking for exact filesizes as we know with hashsums how large a file should be, so if we get a file which has a size we do not expect we can drop it directly, regardless of if the file is larger or smaller than what we expect which should catch most cases which would end up as hashsum errors later now a lot sooner. --- test/integration/framework | 14 ++++++++++ test/integration/test-apt-update-expected-size | 22 +++++++++++++-- test/integration/test-apt-update-filesize-mismatch | 6 +++- test/integration/test-apt-update-hashsum-mismatch | 11 ++------ test/integration/test-apt-update-not-modified | 32 +++++++--------------- test/integration/test-apt-update-stale | 4 +-- test/integration/test-apt-update-transactions | 10 ++----- test/integration/test-pdiff-usage | 4 +-- .../test-ubuntu-bug-1098738-apt-get-source-md5sum | 16 +++-------- 9 files changed, 61 insertions(+), 58 deletions(-) (limited to 'test/integration') diff --git a/test/integration/framework b/test/integration/framework index 12c80b96c..58e56344e 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -2002,6 +2002,20 @@ forallsupportedcompressors() { done } +breakfiles() { + while [ -n "$1" ]; do + mv -f "${1}" "${1}.bak" + testsuccess dd if=/dev/zero of="${1}" bs="$(stat -c %s "${1}.bak")" count=1 + shift + done +} +unbreakfiles() { + while [ -n "$1" ]; do + mv -f "${1}.bak" "${1}" + shift + done +} + ### convenience hacks ### mkdir() { # creating some directories by hand is a tedious task, so make it look simple diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 5c73a2396..32fa03973 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -21,7 +21,7 @@ test_inreleasetoobig() { testsuccess aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 msgtest 'Check that the max write warning is triggered' cp rootdir/tmp/testsuccess.output update.output - testsuccess --nomsg grep -q 'File is larger than expected' update.output + testsuccess --nomsg grep -q 'File has unexpected size' update.output rm -f update.output # ensure the failed InRelease file got renamed testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED @@ -39,12 +39,30 @@ test_packagestoobig() { touch -d '+1hour' "$pkg" done NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages.gz)" - testfailuremsg "E: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages.gz File is larger than expected ($NEW_SIZE > $SIZE). Mirror sync in progress? + testfailuremsg "E: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages.gz File has unexpected size ($NEW_SIZE != $SIZE). Mirror sync in progress? E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::Transaction=0 testsuccess ls rootdir/var/lib/apt/lists/partial/*Packages*.FAILED testfailure test -e rootdir/var/lib/apt/lists/partial/Old.FAILED } +test_packagestoosmall() { + insertpackage 'unstable' 'foo' 'i386' '1.0' + buildaptarchivefromfiles '+1 hour' + signreleasefiles + # replace Packages.gz/Packages with short junk + SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages.gz)" + find aptarchive/dists -name 'Packages*' | while read pkg; do + echo "1234567890" > "$pkg" + touch -d '+1hour' "$pkg" + done + NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages.gz)" + testfailuremsg "E: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages.gz File is smaller than expected ($NEW_SIZE < $SIZE). Mirror sync in progress? +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::Transaction=0 + testsuccess ls rootdir/var/lib/apt/lists/partial/*Packages*.FAILED + testfailure test -e rootdir/var/lib/apt/lists/partial/Old.FAILED +} + + methodtest() { # less complicated test setup this way webserverconfig 'aptwebserver::support::modified-since' 'false' "$1" diff --git a/test/integration/test-apt-update-filesize-mismatch b/test/integration/test-apt-update-filesize-mismatch index 9467e77b6..26b670d8f 100755 --- a/test/integration/test-apt-update-filesize-mismatch +++ b/test/integration/test-apt-update-filesize-mismatch @@ -40,7 +40,11 @@ for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver. testfailure aptget update -o Debug::pkgAcquire::Worker=1 cp rootdir/tmp/testfailure.output rootdir/tmp/update.output - testsuccess grep -E "$(basename "$COMPRESSFILE" '.gz').*Hash Sum mismatch" rootdir/tmp/update.output + if [ -z "$ext" ]; then + testsuccess grep -E "$(basename "$COMPRESSFILE" '.gz').*Hash Sum mismatch" rootdir/tmp/update.output + else + testsuccess grep -E "$(basename "$COMPRESSFILE" '.gz').*File has unexpected size" rootdir/tmp/update.output + fi testfailure aptcache show foo testfailure aptget install foo -s diff --git a/test/integration/test-apt-update-hashsum-mismatch b/test/integration/test-apt-update-hashsum-mismatch index 4d4c33286..48d041a15 100755 --- a/test/integration/test-apt-update-hashsum-mismatch +++ b/test/integration/test-apt-update-hashsum-mismatch @@ -15,12 +15,6 @@ insertsource 'testing' 'foo2' 'all' '1' setupaptarchive --no-update changetowebserver -echo 'Package: bar -Maintainer: Doctor Evil -Description: come to the dark side -' > aptarchive/DoctorEvil -compressfile aptarchive/DoctorEvil - find aptarchive \( -name 'Packages' -o -name 'Sources' -o -name 'Translation-en' \) -delete testsuccess aptget update @@ -29,9 +23,8 @@ testsuccess aptget install foo -s for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver.log.client*.log); do msgmsg 'Test hashsum mismatch with file' "$get" + breakfiles "aptarchive/${get}" rm -rf rootdir/var/lib/apt/lists - webserverconfig 'aptwebserver::overwrite' '' - webserverconfig "aptwebserver::overwrite::$(printf '%s' "${get}" | sed 's#/#%2F#g' )::filename" '%2FDoctorEvil.gz' testfailure aptget update cp rootdir/tmp/testfailure.output rootdir/tmp/update.output @@ -41,4 +34,6 @@ for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver. testfailure aptcache show bar testfailure aptget install bar -s + + unbreakfiles "aptarchive/${get}" done diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified index cb42e8954..c6dbb8d55 100755 --- a/test/integration/test-apt-update-not-modified +++ b/test/integration/test-apt-update-not-modified @@ -37,20 +37,14 @@ Reading package lists..." aptget update configarchitecture 'amd64' 'i386' # … but oh noes, hashsum mismatch! SIZE=$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') - mv aptarchive/dists/unstable/main/binary-amd64/Packages.gz aptarchive/dists/unstable/main/binary-amd64/Packages.gz.orig - cat > aptarchive/dists/unstable/main/binary-amd64/Packages < aptarchive/dists/unstable/main/binary-amd64/Packages < "$1" < "$PATCHINDEX" - # needs to look like a valid command, otherwise the parser will fail before hashes are checked - echo '1d' > "$PATCHFILE" - cat "$PATCHFILE" | gzip > "${PATCHFILE}.gz" + breakfiles "$PATCHFILE" "${PATCHFILE}.gz" generatereleasefiles '+1hour' signreleasefiles testsuccess apt update "$@" diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum index 14e5a20b8..f0a8835a2 100755 --- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -216,32 +216,24 @@ testmismatch() { Need to get 6 B of source archives. Get:1 http://localhost:${APTHTTPPORT} $1 1.0 (dsc) [2 B] Err:1 http://localhost:${APTHTTPPORT} $1 1.0 (dsc) - File is larger than expected (3 > 2). Mirror sync in progress? + File has unexpected size (3 != 2). Mirror sync in progress? Hashes of expected file: - SHA256:943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a - Filesize:2 [weak] Get:2 http://localhost:${APTHTTPPORT} $1 1.0 (tar) [4 B] Err:2 http://localhost:${APTHTTPPORT} $1 1.0 (tar) - Hash Sum mismatch + File has unexpected size (3 != 4). Mirror sync in progress? Hashes of expected file: - SHA256:90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb - Filesize:4 [weak] - Hashes of received file: - - SHA256:90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb - - Filesize:3 [weak] - Last modification reported: $(lastmodification "aptarchive/${1}_1.0.dsc") -E: Failed to fetch http://localhost:${APTHTTPPORT}/${1}_1.0.dsc File is larger than expected (3 > 2). Mirror sync in progress? +E: Failed to fetch http://localhost:${APTHTTPPORT}/${1}_1.0.dsc File has unexpected size (3 != 2). Mirror sync in progress? Hashes of expected file: - SHA256:943d3bf22ac661fb0f59bc4ff68cc12b04ff17a838dfcc2537008eb9c7f3770a - Filesize:2 [weak] -E: Failed to fetch http://localhost:${APTHTTPPORT}/${1}_1.0.tar.gz Hash Sum mismatch +E: Failed to fetch http://localhost:${APTHTTPPORT}/${1}_1.0.tar.gz File has unexpected size (3 != 4). Mirror sync in progress? Hashes of expected file: - SHA256:90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb - Filesize:4 [weak] - Hashes of received file: - - SHA256:90aebae315675cbf04612de4f7d5874850f48e0b8dd82becbeaa47ca93f5ebfb - - Filesize:3 [weak] - Last modification reported: $(lastmodification "aptarchive/${1}_1.0.dsc") E: Failed to fetch some archives." elif [ "$1" = 'pkg-md5-bad' ]; then FAILURE="Reading package lists... -- cgit v1.2.3-70-g09d2