condense parallel requests with the same hashes to one

It shouldn't be too common, but sometimes people have multiple mirrors
in the sources or otherwise repositories with the same content. Now that
we gracefully can handle multiple requests to the same URI, we can also
fold multiple requests with the same expected hashes into one. Note that
this isn't trying to find oppertunities for merging, but just merges if
it happens to encounter the oppertunity for it.

This is most obvious in the new testcase actually as it needs to delay
the action to give the acquire system enough time to figure out that
they can be merged.

12 files changed, 193 insertions, 95 deletions

diff --git a/test/integration/framework b/test/integration/framework
index 1b99929e2..d8f7567d9 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -245,7 +245,7 @@ setupenvironment() {
 	echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf
 	echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf
 	echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf
-	echo "Dir::Bin::Methods \"${METHODSDIR}\";" >> aptconfig.conf
+	echo "Dir::Bin::Methods \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/methods\";" >> aptconfig.conf
 	# store apt-key were we can access it, even if we run it as a different user
 	# destroys coverage reporting though, so just do it for root for now
 	if [ "$(id -u)" = '0' ]; then
@@ -786,6 +786,8 @@ insertsource() {
 	local SPATH="aptarchive/dists/${RELEASE}/main/source"
 	mkdir -p $SPATH
 	local FILE="${SPATH}/Sources"
+	local DSCFILE="${NAME}_${VERSION}.dsc"
+	local TARFILE="${NAME}_${VERSION}.tar.gz"
 	echo "Package: $NAME
 Binary: $NAME
 Version: $VERSION
@@ -793,8 +795,8 @@ Maintainer: Joe Sixpack <joe@example.org>
 Architecture: $ARCH" >> $FILE
 	test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE
 	echo "Files:
- d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.tar.gz
+ $(echo -n "$DSCFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$DSCFILE" | wc -c) $DSCFILE
+ $(echo -n "$TARFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$TARFILE" | wc -c) $TARFILE
 " >> $FILE
 }
 
diff --git a/test/integration/skip-aptwebserver b/test/integration/skip-aptwebserver
deleted file mode 100755
index 0622941ce..000000000
--- a/test/integration/skip-aptwebserver
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/sh
-set -e
-
-TESTDIR=$(readlink -f $(dirname $0))
-. $TESTDIR/framework
-
-setupenvironment
-configarchitecture 'amd64'
-
-buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
-
-setupaptarchive
-changetowebserver
-
-rm -rf rootdir/var/lib/apt/lists
-aptget update -qq
-testequal 'Hit http://localhost stable InRelease
-Hit http://localhost stable/main Sources
-Hit http://localhost stable/main amd64 Packages
-Hit http://localhost stable/main Translation-en
-Reading package lists...' aptget update
-
-mv rootdir/var/lib/apt/lists/localhost* rootdir/var/lib/apt/lists/partial
-aptget update
-
diff --git a/test/integration/test-acquire-same-file-multiple-times b/test/integration/test-acquire-same-file-multiple-times
index d329a39cb..526765521 100755
--- a/test/integration/test-acquire-same-file-multiple-times
+++ b/test/integration/test-acquire-same-file-multiple-times
@@ -45,7 +45,7 @@ changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo
 
 httpdown() {
 	msgtest 'Downloading the same URI to different files' 'twice over http'
-	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
 	cp rootdir/tmp/testsuccess.output download.log
 	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
 	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
@@ -57,7 +57,7 @@ testrun 'httpdown'
 
 httpredirectdown() {
 	msgtest 'Redirect leads' 'first URI to the second URI'
-	testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
 	cp rootdir/tmp/testsuccess.output download.log
 	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
 	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
diff --git a/test/integration/test-acquire-same-repository-multiple-times b/test/integration/test-acquire-same-repository-multiple-times
new file mode 100755
index 000000000..bfeaf88db
--- /dev/null
+++ b/test/integration/test-acquire-same-repository-multiple-times
@@ -0,0 +1,81 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+TESTFILE="$TESTDIR/framework"
+cp $TESTFILE aptarchive/foo
+APTARCHIVE="$(readlink -f ./aptarchive)"
+
+getcodenamefromsuite() { echo "jessie"; }
+buildsimplenativepackage 'foo' 'all' '1.0' 'stable'
+setupaptarchive --no-update
+ln -s "${APTARCHIVE}/dists/stable" "${APTARCHIVE}/dists/jessie"
+for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do
+	sed 's#stable#jessie#g' $FILE > $(echo "$FILE" | sed 's#stable#jessie#g')
+done
+
+# install a slowed down file: otherwise its to fast to reproduce combining
+NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
+OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
+rm $NEWMETHODS
+mkdir $NEWMETHODS
+for METH in $(find $OLDMETHODS ! -type d); do
+	ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS
+done
+rm $NEWMETHODS/file
+cat >$NEWMETHODS/file <<EOF
+#!/bin/sh
+while read line; do
+	echo "\$line"
+	if [ -z "\$line" ]; then
+		sleep 0.2
+	fi
+done | $OLDMETHODS/file
+EOF
+chmod +x $NEWMETHODS/file
+
+tworepos() {
+	msgtest "Downloading the same repository twice over $1" "$3"
+	testsuccess --nomsg aptget update -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	#cat download.log
+	aptget files --format '$(FILENAME)' --no-release-info | sort > file.lst
+	testequal "$(find $(readlink -f ./rootdir/var/lib/apt/lists) -name '*_dists_*' \( ! -name '*InRelease' \) -type f | sort)" cat file.lst
+	testsuccess aptcache policy
+	testequal "foo:
+  Installed: (none)
+  Candidate: 1.0
+  Version table:
+     1.0 0
+        500 $1:$2 jessie/main amd64 Packages
+        500 $1:$2 stable/main amd64 Packages" aptcache policy foo
+	testfailure aptcache show foo/unstable
+	testsuccess aptcache show foo/stable
+	testsuccess aptcache show foo/jessie
+}
+
+tworepos 'file' "$APTARCHIVE" 'no partial'
+testequal '12' grep -c '200%20URI%20Start' ./download.log
+testequal '12' grep -c '201%20URI%20Done' ./download.log
+testequal '6' grep -c '^ @ Queue: Action combined' ./download.log
+tworepos 'file' "$APTARCHIVE" 'hit'
+testequal '6' grep -c '200%20URI%20Start' ./download.log
+testequal '6' grep -c '201%20URI%20Done' ./download.log
+testequal '0' grep -c '^ @ Queue: Action combined' ./download.log
+rm -rf rootdir/var/lib/apt/lists
+
+changetowebserver
+
+tworepos 'http' '//localhost:8080' 'no partial'
+testequal '10' grep -c '200%20URI%20Start' ./download.log
+testequal '10' grep -c '201%20URI%20Done' ./download.log
+testequal '6' grep -c '^ @ Queue: Action combined' ./download.log
+tworepos 'http' '//localhost:8080' 'hit'
+testequal '2' grep -c '200%20URI%20Start' ./download.log
+testequal '4' grep -c '201%20URI%20Done' ./download.log
+testequal '0' grep -c '^ @ Queue: Action combined' ./download.log
+rm -rf rootdir/var/lib/apt/lists
diff --git a/test/integration/test-apt-get-source b/test/integration/test-apt-get-source
index 9db24370f..22f01b997 100755
--- a/test/integration/test-apt-get-source
+++ b/test/integration/test-apt-get-source
@@ -34,43 +34,40 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 # normal operation gets highest version number
 HEADER="Reading package lists...
 Building dependency tree..."
+DOWNLOAD1="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef
+'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184"
+DOWNLOAD2="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55
+'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f"
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo
+$DOWNLOAD2" aptget source -q --print-uris foo
 
 # select by release: suite
 testsuccessequal "$HEADER
 Selected version '1.0' (stable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/stable
+$DOWNLOAD1" aptget source -q --print-uris foo/stable
 testsuccessequal "$HEADER
 Selected version '2.0' (unstable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/unstable
+$DOWNLOAD2" aptget source -q --print-uris foo/unstable
 
 # select by release: codename
 testsuccessequal "$HEADER
 Selected version '2.0' (sid) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/sid
+$DOWNLOAD2" aptget source -q --print-uris foo/sid
 
 # select by version
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo=1.0
+$DOWNLOAD1" aptget source -q --print-uris foo=1.0
 
 # select by release with no binary package (Bug#731102) but ensure to get
 # highest version
+DOWNLOAD01="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af
+'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61"
 testsuccessequal "$HEADER
 Selected version '0.1' (wheezy) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/wheezy
+$DOWNLOAD01" aptget source -q --print-uris foo/wheezy
 
 # unavailable one
 testfailureequal "$HEADER
@@ -78,11 +75,12 @@ E: Can not find version '9.9-not-there' of package 'foo'
 E: Unable to find a source package for foo" aptget source -q --print-uris foo=9.9-not-there
 
 # version and release
+DOWNLOAD001="Need to get 0 B/29 B of source archives.
+'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 13 MD5Sum:6c819ebf0a21b1a480e1dbf6b8edfebd
+'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 16 MD5Sum:a3c7e1ac2159fc0faf522e110d6906fd"
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris -t unstable foo=0.0.1
+$DOWNLOAD001" aptget source -q --print-uris -t unstable foo=0.0.1
 
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
+Need to get 0 B/25 B of source archives.
 Fetch source foo" aptget source -q -s foo
diff --git a/test/integration/test-apt-get-source-arch b/test/integration/test-apt-get-source-arch
index c75798209..f54bb6012 100755
--- a/test/integration/test-apt-get-source-arch
+++ b/test/integration/test-apt-get-source-arch
@@ -28,31 +28,30 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 
 HEADER="Reading package lists...
 Building dependency tree..."
+DOWNLOAD10="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef
+'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184"
 
 # pick :amd64
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:amd64
+$DOWNLOAD10" aptget source -q --print-uris foo:amd64
 
 # pick :i386
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386
+Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55
+'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f" aptget source -q --print-uris foo:i386
 
 # pick :i386 by release
 testsuccessequal "$HEADER
 Selected version '0.1' (oldstable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386/oldstable
+Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af
+'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61" aptget source -q --print-uris foo:i386/oldstable
 
 # pick :i386 by version
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386=1.0
+$DOWNLOAD10" aptget source -q --print-uris foo:i386=1.0
 
 # error on unknown arch
 testfailureequal "$HEADER
diff --git a/test/integration/test-apt-get-source-multisources b/test/integration/test-apt-get-source-multisources
index 03d0400a0..887a30685 100755
--- a/test/integration/test-apt-get-source-multisources
+++ b/test/integration/test-apt-get-source-multisources
@@ -20,11 +20,11 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 HEADER="Reading package lists...
 Building dependency tree..."
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11
+Need to get 0 B/43 B of source archives.
+'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b
+'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11
 
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e"  aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113
+Need to get 0 B/43 B of source archives.
+'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df
+'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b" aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 5b81d56d2..fcabf244a 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -20,8 +20,10 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated files leads to warning
-testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease
+testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease
+Ign:1 file:$APTARCHIVE unstable InRelease
   File not found
+Get:2 file:$APTARCHIVE unstable Release
 Err:2 file:$APTARCHIVE unstable Release
   File not found
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
@@ -36,10 +38,41 @@ filesize() {
 	stat -c%s "$(aptget files --no-release-info --format '$(URI)' "Created-By: $1" | cut -d'/' -f 3- ).gz"
 }
 # allow override
-testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease
+#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1
+#exit
+testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease
+Ign:1 file:$APTARCHIVE unstable InRelease
   File not found
+Get:2 file:$APTARCHIVE unstable Release
 Ign:2 file:$APTARCHIVE unstable Release
   File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
 Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
 Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B]
 Get:5 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd
index 99b4ea050..e30d503cb 100755
--- a/test/integration/test-apt-progress-fd
+++ b/test/integration/test-apt-progress-fd
@@ -16,7 +16,6 @@ setupaptarchive
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -33,7 +32,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -50,7 +48,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -77,7 +74,6 @@ pmstatus:dpkg-exec:75:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)
diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 2b662171c..33b4ed1b9 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -134,6 +134,24 @@ EXPECT="Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
 Ign:2 http://localhost:8080 unstable Release
   404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
 Hit:3 http://localhost:8080 unstable/main Sources
 Hit:4 http://localhost:8080 unstable/main amd64 Packages
 Hit:5 http://localhost:8080 unstable/main Translation-en
diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
index e51d72ccd..1fa94de7d 100755
--- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet
+++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
@@ -21,11 +21,11 @@ testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.d
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris
-testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a
+'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373" aptget source apt -qq --print-uris
 testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris
 
-testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt apt2 -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a
+'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373
+'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 10 MD5Sum:4c572ce45f1e2bedbb30da7f5e1c241c
+'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 13 MD5Sum:2a96fec139f8722d93312a1ff8281232" aptget source apt apt2 -qq --print-uris
diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index 539080200..e86f28824 100755
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -39,18 +39,15 @@ cd - >/dev/null
 testsuccess aptget install apt -y
 testdpkginstalled 'apt'
 
-# create a copy of all methods, expect https
-eval `aptconfig shell METHODS Dir::Bin::Methods/d`
-COPYMETHODS='usr/lib/apt/methods'
-mv rootdir/${COPYMETHODS} rootdir/${COPYMETHODS}.bak
-mkdir -p rootdir/$COPYMETHODS
-cd rootdir/$COPYMETHODS
-find $METHODS \! -type d | while read meth; do
-	ln -s $meth
+# install a slowed down file: otherwise its to fast to reproduce combining
+NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
+OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
+rm $NEWMETHODS
+mkdir $NEWMETHODS
+for METH in $(find $OLDMETHODS ! -type d); do
+	ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS
 done
-rm https
-cd - >/dev/null
-echo "Dir::Bin::Methods \"${COPYMETHODS}\";" >> aptconfig.conf
+rm $NEWMETHODS/https
 
 cd downloaded
 testfailureequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.
@@ -59,8 +56,7 @@ testfailure test -e apt_1.0_all.deb
 cd - >/dev/null
 
 # revert to all methods
-rm -rf rootdir/$COPYMETHODS
-mv rootdir/${COPYMETHODS}.bak rootdir/${COPYMETHODS}
+ln -s $OLDMETHODS/https $NEWMETHODS
 
 # check that downgrades from https to http are not allowed
 webserverconfig 'aptwebserver::support::http' 'true'