Merge branch 'fix/partialfilemirror' into 'main'

Support uncompressed indexes from partial file:/ mirrors

See merge request apt-team/apt!235

5 files changed, 159 insertions, 55 deletions

diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update
index 65c3766d0..a33eb9117 100755
--- a/test/integration/test-apt-by-hash-update
+++ b/test/integration/test-apt-by-hash-update
@@ -13,6 +13,7 @@ insertpackage 'unstable' 'foo' 'all' '1.0'
 insertpackage 'unstable' 'bar' 'i386' '1.0'
 
 setupaptarchive --no-update
+changetowebserver
 
 # make Packages *only* accessible by-hash for this test
 makebyhashonly() {
@@ -100,8 +101,7 @@ msgmsg 'Test InRelease by-hash with' 'no fallback'
 rm -rf aptarchive/dists
 cp -a aptarchive/dists.bak aptarchive/dists
 
-testfailureequal "Get:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease
-Err:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease
-  File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)
+testfailureequal "Err:1 http://localhost:${APTHTTPPORT} unstable InRelease
+  404  Not Found
 Reading package lists...
-E: Failed to fetch file:${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/InRelease  File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)" aptget update
+E: Failed to fetch http://localhost:${APTHTTPPORT}/dists/unstable/InRelease  404  Not Found" aptget update
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 42c4e5a9d..9f252ad9d 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -28,9 +28,7 @@ testwarning aptget update --allow-insecure-repositories
 rm -rf rootdir/var/lib/apt/lists
 find "$APTARCHIVE/dists/unstable" -name '*Release*' -delete
 # update without authenticated files leads to warning
-testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease
-Ign:1 file:$APTARCHIVE unstable InRelease
-Get:2 file:$APTARCHIVE unstable Release
+testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease
 Err:2 file:$APTARCHIVE unstable Release
   File not found - ${APTARCHIVE}/dists/unstable/Release (2: No such file or directory)
 Reading package lists...
@@ -44,46 +42,13 @@ testequal 'auxfiles
 lock
 partial' ls rootdir/var/lib/apt/lists
 
-filesize() {
-	local CREATEDBY="$1"
-	shift
-	stat -c%s "/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 2- ).gz"
-}
 # allow override
-#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1
-#exit
-testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease
-Ign:1 file:$APTARCHIVE unstable InRelease
-Get:2 file:$APTARCHIVE unstable Release
+testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease
 Ign:2 file:$APTARCHIVE unstable Release
 Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
 Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
 Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
 Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
-Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
-Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
-Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
-Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
-Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
-Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
-Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B]
-Get:5 file:$APTARCHIVE unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B]
-Get:6 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
 Reading package lists...
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file.
 N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use.
@@ -92,3 +57,38 @@ N: See apt-secure(8) manpage for repository creation and user configuration deta
 testfailureequal "WARNING: The following packages cannot be authenticated!
   foo
 E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo
+
+rm -rf rootdir/var/lib/apt/lists
+changetowebserver
+
+filesize() {
+	local CREATEDBY="$1"
+	shift
+	stat -c%s "${APTARCHIVE}/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 4- ).gz"
+}
+testwarningequal "Ign:1 http://localhost:$APTHTTPPORT unstable InRelease
+Ign:2 http://localhost:$APTHTTPPORT unstable Release
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Get:3 http://localhost:$APTHTTPPORT unstable/main Sources [$(filesize 'Sources') B]
+Get:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B]
+Get:5 http://localhost:$APTHTTPPORT unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B]
+Get:6 http://localhost:$APTHTTPPORT unstable/main Translation-en [$(filesize 'Translations') B]
+Reading package lists...
+W: The repository 'http://localhost:$APTHTTPPORT unstable Release' does not have a Release file.
+N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use.
+N: See apt-secure(8) manpage for repository creation and user configuration details." aptget update --allow-insecure-repositories
+# ensure we can not install the package
+testfailureequal "WARNING: The following packages cannot be authenticated!
+  foo
+E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index ae1ca7456..4f883b440 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -74,10 +74,11 @@ E: Download Failed"
 setupproxydetect() {
 	local METH="$1"
 	shift
-	{
-		echo '#!/bin/sh -e'
-		echo "$@"
-	} > "${TMPWORKINGDIRECTORY}/apt-proxy-detect"
+	cat >"${TMPWORKINGDIRECTORY}/apt-proxy-detect" <<EOF
+#!/bin/sh
+set -e
+$*
+EOF
 	chmod 755 "${TMPWORKINGDIRECTORY}/apt-proxy-detect"
 	echo "Acquire::${METH}::${CONFNAME} \"${TMPWORKINGDIRECTORY}/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect
 }
@@ -97,15 +98,38 @@ W: ProxyAutoDetect command returned an empty line" apthelper auto-detect-proxy h
     chmod -x "${TMPWORKINGDIRECTORY}/apt-proxy-detect"
     testfailureequal "E: ProxyAutoDetect command '${TMPWORKINGDIRECTORY}/apt-proxy-detect' can not be executed! - access (13: Permission denied)" apthelper auto-detect-proxy http://example.com/
 
-    msgmsg "apt-helper $CONFNAME" 'http proxy'
-    setupproxydetect 'http' 'echo "http://some-proxy"'
-    testsuccessequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com
-    testsuccessequal "Using proxy '' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com
+    for meth in 'http' 'https'; do
+	msgmsg "apt-helper $CONFNAME" "${meth} proxy"
+	for type in 'http' 'https' 'socks5h'; do
+		setupproxydetect "$meth" "echo '${type}://some-proxy'"
+		testsuccessequal "Using proxy '${type}://some-proxy' for URL '${meth}://www.example.com/'" apthelper auto-detect-proxy "${meth}://www.example.com"
+		if [ "$meth" = 'http' ]; then
+			testsuccessequal "Using proxy '' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy 'https://ssl.example.com'
+		else
+			testsuccessequal "Using proxy '' for URL 'http://no-ssl.example.com/'" apthelper auto-detect-proxy 'http://no-ssl.example.com'
+		fi
+	done
+    done
+
+    msgmsg "apt-helper $CONFNAME" 'no strange proxy'
+    for meth in gpgv copy store file; do
+	setupproxydetect "$meth" "echo '${meth}://some-proxy'"
+	testsuccessequal "Using proxy '' for URL '${meth}:/foo/bar'" apthelper auto-detect-proxy "${meth}:/foo/bar"
+	testsuccessequal "Using proxy '' for URL '${meth}://./foo/bar'" apthelper auto-detect-proxy "${meth}://./foo/bar"
+    done
+    for url in 'cdrom://Moo Cow Rom/debian' 'cdrom://[The Debian 1.2 disk, 1/2 R16]/debian/'; do
+	setupproxydetect "${url%%:*}" "echo 'cdrom://some-proxy'"
+	testsuccessequal "Using proxy '' for URL '${url}'" apthelper auto-detect-proxy "$url"
+    done
+    for meth in tor tor+http tor+https; do
+	setupproxydetect "$meth" "echo 'socks5h://some-proxy'"
+	testsuccessequal "Using proxy 'socks5h://some-proxy' for URL '${meth}://example.org/'" apthelper auto-detect-proxy "${meth}://example.org"
+
+	setupproxydetect "$meth" "echo 'DIRECT'"
+	testwarningequal "Using proxy '' for URL '${meth}://example.org/'
+W: ProxyAutoDetect command returned incompatible proxy 'DIRECT' for access type ${meth}" apthelper auto-detect-proxy "${meth}://example.org"
+    done
 
-    msgmsg "apt-helper $CONFNAME" 'https proxy'
-    setupproxydetect 'https' 'echo "https://https-proxy"'
-    testsuccessequal "Using proxy '' for URL 'http://no-ssl.example.com/'" apthelper auto-detect-proxy http://no-ssl.example.com
-    testsuccessequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com
     rm -f rootdir/etc/apt/apt.conf.d/02proxy-detect "${TMPWORKINGDIRECTORY}/apt-proxy-detect"
 }
 
diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror
new file mode 100755
index 000000000..dec9383c9
--- /dev/null
+++ b/test/integration/test-apt-update-incomplete-file-mirror
@@ -0,0 +1,68 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture "amd64"
+configcompression 'gz'
+
+insertpackage 'unstable' 'foo' 'amd64' '1'
+
+setupaptarchive --no-update
+
+testfailure aptcache show foo
+
+# the Release file says gz available, but the mirror has only uncompressed files
+find aptarchive/dists -name '*.gz' -delete
+
+testsuccess apt update
+testsuccess aptcache show foo
+testfailure aptcache show bar
+
+testsuccess apt update
+
+rm -rf rootdir/var/lib/apt/lists
+
+
+msgmsg 'File mirror was hacked'
+mkdir aptarchive2
+cp -a aptarchive/dists aptarchive2/
+rm -rf rootdir/var/lib/apt/lists
+find aptarchive/dists -name 'Packages' | while read FILE; do
+	echo 'hacked' > $FILE
+done
+testfailure apt update -o Debug::pkgAcquire::Worker=1
+testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output
+testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output
+
+echo 'Acquire::By-Hash "force";' > rootdir/etc/apt/apt.conf.d/99force-by-hash.conf
+
+msgmsg 'Fallback over hashsum errors'
+rm -f rootdir/etc/apt/sources.list rootdir/etc/apt/sources.list.d/*
+echo "deb mirror+file:${TMPWORKINGDIRECTORY}/mirror.list unstable main" > rootdir/etc/apt/sources.list
+rm -rf rootdir/var/lib/apt/lists
+cat > mirror.list <<EOF
+copy:${TMPWORKINGDIRECTORY}/aptarchive	priority:1
+file:${TMPWORKINGDIRECTORY}/aptarchive2	priority:2
+EOF
+testsuccess apt update
+cp -a rootdir/tmp/testsuccess.output aptupdate.log
+testsuccessequal '2' grep -c -- '^Ign:' aptupdate.log
+
+rm -rf rootdir/var/lib/apt/lists
+cat > mirror.list <<EOF
+file:${TMPWORKINGDIRECTORY}/aptarchive	priority:1
+copy:${TMPWORKINGDIRECTORY}/aptarchive2	priority:2
+EOF
+testsuccess apt update
+
+rm -rf rootdir/var/lib/apt/lists
+cat > mirror.list <<EOF
+file:${TMPWORKINGDIRECTORY}/aptarchive	priority:1
+file:${TMPWORKINGDIRECTORY}/aptarchive2	priority:2
+EOF
+testsuccess apt update
+cp -a rootdir/tmp/testsuccess.output aptupdate.log
+testsuccessequal '1' grep -c -- '^Ign:' aptupdate.log
diff --git a/test/integration/test-method-mirror b/test/integration/test-method-mirror
index ce44b86e3..79c675455 100755
--- a/test/integration/test-method-mirror
+++ b/test/integration/test-method-mirror
@@ -124,9 +124,22 @@ msgmsg 'all mirrored via file'
 APTARCHIVE="$(readlink -f ./aptarchive)"
 sed -i -e "s#mirror+http://localhost:${APTHTTPPORT}#mirror+file:${APTARCHIVE}#" rootdir/etc/apt/sources.list.d/*
 testrun '*_localhost_*' '*_aptarchive_mirror.txt.gz_*'
+sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/*
+
+mv rootdir/etc/apt/sources.list.d rootdir/etc/apt/sources.list.d.bak
+mkdir rootdir/etc/apt/sources.list.d
+msgmsg 'fail gracefully if mirror uri has no filename'
+echo "deb mirror://localhost:${APTHTTPPORT}/ stable main" > rootdir/etc/apt/sources.list.d/mirrordir.list
+testfailure apt update
+
+msgmsg 'but succeed if it is indeed a mirror list'
+ln -s mirror.txt aptarchive/index.html
+testsuccess apt update
+rm aptarchive/index.html rootdir/etc/apt/sources.list.d/mirrordir.list
+rmdir rootdir/etc/apt/sources.list.d
+mv rootdir/etc/apt/sources.list.d.bak rootdir/etc/apt/sources.list.d
 
 msgmsg 'fallback mirrors are used if needed' 'as usual'
-sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/*
 echo "http://localhost:${APTHTTPPORT}/failure2	priority:3
 http://localhost:${APTHTTPPORT}/redirectme	priority:2
 http://localhost:${APTHTTPPORT}/failure	priority:1" > aptarchive/mirror.txt
@@ -242,7 +255,6 @@ echo "file:/nonexistent/apt/archive	priority:1
 http://localhost:${APTHTTPPORT}/redirectme
 " > aptarchive/mirror.txt
 testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
-Get:2 foo+file:/nonexistent/apt/archive unstable InRelease
 Ign:2 foo+file:/nonexistent/apt/archive unstable InRelease
   File not found - /nonexistent/apt/archive/dists/unstable/InRelease (2: No such file or directory)
 Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease