Read and work with canonical file-URIs from sources.lists

We allow file (and other file-based methods) URIs to either be given
as file:///path or as file:/path, but in various places of the acquire
system we perform string comparisons on URIs which do not handle this
expecting the canonical representation produced by our URI code.

That used to be hidden by us quoting and dequoting the URIs in the
system, but as we don't do this anymore we have to be a bit more careful
on input.

Ideally we would do less of these comparisons, but for now lets be
content with inserting a canonicalisation early on to prevent hangs in
the acquire system.

9 files changed, 54 insertions, 36 deletions

diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index cd7dbce9c..0ac59fcfc 100644
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -85,12 +85,12 @@ bool pkgSourceList::Type::FixupURI(string &URI) const
    if (URI.find(':') == string::npos)
       return false;
 
-   URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
-   
+   URI = ::URI{SubstVar(URI, "$(ARCH)", _config->Find("APT::Architecture"))};
+
    // Make sure that the URI is / postfixed
-   if (URI[URI.size() - 1] != '/')
-      URI += '/';
-   
+   if (URI.back() != '/')
+      URI.push_back('/');
+
    return true;
 }
 									/*}}}*/
diff --git a/test/integration/framework b/test/integration/framework
index 1f942d162..009eed9cd 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1181,7 +1181,7 @@ setupdistsaptarchive() {
 		SECTIONS=$(find "./aptarchive/dists/${DISTS}/" -mindepth 1 -maxdepth 1 -type d | cut -d'/' -f 5 | tr '\n' ' ')
 		msgninfo "\tadd deb and deb-src sources.list lines for ${CCMD}${DISTS} ${SECTIONS}${CINFO}… "
 		echo "deb file://$APTARCHIVE $DISTS $SECTIONS" > "rootdir/etc/apt/sources.list.d/apt-test-${DISTS}-deb.list"
-		echo "deb-src file://$APTARCHIVE $DISTS $SECTIONS" > "rootdir/etc/apt/sources.list.d/apt-test-${DISTS}-deb-src.list"
+		echo "deb-src file:$APTARCHIVE $DISTS $SECTIONS" > "rootdir/etc/apt/sources.list.d/apt-test-${DISTS}-deb-src.list"
 		msgdone "info"
 	done
 }
@@ -1198,7 +1198,7 @@ setupflataptarchive() {
 	fi
 	if [ -f "${APTARCHIVE}/Sources" ]; then
 		msgninfo "\tadd deb-src sources.list line… "
-		echo "deb-src file://$APTARCHIVEURI /" > 'rootdir/etc/apt/sources.list.d/apt-test-archive-deb-src.list'
+		echo "deb-src file:$APTARCHIVEURI /" > 'rootdir/etc/apt/sources.list.d/apt-test-archive-deb-src.list'
 		msgdone 'info'
 	else
 		rm -f 'rootdir/etc/apt/sources.list.d/apt-test-archive-deb-src.list'
@@ -1360,10 +1360,13 @@ webserverconfig() {
 }
 
 rewritesourceslist() {
-	local APTARCHIVE="file://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')"
-	local APTARCHIVE2="copy://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')"
+	local APTARCHIVE="$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive" | sed 's# #%20#g')"
 	for LIST in $(find "${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list.d/" -name 'apt-test-*.list'); do
-		sed -i $LIST -e "s#$APTARCHIVE#${1}#" -e "s#$APTARCHIVE2#${1}#" \
+		sed -i $LIST \
+			-e "s#file://$APTARCHIVE#${1}#" \
+			-e "s#file:$APTARCHIVE#${1}#" \
+			-e "s#copy://$APTARCHIVE#${1}#" \
+			-e "s#copy:$APTARCHIVE#${1}#" \
 			-e "s#http://[^@]*@\?localhost:${APTHTTPPORT}/\?#${1}#" \
 			-e "s#https://[^@]*@\?localhost:${APTHTTPSPORT}/\?#${1}#"
 	done
@@ -1384,10 +1387,9 @@ waitforpidfile() {
 }
 
 changetowebserver() {
-	local REWRITE='no'
-	if [ "$1" != '--no-rewrite' ]; then
-		REWRITE='yes'
-	else
+	local REWRITE='yes'
+	if [ "$1" = '--no-rewrite' ]; then
+		REWRITE='no'
 		shift
 	fi
 	if test -x "${APTTESTHELPERSBINDIR}/aptwebserver"; then
@@ -1412,12 +1414,17 @@ changetowebserver() {
 	else
 		msgdie 'You have to build apt from source to have test/interactive-helper/aptwebserver available for tests requiring a webserver'
 	fi
-	if [ "$REWRTE" != 'yes' ]; then
+	if [ "$REWRITE" != 'no' ]; then
 		rewritesourceslist "http://localhost:${APTHTTPPORT}/"
 	fi
 }
 
 changetohttpswebserver() {
+	local REWRITE='yes'
+	if [ "$1" = '--no-rewrite' ]; then
+		REWRITE='no'
+		shift
+	fi
 	local stunnel4
 	if command -v stunnel4 >/dev/null 2>&1; then
 		stunnel4=stunnel4
@@ -1446,7 +1453,9 @@ connect = $APTHTTPPORT
 	addtrap 'prefix' "kill ${PID};"
 	APTHTTPSPORT="$(lsof -i -n | awk "/^$stunnel4 / && \$2 == \"${PID}\" {print \$9; exit; }" | cut -d':' -f 2)"
 	webserverconfig 'aptwebserver::port::https' "$APTHTTPSPORT" "https://localhost:${APTHTTPSPORT}"
-	rewritesourceslist "https://localhost:${APTHTTPSPORT}/"
+	if [ "$REWRITE" != 'no' ]; then
+		rewritesourceslist "https://localhost:${APTHTTPSPORT}/"
+	fi
 }
 
 changetocdrom() {
diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update
index f4794a84f..65c3766d0 100755
--- a/test/integration/test-apt-by-hash-update
+++ b/test/integration/test-apt-by-hash-update
@@ -104,4 +104,4 @@ testfailureequal "Get:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InReleas
 Err:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease
   File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)
 Reading package lists...
-E: Failed to fetch file://${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/InRelease  File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)" aptget update
+E: Failed to fetch file:${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/InRelease  File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)" aptget update
diff --git a/test/integration/test-apt-get-download b/test/integration/test-apt-get-download
index fc0e6cfa3..ea74ac662 100755
--- a/test/integration/test-apt-get-download
+++ b/test/integration/test-apt-get-download
@@ -73,7 +73,7 @@ testdownload apt_1.0_all.deb apt/stable
 testdownload apt_2.0_all.deb apt
 
 DEBFILE="$(readlink -f ../aptarchive)/pool/apt_2.0_all.deb"
-testequal "'file://${DEBFILE}' apt_2.0_all.deb $(stat -c%s "$DEBFILE") SHA512:$(sha512sum "$DEBFILE" | cut -d' ' -f 1)" aptget download apt --print-uris
+testequal "'file:${DEBFILE}' apt_2.0_all.deb $(stat -c%s "$DEBFILE") SHA512:$(sha512sum "$DEBFILE" | cut -d' ' -f 1)" aptget download apt --print-uris
 
 # deb:677887
 testequal "E: Can't find a source to download version '1.0' of 'vrms:i386'" aptget download vrms --print-uris
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index a0d7a59d9..42c4e5a9d 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -47,7 +47,7 @@ partial' ls rootdir/var/lib/apt/lists
 filesize() {
 	local CREATEDBY="$1"
 	shift
-	stat -c%s "$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 3- ).gz"
+	stat -c%s "/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 2- ).gz"
 }
 # allow override
 #aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1
diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback
index d7e30ba20..637716543 100755
--- a/test/integration/test-apt-update-nofallback
+++ b/test/integration/test-apt-update-nofallback
@@ -178,7 +178,7 @@ test_subvert_inrelease()
     # replace InRelease with something else
     mv "$APTARCHIVE/dists/unstable/Release" "$APTARCHIVE/dists/unstable/InRelease"
 
-    testfailuremsg "E: Failed to fetch file://${APTARCHIVE}/dists/unstable/InRelease  Clearsigned file isn't valid, got 'NOSPLIT' (does the network require authentication?)
+    testfailuremsg "E: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  Clearsigned file isn't valid, got 'NOSPLIT' (does the network require authentication?)
 E: The repository 'file:${APTARCHIVE} unstable InRelease' is no longer signed.
 N: Updating from such a repository can't be done securely, and is therefore disabled by default.
 N: See apt-secure(8) manpage for repository creation and user configuration details." aptget update
diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
index 116e8fa2a..73290a72d 100755
--- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet
+++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
@@ -17,10 +17,10 @@ setupaptarchive
 
 APTARCHIVE=$(readlink -f ./aptarchive)
 
-testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 42 " aptget upgrade -qq --print-uris
-testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 42 " aptget dist-upgrade -qq --print-uris
-testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 42 " aptget install apt -qq --print-uris
-testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 42 SHA256:0000000000000000000000000000000000000000000000000000000000000000" aptget download apt -qq --print-uris
+testsuccessequal "'file:${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 42 " aptget upgrade -qq --print-uris
+testsuccessequal "'file:${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 42 " aptget dist-upgrade -qq --print-uris
+testsuccessequal "'file:${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 42 " aptget install apt -qq --print-uris
+testsuccessequal "'file:${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 42 SHA256:0000000000000000000000000000000000000000000000000000000000000000" aptget download apt -qq --print-uris
 testsuccessequal "'file:${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 SHA256:7776436a6d741497f1cd958014e1a05b352224231428152aae39da3c17fd2fd4
 'file:${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 SHA256:f57f565eabe3fde0ec6e6e0bcc8db1d86fe2b4d6344a380a23520ddbb7728e99" aptget source apt -qq --print-uris
 testsuccessequal "'https://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris
diff --git a/test/integration/test-bug-723705-tagfile-truncates-fields b/test/integration/test-bug-723705-tagfile-truncates-fields
index d167a3f65..15422286f 100755
--- a/test/integration/test-bug-723705-tagfile-truncates-fields
+++ b/test/integration/test-bug-723705-tagfile-truncates-fields
@@ -8,7 +8,7 @@ configarchitecture 'amd64'
 
 setupaptarchive
 
-aptget install --print-uris -y cdebconf-newt-terminal cdebconf-gtk-terminal 2>&1 | sed "s#file://${TMPWORKINGDIRECTORY}#file:///tmp#g" > filename.log
+aptget install --print-uris -y cdebconf-newt-terminal cdebconf-gtk-terminal 2>&1 | sed "s#file:\(//\|\)${TMPWORKINGDIRECTORY}#file:/tmp#g" > filename.log
 
 testfileequal filename.log "Reading package lists...
 Building dependency tree...
@@ -22,12 +22,12 @@ The following NEW packages will be installed:
 0 upgraded, 9 newly installed, 0 to remove and 0 not upgraded.
 Need to get 0 B/4774 kB of archives.
 After this operation, 19.8 MB of additional disk space will be used.
-'file:///tmp/aptarchive/pool/main/c/cdebconf/cdebconf-udeb_0.185_amd64.udeb' cdebconf-udeb_0.185_amd64.udeb 77376 MD5Sum:e3883706fdbf54c2e5ea959c92b2d37f
-'file:///tmp/aptarchive/pool/main/c/cdebconf/cdebconf-gtk-udeb_0.185_amd64.udeb' cdebconf-gtk-udeb_0.185_amd64.udeb 27278 MD5Sum:a1bbbc1d4fb8e0615b5621abac021924
-'file:///tmp/aptarchive/pool/main/c/cdebconf/cdebconf-newt-udeb_0.185_amd64.udeb' cdebconf-newt-udeb_0.185_amd64.udeb 19192 MD5Sum:de27807f56dae2f2403b3322d5fe6bd2
-'file:///tmp/aptarchive/pool/main/g/glib2.0/libglib2.0-udeb_2.36.4-1_amd64.udeb' libglib2.0-udeb_2.36.4-1_amd64.udeb 1714604 MD5Sum:72da029f1bbb36057d874f1f82a5d00a
-'file:///tmp/aptarchive/pool/main/e/eglibc/libc6-udeb_2.17-92%2bb1_amd64.udeb' libc6-udeb_2.17-92+b1_amd64.udeb 1056000 MD5Sum:7fd7032eeeecf7f76eff79a0543fbd72
-'file:///tmp/aptarchive/pool/main/g/gtk%2b2.0/libgtk2.0-0-udeb_2.24.20-1_amd64.udeb' libgtk2.0-0-udeb_2.24.20-1_amd64.udeb 1643046 MD5Sum:25513478eb2e02e5766c0eea0b411ca9
-'file:///tmp/aptarchive/pool/main/v/vte/libvte9-udeb_0.28.2-5_amd64.udeb' libvte9-udeb_1%3a0.28.2-5_amd64.udeb 216968 MD5Sum:7da7201effaf5ced19abd9d0b45aa2c6
-'file:///tmp/aptarchive/pool/main/c/cdebconf-terminal/cdebconf-gtk-terminal_0.22_amd64.udeb' cdebconf-gtk-terminal_0.22_amd64.udeb 14734 MD5Sum:f9c3a7354560cb88e0396e2b7ba54363
-'file:///tmp/aptarchive/pool/main/c/cdebconf-terminal/cdebconf-newt-terminal_0.22_amd64.udeb' cdebconf-newt-terminal_0.22_amd64.udeb 4538 MD5Sum:20db6152fce5081fcbf49c7c08f21246"
+'file:/tmp/aptarchive/pool/main/c/cdebconf/cdebconf-udeb_0.185_amd64.udeb' cdebconf-udeb_0.185_amd64.udeb 77376 MD5Sum:e3883706fdbf54c2e5ea959c92b2d37f
+'file:/tmp/aptarchive/pool/main/c/cdebconf/cdebconf-gtk-udeb_0.185_amd64.udeb' cdebconf-gtk-udeb_0.185_amd64.udeb 27278 MD5Sum:a1bbbc1d4fb8e0615b5621abac021924
+'file:/tmp/aptarchive/pool/main/c/cdebconf/cdebconf-newt-udeb_0.185_amd64.udeb' cdebconf-newt-udeb_0.185_amd64.udeb 19192 MD5Sum:de27807f56dae2f2403b3322d5fe6bd2
+'file:/tmp/aptarchive/pool/main/g/glib2.0/libglib2.0-udeb_2.36.4-1_amd64.udeb' libglib2.0-udeb_2.36.4-1_amd64.udeb 1714604 MD5Sum:72da029f1bbb36057d874f1f82a5d00a
+'file:/tmp/aptarchive/pool/main/e/eglibc/libc6-udeb_2.17-92%2bb1_amd64.udeb' libc6-udeb_2.17-92+b1_amd64.udeb 1056000 MD5Sum:7fd7032eeeecf7f76eff79a0543fbd72
+'file:/tmp/aptarchive/pool/main/g/gtk%2b2.0/libgtk2.0-0-udeb_2.24.20-1_amd64.udeb' libgtk2.0-0-udeb_2.24.20-1_amd64.udeb 1643046 MD5Sum:25513478eb2e02e5766c0eea0b411ca9
+'file:/tmp/aptarchive/pool/main/v/vte/libvte9-udeb_0.28.2-5_amd64.udeb' libvte9-udeb_1%3a0.28.2-5_amd64.udeb 216968 MD5Sum:7da7201effaf5ced19abd9d0b45aa2c6
+'file:/tmp/aptarchive/pool/main/c/cdebconf-terminal/cdebconf-gtk-terminal_0.22_amd64.udeb' cdebconf-gtk-terminal_0.22_amd64.udeb 14734 MD5Sum:f9c3a7354560cb88e0396e2b7ba54363
+'file:/tmp/aptarchive/pool/main/c/cdebconf-terminal/cdebconf-newt-terminal_0.22_amd64.udeb' cdebconf-newt-terminal_0.22_amd64.udeb 4538 MD5Sum:20db6152fce5081fcbf49c7c08f21246"
diff --git a/test/integration/test-method-mirror b/test/integration/test-method-mirror
index 81a5585fd..ce44b86e3 100755
--- a/test/integration/test-method-mirror
+++ b/test/integration/test-method-mirror
@@ -148,7 +148,16 @@ Ign:2 http://localhost:${APTHTTPPORT}/failure2 stable InRelease
 Ign:3 http://localhost:${APTHTTPPORT}/failure unstable InRelease
   404  Not Found" head -n 5 aptupdate.output
 
-changetohttpswebserver
+msgmsg 'do not hang on' 'file:///'
+sed -i -e 's#file:/tmp#file:///tmp#' rootdir/etc/apt/sources.list.d/apt-test-unstable-deb*
+testrun '*_localhost_*' '*_aptarchive_mirror.txt_*'
+testsuccessequal "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Ign:2 http://localhost:${APTHTTPPORT}/failure2 stable InRelease
+  404  Not Found
+Ign:3 http://localhost:${APTHTTPPORT}/failure unstable InRelease
+  404  Not Found" head -n 5 aptupdate.output
+
+changetohttpswebserver --no-rewrite
 rm -f rootdir/etc/apt/sources.list.d/*-stable-*
 msgmsg 'fallback mirrors are used if needed' 'random'
 echo "http://localhost:${APTHTTPPORT}/failure2	priority:1
@@ -241,7 +250,7 @@ Reading package lists...
 Building dependency tree...
 Reading state information...
 All packages are up to date." apt update
-sed -i -e "s#+file:${APTARCHIVE}#+http://localhost:${APTHTTPPORT}#" rootdir/etc/apt/sources.list.d/*
+sed -i -e "s#+file:\(//\|\)${APTARCHIVE}#+http://localhost:${APTHTTPPORT}#" rootdir/etc/apt/sources.list.d/*
 testsuccess apt update
 testrundownload 'foo=2'