handle complete responses to https range requests

Servers might respond with a complete file either because they don't
support Ranges at all or the If-Range condition isn't statisfied, so we
have to parse the headers curl gets ourself to seek or truncate the file
we have so far.

This also finially adds the testcase testing a bunch of partial
situations for both, http and https - which is now all green.

Closes: 617643, 667699
LP: 1157943

3 files changed, 226 insertions, 22 deletions

diff --git a/test/integration/apt.pem b/test/integration/apt.pem
new file mode 100644
index 000000000..f48df054d
--- /dev/null
+++ b/test/integration/apt.pem
@@ -0,0 +1,49 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCt4R1Q2oYF6utL
+19GBhnlHW8L2BI7PRFWge/ZpqIZWsaFcb30FV86Z6aXXZmgfEJ2814ZZYD1IKeCe
+JsJpns7B4vYe1v64r995ZNBQAAKIYjICkKZOBgOphV+ChBfrCctVXgfLbMP6iBdH
+J02wHzSCCdZm0sdVl9tB5l/OyJU8Mb4KB3btBhfZfY2M6lU+FOjcXs1LOduUrv7K
+fZ+DEalvVGkomLtHtD0qb2vkqFrTjVCkziUVWhhxFFflt08oQ01Clxpl+uv7rOQo
+jtkJ1LrMuv7iPfaZ/z3qLiFxZYG1BCGEwTOKCtJo6bgFzXiN3q7Q5FFlmv851x2J
+Dn8C7Qm7AgMBAAECggEAE3q6vAofJZ6Ryadd8zLLd3ESQFl2XkX7icUZb/DPS/sO
+ZrqeuPCDVr7UM3NnisNjyHoktPKRKvp2DYGuGgMOiq4QgJf5ZVten8zpgWze28SU
+cbEe0HLgCifE8Ww2+b/ZJbEpEmMW+YQxh2khzO9SBJdxi4dliXM/vvw+E35pKZsB
+s6glrz6VQAxxa9fY4fLnB2DafHy+pUvRVw8gC6PCM9jXN9tMYAqztsJu7aaanNyT
+HX2UDWa8hxVx6t5UQZuxvst9N+RcEwmVCR2qlfZt/VRBRibBm62crEKbTD00mNHQ
+4AIDn3g6Y3SXpDlgtNpjLyBL3fODPIwqwGdblaSKkQKBgQDYXecu0Eda7kbR5ciW
+IAn8XOxsBIkkh8YVl2gRiiajRVoeiYBHaW9TyuQiaWrftiDQxB/N4G2focTXy/7O
+VJn6e/SUoO/ZGRw2GbTxLUQptgvFsejYCcW9XpC8MCwE/y2swiY7JM0WR8cV2nCk
+a/Cls6f1LjL13aFO0PAorEcahQKBgQDNuth6EHZVwfDgUuqhRw4HIIpfsfiA3UOd
+b5k/NsfQIev1YUqnfucgInNPDq2Jf8eTQw3TKaszo2DCjDffCsEgM09Tym143Bd6
+AsMuqAStsE3IEC7pnmh95l29/7mh4OuG5cp5JUx0Pi5PkuJ6ywA8P1rM1MB9Zf52
+NGJCo1pnPwKBgQCx/n4i+uDYo1DLd/dN2UmdvGwaaJjR3ohMVuQcGcSzaGg82u0W
+0lvtWOnYjVSIeXIBjHaFjW1hd1lSFdWms96AO9z3MHZf6NJWh0tdZNnAXqzMlBFz
+OIbdxJ/Y0OBFtA9FIesFmL7G54GWLr+f49Ry3Jr9jmYJ8au0BRqsux07aQKBgC4q
+CT2KyCMCO/z6XjAGc71hres/UlYIUI3ZZvfqYPfxRLNxO4FOVqq9UEajMomyJKSE
+3WtO5F3YAXRmZnskPKXvHZPdzqbaLGJykD298h7PewSzrPM7WpM1yD9ETPFoOTGy
+CrcYiYlkEpxEh5GqT8k1JjjkXLVG18zKgGoXocedAoGAQyU2DCNfxwzIJfFHKZEG
+zpni72cR68Tu3AhW/38vMR2ZPca4KzXrUA52T+j7vkQC38LHm/mzNXNP7Vya0PJ3
+WoYOcLtg2uFPh0P/35ArEzuNooLsvulgg1jsamPbF8KAvJZKZHr30hlC/JGYSBbV
+bnkzJTShsKzHIUiLtQ8Ja+E=
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIDezCCAmOgAwIBAgIJAJ39xapQo0vLMA0GCSqGSIb3DQEBBQUAMFMxCzAJBgNV
+BAYTAkRFMRMwEQYDVQQIDApTb21lLVN0YXRlMRswGQYDVQQKDBJBUFQgVGVzdGNh
+c2VzIEdtYkgxEjAQBgNVBAMMCWxvY2FsaG9zdDAgFw0xMzA5MTYwODQ4MzVaGA80
+NzUxMDgxMzA4NDgzNVowUzELMAkGA1UEBhMCREUxEzARBgNVBAgMClNvbWUtU3Rh
+dGUxGzAZBgNVBAoMEkFQVCBUZXN0Y2FzZXMgR21iSDESMBAGA1UEAwwJbG9jYWxo
+b3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAreEdUNqGBerrS9fR
+gYZ5R1vC9gSOz0RVoHv2aaiGVrGhXG99BVfOmeml12ZoHxCdvNeGWWA9SCngnibC
+aZ7OweL2Htb+uK/feWTQUAACiGIyApCmTgYDqYVfgoQX6wnLVV4Hy2zD+ogXRydN
+sB80ggnWZtLHVZfbQeZfzsiVPDG+Cgd27QYX2X2NjOpVPhTo3F7NSznblK7+yn2f
+gxGpb1RpKJi7R7Q9Km9r5Kha041QpM4lFVoYcRRX5bdPKENNQpcaZfrr+6zkKI7Z
+CdS6zLr+4j32mf896i4hcWWBtQQhhMEzigrSaOm4Bc14jd6u0ORRZZr/OdcdiQ5/
+Au0JuwIDAQABo1AwTjAdBgNVHQ4EFgQUhd26E7ykEYRTDbgMzkYtFtENhSkwHwYD
+VR0jBBgwFoAUhd26E7ykEYRTDbgMzkYtFtENhSkwDAYDVR0TBAUwAwEB/zANBgkq
+hkiG9w0BAQUFAAOCAQEAWcyMKi0Vc4beGV7w4Qft0/2P68jjMlQRdgkz+gGXbMVr
+//KhqR3PbgFmHHpUsZ718AHeerNNdfFzOUptiAiOqH2muyAGdeWCxJ8KcU0sic8x
+/h3TOzMYfEozhgMSJp9YW1z655uHcb15S7jb4zZwXwGyQzxwXT35SKj2mCqSbjIb
+G987DGI+MtyoGRXhIwnBEsGTI1ck3NoeXBJ/tS/Ma8gUUC2xldMSprtHjeUHvZV2
+iz/HTqGlMLGW96AVeZiFNiC1fJ6pvref2XW5MkkvQm8tOi2cSrwJc9CgnCpCxkLp
+liRsbwAduwkA26XzEomMR7yyYS5pm0Eu0cO9X39FKQ==
+-----END CERTIFICATE-----
diff --git a/test/integration/framework b/test/integration/framework
index 4003d932c..a2bb871cc 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -795,6 +795,13 @@ signreleasefiles() {
 	msgdone "info"
 }
 
+rewritesourceslist() {
+	local APTARCHIVE="file://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive")"
+	for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
+		sed -i $LIST -e "s#$APTARCHIVE#${1}#" -e "s#http://localhost:8080/#${1}#" -e "s#http://localhost:4433/#${1}#"
+	done
+}
+
 changetowebserver() {
 	local LOG='/dev/null'
 	if test -x ${BUILDDIRECTORY}/aptwebserver; then
@@ -806,31 +813,32 @@ changetowebserver() {
 		fi
 		addtrap "kill $PID;"
 		cd - > /dev/null
-	elif [ $# -gt 0 ]; then
-		msgdie 'Need the aptwebserver when passing arguments for the webserver'
-	elif which weborf > /dev/null; then
-		weborf -xb aptarchive/ >$LOG 2>&1 &
-		addtrap "kill $!;"
-	elif which gatling > /dev/null; then
-		cd aptarchive
-		gatling -p 8080 -F -S >$LOG 2>&1 &
-		addtrap "kill $!;"
-		cd - > /dev/null
-	elif which lighttpd > /dev/null; then
-		echo "server.document-root = \"$(readlink -f ./aptarchive)\"
-server.port = 8080
-server.stat-cache-engine = \"disable\"" > lighttpd.conf
-		lighttpd -t -f lighttpd.conf >/dev/null || msgdie 'Can not change to webserver: our lighttpd config is invalid'
-		lighttpd -D -f lighttpd.conf >$LOG 2>&1 &
-		addtrap "kill $!;"
 	else
 		msgdie 'You have to build aptwerbserver or install a webserver'
 	fi
-	local APTARCHIVE="file://$(readlink -f ./aptarchive)"
-	for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
-		sed -i $LIST -e "s#$APTARCHIVE#http://localhost:8080/#"
-	done
-	return 0
+	if [ "$1" != '--no-rewrite' ]; then
+		rewritesourceslist 'http://localhost:8080/'
+	fi
+}
+
+changetohttpswebserver() {
+	if ! which stunnel4 >/dev/null; then
+		msgdie 'You need to install stunnel4 for https testcases'
+	fi
+	if [ ! -e "${TMPWORKINGDIRECTORY}/aptarchive/aptwebserver.pid" ]; then
+		changetowebserver --no-rewrite
+	fi
+	echo "pid = ${TMPWORKINGDIRECTORY}/aptarchive/stunnel.pid
+cert = ${TESTDIRECTORY}/apt.pem
+
+[https]
+accept = 4433
+connect = 8080
+" > ${TMPWORKINGDIRECTORY}/stunnel.conf
+	stunnel4 "${TMPWORKINGDIRECTORY}/stunnel.conf"
+	local PID="$(cat ${TMPWORKINGDIRECTORY}/aptarchive/stunnel.pid)"
+	addtrap 'prefix' "kill ${PID};"
+	rewritesourceslist 'https://localhost:4433/'
 }
 
 changetocdrom() {
@@ -848,6 +856,46 @@ changetocdrom() {
 	find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list' -delete
 }
 
+downloadfile() {
+	PROTO="$(echo "$1" | cut -d':' -f 1)"
+	local DOWNLOG="${TMPWORKINGDIRECTORY}/download.log"
+	rm -f "$DOWNLOG"
+	touch "$DOWNLOG"
+	{
+		echo "601 Configuration
+Config-Item: Acquire::https::CaInfo=${TESTDIR}/apt.pem
+Config-Item: Debug::Acquire::${PROTO}=1
+
+600 Acquire URI
+URI: $1
+Filename: ${2}
+"
+		# simple worker keeping stdin open until we are done (201) or error (400)
+		# and requesting new URIs on try-agains/redirects inbetween
+		{ tail -n 999 -f "$DOWNLOG" & echo "TAILPID: $!"; } | while read f1 f2; do
+			if [ "$f1" = 'TAILPID:' ]; then
+				TAILPID="$f2"
+			elif [ "$f1" = 'New-URI:' ]; then
+				echo "600 Acquire URI
+URI: $f2
+Filename: ${2}
+"
+			elif [ "$f1" = '201' ] || [ "$f1" = '400' ]; then
+				# tail would only die on next read – which never happens
+				test -z "$TAILPID" || kill -s HUP "$TAILPID"
+				break
+			fi
+		done
+	} | LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/methods/${PROTO} 2>&1 | tee "$DOWNLOG"
+	rm "$DOWNLOG"
+	# only if the file exists the download was successful
+	if [ -e "$2" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
 checkdiff() {
 	local DIFFTEXT="$($(which diff) -u $* | sed -e '/^---/ d' -e '/^+++/ d' -e '/^@@/ d')"
 	if [ -n "$DIFFTEXT" ]; then
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
new file mode 100755
index 000000000..8d1c51ae0
--- /dev/null
+++ b/test/integration/test-partial-file-support
@@ -0,0 +1,107 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+changetowebserver
+
+copysource() {
+	dd if="$1" bs=1 count="$2" of="$3" 2>/dev/null
+	touch -d "$(stat --format '%y' "${TESTFILE}")" "$3"
+}
+
+testdownloadfile() {
+	local DOWNLOG='download-testfile.log'
+	rm -f "$DOWNLOG"
+	msgtest "Testing download of file $2 with" "$1"
+	if ! downloadfile "$2" "$3" > "$DOWNLOG"; then
+		cat "$DOWNLOG"
+		msgfail
+	else
+		msgpass
+	fi
+	cat "$DOWNLOG" | while read field hash; do
+		local EXPECTED
+		case "$field" in
+		'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";;
+		'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";;
+		'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";;
+		'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";;
+		*) continue;;
+		esac
+		if [ "$4" = '=' ]; then
+			msgtest 'Test downloaded file for correct' "$field"
+		else
+			msgtest 'Test downloaded file does not match in' "$field"
+		fi
+		if [ "$EXPECTED" "$4" "$hash" ]; then
+			msgpass
+		else
+			cat "$DOWNLOG"
+			msgfail "expected: $EXPECTED ; got: $hash"
+		fi
+	done
+}
+
+testwebserverlaststatuscode() {
+	STATUS="$(mktemp)"
+	addtrap "rm $STATUS;"
+	msgtest 'Test last status code from the webserver was' "$1"
+	downloadfile "http://localhost:8080/_config/find/aptwebserver::last-status-code" "$STATUS" >/dev/null
+	if [ "$(cat "$STATUS")" = "$1" ]; then
+		msgpass
+	else
+		cat download-testfile.log
+		msgfail "Status was $(cat "$STATUS")"
+	fi
+}
+
+
+TESTFILE='aptarchive/testfile'
+cp -a ${TESTDIR}/framework $TESTFILE
+
+testrun() {
+	downloadfile "$1/_config/set/aptwebserver::support::range/true" '/dev/null' >/dev/null
+	testwebserverlaststatuscode '200'
+
+	copysource $TESTFILE 0 ./testfile
+	testdownloadfile 'no data' "${1}/testfile" './testfile' '='
+	testwebserverlaststatuscode '200'
+
+	copysource $TESTFILE 20 ./testfile
+	testdownloadfile 'valid partial data' "${1}/testfile" './testfile' '='
+	testwebserverlaststatuscode '206'
+
+	copysource /dev/zero 20 ./testfile
+	testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
+	testwebserverlaststatuscode '206'
+
+	copysource $TESTFILE 1M ./testfile
+	testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '='
+	testwebserverlaststatuscode '416'
+
+	copysource /dev/zero 1M ./testfile
+	testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='
+	testwebserverlaststatuscode '200'
+
+	copysource /dev/zero 20 ./testfile
+	touch ./testfile
+	testdownloadfile 'old data' "${1}/testfile" './testfile' '='
+	testwebserverlaststatuscode '200'
+
+	downloadfile "$1/_config/set/aptwebserver::support::range/false" '/dev/null' >/dev/null
+	testwebserverlaststatuscode '200'
+
+	copysource $TESTFILE 20 ./testfile
+	testdownloadfile 'no server support' "${1}/testfile" './testfile' '='
+	testwebserverlaststatuscode '200'
+}
+
+testrun 'http://localhost:8080'
+
+changetohttpswebserver
+
+testrun 'https://localhost:4433'