From 9fc0b435593839de47098212f0ae5f15b6263099 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Mon, 17 Nov 2014 15:06:35 +0100
Subject: close leaking slave fd after setting up pty magic

The fd moves out of scope here anyway, so we should close it properly
instead of leaking it which will tickle down to dpkg maintainer scripts.

Closes: 767774
---
 test/integration/framework | 78 +++++++++++++++++++++++++++++++++++++---------
 1 file changed, 64 insertions(+), 14 deletions(-)

(limited to 'test/integration/framework')

diff --git a/test/integration/framework b/test/integration/framework
index 7923e23d9..df1942ff9 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -325,6 +325,59 @@ configdpkg() {
 	fi
 }
 
+configdpkgnoopchroot() {
+	# create a library to noop chroot() and rewrite maintainer script executions
+	# via execvp() as used by dpkg as we don't want our rootdir to be a fullblown
+	# chroot directory dpkg could chroot into to execute the maintainer scripts
+	msgtest 'Building library to preload to make maintainerscript work in' 'dpkg'
+	cat << EOF > noopchroot.c
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <dlfcn.h>
+
+static char * chrootdir = NULL;
+
+int chroot(const char *path) {
+	printf("WARNING: CHROOTing to %s was ignored!\n", path);
+	free(chrootdir);
+	chrootdir = strdup(path);
+	return 0;
+}
+int execvp(const char *file, char *const argv[]) {
+	static int (*func_execvp) (const char *, char * const []) = NULL;
+	if (func_execvp == NULL)
+		func_execvp = (int (*) (const char *, char * const [])) dlsym(RTLD_NEXT, "execvp");
+	if (chrootdir == NULL || strncmp(file, "/var/lib/dpkg/", strlen("/var/lib/dpkg/")) != 0)
+		return func_execvp(file, argv);
+	printf("REWRITE execvp call %s into %s\n", file, chrootdir);
+	char newfile[strlen(chrootdir) + strlen(file)];
+	strcpy(newfile, chrootdir);
+	strcat(newfile, file);
+	return func_execvp(newfile, argv);
+}
+EOF
+	testsuccess --nomsg gcc -fPIC -shared -o noopchroot.so noopchroot.c -ldl
+
+	mkdir -p "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/"
+	DPKG="${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg"
+	echo "#!/bin/sh
+if [ -n \"\$LD_PRELOAD\" ]; then
+	export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so \${LD_PRELOAD}\"
+else
+	export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so\"
+fi
+dpkg \"\$@\"" > $DPKG
+	chmod +x $DPKG
+	sed -ie "s|^DPKG::options:: \"dpkg\";\$|DPKG::options:: \"$DPKG\";|" aptconfig.conf
+}
+
+configallowinsecurerepositories() {
+    echo "Acquire::AllowInsecureRepositories \"$1\";" >  rootdir/etc/apt/apt.conf.d/allow-insecure-repositories.conf
+
+}
+
 configcompression() {
 	while [ -n "$1" ]; do
 		case "$1" in
@@ -442,7 +495,7 @@ buildsimplenativepackage() {
 	fi
 	local BUILDDIR=${TMPWORKINGDIRECTORY}/incoming/${NAME}-${VERSION}
 
-	msgninfo "Build package ${NAME} in ${VERSION} for ${RELEASE} in ${DISTSECTION}… "
+	msgtest "Build source package in version ${VERSION} for ${RELEASE} in ${DISTSECTION}" "$NAME"
 	mkdir -p $BUILDDIR/debian/source
 	echo "* most suckless software product ever" > ${BUILDDIR}/FEATURES
 	echo "#!/bin/sh
@@ -474,7 +527,10 @@ Package: $NAME" >> ${BUILDDIR}/debian/control
 	echo "Description: $DESCRIPTION" >> ${BUILDDIR}/debian/control
 
 	echo '3.0 (native)' > ${BUILDDIR}/debian/source/format
-	(cd ${BUILDDIR}/..; dpkg-source -b ${NAME}-${VERSION} 2>&1) | sed -n 's#^dpkg-source: info: building [^ ]\+ in ##p' \
+	cd ${BUILDDIR}/..
+	testsuccess --nomsg dpkg-source -b ${NAME}-${VERSION}
+	cd - >/dev/null
+	sed -n 's#^dpkg-source: info: building [^ ]\+ in ##p' ${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output \
 		| while read SRC; do
 		echo "pool/${SRC}" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.srclist
 #		if expr match "${SRC}" '.*\.dsc' >/dev/null 2>&1; then
@@ -486,6 +542,7 @@ Package: $NAME" >> ${BUILDDIR}/debian/control
 	done
 
 	for arch in $(getarchitecturesfromcommalist "$ARCH"); do
+		msgtest "Build binary package for ${RELEASE} in ${SECTION}" "$NAME"
 		rm -rf ${BUILDDIR}/debian/tmp
 		mkdir -p ${BUILDDIR}/debian/tmp/DEBIAN ${BUILDDIR}/debian/tmp/usr/share/doc/${NAME} ${BUILDDIR}/debian/tmp/usr/bin
 		cp ${BUILDDIR}/debian/copyright ${BUILDDIR}/debian/changelog ${BUILDDIR}/FEATURES ${BUILDDIR}/debian/tmp/usr/share/doc/${NAME}
@@ -499,11 +556,7 @@ Package: $NAME" >> ${BUILDDIR}/debian/control
 		local LOG="${BUILDDIR}/../${NAME}_${VERSION}_${arch}.dpkg-deb.log"
 		# ensure the right permissions as dpkg-deb ensists
 		chmod 755 ${BUILDDIR}/debian/tmp/DEBIAN
-		if ! dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/.. >$LOG 2>&1; then
-			cat $LOG
-			false
-		fi
-		rm $LOG
+		testsuccess --nomsg dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/..
 		echo "pool/${NAME}_${VERSION}_${arch}.deb" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.pkglist
 	done
 
@@ -521,15 +574,13 @@ buildpackage() {
 	local ARCH=$(getarchitecture $4)
 	local PKGNAME="$(echo "$BUILDDIR" | grep -o '[^/]*$')"
 	local BUILDLOG="$(readlink -f "${BUILDDIR}/../${PKGNAME}_${RELEASE}_${SECTION}.dpkg-bp.log")"
-	msgninfo "Build package ${PKGNAME} for ${RELEASE} in ${SECTION}… "
+	msgtest "Build package for ${RELEASE} in ${SECTION}" "$PKGNAME"
 	cd $BUILDDIR
 	if [ "$ARCH" = "all" ]; then
 		ARCH="$(dpkg-architecture -qDEB_HOST_ARCH 2> /dev/null)"
 	fi
-	if ! dpkg-buildpackage -uc -us -a$ARCH >$BUILDLOG 2>&1 ; then
-		cat $BUILDLOG
-		false
-	fi
+	testsuccess --nomsg dpkg-buildpackage -uc -us -a$ARCH
+	cp ${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output $BUILDLOG
 	local PKGS="$(grep '^dpkg-deb: building package' $BUILDLOG | cut -d'/' -f 2 | sed -e "s#'\.##")"
 	local SRCS="$(grep '^dpkg-source: info: building' $BUILDLOG | grep -o '[a-z0-9._+~-]*$')"
 	cd - > /dev/null
@@ -539,7 +590,6 @@ buildpackage() {
 	for SRC in $SRCS; do
 		echo "pool/${SRC}" >> ${TMPWORKINGDIRECTORY}/incoming/${RELEASE}.${SECTION}.srclist
 	done
-	msgdone "info"
 }
 
 buildaptarchive() {
@@ -1072,7 +1122,7 @@ testequal() {
 	if [ -n "$MSG" ]; then
 		msgtest "$MSG" "$*"
 	fi
-	$* 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
+	"$@" 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
 }
 
 testequalor2() {
-- 
cgit v1.2.3-70-g09d2


From 92e8c1ff287ab829de825e00cdf94744e699ff97 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Sat, 29 Nov 2014 17:59:52 +0100
Subject: dispose http(s) 416 error page as non-content
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Real webservers (like apache) actually send an error page with a 416
response, but our client didn't expect it leaving the page on the socket
to be parsed as response for the next request (http) or as file content
(https), which isn't what we want at all… Symptom is a "Bad header line"
as html usually doesn't parse that well to an http-header.

This manifests itself e.g. if we have a complete file (or larger) in
partial/ which isn't discarded by If-Range as the server doesn't support
it (or it is just newer, think: mirror rotation).
It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a,
which removed the filesize - 1 trick, but this had its own problems…

To properly test this our webserver gains the ability to reply with
transfer-encoding: chunked as most real webservers will use it to send
the dynamically generated error pages.

(The tests and their binary helpers had to be slightly modified to
apply, but the patch to fix the issue itself is unchanged.)

Closes: 768797
---
 cmdline/apt-helper.cc                      |  35 ++++--
 methods/http.cc                            |   2 +
 methods/https.cc                           |  12 +-
 methods/server.cc                          |  26 +++--
 methods/server.h                           |   5 +-
 test/integration/framework                 |   6 +-
 test/integration/test-apt-helper           |  24 ++--
 test/integration/test-partial-file-support |  62 +++++++++-
 test/interactive-helper/aptwebserver.cc    | 181 ++++++++++++++++++-----------
 9 files changed, 241 insertions(+), 112 deletions(-)

(limited to 'test/integration/framework')

diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index dd43ea1bc..63f70983c 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,23 +48,34 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
-
    pkgAcquire Fetcher;
    AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
    Fetcher.Setup(&Stat);
-   std::string download_uri = CmdL.FileList[1];
-   std::string targetfile = CmdL.FileList[2];
-   std::string hash;
-   if (CmdL.FileSize() > 3)
-      hash = CmdL.FileList[3];
-   // we use download_uri as descr and targetfile as short-descr
-   new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, 
-                  "dest-dir-ignored", targetfile);
-   Fetcher.Run();
+
+   size_t fileind = 0;
+   std::vector<std::string> targetfiles;
+   while (fileind + 2 <= CmdL.FileSize())
+   {
+      std::string download_uri = CmdL.FileList[fileind + 1];
+      std::string targetfile = CmdL.FileList[fileind + 2];
+      std::string hash;
+      if (CmdL.FileSize() > fileind + 3)
+	 hash = CmdL.FileList[fileind + 3];
+      // we use download_uri as descr and targetfile as short-descr
+      new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile,
+	    "dest-dir-ignored", targetfile);
+      targetfiles.push_back(targetfile);
+      fileind += 3;
+   }
+
    bool Failed = false;
-   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true ||
-	 FileExists(targetfile) == false)
+   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true)
       return _error->Error(_("Download Failed"));
+   if (targetfiles.empty() == false)
+      for (std::vector<std::string>::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f)
+	 if (FileExists(*f) == false)
+	    return _error->Error(_("Download Failed"));
+
    return true;
 }
 
diff --git a/methods/http.cc b/methods/http.cc
index f2a4a4db6..1b996db98 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -440,6 +440,8 @@ bool HttpServerState::RunData(FileFd * const File)
          loss of the connection means we are done */
       if (Encoding == Closes)
 	 In.Limit(-1);
+      else if (JunkSize != 0)
+	 In.Limit(JunkSize);
       else
 	 In.Limit(Size - StartPos);
       
diff --git a/methods/https.cc b/methods/https.cc
index 0499af0c5..65a744e2a 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -59,6 +59,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       {
          me->Server->Result = 200;
 	 me->Server->StartPos = me->Server->Size;
+	 // the actual size is not important for https as curl will deal with it
+	 // by itself and e.g. doesn't bother us with transport-encoding…
+	 me->Server->JunkSize = std::numeric_limits<unsigned long long>::max();
       }
       else
 	 me->Server->StartPos = 0;
@@ -76,13 +79,18 @@ size_t
 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    HttpsMethod *me = (HttpsMethod *)userp;
+   size_t buffer_size = size * nmemb;
+   // we don't need to count the junk here, just drop anything we get as
+   // we don't always know how long it would be, e.g. in chunked encoding.
+   if (me->Server->JunkSize != 0)
+      return buffer_size;
 
    if (me->Res.Size == 0)
       me->URIStart(me->Res);
-   if(me->File->Write(buffer, size*nmemb) != true)
+   if(me->File->Write(buffer, buffer_size) != true)
       return false;
 
-   return size*nmemb;
+   return buffer_size;
 }
 
 int
diff --git a/methods/server.cc b/methods/server.cc
index 92d94e638..cb0341d5f 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
    Minor = 0; 
    Result = 0; 
    Size = 0; 
+   JunkSize = 0;
    StartPos = 0;
    Encoding = Closes;
    HaveContent = false;
@@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line)
 	 Encoding = Stream;
       HaveContent = true;
 
-      // The length is already set from the Content-Range header
-      if (StartPos != 0)
-	 return true;
+      unsigned long long * SizePtr = &Size;
+      if (Result == 416)
+	 SizePtr = &JunkSize;
 
-      Size = strtoull(Val.c_str(), NULL, 10);
-      if (Size >= std::numeric_limits<unsigned long long>::max())
+      *SizePtr = strtoull(Val.c_str(), NULL, 10);
+      if (*SizePtr >= std::numeric_limits<unsigned long long>::max())
 	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
-      else if (Size == 0)
+      else if (*SizePtr == 0)
 	 HaveContent = false;
       return true;
    }
@@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line)
 
       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
-      {
-	 StartPos = 1; // ignore Content-Length, it would override Size
-	 HaveContent = false;
-      }
+	 ; // we got the expected filesize which is all we wanted
       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
@@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
 	 if ((unsigned long long)SBuf.st_size == Server->Size)
 	 {
 	    // the file is completely downloaded, but was not moved
+	    if (Server->HaveContent == true)
+	    {
+	       // Send to error page to dev/null
+	       FileFd DevNull("/dev/null",FileFd::WriteExists);
+	       Server->RunData(&DevNull);
+	    }
+	    Server->HaveContent = false;
 	    Server->StartPos = Server->Size;
 	    Server->Result = 200;
-	    Server->HaveContent = false;
 	 }
 	 else if (unlink(Queue->DestFile.c_str()) == 0)
 	 {
diff --git a/methods/server.h b/methods/server.h
index f5e68d902..1b81e3549 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,7 +34,8 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
-   unsigned long long Size;
+   unsigned long long Size; // size of the usable content (aka: the file)
+   unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
    time_t Date;
    bool HaveContent;
@@ -71,7 +72,7 @@ struct ServerState
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
-   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
+   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
 		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true;};
    virtual bool WriteResponse(std::string const &Data) = 0;
diff --git a/test/integration/framework b/test/integration/framework
index df1942ff9..ac482a7a0 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1064,8 +1064,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 }
 
 downloadfile() {
-	local PROTO="$(echo "$1" | cut -d':' -f 1 )"
-	apthelper -o Debug::Acquire::${PROTO}=1 \
+	local PROTO="${1%%:*}"
+	apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
 		download-file "$1" "$2" 2>&1 || true
 	# only if the file exists the download was successful
 	if [ -e "$2" ]; then
@@ -1221,7 +1221,7 @@ testsuccess() {
 		msgtest 'Test for successful execution of' "$*"
 	fi
 	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
-	if $@ >${OUTPUT} 2>&1; then
+	if "$@" >${OUTPUT} 2>&1; then
 		msgpass
 	else
 		echo >&2
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index c749224ca..31e471677 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -10,34 +10,36 @@ configarchitecture "i386"
 changetohttpswebserver
 
 test_apt_helper_download() {
-    echo "foo" > aptarchive/foo
+    echo 'foo' > aptarchive/foo
+    echo 'bar' > aptarchive/foo2
 
     msgtest 'apt-file download-file md5sum'
-    apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00
     testfileequal foo2 'foo'
 
     msgtest 'apt-file download-file sha1'
-    apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
     testfileequal foo1 'foo'
 
     msgtest 'apt-file download-file sha256'
-    apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c
     testfileequal foo3 'foo'
 
     msgtest 'apt-file download-file no-hash'
-    apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo4
     testfileequal foo4 'foo'
     
     msgtest 'apt-file download-file wrong hash'
-    if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then
-        msgpass
-    else
-        msgfail
-    fi
-    testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
+    testfailure --nomsg apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc
+    testfileequal rootdir/tmp/testfailure.output 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
 
 E: Download Failed'
     testfileequal foo5.FAILED 'foo'
+
+    msgtest 'apt-file download-file md5sum sha1'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo6 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 http://localhost:8080/foo2 foo7 SHA1:e242ed3bffccdf271b7fbaf34ed72d089537b42f
+    testfileequal foo6 'foo'
+    testfileequal foo7 'bar'
 }
 
 test_apt_helper_detect_proxy() {
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 5ab326def..160d451b6 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -24,13 +24,25 @@ testdownloadfile() {
 	else
 		msgpass
 	fi
-	cat "$DOWNLOADLOG" | while read field hash; do
+	sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log
+	testsuccess test -s receivedhashes.log
+	local HASHES_OK=0
+	local HASHES_BAD=0
+	while read field hash; do
 		local EXPECTED
 		case "$field" in
 		'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";;
+		'Checksum-FileSize-Hash:')
+			#filesize is too weak to check for !=
+			if [ "$4" = '=' ]; then
+				EXPECTED="$(stat -c '%s' "$TESTFILE")"
+			else
+				continue
+			fi
+			;;
 		*) continue;;
 		esac
 		if [ "$4" = '=' ]; then
@@ -40,15 +52,41 @@ testdownloadfile() {
 		fi
 		if [ "$EXPECTED" "$4" "$hash" ]; then
 			msgpass
+			HASHES_OK=$((HASHES_OK+1));
 		else
-			cat >&2 "$DOWNLOADLOG"
 			msgfail "expected: $EXPECTED ; got: $hash"
+			HASHES_BAD=$((HASHES_BAD+1));
 		fi
-	done
+	done < receivedhashes.log
+	msgtest 'At least one good hash and no bad ones'
+	if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then
+		cat >&2 "$DOWNLOADLOG"
+		msgfail
+	else
+		msgpass
+	fi
 }
 
 TESTFILE='aptarchive/testfile'
 cp -a ${TESTDIR}/framework $TESTFILE
+cp -a ${TESTDIR}/framework "${TESTFILE}2"
+
+followuprequest() {
+	local DOWN='./testfile'
+
+	copysource $TESTFILE 1M $DOWN
+	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
+	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+
+	copysource $TESTFILE 1M $DOWN
+	copysource "${TESTFILE}2" 20 "${DOWN}2"
+	msgtest 'Testing download of files with' 'completely downloaded file + partial file'
+	testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \
+		download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2"
+	testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output'
+	testsuccess diff -u "$TESTFILE" "${DOWN}"
+	testsuccess diff -u "${DOWN}" "${DOWN}2"
+}
 
 testrun() {
 	webserverconfig 'aptwebserver::support::range' 'true'
@@ -65,9 +103,11 @@ testrun() {
 	testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
 	testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 
-	copysource $TESTFILE 1M ./testfile
-	testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '='
-	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'true'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
 
 	copysource /dev/zero 1M ./testfile
 	testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='
@@ -85,8 +125,18 @@ testrun() {
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 
+msgmsg 'http: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'http://localhost:8080'
+msgmsg 'http: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'http://localhost:8080'
 
 changetohttpswebserver
 
+msgmsg 'https: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'https://localhost:4433'
+msgmsg 'https: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'https://localhost:4433'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 34476e1af..cd52da692 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -19,6 +19,8 @@
 #include <sys/stat.h>
 #include <time.h>
 #include <unistd.h>
+
+#include <algorithm>
 #include <iostream>
 #include <sstream>
 #include <list>
@@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode)		/*{{{*/
    return NULL;
 }
 									/*}}}*/
+static bool chunkedTransferEncoding(std::list<std::string> const &headers) {
+   if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end())
+      return true;
+   if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true)
+      return true;
+   return false;
+}
 static void addFileHeaders(std::list<std::string> &headers, FileFd &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.FileSize();
-   headers.push_back(contentlength.str());
-
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.FileSize();
+      headers.push_back(contentlength.str());
+   }
    std::string lastmodified("Last-Modified: ");
    lastmodified.append(TimeRFC1123(data.ModificationTime()));
    headers.push_back(lastmodified);
@@ -92,9 +103,12 @@ static void addFileHeaders(std::list<std::string> &headers, FileFd &data)/*{{{*/
 									/*}}}*/
 static void addDataHeaders(std::list<std::string> &headers, std::string &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.size();
-   headers.push_back(contentlength.str());
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.size();
+      headers.push_back(contentlength.str());
+   }
 }
 									/*}}}*/
 static bool sendHead(int const client, int const httpcode, std::list<std::string> &headers)/*{{{*/
@@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list<std::string
    date.append(TimeRFC1123(time(NULL)));
    headers.push_back(date);
 
+   if (chunkedTransferEncoding(headers) == true)
+      headers.push_back("Transfer-Encoding: chunked");
+
    std::clog << ">>> RESPONSE to " << client << " >>>" << std::endl;
    bool Success = true;
    for (std::list<std::string>::const_iterator h = headers.begin();
@@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list<std::string
    return Success;
 }
 									/*}}}*/
-static bool sendFile(int const client, FileFd &data)			/*{{{*/
+static bool sendFile(int const client, std::list<std::string> const &headers, FileFd &data)/*{{{*/
 {
    bool Success = true;
+   bool const chunked = chunkedTransferEncoding(headers);
    char buffer[500];
    unsigned long long actual = 0;
    while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true)
    {
       if (actual == 0)
 	 break;
-      Success &= FileFd::Write(client, buffer, actual);
+
+      if (chunked == true)
+      {
+	 std::string size;
+	 strprintf(size, "%llX\r\n", actual);
+	 Success &= FileFd::Write(client, size.c_str(), size.size());
+	 Success &= FileFd::Write(client, buffer, actual);
+	 Success &= FileFd::Write(client, "\r\n", strlen("\r\n"));
+      }
+      else
+	 Success &= FileFd::Write(client, buffer, actual);
+   }
+   if (chunked == true)
+   {
+      char const * const finish = "0\r\n\r\n";
+      Success &= FileFd::Write(client, finish, strlen(finish));
    }
    if (Success == false)
-      std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl;
+      std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl;
    return Success;
 }
 									/*}}}*/
-static bool sendData(int const client, std::string const &data)		/*{{{*/
+static bool sendData(int const client, std::list<std::string> const &headers, std::string const &data)/*{{{*/
 {
-   if (FileFd::Write(client, data.c_str(), data.size()) == false)
+   if (chunkedTransferEncoding(headers) == true)
+   {
+      unsigned long long const ullsize = data.length();
+      std::string size;
+      strprintf(size, "%llX\r\n", ullsize);
+      char const * const finish = "\r\n0\r\n\r\n";
+      if (FileFd::Write(client, size.c_str(), size.length()) == false ||
+	    FileFd::Write(client, data.c_str(), ullsize) == false ||
+	    FileFd::Write(client, finish, strlen(finish)) == false)
+      {
+	 std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl;
+	 return false;
+      }
+   }
+   else if (FileFd::Write(client, data.c_str(), data.size()) == false)
    {
       std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl;
       return false;
@@ -157,34 +204,38 @@ static bool sendData(int const client, std::string const &data)		/*{{{*/
 }
 									/*}}}*/
 static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list<std::string> &headers)
 {
-   std::list<std::string> headers;
    std::string response("<html><head><title>");
    response.append(httpcodeToStr(httpcode)).append("</title></head>");
    response.append("<body><h1>").append(httpcodeToStr(httpcode)).append("</h1>");
    if (httpcode != 200)
-   {
-      if (error.empty() == false)
-	 response.append("<p><em>Error</em>: ").append(error).append("</p>");
-      response.append("This error is a result of the request: <pre>");
-   }
+      response.append("<p><em>Error</em>: ");
+   else
+      response.append("<p><em>Success</em>: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("</p>This error is a result of the request: <pre>");
    else
-   {
-      if (error.empty() == false)
-	 response.append("<p><em>Success</em>: ").append(error).append("</p>");
       response.append("The successfully executed operation was requested by: <pre>");
-   }
    response.append(request).append("</pre></body></html>");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list<std::string> &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -221,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -263,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list<std::string> &headers)
 {
-   std::list<std::string> headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -311,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string &params, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list<std::string> &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -333,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -345,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -360,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -371,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
       // strip the host from the request to make it an absolute path
@@ -379,7 +429,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -398,7 +448,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list<std::string> headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -434,46 +485,45 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector<std::string> const &parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector<std::string> parts, std::list<std::string> &headers)
 {
    size_t const pcount = parts.size();
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list<std::string> headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list<std::string> headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -482,18 +532,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector<std::string> messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list<std::string> headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector<std::string>::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list<std::string> headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -502,7 +556,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector<std::string> parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -534,7 +588,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -553,7 +607,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -609,17 +663,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -628,22 +681,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From e18f6133b254db9e1dc7b202366b067b15a68123 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Wed, 10 Dec 2014 22:26:59 +0100
Subject: do not make PTY slave the controlling terminal
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If we have no controlling terminal opening a terminal will make this
terminal our controller, which is a serious problem if this happens to
be the pseudo terminal we created to run dpkg in as we will close this
terminal at the end hanging ourself up in the process…

The offending open is the one we do to have at least one slave fd open
all the time, but for good measure, we apply the flag also to the slave
fd opening in the child process as we set the controlling terminal
explicitely here.

This is a regression from 150bdc9ca5d656f9fba94d37c5f4f183b02bd746 with
the slight twist that this usecase was silently broken before in that it
wasn't logging the output in term.log (as a pseudo terminal wasn't
created).

Closes: 772641
---
 apt-pkg/deb/dpkgpm.cc                              |   4 +-
 test/integration/framework                         |   2 +-
 .../test-no-fds-leaked-to-maintainer-scripts       | 109 +++++++++++++--------
 3 files changed, 70 insertions(+), 45 deletions(-)

(limited to 'test/integration/framework')

diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index e36a52c3e..93a007d14 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -1131,7 +1131,7 @@ void pkgDPkgPM::StartPtyMagic()
 	       on kfreebsd we get an incorrect ("step like") output then while it has
 	       no problem with closing all references… so to avoid platform specific
 	       code here we combine both and be happy once more */
-	    d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC);
+	    d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC | O_NOCTTY);
 	 }
       }
    }
@@ -1163,7 +1163,7 @@ void pkgDPkgPM::SetupSlavePtyMagic()
    if (setsid() == -1)
       _error->FatalE("setsid", "Starting a new session for child failed!");
 
-   int const slaveFd = open(d->slave, O_RDWR);
+   int const slaveFd = open(d->slave, O_RDWR | O_NOCTTY);
    if (slaveFd == -1)
       _error->FatalE("open", _("Can not write log (%s)"), _("Is /dev/pts mounted?"));
    else if (ioctl(slaveFd, TIOCSCTTY, 0) < 0)
diff --git a/test/integration/framework b/test/integration/framework
index ac482a7a0..9e183057f 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -102,7 +102,7 @@ runapt() {
 	local CMD="$1"
 	shift
 	case $CMD in
-	sh|aptitude|*/*) ;;
+	sh|aptitude|*/*|command) ;;
 	*) CMD="${BUILDDIRECTORY}/$CMD";;
 	esac
 	MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH=${BUILDDIRECTORY} $CMD "$@"
diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts
index 3c6457cab..6eb033055 100755
--- a/test/integration/test-no-fds-leaked-to-maintainer-scripts
+++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts
@@ -5,7 +5,7 @@ TESTDIR=$(readlink -f $(dirname $0))
 . $TESTDIR/framework
 
 setupenvironment
-configarchitecture 'native'
+configarchitecture 'amd64' 'i386'
 configdpkgnoopchroot
 
 setupsimplenativepackage "fdleaks" 'all' '1.0' 'unstable'
@@ -17,58 +17,83 @@ done
 buildpackage "$BUILDDIR" 'unstable' 'main' 'native'
 rm -rf "$BUILDDIR"
 
+PKGNAME='fdleaks:all'
+if ! dpkg-checkbuilddeps -d 'dpkg (>= 1.16.2)' /dev/null >/dev/null 2>&1; then
+	PKGNAME='fdleaks'
+fi
+
 setupaptarchive
 
 rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
 testsuccess aptget install -y fdleaks -qq < /dev/null
-msgtest 'Check if fds were not' 'leaked'
-if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '8' ]; then
-	msgpass
-else
-	echo
-	cat rootdir/tmp/testsuccess.output
-	msgfail
-fi
 
-cp rootdir/tmp/testsuccess.output terminal.output
-tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
-testfileequal 'terminal.log' "$(cat terminal.output)"
+checkfdleak() {
+	msgtest 'Check if fds were not' 'leaked'
+	if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = "$1" ]; then
+		msgpass
+	else
+		echo
+		cat rootdir/tmp/testsuccess.output
+		msgfail
+	fi
+}
+checkinstall() {
+	checkfdleak 8
+
+	cp rootdir/tmp/testsuccess.output terminal.output
+	tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
+	testfileequal 'terminal.log' "$(cat terminal.output)"
 
-testequal 'startup archives unpack
-install fdleaks:all <none> 1.0
-status half-installed fdleaks:all 1.0
-status unpacked fdleaks:all 1.0
-status unpacked fdleaks:all 1.0
+	testequal "startup archives unpack
+install $PKGNAME <none> 1.0
+status half-installed $PKGNAME 1.0
+status unpacked $PKGNAME 1.0
+status unpacked $PKGNAME 1.0
 startup packages configure
-configure fdleaks:all 1.0 <none>
-status unpacked fdleaks:all 1.0
-status half-configured fdleaks:all 1.0
-status installed fdleaks:all 1.0' cut -f 3- -d' ' rootdir/var/log/dpkg.log
+configure $PKGNAME 1.0 <none>
+status unpacked $PKGNAME 1.0
+status half-configured $PKGNAME 1.0
+status installed $PKGNAME 1.0" cut -f 3- -d' ' rootdir/var/log/dpkg.log
+}
+checkinstall
 
 rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
 testsuccess aptget purge -y fdleaks -qq
-msgtest 'Check if fds were not' 'leaked'
-if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '12' ]; then
+checkpurge() {
+	checkfdleak 12
+
+	cp rootdir/tmp/testsuccess.output terminal.output
+	tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
+	testfileequal 'terminal.log' "$(cat terminal.output)"
+
+	testequal "startup packages purge
+status installed $PKGNAME 1.0
+remove $PKGNAME 1.0 <none>
+status half-configured $PKGNAME 1.0
+status half-installed $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+purge $PKGNAME 1.0 <none>
+status config-files $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+status not-installed $PKGNAME <none>" cut -f 3- -d' ' rootdir/var/log/dpkg.log
+}
+checkpurge
+
+msgtest 'setsid provided is new enough to support' '-w'
+if dpkg-checkbuilddeps -d 'util-linux (>= 2.24.2-1)' /dev/null >/dev/null 2>&1; then
 	msgpass
 else
-	echo
-	cat rootdir/tmp/testsuccess.output
-	msgfail
+	msgskip "$(command dpkg -l util-linux)"
+	exit
 fi
-cp rootdir/tmp/testsuccess.output terminal.output
-tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
-testfileequal 'terminal.log' "$(cat terminal.output)"
 
-testequal 'startup packages purge
-status installed fdleaks:all 1.0
-remove fdleaks:all 1.0 <none>
-status half-configured fdleaks:all 1.0
-status half-installed fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-purge fdleaks:all 1.0 <none>
-status config-files fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-status not-installed fdleaks:all <none>' cut -f 3- -d' ' rootdir/var/log/dpkg.log
+rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
+testsuccess runapt command setsid -w "${BUILDDIRECTORY}/apt-get" install -y fdleaks -qq < /dev/null
+checkinstall
+
+rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
+testsuccess runapt command setsid -w "${BUILDDIRECTORY}/apt-get" purge -y fdleaks -qq
+checkpurge
-- 
cgit v1.2.3-70-g09d2


From a2a75ff4516f7609f4c55b42270abb8d08943c60 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Tue, 18 Nov 2014 19:53:56 +0100
Subject: always run 'dpkg --configure -a' at the end of our dpkg callings

dpkg checks now for dependencies before running triggers, so that
packages can now end up in trigger states (especially those we are not
touching at all with our calls) after apt is done running.

The solution to this is trivial: Just tell dpkg to configure everything
after we have (supposely) configured everything already. In the worst
case this means dpkg will have to run a bunch of triggers, usually it
will just do nothing though.

The code to make this happen was already available, so we just flip a
config option here to cause it to be run. This way we can keep
pretending that triggers are an implementation detail of dpkg.
--triggers-only would supposely work as well, but --configure is more
robust in regards to future changes to dpkg and something we will
hopefully make use of in future versions anyway (as it was planed at the
time this and related options were implemented).

Note that dpkg currently has a workaround implemented to allow upgrades
to jessie to be clean, so that the test works before and after. Also
note that test (compared to the one in the bug) drops the await test as
its is considered a loop by dpkg now.

Closes: 769609
---
 apt-pkg/deb/dpkgpm.cc                              |  9 ++-
 test/integration/framework                         | 25 ++++----
 test/integration/test-apt-progress-fd              | 67 ++++++++++---------
 test/integration/test-apt-progress-fd-deb822       | 18 ++++--
 test/integration/test-apt-progress-fd-error        |  2 +-
 ...est-bug-769609-triggers-still-pending-after-run | 75 ++++++++++++++++++++++
 .../test-no-fds-leaked-to-maintainer-scripts       |  6 +-
 7 files changed, 146 insertions(+), 56 deletions(-)
 create mode 100755 test/integration/test-bug-769609-triggers-still-pending-after-run

(limited to 'test/integration/framework')

diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index 93a007d14..d54b7b50f 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -1047,6 +1047,12 @@ void pkgDPkgPM::BuildPackagesProgressMap()
 	 PackagesTotal++;
       }
    }
+   /* one extra: We don't want the progress bar to reach 100%, especially not
+      if we call dpkg --configure --pending and process a bunch of triggers
+      while showing 100%. Also, spindown takes a while, so never reaching 100%
+      is way more correct than reaching 100% while still doing stuff even if
+      doing it this way is slightly bending the rules */
+   ++PackagesTotal;
 }
                                                                         /*}}}*/
 #if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR < 13)
@@ -1280,9 +1286,8 @@ bool pkgDPkgPM::GoNoABIBreak(APT::Progress::PackageManager *progress)
 
    // support subpressing of triggers processing for special
    // cases like d-i that runs the triggers handling manually
-   bool const SmartConf = (_config->Find("PackageManager::Configure", "all") != "all");
    bool const TriggersPending = _config->FindB("DPkg::TriggersPending", false);
-   if (_config->FindB("DPkg::ConfigurePending", SmartConf) == true)
+   if (_config->FindB("DPkg::ConfigurePending", true) == true)
       List.push_back(Item(Item::ConfigurePending, PkgIterator()));
 
    // for the progress
diff --git a/test/integration/framework b/test/integration/framework
index 9e183057f..c9445065b 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1178,10 +1178,13 @@ testnopackage() {
 	fi
 }
 
-testdpkginstalled() {
-	msgtest "Test for correctly installed package(s) with" "dpkg -l $*"
-	local PKGS="$(dpkg -l "$@" 2>/dev/null | grep '^i' | wc -l)"
-	if [ "$PKGS" != $# ]; then
+testdpkgstatus() {
+	local STATE="$1"
+	local NR="$2"
+	shift 2
+	msgtest "Test that $NR package(s) are in state $STATE with" "dpkg -l $*"
+	local PKGS="$(dpkg -l "$@" 2>/dev/null | grep "^${STATE}" | wc -l)"
+	if [ "$PKGS" != $NR ]; then
 		echo >&2 $PKGS
 		dpkg -l "$@" | grep '^[a-z]' >&2
 		msgfail
@@ -1190,16 +1193,12 @@ testdpkginstalled() {
 	fi
 }
 
+testdpkginstalled() {
+	testdpkgstatus 'ii' "$#" "$@"
+}
+
 testdpkgnotinstalled() {
-	msgtest "Test for correctly not-installed package(s) with" "dpkg -l $*"
-	local PKGS="$(dpkg -l "$@" 2> /dev/null | grep '^i' | wc -l)"
-	if [ "$PKGS" != 0 ]; then
-		echo
-		dpkg -l "$@" | grep '^[a-z]' >&2
-		msgfail
-	else
-		msgpass
-	fi
+	testdpkgstatus 'ii' '0' "$@"
 }
 
 testmarkedauto() {
diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd
index d72e7e72d..68cc0439c 100755
--- a/test/integration/test-apt-progress-fd
+++ b/test/integration/test-apt-progress-fd
@@ -19,13 +19,14 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:0:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
-pmstatus:testing:20:Preparing testing (amd64)
-pmstatus:testing:40:Unpacking testing (amd64)
-pmstatus:testing:60:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:60:Running dpkg
-pmstatus:testing:60:Configuring testing (amd64)
-pmstatus:testing:80:Configuring testing (amd64)
-pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log
+pmstatus:testing:16.6667:Preparing testing (amd64)
+pmstatus:testing:33.3333:Unpacking testing (amd64)
+pmstatus:testing:50:Preparing to configure testing (amd64)
+pmstatus:dpkg-exec:50:Running dpkg
+pmstatus:testing:50:Configuring testing (amd64)
+pmstatus:testing:66.6667:Configuring testing (amd64)
+pmstatus:testing:83.3333:Installed testing (amd64)
+pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
 
 # upgrade
 exec 3> apt-progress.log
@@ -34,13 +35,14 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:0:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
-pmstatus:testing:20:Preparing testing (amd64)
-pmstatus:testing:40:Unpacking testing (amd64)
-pmstatus:testing:60:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:60:Running dpkg
-pmstatus:testing:60:Configuring testing (amd64)
-pmstatus:testing:80:Configuring testing (amd64)
-pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log
+pmstatus:testing:16.6667:Preparing testing (amd64)
+pmstatus:testing:33.3333:Unpacking testing (amd64)
+pmstatus:testing:50:Preparing to configure testing (amd64)
+pmstatus:dpkg-exec:50:Running dpkg
+pmstatus:testing:50:Configuring testing (amd64)
+pmstatus:testing:66.6667:Configuring testing (amd64)
+pmstatus:testing:83.3333:Installed testing (amd64)
+pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
 
 # reinstall
 exec 3> apt-progress.log
@@ -49,22 +51,24 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:0:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
-pmstatus:testing:20:Preparing testing (amd64)
-pmstatus:testing:40:Unpacking testing (amd64)
-pmstatus:testing:60:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:60:Running dpkg
-pmstatus:testing:60:Configuring testing (amd64)
-pmstatus:testing:80:Configuring testing (amd64)
-pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log
+pmstatus:testing:16.6667:Preparing testing (amd64)
+pmstatus:testing:33.3333:Unpacking testing (amd64)
+pmstatus:testing:50:Preparing to configure testing (amd64)
+pmstatus:dpkg-exec:50:Running dpkg
+pmstatus:testing:50:Configuring testing (amd64)
+pmstatus:testing:66.6667:Configuring testing (amd64)
+pmstatus:testing:83.3333:Installed testing (amd64)
+pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
 
 # and remove
 exec 3> apt-progress.log
 testsuccess aptget remove testing -y -o APT::Status-Fd=3
 testequal "pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Removing testing (amd64)
-pmstatus:testing:33.3333:Preparing for removal of testing (amd64)
-pmstatus:testing:66.6667:Removing testing (amd64)
-pmstatus:testing:100:Removed testing (amd64)" cat apt-progress.log
+pmstatus:testing:25:Preparing for removal of testing (amd64)
+pmstatus:testing:50:Removing testing (amd64)
+pmstatus:testing:75:Removed testing (amd64)
+pmstatus:dpkg-exec:75:Running dpkg" cat apt-progress.log
 
 # install non-native and ensure we get proper progress info
 exec 3> apt-progress.log
@@ -75,12 +79,13 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:0:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)
-pmstatus:testing2:20:Preparing testing2 (i386)
-pmstatus:testing2:40:Unpacking testing2 (i386)
-pmstatus:testing2:60:Preparing to configure testing2 (i386)
-pmstatus:dpkg-exec:60:Running dpkg
-pmstatus:testing2:60:Configuring testing2 (i386)
-pmstatus:testing2:80:Configuring testing2 (i386)
-pmstatus:testing2:100:Installed testing2 (i386)" cat apt-progress.log
+pmstatus:testing2:16.6667:Preparing testing2 (i386)
+pmstatus:testing2:33.3333:Unpacking testing2 (i386)
+pmstatus:testing2:50:Preparing to configure testing2 (i386)
+pmstatus:dpkg-exec:50:Running dpkg
+pmstatus:testing2:50:Configuring testing2 (i386)
+pmstatus:testing2:66.6667:Configuring testing2 (i386)
+pmstatus:testing2:83.3333:Installed testing2 (i386)
+pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
 
 rm -f apt-progress*.log
diff --git a/test/integration/test-apt-progress-fd-deb822 b/test/integration/test-apt-progress-fd-deb822
index 9d227942d..badc985e4 100755
--- a/test/integration/test-apt-progress-fd-deb822
+++ b/test/integration/test-apt-progress-fd-deb822
@@ -27,37 +27,41 @@ Message: Installing testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 20
+Percent: 16.6667
 Message: Preparing testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 40
+Percent: 33.3333
 Message: Unpacking testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 60
+Percent: 50
 Message: Preparing to configure testing (amd64)
 
 Status: progress
-Percent: 60
+Percent: 50
 Message: Running dpkg
 
 Status: progress
 Package: testing:amd64
-Percent: 60
+Percent: 50
 Message: Configuring testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 80
+Percent: 66.6667
 Message: Configuring testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 100
+Percent: 83.3333
 Message: Installed testing (amd64)
+
+Status: progress
+Percent: 83.3333
+Message: Running dpkg
 " cat apt-progress.log
 
 
diff --git a/test/integration/test-apt-progress-fd-error b/test/integration/test-apt-progress-fd-error
index a47095b9b..632300765 100755
--- a/test/integration/test-apt-progress-fd-error
+++ b/test/integration/test-apt-progress-fd-error
@@ -18,7 +18,7 @@ setupaptarchive
 exec 3> apt-progress.log
 testfailure aptget install foo1 foo2 -y -o APT::Status-Fd=3
 msgtest "Ensure correct error message"
-if grep -q "aptarchive/pool/foo2_0.8.15_amd64.deb:40:trying to overwrite '/usr/bin/file-conflict', which is also in package foo1 0.8.15" apt-progress.log; then
+if grep -q "aptarchive/pool/foo2_0.8.15_amd64.deb:36.3636:trying to overwrite '/usr/bin/file-conflict', which is also in package foo1 0.8.15" apt-progress.log; then
 	msgpass
 else
 	cat apt-progress.log
diff --git a/test/integration/test-bug-769609-triggers-still-pending-after-run b/test/integration/test-bug-769609-triggers-still-pending-after-run
new file mode 100755
index 000000000..146fa766b
--- /dev/null
+++ b/test/integration/test-bug-769609-triggers-still-pending-after-run
@@ -0,0 +1,75 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'amd64'
+
+msgtest 'Check if installed dpkg supports' 'noawait trigger'
+if dpkg-checkbuilddeps -d 'dpkg (>= 1.16.1)' /dev/null; then
+	msgpass
+else
+	msgskip 'dpkg version too old'
+	exit 0
+fi
+configdpkgnoopchroot
+
+buildtriggerpackages() {
+	local TYPE="$1"
+	setupsimplenativepackage "triggerable-$TYPE" 'all' '1.0' 'unstable' "Depends: trigdepends-$TYPE"
+	BUILDDIR="incoming/triggerable-${TYPE}-1.0"
+	cat >${BUILDDIR}/debian/postinst <<EOF
+#!/bin/sh
+if [ "\$1" = 'triggered' ]; then
+	ls -l /proc/self/fd/
+fi
+EOF
+	echo "$TYPE /usr/share/doc" > ${BUILDDIR}/debian/triggers
+	buildpackage "$BUILDDIR" 'unstable' 'main' 'native'
+	rm -rf "$BUILDDIR"
+	buildsimplenativepackage "trigdepends-$TYPE" 'all' '1.0' 'unstable'
+}
+
+#buildtriggerpackages 'interest'
+buildtriggerpackages 'interest-noawait'
+buildsimplenativepackage "trigstuff" 'all' '1.0' 'unstable'
+
+setupaptarchive
+
+runtests() {
+	local TYPE="$1"
+	msgmsg 'Working with trigger type' "$TYPE"
+	testsuccess aptget install triggerable-$TYPE -y
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testsuccess grep '^REWRITE ' terminal.output
+	testdpkginstalled triggerable-$TYPE trigdepends-$TYPE
+
+	testsuccess aptget install trigdepends-$TYPE -y --reinstall
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testsuccess grep '^REWRITE ' terminal.output
+	testsuccess grep ' root root ' terminal.output
+	testdpkginstalled triggerable-$TYPE trigdepends-$TYPE
+
+	testsuccess aptget install trigstuff -y
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testsuccess grep '^REWRITE ' terminal.output
+	testsuccess grep ' root root ' terminal.output
+	testdpkginstalled triggerable-$TYPE trigdepends-$TYPE trigstuff
+
+	testsuccess aptget purge trigstuff -y
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testsuccess grep '^REWRITE ' terminal.output
+	testsuccess grep ' root root ' terminal.output
+	testdpkginstalled triggerable-$TYPE trigdepends-$TYPE
+	testdpkgnotinstalled trigstuff
+
+	testsuccess aptget purge trigdepends-$TYPE -y
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testfailure grep '^REWRITE ' terminal.output
+	testfailure grep ' root root ' terminal.output
+	testdpkgnotinstalled triggerable-$TYPE trigdepends-$TYPE
+}
+#runtests 'interest'
+runtests 'interest-noawait'
diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts
index 6eb033055..7d0c1c6c1 100755
--- a/test/integration/test-no-fds-leaked-to-maintainer-scripts
+++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts
@@ -53,7 +53,8 @@ startup packages configure
 configure $PKGNAME 1.0 <none>
 status unpacked $PKGNAME 1.0
 status half-configured $PKGNAME 1.0
-status installed $PKGNAME 1.0" cut -f 3- -d' ' rootdir/var/log/dpkg.log
+status installed $PKGNAME 1.0
+startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log
 }
 checkinstall
 
@@ -78,7 +79,8 @@ status config-files $PKGNAME 1.0
 status config-files $PKGNAME 1.0
 status config-files $PKGNAME 1.0
 status config-files $PKGNAME 1.0
-status not-installed $PKGNAME <none>" cut -f 3- -d' ' rootdir/var/log/dpkg.log
+status not-installed $PKGNAME <none>
+startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log
 }
 checkpurge
 
-- 
cgit v1.2.3-70-g09d2


From 31be38d205406d4c756684e20b93d62c4701e091 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <david@kalnischkies.de>
Date: Fri, 9 Jan 2015 01:03:31 +0100
Subject: 128 KiB DSC files ought to be enough for everyone

Your mileage may vary, but don't worry: There is more than one way to
do it, but our one size fits all is not a bigger hammer, but an entire
roundhouse kick! So brace yourself for the tl;dr: The limit is gone.*

Beware: This fixes also the problem that a double newline is
unconditionally added 'later' which is an overcommitment in case
the dsc filesize is limit-2 <= x <= limit.

* limited to numbers fitting into an unsigned long long.

Closes: 774893
---
 ftparchive/cachedb.cc                            |  6 ++--
 ftparchive/sources.cc                            | 41 ++++++++++++++++--------
 ftparchive/sources.h                             | 26 ++++++---------
 ftparchive/writer.cc                             | 12 ++-----
 test/integration/framework                       |  2 +-
 test/integration/test-apt-ftparchive-src-cachedb |  4 ---
 6 files changed, 42 insertions(+), 49 deletions(-)

(limited to 'test/integration/framework')

diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc
index 0901492f7..c73a64fb7 100644
--- a/ftparchive/cachedb.cc
+++ b/ftparchive/cachedb.cc
@@ -328,12 +328,12 @@ bool CacheDB::LoadSource()
    if (Dsc.Read(FileName) == false)
       return false;
 
-   if (Dsc.Data == 0)
+   if (Dsc.Length == 0)
       return _error->Error(_("Failed to read .dsc"));
-   
+
    // Write back the control information
    InitQuerySource();
-   if (Put(Dsc.Data, Dsc.Length) == true)
+   if (Put(Dsc.Data.c_str(), Dsc.Length) == true)
       CurStat.Flags |= FlSource;
 
    return true;
diff --git a/ftparchive/sources.cc b/ftparchive/sources.cc
index d0878a70a..ab976b490 100644
--- a/ftparchive/sources.cc
+++ b/ftparchive/sources.cc
@@ -1,5 +1,5 @@
 #include <string>
-#include <iostream>
+#include <sstream>
 
 // for memcpy
 #include <cstring>
@@ -9,17 +9,19 @@
 
 #include "sources.h"
 
-bool DscExtract::TakeDsc(const void *newData, unsigned long newSize)
+bool DscExtract::TakeDsc(const void *newData, unsigned long long newSize)
 {
-   if(newSize > maxSize)
-     return _error->Error("DSC data is too large %lu!", newSize);
-
    if (newSize == 0)
    {
+      // adding two newlines 'off record' for pkgTagSection.Scan() calls
+      Data = "\n\n";
       Length = 0;
       return true;
    }
-   memcpy(Data, newData, newSize);
+
+   Data = std::string((const char*)newData, newSize);
+   // adding two newlines 'off record' for pkgTagSection.Scan() calls
+   Data.append("\n\n");
    Length = newSize;
 
    return true;
@@ -27,20 +29,31 @@ bool DscExtract::TakeDsc(const void *newData, unsigned long newSize)
 
 bool DscExtract::Read(std::string FileName)
 {
+   Data.clear();
+   Length = 0;
+
    FileFd F;
    if (OpenMaybeClearSignedFile(FileName, F) == false)
       return false;
-   
-   unsigned long long const FSize = F.FileSize();
-   if(FSize > maxSize)
-     return _error->Error("DSC file '%s' is too large!",FileName.c_str());
-
-   if (F.Read(Data, FSize) == false)
-      return false;
-   Length = FSize;
 
    IsClearSigned = (FileName != F.Name());
 
+   std::ostringstream data;
+   char buffer[1024];
+   do {
+      unsigned long long actual = 0;
+      if (F.Read(buffer, sizeof(buffer)-1, &actual) == false)
+	 return _error->Errno("read", "Failed to read dsc file %s", FileName.c_str());
+      if (actual == 0)
+	 break;
+      Length += actual;
+      buffer[actual] = '\0';
+      data << buffer;
+   } while(true);
+
+   // adding two newlines 'off record' for pkgTagSection.Scan() calls
+   data << "\n\n";
+   Data = data.str();
    return true;
 }
 
diff --git a/ftparchive/sources.h b/ftparchive/sources.h
index 91e0b1376..a125ec6a4 100644
--- a/ftparchive/sources.h
+++ b/ftparchive/sources.h
@@ -3,29 +3,21 @@
 
 #include <apt-pkg/tagfile.h>
 
-class DscExtract 
+#include <string>
+
+class DscExtract
 {
  public:
-   //FIXME: do we really need to enforce a maximum size of the dsc file?
-   static const int maxSize = 128*1024;
-
-   char *Data;
+   std::string Data;
    pkgTagSection Section;
-   unsigned long Length;
+   unsigned long long Length;
    bool IsClearSigned;
 
-   bool TakeDsc(const void *Data, unsigned long Size);
+   bool TakeDsc(const void *Data, unsigned long long Size);
    bool Read(std::string FileName);
-   
-   DscExtract() : Data(0), Length(0) {
-     Data = new char[maxSize];
-   };
-   ~DscExtract() { 
-      if(Data != NULL) {
-         delete [] Data;
-         Data = NULL;
-      } 
-   };
+
+   DscExtract() : Length(0), IsClearSigned(false) {};
+   ~DscExtract() {};
 };
 
 
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index 7c1c9cc03..0f6cc177b 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -634,18 +634,10 @@ bool SourcesWriter::DoPackage(string FileName)
    // the "db cursor"
    Db.Finish();
 
-   // read stuff
-   char *Start = Db.Dsc.Data;
-   char *BlkEnd = Db.Dsc.Data + Db.Dsc.Length;
-
-   // Add extra \n to the end, just in case (as in clearsigned they are missing)
-   *BlkEnd++ = '\n';
-   *BlkEnd++ = '\n';
-
    pkgTagSection Tags;
-   if (Tags.Scan(Start,BlkEnd - Start) == false)
+   if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
-   
+
    if (Tags.Exists("Source") == false)
       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
    Tags.Trim();
diff --git a/test/integration/framework b/test/integration/framework
index c9445065b..70ad381e9 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -780,7 +780,7 @@ buildaptarchivefromincoming() {
 	[ -e ftparchive.conf ] || createaptftparchiveconfig
 	[ -e dists ] || buildaptftparchivedirectorystructure
 	msgninfo "\tGenerate Packages, Sources and Contents files… "
-	aptftparchive -qq generate ftparchive.conf
+	testsuccess aptftparchive generate ftparchive.conf
 	cd - > /dev/null
 	msgdone "info"
 	generatereleasefiles
diff --git a/test/integration/test-apt-ftparchive-src-cachedb b/test/integration/test-apt-ftparchive-src-cachedb
index adcca6217..0ac4d558f 100755
--- a/test/integration/test-apt-ftparchive-src-cachedb
+++ b/test/integration/test-apt-ftparchive-src-cachedb
@@ -180,10 +180,6 @@ testequal "
 E: Could not find a Source entry in the DSC 'aptarchive/pool/invalid/invalid_1.0.dsc'" aptftparchive sources aptarchive/pool/invalid
 rm -f aptarchive/pool/invalid/invalid_1.0.dsc
 
-dd if=/dev/zero of="aptarchive/pool/invalid/toobig_1.0.dsc" bs=1k count=129 2>/dev/null
-testequal "
-E: DSC file 'aptarchive/pool/invalid/toobig_1.0.dsc' is too large!" aptftparchive sources aptarchive/pool/invalid
-
 # ensure clean works
 rm -f aptarchive/pool/main/*
 aptftparchive clean apt-ftparchive.conf -o Debug::APT::FTPArchive::Clean=1 > clean-out.txt 2>&1 
-- 
cgit v1.2.3-70-g09d2