* apt-pkg/acquire-item.cc:

  - don't use ReadOnlyGzip mode for PDiffs as this mode doesn't work
    in combination with the AddFd methods of our hashclasses

Add also 2 testcases: one to test pdiffs in general and
one to test the handling of compressed indexes.

7 files changed, 303 insertions, 19 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index a289fb7ba..752bc6a99 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -280,7 +280,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
       ss >> ServerSha1 >> size;
       unsigned long const ServerSize = atol(size.c_str());
 
-      FileFd fd(CurrentPackagesFile, FileFd::ReadOnlyGzip);
+      FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
       SHA1Summation SHA1;
       SHA1.AddFD(fd.Fd(), fd.Size());
       string const local_sha1 = SHA1.Result();
@@ -297,7 +297,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
       else 
       {
 	 if(Debug)
-	    std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
+	    std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 
 	 // check the historie and see what patches we need
 	 string const history = Tags.FindS("SHA1-History");
@@ -511,7 +511,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff()					/*{{{*/
    string FinalFile = _config->FindDir("Dir::State::lists");
    FinalFile += URItoFileName(RealURI);
 
-   FileFd fd(FinalFile, FileFd::ReadOnlyGzip);
+   FileFd fd(FinalFile, FileFd::ReadOnly);
    SHA1Summation SHA1;
    SHA1.AddFD(fd.Fd(), fd.Size());
    string local_sha1 = string(SHA1.Result());
diff --git a/debian/changelog b/debian/changelog
index c2c0b605d..c05f9439c 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,8 +19,11 @@ apt (0.8.0~pre2) UNRELEASED; urgency=low
     - some http servers violate HTTP1.1 by not issuing a Reason-Phrase
       (or at least a space after the code) especially for 200, but lets
       be nice and ignore it as we don't need the reason in general
+  * apt-pkg/acquire-item.cc:
+    - don't use ReadOnlyGzip mode for PDiffs as this mode doesn't work
+      in combination with the AddFd methods of our hashclasses
 
- -- David Kalnischkies <kalnischkies@gmail.com>  Fri, 20 Aug 2010 23:00:54 +0200
+ -- David Kalnischkies <kalnischkies@gmail.com>  Sat, 21 Aug 2010 22:18:41 +0200
 
 apt (0.8.0~pre1) experimental; urgency=low
 
diff --git a/test/integration/Packages-pdiff-usage b/test/integration/Packages-pdiff-usage
new file mode 100644
index 000000000..d1530a95c
--- /dev/null
+++ b/test/integration/Packages-pdiff-usage
@@ -0,0 +1,34 @@
+Package: apt
+Version: 0.7.25.3
+Architecture: i386
+Maintainer: APT Development Team <deity@lists.debian.org>
+Installed-Size: 5244
+Replaces: libapt-pkg-dev (<< 0.3.7), libapt-pkg-doc (<< 0.3.7)
+Provides: libapt-pkg-libc6.9-6-4.8
+Depends: libc6 (>= 2.3.4), libgcc1 (>= 1:4.1.1), libstdc++6 (>= 4.4.0), debian-archive-keyring
+Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt
+Filename: pool/main/a/apt/apt_0.7.25.3_i386.deb
+Size: 1750610
+MD5sum: 311aadc67d1b72428b54c9b4e1f76671
+SHA1: 3c695e028f74d5c55226f9ef30000bbbd881088c
+SHA256: b46fd1546151c545fe4bfa56a5cc0e7811a9f68826326a529777fd660f28f050
+Description: Advanced front-end for dpkg
+ This is Debian's next generation front-end for the dpkg package manager.
+ It provides the apt-get utility and APT dselect method that provides a
+ simpler, safer way to install and upgrade packages.
+ .
+ APT features complete installation ordering, multiple source capability
+ and several other unique features, see the Users Guide in apt-doc.
+
+Package: oldstuff
+Version: 1.0
+Architecture: i386
+Maintainer: Joe Sixpack <joe@example.org>
+Installed-Size: 100
+Filename: pool/oldstuff_1.0_i386.deb
+Size: 100000
+MD5sum: 311aeeadf78324aaff54c9b4e1f76671
+SHA1: 3c695e028f74d5c544deeddaaa1242desa81088c
+SHA256: b46fd1546151c545fe4bfa56a5cc0e7deaef23e2da3e4f129727fd660f28f050
+Description: some cool but old stuff
+ This package will disappear in the next mirror update
diff --git a/test/integration/Packages-pdiff-usage-new b/test/integration/Packages-pdiff-usage-new
new file mode 100644
index 000000000..9157596a7
--- /dev/null
+++ b/test/integration/Packages-pdiff-usage-new
@@ -0,0 +1,37 @@
+Package: newstuff
+Version: 1.0
+Architecture: i386
+Maintainer: Joe Sixpack <joe@example.org>
+Installed-Size: 101
+Filename: pool/newstuff_1.0_i386.deb
+Size: 101100
+MD5sum: 311aeeadf78324aaff1ceaf3e1f76671
+SHA1: 3c695e028f7a1ae324deeddaaa1242desa81088c
+SHA256: b46fd154615edefab321cc56a5cc0e7deaef23e2da3e4f129727fd660f28f050
+Description: some cool and shiny new stuff
+ This package will appear in the next mirror update
+
+Package: apt
+Priority: important
+Section: admin
+Installed-Size: 5672
+Maintainer: APT Development Team <deity@lists.debian.org>
+Architecture: i386
+Version: 0.8.0~pre1
+Replaces: manpages-pl (<< 20060617-3~)
+Provides: libapt-pkg4.10
+Depends: libc6 (>= 2.3.4), libgcc1 (>= 1:4.1.1), libstdc++6 (>= 4.4.0), zlib1g (>= 1:1.1.4), debian-archive-keyring, gnupg
+Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt
+Conflicts: python-apt (<< 0.7.93.2~)
+Filename: pool/main/a/apt/apt_0.8.0~pre1_i386.deb
+Size: 2013046
+MD5sum: 6786ca6270c988f2c201716ededaedec
+SHA1: fe26559e745d4c2c977c27170938852041c9adff
+SHA256: a12f968467e1e3cec24191b72bfe84f7aeed3ce422e1a60bb4f1454f2b89b8ee
+Description: Advanced front-end for dpkg
+ This is Debian's next generation front-end for the dpkg package manager.
+ It provides the apt-get utility and APT dselect method that provides a
+ simpler, safer way to install and upgrade packages.
+ .
+ APT features complete installation ordering, multiple source capability
+ and several other unique features, see the Users Guide in apt-doc.
diff --git a/test/integration/framework b/test/integration/framework
index 82fca2046..8efe47330 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -80,7 +80,8 @@ setupenvironment() {
 	BUILDDIRECTORY="${TESTDIR}/../../build/bin"
 	test -x "${BUILDDIRECTORY}/apt-get" || msgdie "You need to build tree first"
 	local OLDWORKINGDIRECTORY=$(pwd)
-# 	trap "cd /; rm -rf $TMPWORKINGDIRECTORY; cd $OLDWORKINGDIRECTORY" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
+	CURRENTTRAP="cd /; rm -rf $TMPWORKINGDIRECTORY; cd $OLDWORKINGDIRECTORY"
+	trap "$CURRENTTRAP" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
 	cd $TMPWORKINGDIRECTORY
 	mkdir rootdir aptarchive keys
 	cd rootdir
@@ -373,17 +374,44 @@ setupaptarchive() {
 	else
 		setupflataptarchive
 	fi
-	msgninfo "\tSign archive with Joe Sixpack key… "
-	for RELEASE in $(find aptarchive/ -name Release); do
-		gpg --no-default-keyring --secret-keyring ./keys/joesixpack.sec --keyring ./keys/joesixpack.pub --default-key Joe -abs -o ${RELEASE}.gpg ${RELEASE}
-	done
-	msgdone "info"
+	signreleasefiles
 	msgninfo "\tSync APT's cache with the archive… "
 	aptget update -qq
 	msgdone "info"
 }
 
-diff() {
+signreleasefiles() {
+	local SIGNER="${1:-Joe Sixpack}"
+	msgninfo "\tSign archive with $SIGNER key… "
+	local SECKEYS=""
+	for KEY in $(find keys/ -name '*.sec'); do
+		SECKEYS="$SECKEYS --secret-keyring $KEY"
+	done
+	local PUBKEYS=""
+	for KEY in $(find keys/ -name '*.pub'); do
+		PUBKEYS="$PUBKEYS --keyring $KEY"
+	done
+	for RELEASE in $(find aptarchive/ -name Release); do
+		gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" -abs -o ${RELEASE}.gpg ${RELEASE}
+	done
+	msgdone "info"
+}
+
+changetowebserver() {
+	if which weborf > /dev/null; then
+		weborf -xb aptarchive/ 2>&1 > /dev/null &
+		CURRENTTRAP="kill $(ps | grep weborf | sed -e 's#^[ ]*##' | cut -d' ' -f 1); $CURRENTTRAP"
+		trap "$CURRENTTRAP" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
+		local APTARCHIVE="file://$(readlink -f ./aptarchive)"
+		for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
+			sed -i $LIST -e "s#$APTARCHIVE#http://localhost:8080/#"
+		done
+		return 0
+	fi
+	return 1
+}
+
+checkdiff() {
 	local DIFFTEXT="$($(which diff) -u $* | sed -e '/^---/ d' -e '/^+++/ d' -e '/^@@/ d')"
 	if [ -n "$DIFFTEXT" ]; then
 		echo
@@ -399,9 +427,9 @@ testfileequal() {
 	shift
 	msgtest "Test for correctness of file" "$FILE"
 	if [ -z "$*" ]; then
-		echo -n "" | diff $FILE - && msgpass || msgfail
+		echo -n "" | checkdiff $FILE - && msgpass || msgfail
 	else
-		echo "$*" | diff $FILE - && msgpass || msgfail
+		echo "$*" | checkdiff $FILE - && msgpass || msgfail
 	fi
 }
 
@@ -410,7 +438,7 @@ testequal() {
 	echo "$1" > $COMPAREFILE
 	shift
 	msgtest "Test for equality of" "$*"
-	$* 2>&1 | diff $COMPAREFILE - && msgpass || msgfail
+	$* 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
 	rm $COMPAREFILE
 }
 
@@ -423,10 +451,10 @@ testequalor2() {
 	shift 2
 	msgtest "Test for equality OR of" "$*"
 	$* 2>&1 1> $COMPAREAGAINST
-	(diff $COMPAREFILE1 $COMPAREAGAINST 1> /dev/null ||
-		diff $COMPAREFILE2 $COMPAREAGAINST 1> /dev/null) && msgpass ||
-		( echo "\n${CINFO}Diff against OR 1${CNORMAL}" "$(diff $COMPAREFILE1 $COMPAREAGAINST)" \
-		       "\n${CINFO}Diff against OR 2${CNORMAL}" "$(diff $COMPAREFILE2 $COMPAREAGAINST)" &&
+	(checkdiff $COMPAREFILE1 $COMPAREAGAINST 1> /dev/null ||
+		checkdiff $COMPAREFILE2 $COMPAREAGAINST 1> /dev/null) && msgpass ||
+		( echo "\n${CINFO}Diff against OR 1${CNORMAL}" "$(checkdiff $COMPAREFILE1 $COMPAREAGAINST)" \
+		       "\n${CINFO}Diff against OR 2${CNORMAL}" "$(checkdiff $COMPAREFILE2 $COMPAREAGAINST)" &&
 		  msgfail )
 	rm $COMPAREFILE1 $COMPAREFILE2 $COMPAREAGAINST
 }
@@ -448,7 +476,7 @@ N: No packages found"
 	local ARCH=$(dpkg-architecture -qDEB_HOST_ARCH_CPU)
 	eval `apt-config shell ARCH APT::Architecture`
 	echo "$VIRTUAL" | sed -e "s/:$ARCH//" -e 's/:all//' > $COMPAREFILE
-	aptcache show $PACKAGE 2>&1 | diff $COMPAREFILE - && msgpass || msgfail
+	aptcache show $PACKAGE 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
 	rm $COMPAREFILE
 }
 
diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
new file mode 100755
index 000000000..0b73f4c35
--- /dev/null
+++ b/test/integration/test-compressed-indexes
@@ -0,0 +1,131 @@
+#!/bin/sh
+set -e
+
+local TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+buildsimplenativepackage "testpkg" "i386" "1.0"
+setupaptarchive
+
+local GOODSHOW="$(aptcache show testpkg)
+"
+local GOODPOLICY="$(aptcache policy testpkg)"
+local GOODSHOWSRC="$(aptcache showsrc testpkg)
+"
+
+test $(echo "$GOODSHOW" | grep -e '^Package: testpkg' -e '^Version: 1.0' -e '^Architecture: i386' | wc -l) -eq 3
+testequal "$GOODSHOW" aptcache show testpkg
+test $(echo "$GOODPOLICY" | grep -e '^testpkg:' -e '^  Candidate:' -e '^  Installed: (none)' -e '500 file:/' | wc -l) -eq 4
+testequal "$GOODPOLICY" aptcache policy testpkg
+test $(echo "$GOODSHOWSRC" | grep -e '^Package: testpkg' -e '^Format: 3.0 (native)' -e '^Files:' -e '^Checksums-Sha256:' | wc -l) -eq 4
+testequal "$GOODSHOWSRC" aptcache showsrc testpkg
+
+
+testrun() {
+	local F
+	if [ -e rootdir/var/lib/apt/lists/*localhost*Release ]; then
+		msgtest "Check if all index files are" "${1:-uncompressed}"
+		if [ "$1" = "compressed" ]; then
+			! test -e rootdir/var/lib/apt/lists/*_Packages || F=1
+			! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
+			test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1
+			test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1
+		else
+			test -e rootdir/var/lib/apt/lists/*_Packages || F=1
+			test -e rootdir/var/lib/apt/lists/*_Sources || F=1
+			! test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1
+			! test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1
+		fi
+		if [ -n "$F" ]; then
+			ls -laR rootdir/var/lib/apt/lists/
+			msgfail
+		else
+			msgpass
+		fi
+		msgtest "Check if package is downloadable"
+		aptget install -d testpkg -qq && msgpass || msgfail
+		msgtest "\tdeb file is present"; test -f rootdir/var/cache/apt/archives/testpkg_1.0_i386.deb && msgpass || msgfail
+		aptget clean
+		msgtest "\tdeb file is gone"; ! test -f rootdir/var/cache/apt/archives/testpkg_1.0_i386.deb && msgpass || msgfail
+	fi
+	rm rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
+	testequal "$GOODSHOW" aptcache show testpkg
+	testequal "$GOODSHOW" aptcache show testpkg
+	rm rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
+	testequal "$GOODPOLICY" aptcache policy testpkg
+	testequal "$GOODPOLICY" aptcache policy testpkg
+	rm rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
+	testequal "$GOODSHOWSRC" aptcache showsrc testpkg
+	testequal "$GOODSHOWSRC" aptcache showsrc testpkg
+	rm -f rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
+	rm -rf rootdir/var/cache/apt/archives
+	msgtest "Check if the source is aptgetable"
+	aptget source testpkg -qq 2> /dev/null > /dev/null && msgpass || msgfail
+	msgtest "\tdsc file is present"; test -f testpkg_1.0.dsc && msgpass || msgfail
+	msgtest "\tdirectory is present"; test -d testpkg-1.0 && msgpass || msgfail
+	rm -rf testpkg-1.0
+}
+
+msgmsg "File: Test with uncompressed indexes"
+testrun
+
+aptget update -qq -o Acquire::Pdiffs=1
+msgmsg "File: Test with uncompressed indexes (update unchanged with pdiffs)"
+testrun
+
+aptget update -qq -o Acquire::Pdiffs=0
+msgmsg "File: Test with uncompressed indexes (update unchanged without pdiffs)"
+testrun
+
+rm -rf rootdir/var/lib/apt/lists
+echo 'Acquire::CompressionTypes::Order:: "gz";
+Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
+
+aptget update -qq
+msgmsg "File: Test with compressed indexes"
+testrun "compressed"
+
+aptget update -qq -o Acquire::Pdiffs=1
+msgmsg "File: Test with compressed indexes (update unchanged with pdiffs)"
+testrun "compressed"
+
+aptget update -qq -o Acquire::Pdiffs=0
+msgmsg "File: Test with compressed indexes (update unchanged without pdiffs)"
+testrun "compressed"
+
+rm rootdir/etc/apt/apt.conf.d/02compressindex
+changetowebserver
+aptget update -qq
+local GOODPOLICY="$(aptcache policy testpkg)"
+test $(echo "$GOODPOLICY" | grep -e '^testpkg:' -e '^  Candidate:' -e '^  Installed: (none)' -e '500 http://' | wc -l) -eq 4
+testequal "$GOODPOLICY" aptcache policy testpkg
+
+msgmsg "HTTP: Test with uncompressed indexes"
+testrun
+
+aptget update -qq -o Acquire::Pdiffs=1
+msgmsg "HTTP: Test with uncompressed indexes (update unchanged with pdiffs)"
+testrun
+
+aptget update -qq -o Acquire::Pdiffs=0
+msgmsg "HTTP: Test with uncompressed indexes (update unchanged without pdiffs)"
+testrun
+
+rm -rf rootdir/var/lib/apt/lists
+echo 'Acquire::CompressionTypes::Order:: "gz";
+Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
+
+aptget update -qq
+msgmsg "HTTP: Test with compressed indexes"
+testrun "compressed"
+
+aptget update -qq -o Acquire::Pdiffs=1
+msgmsg "HTTP: Test with compressed indexes (update unchanged with pdiffs)"
+testrun "compressed"
+
+aptget update -qq -o Acquire::Pdiffs=0
+msgmsg "HTTP: Test with compressed indexes (update unchanged without pdiffs)"
+testrun "compressed"
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
new file mode 100755
index 000000000..85cbe9082
--- /dev/null
+++ b/test/integration/test-pdiff-usage
@@ -0,0 +1,51 @@
+#!/bin/sh
+set -e
+
+local TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+buildaptarchive
+setupflataptarchive
+changetowebserver
+signreleasefiles
+aptget update -qq
+
+testnopackage newstuff
+PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')"
+testequal "$(cat ${PKGFILE})
+" aptcache show apt oldstuff
+
+cp ${PKGFILE}-new aptarchive/Packages
+cat aptarchive/Packages | gzip > aptarchive/Packages.gz
+cat aptarchive/Packages | bzip2 > aptarchive/Packages.bz2
+cat aptarchive/Packages | lzma > aptarchive/Packages.lzma
+rm -rf aptarchive/Packages.diff
+aptftparchive release aptarchive/ > aptarchive/Release
+mkdir -p aptarchive/Packages.diff
+PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
+diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true
+cat $PATCHFILE | gzip > ${PATCHFILE}.gz
+PATCHINDEX="aptarchive/Packages.diff/Index"
+echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new)
+SHA1-History:
+ 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28
+ $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE)
+SHA1-Patches:
+ 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28
+ $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+sed -i aptarchive/Release \
+ -e "/^MD5Sum:/ a\
+ \ $(md5sum $PATCHINDEX | cut -d' ' -f 1) $(stat -c%s $PATCHINDEX) Packages.diff/Index" \
+ -e "/^SHA1:/ a\
+ \ $(sha1sum $PATCHINDEX | cut -d' ' -f 1) $(stat -c%s $PATCHINDEX) Packages.diff/Index" \
+ -e "/^SHA256:/ a\
+ \ $(sha256sum $PATCHINDEX | cut -d' ' -f 1) $(stat -c%s $PATCHINDEX) Packages.diff/Index"
+signreleasefiles
+aptget update -qq
+
+testnopackage oldstuff
+testequal "$(cat ${PKGFILE}-new)
+" aptcache show apt newstuff