Merge branch 'feature/acq-trans' into feature/expected-size

24 files changed, 711 insertions, 41 deletions

diff --git a/test/integration/framework b/test/integration/framework
index e83606fae..0aa648fb6 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -47,6 +47,9 @@ msgskip() { printf "${CWARNING}SKIP${CNORMAL}\n" >&2; }
 msgfail() {
 	if [ $# -gt 0 ]; then printf "${CFAIL}FAIL: $*${CNORMAL}\n" >&2;
 	else printf "${CFAIL}FAIL${CNORMAL}\n" >&2; fi
+        if [ -n "$APT_DEBUG_TESTS" ]; then
+            bash
+        fi
 	EXIT_CODE=$((EXIT_CODE+1));
 }
 
@@ -164,9 +167,10 @@ addtrap() {
 
 setupenvironment() {
 	TMPWORKINGDIRECTORY=$(mktemp -d)
-	TESTDIRECTORY=$(readlink -f $(dirname $0))
+	addtrap "cd /; rm -rf $TMPWORKINGDIRECTORY;"
 	msgninfo "Preparing environment for ${CCMD}$(basename $0)${CINFO} in ${TMPWORKINGDIRECTORY}… "
 
+	TESTDIRECTORY=$(readlink -f $(dirname $0))
         # allow overriding the default BUILDDIR location
 	BUILDDIRECTORY=${APT_INTEGRATION_TESTS_BUILD_DIR:-"${TESTDIRECTORY}/../../build/bin"}
 	LIBRARYPATH=${APT_INTEGRATION_TESTS_LIBRARY_PATH:-"${BUILDDIRECTORY}"}
@@ -177,7 +181,6 @@ setupenvironment() {
 	test -x "${BUILDDIRECTORY}/apt-get" || msgdie "You need to build tree first"
         # -----
 
-	addtrap "cd /; rm -rf $TMPWORKINGDIRECTORY;"
 	cd $TMPWORKINGDIRECTORY
 	mkdir rootdir aptarchive keys
 	cd rootdir
@@ -210,6 +213,7 @@ setupenvironment() {
 		cp "${TESTDIRECTORY}/${SOURCESSFILE}" aptarchive/Sources
 	fi
 	cp $(find $TESTDIRECTORY -name '*.pub' -o -name '*.sec') keys/
+	chmod 644 $(find keys -name '*.pub' -o -name '*.sec')
 	ln -s ${TMPWORKINGDIRECTORY}/keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf
 	echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf
@@ -714,7 +718,7 @@ buildaptarchivefromincoming() {
 	aptftparchive -qq generate ftparchive.conf
 	cd - > /dev/null
 	msgdone "info"
-	generatereleasefiles
+	generatereleasefiles "$@"
 }
 
 buildaptarchivefromfiles() {
@@ -829,17 +833,20 @@ setupflataptarchive() {
 }
 
 setupaptarchive() {
-	buildaptarchive
+	local NOUPDATE=0
+	if [ "$1" = '--no-update' ]; then
+		NOUPDATE=1
+		shift
+	fi
+	buildaptarchive "$@"
 	if [ -e aptarchive/dists ]; then
 		setupdistsaptarchive
 	else
 		setupflataptarchive
 	fi
-	signreleasefiles
-	if [ "$1" != '--no-update' ]; then
-		msgninfo "\tSync APT's cache with the archive… "
-		aptget update -qq
-		msgdone "info"
+	signreleasefiles 'Joe Sixpack' "$@"
+	if [ "1" != "$NOUPDATE" ]; then
+		testsuccess aptget update -o Debug::pkgAcquire::Worker=true -o Debug::Acquire::gpgv=true
 	fi
 }
 
@@ -1175,6 +1182,19 @@ testfailure() {
 	fi
 }
 
+testaccessrights() {
+	msgtest "Test that file $1 has access rights set to" "$2"
+	if [ "$2" = "$(stat --format '%a' "$1")" ]; then
+		msgpass
+	else
+		echo >&2
+		ls -l >&2 "$1"
+		echo -n >&2 "stat(1) reports access rights: "
+		stat --format '%a' "$1"
+		msgfail
+	fi
+}
+
 testwebserverlaststatuscode() {
 	local DOWNLOG='rootdir/tmp/webserverstatus-testfile.log'
 	local STATUS='rootdir/tmp/webserverstatus-statusfile.log'
diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update
index 23282bf86..6e1ecdaff 100755
--- a/test/integration/test-apt-by-hash-update
+++ b/test/integration/test-apt-by-hash-update
@@ -34,7 +34,7 @@ Building dependency tree...
 E: Unable to locate package foo" aptget install -q -s foo
 
 # ensure we can apt-get update by hash
-testsuccess aptget update -o APT::Acquire::By-Hash=1
+testsuccess aptget update -o APT::Acquire::By-Hash=1 
 
 # ensure it works
 testequal "Inst foo (1.0 unstable [all])
diff --git a/test/integration/test-apt-get-clean b/test/integration/test-apt-get-clean
index 646ea31be..98f7c84d0 100755
--- a/test/integration/test-apt-get-clean
+++ b/test/integration/test-apt-get-clean
@@ -18,6 +18,7 @@ testsuccess aptget clean
 
 # generate some dirt and clean it up
 touch rootdir/var/lib/apt/lists/partial/http.debian.net_debian_dists_sid_main_i18n_Translation-en
+mkdir -p rootdir/var/cache/apt/archives
 touch rootdir/var/cache/apt/archives/foo_1_all.deb
 touch rootdir/var/cache/apt/archives/foo_2_all.deb
 touch rootdir/var/cache/apt/archives/foo_3_all.deb
diff --git a/test/integration/test-apt-get-download b/test/integration/test-apt-get-download
index 58ed44f8f..0514542b3 100755
--- a/test/integration/test-apt-get-download
+++ b/test/integration/test-apt-get-download
@@ -11,8 +11,23 @@ buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
 buildsimplenativepackage 'apt' 'all' '2.0' 'unstable'
 insertinstalledpackage 'vrms' 'all' '1.0'
 
+umask 0027
+
 setupaptarchive
 
+# apt-ftparchive knows how to chmod files
+find aptarchive/dists -name '*Packages*' -type f | while read file; do
+	testaccessrights "$file" '644'
+done
+# created by the framework without special care
+find aptarchive/dists -name '*Release*' -type f | while read file; do
+	testaccessrights "$file" '640'
+done
+# all copied files are properly chmodded
+find rootdir/var/lib/apt/lists -type f | while read file; do
+	testaccessrights "$file" '644'
+done
+
 testdownload() {
 	local APT="$2"
 	if [ -n "$3" ]; then
@@ -65,6 +80,7 @@ testsuccess aptget update
 # test with already stored deb
 testsuccess aptget install -d apt
 testsuccess test -s rootdir/var/cache/apt/archives/apt_2.0_all.deb
+testaccessrights 'aptarchive/pool/apt_2.0_all.deb' '644'
 mv aptarchive/pool/apt_2.0_all.deb aptarchive/pool/apt_2.0_all.deb.gone
 testdownload apt_2.0_all.deb apt
 mv aptarchive/pool/apt_2.0_all.deb.gone aptarchive/pool/apt_2.0_all.deb
diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated
index 2cee13923..d833ddd85 100755
--- a/test/integration/test-apt-get-source-authenticated
+++ b/test/integration/test-apt-get-source-authenticated
@@ -21,7 +21,7 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated InRelease file
-testsuccess aptget update
+testsuccess aptget update --allow-insecure-repositories
 
 # this all should fail
 testfailure aptget install -y foo
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
new file mode 100755
index 000000000..8e212a3c4
--- /dev/null
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -0,0 +1,46 @@
+#!/bin/sh
+#
+# ensure we print warnings for unauthenticated repositories
+# 
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+# a "normal" package with source and binary
+buildsimplenativepackage 'foo' 'all' '2.0'
+
+setupaptarchive --no-update
+
+APTARCHIVE=$(readlink -f ./aptarchive)
+rm -f $APTARCHIVE/dists/unstable/*Release*
+
+# update without authenticated files leads to warning
+testequal "Ign file: unstable InRelease
+  File not found
+Err file: unstable Release
+  File not found
+W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
+W: Use --allow-insecure-repositories to force the update
+W: Failed to fetch file:$APTARCHIVE/dists/unstable/Release  File not found
+
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update
+
+# no package foo
+testequal "Listing..." apt list foo
+testequal "partial" ls rootdir/var/lib/apt/lists
+
+# allow override
+testequal "Ign file: unstable InRelease
+  File not found
+Ign file: unstable Release
+  File not found
+Reading package lists...
+W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories
+# ensure we can not install the package
+testequal "WARNING: The following packages cannot be authenticated!
+  foo
+E: There are problems and -y was used without --force-yes" aptget install -qq -y foo
diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
new file mode 100755
index 000000000..8aa5a7262
--- /dev/null
+++ b/test/integration/test-apt-update-ims
@@ -0,0 +1,85 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
+
+setupaptarchive
+changetowebserver
+
+runtest() {
+    rm -f rootdir/var/lib/apt/lists/localhost*
+
+    testsuccess aptget update
+
+    # ensure no leftovers in partial
+    testfailure ls "rootdir/var/lib/apt/lists/partial/*"
+
+    # check that I-M-S header is kept in redirections
+    testequal "$EXPECT" aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
+    
+    # ensure that we still do a hash check on ims hit
+    msgtest 'Test I-M-S reverify'
+    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
+
+    # ensure no leftovers in partial
+    testfailure ls "rootdir/var/lib/apt/lists/partial/*"
+}
+
+msgmsg "InRelease"
+EXPECT="Hit http://localhost:8080 unstable InRelease
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists..."
+# with InRelease
+runtest
+
+# with gzip
+echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
+
+msgmsg "Release/Release.gpg"
+# with Release/Release.gpg
+EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
+Hit http://localhost:8080 unstable Release
+Hit http://localhost:8080 unstable Release.gpg
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists..."
+
+find aptarchive -name "InRelease" | xargs rm -f
+
+echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
+
+echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
+
+
+# no Release.gpg or InRelease
+msgmsg "Release only"
+EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
+Hit http://localhost:8080 unstable Release
+Ign http://localhost:8080 unstable Release.gpg
+  404  Not Found
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists..."
+
+find aptarchive -name "Release.gpg" | xargs rm -f
+
+echo 'Acquire::AllowInsecureRepositories "1";' > rootdir/etc/apt/apt.conf.d/insecure.conf
+echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
+
+echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest
diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback
new file mode 100755
index 000000000..321472c2e
--- /dev/null
+++ b/test/integration/test-apt-update-nofallback
@@ -0,0 +1,234 @@
+#!/bin/sh
+#
+# ensure we never fallback from a signed to a unsigned repo
+# 
+# hash checks are done in 
+# 
+set -e
+
+simulate_mitm_and_inject_evil_package()
+{
+    rm -f $APTARCHIVE/dists/unstable/InRelease
+    rm -f $APTARCHIVE/dists/unstable/Release.gpg
+    inject_evil_package
+}
+
+inject_evil_package()
+{
+    cat > $APTARCHIVE/dists/unstable/main/binary-i386/Packages <<EOF
+Package: evil
+Installed-Size: 29
+Maintainer: Joe Sixpack <joe@example.org>
+Architecture: all
+Version: 1.0
+Filename: pool/evil_1.0_all.deb
+Size: 1270
+Description: an autogenerated evil package
+EOF
+    # avoid ims hit
+    touch -d '+1hour' aptarchive/dists/unstable/main/binary-i386/Packages
+}
+
+assert_update_is_refused_and_last_good_state_used()
+{
+    testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
+
+    assert_repo_is_intact
+}
+
+assert_repo_is_intact()
+{
+    testequal "foo/unstable 2.0 all" apt list -q
+    testsuccess "" aptget install -y -s foo
+    testfailure "" aptget install -y evil
+
+    LISTDIR=rootdir/var/lib/apt/lists
+    if ! ( ls $LISTDIR/*InRelease >/dev/null 2>&1 || 
+            ls $LISTDIR/*Release.gpg >/dev/null 2>&1 ); then
+        echo "Can not find InRelease/Release.gpg in $(ls $LISTDIR)"
+        msgfail
+    fi
+}
+
+setupaptarchive_with_lists_clean()
+{
+    setupaptarchive --no-update
+    rm -f rootdir/var/lib/apt/lists/_*
+    #rm -rf rootdir/var/lib/apt/lists
+}
+
+test_from_inrelease_to_unsigned()
+{
+    # setup archive with InRelease file
+    setupaptarchive_with_lists_clean
+    testsuccess aptget update
+
+    simulate_mitm_and_inject_evil_package
+    assert_update_is_refused_and_last_good_state_used
+}
+
+test_from_release_gpg_to_unsigned()
+{
+    # setup archive with Release/Release.gpg (but no InRelease)
+    setupaptarchive_with_lists_clean
+    rm $APTARCHIVE/dists/unstable/InRelease
+    testsuccess aptget update
+
+    simulate_mitm_and_inject_evil_package
+    assert_update_is_refused_and_last_good_state_used
+}
+
+test_from_inrelease_to_unsigned_with_override()
+{
+    # setup archive with InRelease file
+    setupaptarchive_with_lists_clean
+    testsuccess aptget update
+
+    # simulate moving to a unsigned but otherwise valid repo
+    simulate_mitm_and_inject_evil_package
+    generatereleasefiles
+
+    # and ensure we can update to it (with enough force) 
+    testsuccess aptget update --allow-insecure-repositories \
+        -o Acquire::AllowDowngradeToInsecureRepositories=1
+    # but that the individual packages are still considered untrusted
+    testequal "WARNING: The following packages cannot be authenticated!
+  evil
+E: There are problems and -y was used without --force-yes" aptget install -qq -y evil
+}
+
+test_cve_2012_0214()
+{
+    # see https://bugs.launchpad.net/ubuntu/+source/apt/+bug/947108
+    #
+    # it was possible to MITM the download so that InRelease/Release.gpg
+    # are not delivered (404) and a altered Release file was send
+    #
+    # apt left the old InRelease file in /var/lib/apt/lists and downloaded
+    # the unauthenticated Release file too giving the false impression that
+    # Release was authenticated
+    #
+    # Note that this is pretty much impossible nowdays because:
+    # a) InRelease is left as is, not split to InRelease/Release as it was 
+    #    in the old days
+    # b) we refuse to go from signed->unsigned
+    #
+    # Still worth having a regression test the simulates the condition
+
+    # setup archive with InRelease
+    setupaptarchive_with_lists_clean
+    testsuccess aptget update
+
+    # do what CVE-2012-0214 did
+    rm $APTARCHIVE/dists/unstable/InRelease
+    rm $APTARCHIVE/dists/unstable/Release.gpg
+    inject_evil_package
+    # build valid Release file
+    aptftparchive -qq release ./aptarchive > aptarchive/dists/unstable/Release 
+
+    assert_update_is_refused_and_last_good_state_used
+
+    # ensure there is no _Release file downloaded
+    testfailure ls rootdir/var/lib/apt/lists/*_Release
+}
+
+test_subvert_inrelease()
+{
+    # setup archive with InRelease
+    setupaptarchive_with_lists_clean
+    testsuccess aptget update
+
+    # replace InRelease with something else
+    mv $APTARCHIVE/dists/unstable/Release $APTARCHIVE/dists/unstable/InRelease
+
+    testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  Does not start with a cleartext signature
+
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+
+    # ensure we keep the repo
+    assert_repo_is_intact
+}
+
+test_inrelease_to_invalid_inrelease()
+{
+    # setup archive with InRelease
+    setupaptarchive_with_lists_clean
+    testsuccess aptget update
+
+    # now remove InRelease and subvert Release do no longer verify
+    sed -i 's/Codename.*/Codename: evil!'/ $APTARCHIVE/dists/unstable/InRelease
+    inject_evil_package
+
+    testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) <joe@example.org>
+
+W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) <joe@example.org>
+
+W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+
+    # ensure we keep the repo
+    assert_repo_is_intact
+    testfailure grep "evil" rootdir/var/lib/apt/lists/*InRelease
+}
+
+test_release_gpg_to_invalid_release_release_gpg()
+{
+    # setup archive with InRelease
+    setupaptarchive_with_lists_clean
+    rm $APTARCHIVE/dists/unstable/InRelease
+    testsuccess aptget update
+
+    # now subvert Release do no longer verify
+    echo "Some evil data" >>  $APTARCHIVE/dists/unstable/Release
+    inject_evil_package
+
+    testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release.gpg: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) <joe@example.org>
+
+W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg  
+
+W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+
+    assert_repo_is_intact
+    testfailure grep "evil" rootdir/var/lib/apt/lists/*Release
+}
+
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+# a "normal" package with source and binary
+buildsimplenativepackage 'foo' 'all' '2.0'
+
+# setup the archive and ensure we have a single package that installs fine
+setupaptarchive
+APTARCHIVE=$(readlink -f ./aptarchive)
+assert_repo_is_intact
+
+# test the various cases where a repo may go from signed->unsigned
+msgmsg "test_from_inrelease_to_unsigned"
+test_from_inrelease_to_unsigned
+
+msgmsg "test_from_release_gpg_to_unsigned"
+test_from_release_gpg_to_unsigned
+
+# ensure we do not regress on CVE-2012-0214
+msgmsg "test_cve_2012_0214"
+test_cve_2012_0214
+
+# ensure InRelase can not be subverted
+msgmsg "test_subvert_inrelease"
+test_subvert_inrelease
+
+# ensure we revert to last good state if InRelease does not verify
+msgmsg "test_inrelease_to_invalid_inrelease"
+test_inrelease_to_invalid_inrelease
+
+# ensure we revert to last good state if Release/Release.gpg does not verify
+msgmsg "test_release_gpg_to_invalid_release_release_gpg"
+test_release_gpg_to_invalid_release_release_gpg
+
+# ensure we can ovveride the downgrade error
+msgmsg "test_from_inrelease_to_unsigned"
+test_from_inrelease_to_unsigned_with_override
diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback
new file mode 100755
index 000000000..5b9c200fe
--- /dev/null
+++ b/test/integration/test-apt-update-rollback
@@ -0,0 +1,195 @@
+#!/bin/sh
+#
+# test that apt-get update is transactional
+# 
+set -e
+
+avoid_ims_hit() {
+    touch -d '+1hour' aptarchive/dists/unstable/main/binary-i386/Packages*
+    touch -d '+1hour' aptarchive/dists/unstable/main/source/Sources*
+    touch -d '+1hour' aptarchive/dists/unstable/*Release*
+
+    touch -d '-1hour' rootdir/var/lib/apt/lists/*
+}
+
+create_fresh_archive()
+{
+    rm -rf aptarchive/*
+    rm -f rootdir/var/lib/apt/lists/_* rootdir/var/lib/apt/lists/partial/*
+
+    insertpackage 'unstable' 'old' 'all' '1.0'
+
+    setupaptarchive --no-update
+}
+
+add_new_package() {
+    insertpackage "unstable" "new" "all" "1.0"
+    insertsource "unstable" "new" "all" "1.0"
+
+    setupaptarchive --no-update "$@"
+}
+
+break_repository_sources_index() {
+    printf 'xxx' > $APTARCHIVE/dists/unstable/main/source/Sources
+    compressfile "$APTARCHIVE/dists/unstable/main/source/Sources" "$@"
+}
+
+start_with_good_inrelease() {
+    create_fresh_archive
+    testsuccess aptget update
+    testequal "old/unstable 1.0 all" apt list -q
+}
+
+test_inrelease_to_new_inrelease() {
+    msgmsg 'Test InRelease to new InRelease works fine'
+    start_with_good_inrelease
+
+    add_new_package '+1hour'
+    testsuccess aptget update -o Debug::Acquire::Transaction=1
+    testequal "new/unstable 1.0 all
+old/unstable 1.0 all" apt list -q
+}
+
+test_inrelease_to_broken_hash_reverts_all() {
+    msgmsg 'Test InRelease to broken InRelease reverts everything'
+    start_with_good_inrelease
+
+    add_new_package '+1hour'
+    # break the Sources file
+    break_repository_sources_index '+1hour'
+
+    # test the error condition
+    testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources  Hash Sum mismatch
+
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+    # ensure that the Packages file is also rolled back
+    testequal "E: Unable to locate package new" aptget install new -s -qq
+}
+
+test_inrelease_to_valid_release() {
+    msgmsg 'Test InRelease to valid Release'
+    start_with_good_inrelease
+
+    add_new_package '+1hour'
+    # switch to a unsigned repo now
+    rm $APTARCHIVE/dists/unstable/InRelease
+    rm $APTARCHIVE/dists/unstable/Release.gpg
+
+    # update fails
+    testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
+
+    # test that we can install the new packages but do no longer have a sig
+    testsuccess aptget install old -s
+    testfailure aptget install new -s
+    testsuccess ls $ROOTDIR/var/lib/apt/lists/*_InRelease
+    testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release
+}
+
+test_inrelease_to_release_reverts_all() {
+    msgmsg 'Test InRelease to broken Release reverts everything'
+    start_with_good_inrelease
+
+    # switch to a unsigned repo now
+    add_new_package '+1hour'
+    rm $APTARCHIVE/dists/unstable/InRelease
+    rm $APTARCHIVE/dists/unstable/Release.gpg
+
+    # break it
+    break_repository_sources_index '+1hour'
+
+    # ensure error
+    testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1
+
+    # ensure that the Packages file is also rolled back
+    testsuccess aptget install old -s
+    testfailure aptget install new -s
+    testsuccess ls $ROOTDIR/var/lib/apt/lists/*_InRelease
+    testfailure ls $ROOTDIR/var/lib/apt/lists/*_Release
+}
+
+test_unauthenticated_to_invalid_inrelease() {
+    msgmsg 'Test UnAuthenticated to invalid InRelease reverts everything'
+    create_fresh_archive
+    rm $APTARCHIVE/dists/unstable/InRelease
+    rm $APTARCHIVE/dists/unstable/Release.gpg
+
+    testsuccess aptget update --allow-insecure-repositories
+    testequal "WARNING: The following packages cannot be authenticated!
+  old
+E: There are problems and -y was used without --force-yes" aptget install -qq -y old
+
+    # go to authenticated but not correct
+    add_new_package '+1hour'
+    break_repository_sources_index '+1hour'
+
+    testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources  Hash Sum mismatch
+
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+
+    testfailure ls rootdir/var/lib/apt/lists/*_InRelease
+    testequal "WARNING: The following packages cannot be authenticated!
+  old
+E: There are problems and -y was used without --force-yes" aptget install -qq -y old
+}
+
+test_inrelease_to_unauth_inrelease() {
+    msgmsg 'Test InRelease to InRelease without good sig'
+    start_with_good_inrelease
+
+    signreleasefiles 'Marvin Paranoid' '+1hour'
+
+    testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
+
+W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease  The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
+
+W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+
+    testsuccess ls rootdir/var/lib/apt/lists/*_InRelease
+}
+
+test_inrelease_to_broken_gzip() {
+    msgmsg "Test InRelease to broken gzip"
+    start_with_good_inrelease
+
+    # append junk at the end of the compressed file
+    echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz
+    touch -d '+2min' $APTARCHIVE/dists/unstable/main/source/Sources.gz
+    # remove uncompressed file to avoid fallback
+    rm $APTARCHIVE/dists/unstable/main/source/Sources
+
+    testfailure aptget update
+}
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+# setup the archive and ensure we have a single package that installs fine
+setupaptarchive
+APTARCHIVE=$(readlink -f ./aptarchive)
+ROOTDIR=${TMPWORKINGDIRECTORY}/rootdir
+APTARCHIVE_LISTS="$(echo $APTARCHIVE | tr "/" "_" )"
+
+# test the following cases:
+# - InRelease -> broken InRelease revert to previous state
+# - empty lists dir and broken remote leaves nothing on the system
+# - InRelease -> hashsum mismatch for one file reverts all files to previous state
+# - Release/Release.gpg -> hashsum mismatch
+# - InRelease -> Release with hashsum mismatch revert entire state and kills Release
+# - Release -> InRelease with broken Sig/Hash removes InRelease
+# going from Release/Release.gpg -> InRelease and vice versa
+# - unauthenticated -> invalid InRelease
+
+# stuff to do:
+# - ims-hit
+# - gzip-index tests
+
+test_inrelease_to_new_inrelease
+test_inrelease_to_broken_hash_reverts_all
+test_inrelease_to_valid_release
+test_inrelease_to_release_reverts_all
+test_unauthenticated_to_invalid_inrelease
+test_inrelease_to_unauth_inrelease
+test_inrelease_to_broken_gzip
diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions
new file mode 100755
index 000000000..247334991
--- /dev/null
+++ b/test/integration/test-apt-update-transactions
@@ -0,0 +1,24 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+insertpackage 'unstable' 'foo' 'all' '1.0'
+
+setupaptarchive --no-update
+changetowebserver
+
+# break package file
+cat > aptarchive/dists/unstable/main/binary-i386/Packages <<EOF
+Package: bar
+EOF
+compressfile aptarchive/dists/unstable/main/binary-i386/Packages '+1hour'
+
+# ensure that a update will only succeed entirely or not at all
+testfailure aptget update
+testequal "partial" ls rootdir/var/lib/apt/lists
+
diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth
index 13487603c..b7ccd6cf3 100755
--- a/test/integration/test-apt-update-unauth
+++ b/test/integration/test-apt-update-unauth
@@ -8,6 +8,8 @@ set -e
 TESTDIR=$(readlink -f $(dirname $0))
 . $TESTDIR/framework
 
+umask 022
+
 setupenvironment
 configarchitecture "i386"
 
@@ -17,23 +19,53 @@ insertsource 'unstable' 'foo' 'all' '1.0'
 setupaptarchive
 changetowebserver
 
+# FIXME: 
+#  - also check the unauth -> auth success case, i.e. that all files are
+#    reverified
 runtest() {
     # start unauthenticated
     find rootdir/var/lib/apt/lists/ -type f | xargs rm -f
     rm -f aptarchive/dists/unstable/*Release*
-    aptget update -qq
+
+    testsuccess aptget update -qq --allow-insecure-repositories
+
+    # FIXME: this really shouldn't be needed
+    rm -f rootdir/var/lib/apt/lists/partial/*
 
     # become authenticated
     generatereleasefiles
     signreleasefiles
 
-    # and ensure we do download the data again
-    msgtest "Check that the data is check when going to authenticated"
-    if aptget update |grep -q Hit; then
-        msgfail
-    else
+    # move uncompressed away
+    mv aptarchive/dists/unstable/main/binary-i386/Packages \
+        aptarchive/dists/unstable/main/binary-i386/Packages.uncompressed
+
+    # and ensure we re-check the downloaded data
+
+    # change the local packages file
+    PKGS=$(ls rootdir/var/lib/apt/lists/*Packages*)
+    echo "meep" > $PKGS
+    ls rootdir/var/lib/apt/lists/ > lists.before
+
+    # update and ensure all is reverted on the hashsum failure
+    testfailure aptget update -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Auth=1 -o Debug::pkgAcquire::worker=0 -o Debug::acquire::http=0
+
+    # ensure we have before what we have after
+    msgtest 'Check rollback on going from' 'unauth -> auth'
+    ls rootdir/var/lib/apt/lists/ > lists.after
+    if cmp lists.before lists.after; then
         msgpass
+    else
+	echo >&2 '### Output of previous apt-get update ###'
+	cat >&2 rootdir/tmp/testfailure.output
+	echo >&2 '### Changes in the lists-directory: ###'
+	diff -u >&2 lists.before lists.after
+	msgfail
     fi
+
+    # move uncompressed back for release file
+    mv aptarchive/dists/unstable/main/binary-i386/Packages.uncompressed \
+        aptarchive/dists/unstable/main/binary-i386/Packages
 }
 
 for COMPRESSEDINDEXES in 'false' 'true'; do
@@ -43,6 +75,5 @@ for COMPRESSEDINDEXES in 'false' 'true'; do
 	else
 		msgmsg 'Run tests with GzipIndexes disabled'
 	fi
-
-        runtest
+	runtest
 done
diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index 683c174bd..fedf82c92 100755
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -13,7 +13,7 @@ setupflataptarchive
 testaptgetupdate() {
 	rm -rf rootdir/var/lib/apt
 	aptget update 2>> testaptgetupdate.diff >> testaptgetupdate.diff || true
-	sed -i -e '/Ign / d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
+	sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
 	GIVEN="$1"
 	shift
 	msgtest "Test for correctness of" "apt-get update with $*"
diff --git a/test/integration/test-bug-596498-trusted-unsigned-repo b/test/integration/test-bug-596498-trusted-unsigned-repo
index 06c9c8285..3104a70c2 100755
--- a/test/integration/test-bug-596498-trusted-unsigned-repo
+++ b/test/integration/test-bug-596498-trusted-unsigned-repo
@@ -12,7 +12,7 @@ setupaptarchive
 
 aptgetupdate() {
 	rm -rf rootdir/var/lib/apt/ rootdir/var/cache/apt/*.bin
-	aptget update -qq
+	aptget update -qq --allow-insecure-repositories
 }
 
 PKGTEXT="$(aptget install cool --assume-no -d | head -n 7)"
diff --git a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted
index f93510fd7..0736bb6dc 100755
--- a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted
+++ b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted
@@ -24,15 +24,18 @@ testfilemissing() {
 
 testrun() {
 	rm -rf rootdir/var/lib/apt
-	testsuccess aptget update
 
 	if [ "$1" = 'trusted' ]; then
+	        testsuccess aptget update
+
 		testsuccess aptget download cool
 		testfileexists 'cool_1.0_i386.deb'
 
 		testsuccess aptget download cool --allow-unauthenticated
 		testfileexists 'cool_1.0_i386.deb'
 	else
+                testsuccess aptget update --allow-insecure-repositories
+
 		testfailure aptget download cool
 		testfilemissing 'cool_1.0_i386.deb'
 
diff --git a/test/integration/test-bug-717891-abolute-uris-for-proxies b/test/integration/test-bug-717891-abolute-uris-for-proxies
index ac1d6ec11..54a616686 100755
--- a/test/integration/test-bug-717891-abolute-uris-for-proxies
+++ b/test/integration/test-bug-717891-abolute-uris-for-proxies
@@ -12,7 +12,7 @@ setupaptarchive
 changetowebserver --request-absolute='uri'
 
 msgtest 'Check that absolute paths are' 'not accepted'
-testfailure --nomsg aptget update
+testfailure --nomsg aptget update --allow-insecure-repositories
 
 echo 'Acquire::http::Proxy "http://localhost:8080";' > rootdir/etc/apt/apt.conf.d/99proxy
 
diff --git a/test/integration/test-bug-728500-tempdir b/test/integration/test-bug-728500-tempdir
index bdc38c3ca..37e5a013e 100755
--- a/test/integration/test-bug-728500-tempdir
+++ b/test/integration/test-bug-728500-tempdir
@@ -17,7 +17,7 @@ msgtest 'Test apt-get update with incorrect' 'TMPDIR'
 OUTPUT=$(mktemp)
 addtrap "rm $OUTPUT;"
 export TMPDIR=/does-not-exists
-if aptget update >${OUTPUT} 2>&1; then
+if aptget update -o Debug::Acquire::gpg=1 >${OUTPUT} 2>&1; then
 	msgpass
 else
 	echo
diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index 1e5748eae..f81bba4b9 100755
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -60,4 +60,4 @@ mv rootdir/${COPYMETHODS}.bak rootdir/${COPYMETHODS}
 # check that downgrades from https to http are not allowed
 webserverconfig 'aptwebserver::support::http' 'true'
 sed -i -e 's#:8080/redirectme#:4433/downgrademe#' -e 's# http:# https:#' rootdir/etc/apt/sources.list.d/*
-testfailure aptget update
+testfailure aptget update --allow-insecure-repositories
diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
index 805ed5964..1beb5d831 100755
--- a/test/integration/test-compressed-indexes
+++ b/test/integration/test-compressed-indexes
@@ -39,10 +39,10 @@ testrun() {
 		test -e rootdir/var/lib/apt/lists/*_Translation-en.${COMPRESS} || F=1
 		# there is no point in trying pdiff if we have compressed indexes
 		# as we can't patch compressed files (well, we can, but what is the point?)
-		! test -e rootdir/var/lib/apt/lists/*.IndexDiff || F=1
+		! test -e rootdir/var/lib/apt/lists/*diff_Index || F=1
 	else
 		# clear the faked pdiff indexes so the glob below works
-		rm -f rootdir/var/lib/apt/lists/*.IndexDiff
+		rm -f rootdir/var/lib/apt/lists/*diff_Index
 		test -e rootdir/var/lib/apt/lists/*_Packages || F=1
 		test -e rootdir/var/lib/apt/lists/*_Sources || F=1
 		test -e rootdir/var/lib/apt/lists/*_Translation-en || F=1
diff --git a/test/integration/test-hashsum-verification b/test/integration/test-hashsum-verification
index 2a400dcb4..5f88110b3 100755
--- a/test/integration/test-hashsum-verification
+++ b/test/integration/test-hashsum-verification
@@ -70,9 +70,13 @@ runtest() {
         rm -rf rootdir/var/lib/apt/lists
         rm aptarchive/InRelease aptarchive/Release.gpg
         msgtest 'unsigned apt-get update gets the expected hashsum mismatch'
-	aptget update 2>&1 | grep "Hash Sum mismatch" > /dev/null && msgpass || msgfail
-
-
+	aptget update --allow-insecure-repositories >output.log 2>&1 || true
+        if grep -q "Hash Sum mismatch" output.log; then
+            msgpass
+        else
+            cat output.log
+            msgfail
+        fi
 }
 
 for COMPRESSEDINDEXES in 'false' 'true'; do
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 0d5261429..5bad90214 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -94,7 +94,7 @@ SHA256-Patches:
 " aptcache show apt newstuff
 
 	msgmsg "Testcase: index is already up-to-date: $*"
-	find rootdir/var/lib/apt/lists -name '*.IndexDiff' -type f -delete
+	find rootdir/var/lib/apt/lists -name '*diff_Index' -type f -delete
 	testsuccess aptget update "$@"
 	testequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
@@ -182,6 +182,7 @@ SHA256-Patches:
 " aptcache show apt newstuff
 }
 echo 'Debug::pkgAcquire::Diffs "true";
+Debug::Acquire::Transaction "true";
 Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
 
 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning
index 8eb4bcbad..c08a2f103 100755
--- a/test/integration/test-policy-pinning
+++ b/test/integration/test-policy-pinning
@@ -28,7 +28,7 @@ Pinned packages:" aptcache policy $*
 aptgetupdate() {
 	# just to be sure that no old files are used
 	rm -rf rootdir/var/lib/apt
-	if aptget update -qq 2>&1 | grep '^E: '; then
+	if aptget update  --allow-insecure-repositories -qq 2>&1 | grep '^E: '; then
 		msgwarn 'apt-get update failed with an error'
 	fi
 }
@@ -36,6 +36,7 @@ aptgetupdate() {
 ### not signed archive
 
 aptgetupdate
+
 testequalpolicy 100 500
 testequalpolicy 990 500 -t now
 
diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification
index e558b83e8..3765a4b1f 100755
--- a/test/integration/test-releasefile-verification
+++ b/test/integration/test-releasefile-verification
@@ -235,10 +235,21 @@ runtest2() {
 " aptcache show apt
 	failaptnew
 }
-runtest2
 
+# diable some protection by default and ensure we still do the verification
+# correctly
+cat > rootdir/etc/apt/apt.conf.d/weaken-security <<EOF
+Acquire::AllowInsecureRepositories "1";
+Acquire::AllowDowngradeToInsecureRepositories "1";
+EOF
+
+msgmsg "Runing base test"
+runtest2
 
 DELETEFILE="InRelease"
+msgmsg "Running test with deletion of $DELETEFILE"
 runtest
+
 DELETEFILE="Release.gpg"
+msgmsg "Running test with deletion of $DELETEFILE"
 runtest
diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall
index f655ae2d8..df2c69cf6 100755
--- a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall
+++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall
@@ -36,8 +36,8 @@ ensure_n_canary_strings_in_dir() {
 
 LISTS='rootdir/var/lib/apt/lists'
 rm -rf rootdir/var/lib/apt/lists
-msgtest 'Got expected NODATA failure in' 'apt-get update'
-aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail
+msgtest 'Got expected failure message' 'apt-get update'
+aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail
 
 ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0
 testequal 'partial' ls $LISTS
@@ -47,8 +47,8 @@ for f in Release Release.gpg main_binary-amd64_Packages main_source_Sources; do
     echo 'peng neee-wom' > $LISTS/localhost:8080_dists_stable_${f}
 done
 
-msgtest 'Got expected NODATA failure in' 'apt-get update'
-aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail
+msgtest 'Got expected failure message in' 'apt-get update'
+aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail
 
 ensure_n_canary_strings_in_dir $LISTS 'peng neee-wom' 4
 ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0
@@ -57,7 +57,7 @@ ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0
 echo 'peng neee-wom' > $LISTS/localhost:8080_dists_stable_InRelease
 rm -f $LISTS/localhost:8080_dists_stable_Release $LISTS/localhost:8080_dists_stable_Release.gpg
 msgtest 'excpected failure of' 'apt-get update'
-aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail
+aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail
 
 ensure_n_canary_strings_in_dir $LISTS 'peng neee-wom' 3
 ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0
diff --git a/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only b/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only
index 50ca2bf57..09315868b 100755
--- a/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only
+++ b/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only
@@ -28,12 +28,10 @@ MD5Sum:
 done
 
 msgtest 'The unsigned garbage before signed block is' 'ignored'
-testsuccess --nomsg aptget update
+aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail
 
 ROOTDIR="$(readlink -f .)"
 testequal "Package files:
  100 ${ROOTDIR}/rootdir/var/lib/dpkg/status
      release a=now
- 500 file:${ROOTDIR}/aptarchive/ unstable/main i386 Packages
-     release a=unstable,n=sid,c=main
 Pinned packages:" aptcache policy