From c355ea30383486e74635cd0248d3b53ed9759c39 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 5 Nov 2014 18:04:29 +0100 Subject: reenable patchsize limit option for pdiffs One word: "doh!" Commit f6d4ab9ad8a2cfe52737ab620dd252cf8ceec43d disabled the check to prevent apt from downloading bigger patches than the index it tries to patch. Happens rarly of course, but still. Detected by scan-build complaining about a dead assignment. To make up for the mistake a test is included as well. --- test/integration/test-pdiff-usage | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'test') diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index 5bad90214..d773dcd66 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -180,6 +180,40 @@ SHA256-Patches: testnopackage oldstuff testequal "$(cat ${PKGFILE}-new) " aptcache show apt newstuff + + msgmsg "Testcase: pdiff patch bigger than index itself: $*" + rm -rf rootdir/var/lib/apt/lists + cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists + cp ${PKGFILE}-new aptarchive/Packages + compressfile 'aptarchive/Packages' + mkdir -p aptarchive/Packages.diff + PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)" + diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true + cat $PATCHFILE | gzip > ${PATCHFILE}.gz + PATCHINDEX='aptarchive/Packages.diff/Index' + echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) +SHA1-History: + 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28 + $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) +SHA1-Patches: + 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-2013.28 + $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE) +SHA256-Current: $(sha256sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new) +SHA256-History: + 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b 33053002 2010-08-18-2013.28 + $(sha256sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE) +SHA256-Patches: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28 + $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE)" > $PATCHINDEX + generatereleasefiles '+1hour' + signreleasefiles + #find aptarchive -name 'Packages*' -type f -delete + testsuccess aptget update -o Debug::pkgAcquire::Diffs=1 "$@" + cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output + testsuccess grep 'bytes (Limit is' rootdir/tmp/aptgetupdate.output + testnopackage oldstuff + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt newstuff } echo 'Debug::pkgAcquire::Diffs "true"; Debug::Acquire::Transaction "true"; -- cgit v1.2.3-70-g09d2 From 4234d90d58b684824bf217d9226c50b7c9583346 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 5 Nov 2014 18:14:04 +0100 Subject: tests: silence clang on uninitilized variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The testcases have far worse problems if these ever end up being NULL and/or are not given a value by the method called, but clang is right to warn about it, just that we don't want to fix it in testcases… Git-Dch: Ignore --- test/libapt/cdrom_test.cc | 3 ++- test/libapt/sourcelist_test.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/libapt/cdrom_test.cc b/test/libapt/cdrom_test.cc index 5cf3b353c..7257eaf1b 100644 --- a/test/libapt/cdrom_test.cc +++ b/test/libapt/cdrom_test.cc @@ -109,6 +109,7 @@ TEST(CDROMTest, FindMountPointForDevice) EXPECT_EQ("/boot/efi", FindMountPointForDevice("/dev/sda1")); EXPECT_EQ("/tmp", FindMountPointForDevice("tmpfs")); - unlink(tempfile); + if (tempfile != NULL) + unlink(tempfile); free(tempfile); } diff --git a/test/libapt/sourcelist_test.cc b/test/libapt/sourcelist_test.cc index eb2d76c43..747ab4957 100644 --- a/test/libapt/sourcelist_test.cc +++ b/test/libapt/sourcelist_test.cc @@ -20,7 +20,7 @@ class SourceList : public pkgSourceList { TEST(SourceListTest,ParseFileDeb822) { FileFd fd; - char * tempfile; + char * tempfile = NULL; createTemporaryFile("parsefiledeb822", fd, &tempfile, "Types: deb\n" "URIs: http://ftp.debian.org/debian\n" -- cgit v1.2.3-70-g09d2 From ad5ffe1ffb203eb58e5e25fe6f4c0dabc53253c9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 5 Nov 2014 18:42:56 +0100 Subject: Division by result of sizeof(). memset() expects a size in bytes "did you intend to multiply instead?" is what cppcheck helpful says and it is absolutely right. Doesn't make a whole lot of a difference though as we are talking about 'char' in this testcase, but just to be sure. Reported-By: cppcheck Git-Dch: Ignore --- test/libapt/fileutl_test.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/libapt/fileutl_test.cc b/test/libapt/fileutl_test.cc index 8d47c5098..57d5bbdc2 100644 --- a/test/libapt/fileutl_test.cc +++ b/test/libapt/fileutl_test.cc @@ -53,11 +53,16 @@ static void TestFileFd(mode_t const a_umask, mode_t const ExpectedFilePermission // ensure the memory is as predictably messed up #define APT_INIT_READBACK \ char readback[20]; \ - memset(readback, 'D', sizeof(readback)/sizeof(readback[0])); \ + memset(readback, 'D', sizeof(readback)*sizeof(readback[0])); \ readback[19] = '\0'; #define EXPECT_N_STR(expect, actual) \ EXPECT_EQ(0, strncmp(expect, actual, strlen(expect))); - + { + APT_INIT_READBACK + char const * const expect = "DDDDDDDDDDDDDDDDDDD"; + EXPECT_STREQ(expect,readback); + EXPECT_N_STR(expect, readback); + } { APT_INIT_READBACK char const * const expect = "This"; -- cgit v1.2.3-70-g09d2 From 23fb7b2cbbda62c99a199d29ad62205b23d35af4 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 5 Nov 2014 18:56:12 +0100 Subject: (style) Variable 'res' is assigned a value that is never used Checking the return value of this (and many other calls) in this testcase is a good idea, so we do it now. Reported-By: cppcheck Git-Dch: Ignore --- test/libapt/fileutl_test.cc | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'test') diff --git a/test/libapt/fileutl_test.cc b/test/libapt/fileutl_test.cc index 57d5bbdc2..a2c303768 100644 --- a/test/libapt/fileutl_test.cc +++ b/test/libapt/fileutl_test.cc @@ -252,37 +252,40 @@ TEST(FileUtlTest, Popen) // output something const char* Args[10] = {"/bin/echo", "meepmeep", NULL}; - bool res = Popen(Args, Fd, Child, FileFd::ReadOnly); - Fd.Read(buf, sizeof(buf)-1, &n); + EXPECT_TRUE(Popen(Args, Fd, Child, FileFd::ReadOnly)); + EXPECT_TRUE(Fd.Read(buf, sizeof(buf)-1, &n)); buf[n] = 0; EXPECT_NE(n, 0); - EXPECT_EQ(res, true); EXPECT_STREQ(buf, "meepmeep\n"); // wait for the child to exit and cleanup - ExecWait(Child, "PopenRead"); - Fd.Close(); + EXPECT_TRUE(ExecWait(Child, "PopenRead")); + EXPECT_TRUE(Fd.Close()); // ensure that after a close all is good again if(FileExists("/proc/self/fd")) EXPECT_EQ(Glob("/proc/self/fd/*").size(), OpenFds.size()); - // ReadWrite is not supported - res = Popen(Args, Fd, Child, FileFd::ReadWrite); - EXPECT_EQ(res, false); - _error->Discard(); + _error->PushToStack(); + EXPECT_FALSE(Popen(Args, Fd, Child, FileFd::ReadWrite)); + EXPECT_FALSE(Fd.IsOpen()); + EXPECT_FALSE(Fd.Failed()); + EXPECT_TRUE(_error->PendingError()); + _error->RevertToStack(); // write something Args[0] = "/bin/bash"; Args[1] = "-c"; Args[2] = "read"; Args[3] = NULL; - res = Popen(Args, Fd, Child, FileFd::WriteOnly); + EXPECT_TRUE(Popen(Args, Fd, Child, FileFd::WriteOnly)); s = "\n"; - Fd.Write(s.c_str(), s.size()); - Fd.Close(); - ExecWait(Child, "PopenWrite"); + EXPECT_TRUE(Fd.Write(s.c_str(), s.length())); + EXPECT_TRUE(Fd.Close()); + EXPECT_FALSE(Fd.IsOpen()); + EXPECT_FALSE(Fd.Failed()); + EXPECT_TRUE(ExecWait(Child, "PopenWrite")); } TEST(FileUtlTest, flAbsPath) { -- cgit v1.2.3-70-g09d2 From 20801f613690b330c79b4f7a30dc3ff52b722468 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 14:23:18 +0100 Subject: fix variable naming typo used in debug output Git-Dch: Ignore --- apt-pkg/acquire-worker.cc | 10 +++++----- test/integration/test-apt-update-ims | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'test') diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 724bdfd49..f4d1ad412 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -346,28 +346,28 @@ bool pkgAcquire::Worker::RunMessages() Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize); // see if there is a hash to verify - HashStringList RecivedHashes; + HashStringList ReceivedHashes; HashStringList expectedHashes = Owner->HashSums(); for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs) { std::string const tagname = hs->HashType() + "-Hash"; std::string const hashsum = LookupTag(Message, tagname.c_str()); if (hashsum.empty() == false) - RecivedHashes.push_back(HashString(hs->HashType(), hashsum)); + ReceivedHashes.push_back(HashString(hs->HashType(), hashsum)); } if(_config->FindB("Debug::pkgAcquire::Auth", false) == true) { std::clog << "201 URI Done: " << Owner->DescURI() << endl - << "RecivedHash:" << endl; - for (HashStringList::const_iterator hs = RecivedHashes.begin(); hs != RecivedHashes.end(); ++hs) + << "ReceivedHash:" << endl; + for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs) std::clog << "\t- " << hs->toStr() << std::endl; std::clog << "ExpectedHash:" << endl; for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs) std::clog << "\t- " << hs->toStr() << std::endl; std::clog << endl; } - Owner->Done(Message, ServerSize, RecivedHashes, Config); + Owner->Done(Message, ServerSize, ReceivedHashes, Config); ItemDone(); // Log that we are done diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index 5394a9f30..0fa882d78 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -30,7 +30,7 @@ runtest() { # ensure that we still do a hash check on ims hit msgtest 'Test I-M-S' 'reverify' - aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A2 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail + aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A2 'ReceivedHash:' | grep -q -- '- SHA' && msgpass || msgfail # ensure no leftovers in partial testfailure ls "rootdir/var/lib/apt/lists/partial/*" -- cgit v1.2.3-70-g09d2 From ad7e0941b376d792911f240377094a2e78ca8756 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 18:14:46 +0100 Subject: streamline display of --help in all tools By convention, if I run a tool with --help or --version I expect it to exit successfully with the usage, while if I do call it wrong (like without any parameters) I expect the usage message shown with a non-zero exit. --- apt-pkg/contrib/cmndline.cc | 3 ++ apt-pkg/contrib/cmndline.h | 1 + apt-private/private-cmndline.cc | 33 ++++++++++++++++++ apt-private/private-cmndline.h | 6 ++++ cmdline/apt-cache.cc | 18 ++-------- cmdline/apt-cdrom.cc | 15 ++------- cmdline/apt-config.cc | 15 ++------- cmdline/apt-extracttemplates.cc | 32 +++++++----------- cmdline/apt-get.cc | 22 ++---------- cmdline/apt-helper.cc | 21 ++---------- cmdline/apt-internal-solver.cc | 17 +++------- cmdline/apt-mark.cc | 22 ++---------- cmdline/apt-sortpkgs.cc | 24 +++++-------- cmdline/apt.cc | 20 ++--------- cmdline/makefile | 8 ++--- debian/tests/run-tests | 1 + ftparchive/apt-ftparchive.cc | 19 +++-------- ftparchive/makefile | 4 +-- prepare-release | 4 +-- test/integration/framework | 2 ++ test/integration/test-00-commands-have-help | 52 +++++++++++++++++++++++++++++ 21 files changed, 149 insertions(+), 190 deletions(-) create mode 100755 test/integration/test-00-commands-have-help (limited to 'test') diff --git a/apt-pkg/contrib/cmndline.cc b/apt-pkg/contrib/cmndline.cc index 93c1f4664..ff8b09ebc 100644 --- a/apt-pkg/contrib/cmndline.cc +++ b/apt-pkg/contrib/cmndline.cc @@ -33,6 +33,9 @@ using namespace std; CommandLine::CommandLine(Args *AList,Configuration *Conf) : ArgList(AList), Conf(Conf), FileList(0) { +} +CommandLine::CommandLine() : ArgList(NULL), Conf(NULL), FileList(0) +{ } /*}}}*/ // CommandLine::~CommandLine - Destructor /*{{{*/ diff --git a/apt-pkg/contrib/cmndline.h b/apt-pkg/contrib/cmndline.h index 143df58b2..58cbaa8c3 100644 --- a/apt-pkg/contrib/cmndline.h +++ b/apt-pkg/contrib/cmndline.h @@ -91,6 +91,7 @@ class CommandLine static CommandLine::Args MakeArgs(char ShortOpt, char const *LongOpt, char const *ConfName, unsigned long Flags) APT_CONST; + CommandLine(); CommandLine(Args *AList,Configuration *Conf); ~CommandLine(); }; diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc index aa01be757..bb9a00803 100644 --- a/apt-private/private-cmndline.cc +++ b/apt-private/private-cmndline.cc @@ -2,12 +2,17 @@ #include #include +#include +#include +#include +#include #include #include #include #include +#include #include /*}}}*/ @@ -287,3 +292,31 @@ std::vector getCommandArgs(char const * const Program, char c /*}}}*/ #undef CmdMatches #undef addArg +void ParseCommandLine(CommandLine &CmdL, CommandLine::Dispatch * const Cmds, CommandLine::Args * const Args,/*{{{*/ + Configuration * const * const Cnf, pkgSystem ** const Sys, int const argc, const char *argv[], bool(*ShowHelp)(CommandLine &CmdL)) +{ + CmdL = CommandLine(Args,_config); + if ((Cnf != NULL && pkgInitConfig(**Cnf) == false) || + CmdL.Parse(argc,argv) == false || + (Sys != NULL && pkgInitSystem(*_config, *Sys) == false)) + { + if (_config->FindB("version") == true) + ShowHelp(CmdL); + + _error->DumpErrors(); + exit(100); + } + + // See if the help should be shown + if (_config->FindB("help") == true || _config->FindB("version") == true) + { + ShowHelp(CmdL); + exit(0); + } + if (Cmds != NULL && CmdL.FileSize() == 0) + { + ShowHelp(CmdL); + exit(1); + } +} + /*}}}*/ diff --git a/apt-private/private-cmndline.h b/apt-private/private-cmndline.h index d0af16782..7b468456b 100644 --- a/apt-private/private-cmndline.h +++ b/apt-private/private-cmndline.h @@ -6,6 +6,12 @@ #include +class Configuration; +class pkgSystem; + APT_PUBLIC std::vector getCommandArgs(char const * const Program, char const * const Cmd); +APT_PUBLIC void ParseCommandLine(CommandLine &CmdL, CommandLine::Dispatch * const Cmds, CommandLine::Args * const Args, + Configuration * const * const Cnf, pkgSystem ** const Sys, int const argc, const char * argv[], + bool(*ShowHelp)(CommandLine &CmdL)); #endif diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 9bac45029..12ed4f719 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1896,23 +1896,9 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - // Deal with stdout not being a tty if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) _config->Set("quiet","1"); diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc index 53efe65b8..8ac73fd7e 100644 --- a/cmdline/apt-cdrom.cc +++ b/cmdline/apt-cdrom.cc @@ -249,19 +249,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || _config->FindB("version") == true || - CmdL.FileSize() == 0) - return ShowHelp(CmdL); + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // Deal with stdout not being a tty if (isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) diff --git a/cmdline/apt-config.cc b/cmdline/apt-config.cc index 40ba468eb..e0b8a624e 100644 --- a/cmdline/apt-config.cc +++ b/cmdline/apt-config.cc @@ -115,19 +115,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(CmdL); + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); std::vector const langs = APT::Configuration::getLanguages(true); _config->Clear("Acquire::Languages"); diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc index f95b9e5ba..5211ee027 100644 --- a/cmdline/apt-extracttemplates.cc +++ b/cmdline/apt-extracttemplates.cc @@ -33,6 +33,8 @@ #include #include +#include + #include #include #include @@ -215,15 +217,15 @@ bool DebFile::ParseInfo() // ShowHelp - show a short help text /*{{{*/ // --------------------------------------------------------------------- /* */ -static int ShowHelp(void) +static bool ShowHelp(CommandLine &) { - ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, + ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, COMMON_ARCH,__DATE__,__TIME__); - if (_config->FindB("version") == true) - return 0; + if (_config->FindB("version") == true) + return true; - cout << + cout << _("Usage: apt-extracttemplates file1 [file2 ...]\n" "\n" "apt-extracttemplates is a tool to extract config and template info\n" @@ -234,7 +236,7 @@ static int ShowHelp(void) " -t Set the temp dir\n" " -c=? Read this configuration file\n" " -o=? Set an arbitrary configuration option, eg -o dir::cache=/tmp\n"); - return 0; + return true; } /*}}}*/ // WriteFile - write the contents of the passed string to a file /*{{{*/ @@ -356,20 +358,10 @@ int main(int argc, const char **argv) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(); - + CommandLine::Dispatch Cmds[] = {{NULL, NULL}}; + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args, &_config, &_system, argc, argv, ShowHelp); + Go(CmdL); // Print any errors or warnings found during operation diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index eca4a723b..a9053bdfd 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1746,26 +1746,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // see if we are in simulate mode CheckSimulateMode(CmdL); diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index 27abb2013..1b832f165 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -108,25 +108,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); InitOutput(); diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index 92a4429e5..4fabeb02f 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -24,7 +24,9 @@ #include #include #include + #include +#include #include #include @@ -79,19 +81,8 @@ int main(int argc,const char *argv[]) /*{{{*/ // we really don't need anything DropPrivileges(); - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false) { - _error->DumpErrors(); - return 2; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true) { - ShowHelp(CmdL); - return 1; - } + CommandLine CmdL; + ParseCommandLine(CmdL, NULL, Args, &_config, NULL, argc, argv, ShowHelp); if (CmdL.FileList[0] != 0 && strcmp(CmdL.FileList[0], "scenario") == 0) { diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index ed348358a..487f3d8a1 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -441,26 +441,8 @@ int main(int argc,const char *argv[]) /*{{{*/ setlocale(LC_ALL,""); textdomain(PACKAGE); - // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // Deal with stdout not being a tty if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc index c2b11890a..9b66ad4db 100644 --- a/cmdline/apt-sortpkgs.cc +++ b/cmdline/apt-sortpkgs.cc @@ -23,6 +23,8 @@ #include #include +#include + #include #include #include @@ -142,12 +144,12 @@ static bool DoIt(string InFile) // ShowHelp - Show the help text /*{{{*/ // --------------------------------------------------------------------- /* */ -static int ShowHelp() +static bool ShowHelp(CommandLine &) { ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, COMMON_ARCH,__DATE__,__TIME__); if (_config->FindB("version") == true) - return 0; + return true; cout << _("Usage: apt-sortpkgs [options] file1 [file2 ...]\n" @@ -161,7 +163,7 @@ static int ShowHelp() " -c=? Read this configuration file\n" " -o=? Set an arbitrary configuration option, eg -o dir::cache=/tmp\n"); - return 0; + return true; } /*}}}*/ int main(int argc,const char *argv[]) /*{{{*/ @@ -179,19 +181,9 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(); + CommandLine::Dispatch Cmds[] = {{NULL, NULL}}; + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args, &_config, &_system, argc, argv, ShowHelp); // Match the operation for (unsigned int I = 0; I != CmdL.FileSize(); I++) diff --git a/cmdline/apt.cc b/cmdline/apt.cc index 2cfdf8e8e..056cd213f 100644 --- a/cmdline/apt.cc +++ b/cmdline/apt.cc @@ -119,15 +119,10 @@ int main(int argc, const char *argv[]) /*{{{*/ _config->CndSet("APT::Cmd::Show-Update-Stats", true); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(), _config); - if (CmdL.Parse(argc, argv) == false || - pkgInitSystem(*_config, _system) == false) - { - _error->DumpErrors(); - return 100; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), NULL, &_system, argc, argv, ShowHelp); - if(!isatty(STDOUT_FILENO) && + if(!isatty(STDOUT_FILENO) && _config->FindB("Apt::Cmd::Disable-Script-Warning", false) == false) { std::cerr << std::endl @@ -138,15 +133,6 @@ int main(int argc, const char *argv[]) /*{{{*/ << std::endl; } - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - // see if we are in simulate mode CheckSimulateMode(CmdL); diff --git a/cmdline/makefile b/cmdline/makefile index b7c35ddd1..816038c3b 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -67,15 +67,15 @@ APT_DOMAIN:=apt-utils # The apt-sortpkgs program PROGRAM=apt-sortpkgs -SLIBS = -lapt-pkg $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-sortpkgs.cc include $(PROGRAM_H) # The apt-extracttemplates program PROGRAM=apt-extracttemplates -SLIBS = -lapt-pkg -lapt-inst $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile apt-inst/makefile +SLIBS = -lapt-pkg -lapt-inst -lapt-private $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-inst/makefile apt-private/makefile SOURCE = apt-extracttemplates.cc include $(PROGRAM_H) diff --git a/debian/tests/run-tests b/debian/tests/run-tests index 9d7c484fc..7cc37f618 100644 --- a/debian/tests/run-tests +++ b/debian/tests/run-tests @@ -15,6 +15,7 @@ APT_INTEGRATION_TESTS_WEBSERVER_BIN_DIR=$(pwd)/build/bin \ APT_INTEGRATION_TESTS_METHODS_DIR=/usr/lib/apt/methods \ APT_INTEGRATION_TESTS_LIBEXEC_DIR=/usr/lib/apt/ \ APT_INTEGRATION_TESTS_INTERNAL_SOLVER=/usr/lib/apt/solvers/apt \ +APT_INTEGRATION_TESTS_DUMP_SOLVER=/usr/lib/apt/solvers/dump \ APT_INTEGRATION_TESTS_BUILD_DIR=/usr/bin \ APT_INTEGRATION_TESTS_LIBRARY_PATH=/dev/null/does/not/exist \ ./test/integration/run-tests diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index ebf99a8f8..7d9af4178 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -19,6 +19,8 @@ #include #include +#include + #include #include #include @@ -1060,21 +1062,8 @@ int main(int argc, const char *argv[]) // Parse the command line and initialize the package library CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - + ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp); + // Setup the output streams c0out.rdbuf(clog.rdbuf()); c1out.rdbuf(clog.rdbuf()); diff --git a/ftparchive/makefile b/ftparchive/makefile index d1ffe182a..e67272e1e 100644 --- a/ftparchive/makefile +++ b/ftparchive/makefile @@ -9,8 +9,8 @@ include ../buildlib/defaults.mak ifdef BDBLIB APT_DOMAIN:=apt-utils PROGRAM=apt-ftparchive -SLIBS = -lapt-pkg -lapt-inst $(BDBLIB) $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile apt-inst/makefile +SLIBS = -lapt-pkg -lapt-inst -lapt-private $(BDBLIB) $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-inst/makefile apt-private/makefile SOURCE = apt-ftparchive.cc cachedb.cc writer.cc contents.cc override.cc \ multicompress.cc sources.cc include $(PROGRAM_H) diff --git a/prepare-release b/prepare-release index e61266eef..67f4638d6 100755 --- a/prepare-release +++ b/prepare-release @@ -130,8 +130,8 @@ elif [ "$1" = 'coverage' ]; then LCOVRC='--rc geninfo_checksum=1 --rc lcov_branch_coverage=1' mkdir "$DIR" lcov --no-external --directory . --capture --initial --output-file "${DIR}/apt.coverage.init" ${LCOVRC} - make test - ./test/integration/run-tests -q + make test || true + ./test/integration/run-tests -q || true lcov --no-external --directory . --capture --output-file "${DIR}/apt.coverage.run" ${LCOVRC} lcov -a "${DIR}/apt.coverage.init" -a "${DIR}/apt.coverage.run" -o "${DIR}/apt.coverage.total" ${LCOVRC} cp "${DIR}/apt.coverage.total" "${DIR}/apt.coverage.fixed" diff --git a/test/integration/framework b/test/integration/framework index 153c5bb25..dd66f2a0c 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -128,6 +128,7 @@ aptwebserver() { runapt "${APTWEBSERVERBINDIR}/aptwebserver" "$@"; } aptitude() { runapt aptitude "$@"; } aptextracttemplates() { runapt apt-extracttemplates "$@"; } aptinternalsolver() { runapt "${APTINTERNALSOLVER}" "$@"; } +aptdumpsolver() { runapt "${APTDUMPSOLVER}" "$@"; } dpkg() { command dpkg --root=${TMPWORKINGDIRECTORY}/rootdir --force-not-root --force-bad-path --log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log "$@" @@ -194,6 +195,7 @@ setupenvironment() { APTHELPERBINDIR=${APT_INTEGRATION_TESTS_LIBEXEC_DIR:-"${BUILDDIRECTORY}"} APTWEBSERVERBINDIR=${APT_INTEGRATION_TESTS_WEBSERVER_BIN_DIR:-"${BUILDDIRECTORY}"} APTINTERNALSOLVER=${APT_INTEGRATION_TESTS_INTERNAL_SOLVER:-"${BUILDDIRECTORY}/apt-internal-solver"} + APTDUMPSOLVER=${APT_INTEGRATION_TESTS_DUMP_SOLVER:-"${BUILDDIRECTORY}/apt-dump-solver"} test -x "${BUILDDIRECTORY}/apt-get" || msgdie "You need to build tree first" # ----- diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help new file mode 100755 index 000000000..ebf8b8cfa --- /dev/null +++ b/test/integration/test-00-commands-have-help @@ -0,0 +1,52 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' + +# this test does double duty: The obvious is checking for --help and co, +# but it also checks if the binary can find all methods in the library. +# The later is quite handy for manual testing of non-abibreaking changes +export LD_BIND_NOW=1 + +checkversionmessage() { + testsuccess grep '^apt .* compiled on ' ${1}-help.output +} + +checkhelpmessage() { + checkversionmessage "$1" + testsuccess grep '^Usage:' ${1}-help.output +} + +checkoptions() { + testsuccess $1 --help + cp -f rootdir/tmp/testsuccess.output ${1}-help.output + checkhelpmessage "$1" + + testsuccess $1 --version + cp -f rootdir/tmp/testsuccess.output ${1}-help.output + checkversionmessage "$1" +} + +for CMD in 'apt-cache' 'apt-cdrom' 'apt-config' \ + 'apt-extracttemplates' 'apt-get' 'apt-helper' \ + 'apt-mark' 'apt-sortpkgs' 'apt' 'apt-ftparchive'; do + cmd="$(echo "$CMD" | tr -d '-')" + msgtest 'Test for failure with no parameters calling' "$CMD" + if $cmd > ${cmd}-help.output 2>&1; then + echo + cat ${cmd}-help.output + msgfail 'zero exit' + else + msgpass + fi + checkhelpmessage "$cmd" + checkoptions "$cmd" +done + +for CMD in 'apt-dump-solver' 'apt-internal-solver'; do + checkoptions "$(echo "$CMD" | tr -d '-')" +done -- cgit v1.2.3-70-g09d2 From d9e518c6f7dc0ad464495b586d1b8e115d54d41a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 20:44:44 +0100 Subject: use the same code to detect quiet setting in all tools Git-Dch: Ignore --- apt-private/private-output.cc | 8 ++-- apt-private/private-output.h | 3 +- cmdline/apt-cache.cc | 4 +- cmdline/apt-cdrom.cc | 7 ++- cmdline/apt-mark.cc | 18 +------ ftparchive/apt-ftparchive.cc | 19 ++------ test/integration/test-apt-ftparchive-cachedb | 30 +++++++----- test/integration/test-apt-ftparchive-src-cachedb | 60 +++++++++++------------- 8 files changed, 63 insertions(+), 86 deletions(-) (limited to 'test') diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc index fc76a05bc..4e18030ab 100644 --- a/apt-private/private-output.cc +++ b/apt-private/private-output.cc @@ -52,14 +52,14 @@ static void SigWinch(int) #endif } /*}}}*/ -bool InitOutput() /*{{{*/ +bool InitOutput(std::basic_streambuf * const out) /*{{{*/ { if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) _config->Set("quiet","1"); - c0out.rdbuf(cout.rdbuf()); - c1out.rdbuf(cout.rdbuf()); - c2out.rdbuf(cout.rdbuf()); + c0out.rdbuf(out); + c1out.rdbuf(out); + c2out.rdbuf(out); if (_config->FindI("quiet",0) > 0) c0out.rdbuf(devnull.rdbuf()); if (_config->FindI("quiet",0) > 1) diff --git a/apt-private/private-output.h b/apt-private/private-output.h index e0dc9bf62..d5b57adec 100644 --- a/apt-private/private-output.h +++ b/apt-private/private-output.h @@ -6,6 +6,7 @@ #include #include +#include // forward declaration class pkgCacheFile; @@ -20,7 +21,7 @@ APT_PUBLIC extern std::ostream c2out; APT_PUBLIC extern std::ofstream devnull; APT_PUBLIC extern unsigned int ScreenWidth; -APT_PUBLIC bool InitOutput(); +APT_PUBLIC bool InitOutput(std::basic_streambuf * const out = std::cout.rdbuf()); void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records, pkgCache::VerIterator const &V, std::ostream &out, diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 12ed4f719..342ad1858 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1899,9 +1899,7 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); + InitOutput(); if (_config->Exists("APT::Cache::Generate") == true) _config->Set("pkgCacheFile::Generate", _config->FindB("APT::Cache::Generate", true)); diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc index 8ac73fd7e..327039e00 100644 --- a/cmdline/apt-cdrom.cc +++ b/cmdline/apt-cdrom.cc @@ -31,6 +31,7 @@ #include #include +#include #include /*}}}*/ @@ -252,10 +253,8 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); - + InitOutput(); + // Match the operation bool returned = CmdL.DispatchArg(Cmds); diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 487f3d8a1..2702dbbd3 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -41,10 +42,6 @@ /*}}}*/ using namespace std; -ostream c0out(0); -ostream c1out(0); -ostream c2out(0); -ofstream devnull("/dev/null"); /* DoAuto - mark packages as automatically/manually installed {{{*/ static bool DoAuto(CommandLine &CmdL) { @@ -444,18 +441,7 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); - - // Setup the output streams - c0out.rdbuf(cout.rdbuf()); - c1out.rdbuf(cout.rdbuf()); - c2out.rdbuf(cout.rdbuf()); - if (_config->FindI("quiet",0) > 0) - c0out.rdbuf(devnull.rdbuf()); - if (_config->FindI("quiet",0) > 1) - c1out.rdbuf(devnull.rdbuf()); + InitOutput(); // Match the operation CmdL.DispatchArg(Cmds); diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index 7d9af4178..adf1b6d73 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -42,11 +43,7 @@ #include /*}}}*/ -using namespace std; -ostream c0out(0); -ostream c1out(0); -ostream c2out(0); -ofstream devnull("/dev/null"); +using namespace std; unsigned Quiet = 0; // struct PackageMap - List of all package files in the config file /*{{{*/ @@ -1064,16 +1061,10 @@ int main(int argc, const char *argv[]) CommandLine CmdL(Args,_config); ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp); - // Setup the output streams - c0out.rdbuf(clog.rdbuf()); - c1out.rdbuf(clog.rdbuf()); - c2out.rdbuf(clog.rdbuf()); + _config->CndSet("quiet",0); Quiet = _config->FindI("quiet",0); - if (Quiet > 0) - c0out.rdbuf(devnull.rdbuf()); - if (Quiet > 1) - c1out.rdbuf(devnull.rdbuf()); - + InitOutput(clog.rdbuf()); + // Match the operation CmdL.DispatchArg(Cmds); diff --git a/test/integration/test-apt-ftparchive-cachedb b/test/integration/test-apt-ftparchive-cachedb index 0e1986bcd..866e5a469 100755 --- a/test/integration/test-apt-ftparchive-cachedb +++ b/test/integration/test-apt-ftparchive-cachedb @@ -69,32 +69,38 @@ buildsimplenativepackage 'foo' 'i386' '1' 'test' mv incoming/* aptarchive/pool/main/ # generate (empty cachedb) -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 2 - dists/test/Contents-i386: New 402 B Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 2 + dists/test/Contents-i386: New 402 B Misses in Cache: 0' # generate again -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 0 - dists/test/Contents-i386: Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 0 + dists/test/Contents-i386: Misses in Cache: 0' # and again (with removing the Packages file) rm -f ./aptarchive/dists/test/main/binary-i386/* rm -f ./aptarchive/dists/test/Contents-i386 -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 0 - dists/test/Contents-i386: New 402 B Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 0 + dists/test/Contents-i386: New 402 B Misses in Cache: 0' # and clean rm -rf aptarchive/pool/main/* -testequal "packages-main-i386.db" aptftparchive clean ftparchive.conf -aptftparchive clean ftparchive.conf -o Debug::APT::FTPArchive::Clean=1 > clean-out.txt 2>&1 +testequal "packages-main-i386.db" aptftparchive clean ftparchive.conf -q=0 +testsuccess aptftparchive clean ftparchive.conf -q=0 -o Debug::APT::FTPArchive::Clean=1 +cp rootdir/tmp/testsuccess.output clean-out.txt testequal "0 Number of unique keys in the tree" grep unique clean-out.txt testequal "packages-main-i386.db" grep packages-main-i386.db clean-out.txt - diff --git a/test/integration/test-apt-ftparchive-src-cachedb b/test/integration/test-apt-ftparchive-src-cachedb index e7b148530..6e857c0cb 100755 --- a/test/integration/test-apt-ftparchive-src-cachedb +++ b/test/integration/test-apt-ftparchive-src-cachedb @@ -106,28 +106,22 @@ mkdir -p aptarchive/dists/test/main/source/ mkdir aptarchive-overrides mkdir aptarchive-cache - - -# generate with --db option -(cd aptarchive && aptftparchive --db ./test.db sources pool/main/ \ - -o APT::FTPArchive::ShowCacheMisses=1 \ - > dists/test/main/source/Sources \ - 2> stats-out.txt - testequal " Misses in Cache: 2" grep Misses stats-out.txt -) +msgtest 'generate with --db option' +cd aptarchive +aptftparchive --db ./test.db sources pool/main/ -q=0 -o APT::FTPArchive::ShowCacheMisses=1 > dists/test/main/source/Sources 2>stats-out.txt && msgpass || msgfail +testsuccess grep Misses stats-out.txt +testfileequal '../rootdir/tmp/testsuccess.output' ' Misses in Cache: 2' +cd .. assert_correct_sources_file -# generate with --db option (again to ensure its in the cache) -(cd aptarchive && aptftparchive --db ./test.db sources pool/main/ \ - -o APT::FTPArchive::ShowCacheMisses=1 \ - > dists/test/main/source/Sources \ - 2> stats-out.txt - testequal " Misses in Cache: 0" grep Misses stats-out.txt -) +msgtest 'generate with --db option (again to ensure its in the cache)' +cd aptarchive +aptftparchive --db ./test.db sources pool/main/ -q=0 -o APT::FTPArchive::ShowCacheMisses=1 > dists/test/main/source/Sources 2>stats-out.txt && msgpass || msgfail +testsuccess grep Misses stats-out.txt +testfileequal '../rootdir/tmp/testsuccess.output' ' Misses in Cache: 0' +cd .. assert_correct_sources_file - - # get ready for the "apt-ftparchive generate" command cat > apt-ftparchive.conf <<"EOF" Dir { @@ -159,20 +153,21 @@ Tree "dists/test" { }; EOF -# generate (empty cachedb) -aptftparchive generate apt-ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt -testequal " Misses in Cache: 2" grep Misses stats-out.txt +msgtest 'generate (empty cachedb)' +testsuccess aptftparchive generate apt-ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal rootdir/tmp/testsuccess.output ' Misses in Cache: 2' assert_correct_sources_file - -# generate again out of the cache +msgtest 'generate again out of the cache' rm -f ./aptarchive/dists/test/main/source/Sources -aptftparchive generate apt-ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt -testequal " Misses in Cache: 0" grep Misses stats-out.txt +testsuccess aptftparchive generate apt-ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal rootdir/tmp/testsuccess.output ' Misses in Cache: 0' assert_correct_sources_file - - # generate invalid files mkdir aptarchive/pool/invalid printf "meep" > aptarchive/pool/invalid/invalid_1.0.dsc @@ -191,8 +186,9 @@ E: DSC file 'aptarchive/pool/invalid/toobig_1.0.dsc' is too large!" aptftparchiv # ensure clean works rm -f aptarchive/pool/main/* -aptftparchive clean apt-ftparchive.conf -o Debug::APT::FTPArchive::Clean=1 > clean-out.txt 2>&1 -testequal "0 Number of unique keys in the tree" grep unique clean-out.txt -testequal "sources-main.db" grep sources-main.db clean-out.txt - - +testsuccess aptftparchive clean apt-ftparchive.conf -q=0 -o Debug::APT::FTPArchive::Clean=1 +cp rootdir/tmp/testsuccess.output clean-out.txt +testsuccess grep unique clean-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' "0 Number of unique keys in the tree" +testsuccess grep sources-main.db clean-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' "sources-main.db" -- cgit v1.2.3-70-g09d2 From 061ee5f0efa9d8e8fe4796c13af68fe282a21073 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 22:23:19 +0100 Subject: enhance apt-extracttemplates test The tool checks for debconf version before printing the info it extracts, so that it doesn't extract data which can't be interpreted before debconf is upgraded. It is only fair to check for this behaviour in the tests. Git-Dch: Ignore --- test/integration/framework | 1 + test/integration/test-apt-extracttemplates | 69 ++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 22 deletions(-) (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index dd66f2a0c..0da9e2aab 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1110,6 +1110,7 @@ testempty() { if "$@" >$COMPAREFILE 2>&1 && test ! -s $COMPAREFILE; then msgpass else + echo cat $COMPAREFILE msgfail fi diff --git a/test/integration/test-apt-extracttemplates b/test/integration/test-apt-extracttemplates index ae2cc8bc2..276862464 100755 --- a/test/integration/test-apt-extracttemplates +++ b/test/integration/test-apt-extracttemplates @@ -8,38 +8,63 @@ setupenvironment configarchitecture 'amd64' # apt-extracttemplates needs this -insertinstalledpackage 'debconf' 'amd64' '1.5' insertinstalledpackage 'pkg-with-template' 'amd64' '1.0' # build a simple package that contains a config and a tempalte mkdir -p DEBIAN -TEMPLATE_STR="Template: foo/bar -Type: string -Description: Some bar var -" -echo "$TEMPLATE_STR" > DEBIAN/templates - CONFIG_STR="#!/bin/sh random shell stuff " echo "$CONFIG_STR" > DEBIAN/config -buildsimplenativepackage 'pkg-with-template' 'amd64' '0.8.15' 'stable' '' 'pkg with template' '' '' './DEBIAN' +testrun() { + local TEMPLATE_STR='Template: foo/bar +Type: string +Description: Some bar var +' + echo "$TEMPLATE_STR" > DEBIAN/templates + buildsimplenativepackage "$1" 'amd64' '0.8.15' 'stable' "$2" 'pkg with template' '' '' './DEBIAN' + + cp dpkg.status rootdir/var/lib/dpkg/status + insertinstalledpackage 'debconf' 'amd64' '3' + + # ensure we get the right stuff out of the file + rm -rf extracttemplates-out rootdir/var/cache/apt + mkdir extracttemplates-out + testsuccess aptextracttemplates -t ./extracttemplates-out incoming/${1}*.deb + OUT='rootdir/tmp/testsuccess.output' + testequal "$1" cut -f1 -d' ' $OUT + if [ -n "$2" ]; then + testequal '' cut -f2 -d' ' $OUT + else + testequal '1.0' cut -f2 -d' ' $OUT + fi + TEMPLATE=$(cut -f3 -d' ' $OUT) + testfileequal "$TEMPLATE" "$TEMPLATE_STR" + CONFIG=$(cut -f4 -d' ' $OUT) + testfileequal "$CONFIG" "$CONFIG_STR" -# ensure we get the right stuff out of the file -mkdir extracttemplates-out -OUT="$(aptextracttemplates -t ./extracttemplates-out incoming/pkg-with-template*.deb)" + # ensure that the format of the output string has the right number of dots + for s in "$CONFIG" "$TEMPLATE"; do + NR_DOTS=$(basename "$s" | tr -c -d '.') + testequal '..' echo $NR_DOTS + done -PKG=$(printf "$OUT" | cut -f1 -d' ') -INSTALLED_VER=$(printf "$OUT" | cut -f2 -d' ') -TEMPLATE=$(printf "$OUT" | cut -f3 -d' ') -CONFIG=$(printf "$OUT" | cut -f4 -d' ') + if [ -n "$2" ]; then + rm -rf extracttemplates-out rootdir/var/cache/apt + mkdir extracttemplates-out + cp dpkg.status rootdir/var/lib/dpkg/status + insertinstalledpackage 'debconf' 'amd64' '1' + testempty aptextracttemplates -t ./extracttemplates-out incoming/${1}*.deb + fi +} -testequal "$CONFIG_STR" cat $CONFIG -testequal "$TEMPLATE_STR" cat $TEMPLATE +cp rootdir/var/lib/dpkg/status dpkg.status +testrun 'pkg-with-template' '' +testrun 'pkg-with-template-depends' 'Depends: debconf (>= 2)' +testrun 'pkg-with-template-predepends' 'Pre-Depends: debconf (>= 2)' -# ensure that the format of the output string has the right number of dots -for s in "$CONFIG" "$TEMPLATE"; do - NR_DOTS=$(basename "$s" | tr -c -d .) - testequal ".." echo $NR_DOTS -done +# test with no debconf installed +cp dpkg.status rootdir/var/lib/dpkg/status +testfailure aptextracttemplates -t ./extracttemplates-out incoming/pkg-with-template-depends*.deb +testfileequal 'rootdir/tmp/testfailure.output' 'E: Cannot get debconf version. Is debconf installed?' -- cgit v1.2.3-70-g09d2 From 7824564bfde681eddf6969174bba7604d6f34848 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 23:59:31 +0100 Subject: fix test to not spoil output with warnings Git-Dch: Ignore --- .../test-bug-624218-Translation-file-handling | 57 +++++++++++++++------- 1 file changed, 39 insertions(+), 18 deletions(-) (limited to 'test') diff --git a/test/integration/test-bug-624218-Translation-file-handling b/test/integration/test-bug-624218-Translation-file-handling index d32bd513b..652386892 100755 --- a/test/integration/test-bug-624218-Translation-file-handling +++ b/test/integration/test-bug-624218-Translation-file-handling @@ -8,80 +8,101 @@ configarchitecture 'i386' buildsimplenativepackage 'coolstuff' 'all' '1.0' 'unstable' -setupaptarchive +setupaptarchive --no-update changetowebserver - rm -rf rootdir/var/lib/apt/lists translationslisted() { msgtest 'No download of non-existent locals' "$1" - LC_ALL="" aptget update -o Acquire::Languages=en | grep -q -e 'Translation-[^e][^n] ' && msgfail || msgpass + export LC_ALL="" + testsuccess --nomsg aptget update -o Acquire::Languages=en + testfailure grep -q -e 'Translation-[^e][^n] ' rootdir/tmp/testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download of existent locals' "$1" - LC_ALL="" aptget update | grep -q -e 'Translation-en ' && msgpass || msgfail + testsuccess --nomsg aptget update + cp rootdir/tmp/testsuccess.output testsuccess.output + testsuccess grep -q -e 'Translation-en ' testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download of en in LC_ALL=C' "$1" - LC_ALL=C aptget update | grep -q -e 'Translation-en ' && msgpass || msgfail + export LC_ALL=C + testsuccess --nomsg aptget update + cp rootdir/tmp/testsuccess.output testsuccess.output + testsuccess grep -q -e 'Translation-en ' testsuccess.output rm -rf rootdir/var/lib/apt/lists + unset LC_ALL msgtest 'Download of en as forced language' "$1" - aptget update -o Acquire::Languages=en | grep -q -e 'Translation-en ' && msgpass || msgfail + testsuccess --nomsg aptget update -o Acquire::Languages=en + cp rootdir/tmp/testsuccess.output testsuccess.output + testsuccess grep -q -e 'Translation-en ' testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download of nothing else in forced language' "$1" - aptget update -o Acquire::Languages=en | grep -q -e 'Translation-[^e][^n] ' && msgfail || msgpass + testsuccess --nomsg aptget update -o Acquire::Languages=en + testfailure grep -q -e 'Translation-[^e][^n] ' rootdir/tmp/testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download no Translation- if forced language is non-existent' "$1" - aptget update -o Acquire::Languages=ast_DE | grep -q -e 'Translation-' && msgfail || msgpass + testsuccess --nomsg aptget update -o Acquire::Languages=ast_DE + testfailure grep -q -e 'Translation-' rootdir/tmp/testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download of nothing if none is forced' "$1" - aptget update -o Acquire::Languages=none | grep -q -e 'Translation' && msgfail || msgpass + testsuccess --nomsg aptget update -o Acquire::Languages=none + testfailure grep -q -e 'Translation' rootdir/tmp/testsuccess.output rm -rf rootdir/var/lib/apt/lists } translationslisted 'with full Index' - # No Release file at all, so no records about Translation files # (fallback to guessing) -find aptarchive -name 'Release' -or -name 'InRelease' | xargs rm -f +find aptarchive \( -name 'Release' -o -name 'InRelease' \) -delete configallowinsecurerepositories "true"; msgtest 'Download of en as forced language' 'without Index' -aptget update -o Acquire::Languages=en | grep -q -e 'Translation-en ' && msgpass || msgfail +testwarning --nomsg aptget update -o Acquire::Languages=en +cp rootdir/tmp/testsuccess.output testsuccess.output +testsuccess grep -q -e 'Translation-en ' testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download of nothing else in forced language' 'without Index' -aptget update -o Acquire::Languages=en | grep -q -e 'Translation-[^e][^n] ' && msgfail || msgpass +testwarning --nomsg aptget update -o Acquire::Languages=en +testfailure grep -q -e 'Translation-[^e][^n] ' rootdir/tmp/testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download of ast_DE as forced language' 'without Index' -aptget update -o Acquire::Languages=ast_DE | grep -q -e 'Translation-ast_DE$' && msgpass || msgfail +testwarning --nomsg aptget update -o Acquire::Languages=ast_DE +cp rootdir/tmp/testsuccess.output testsuccess.output +testsuccess grep -q -e 'Translation-ast_DE$' testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download of nothing else in forced language' 'without Index' -aptget update -o Acquire::Languages=ast_DE | grep -q -e 'Translation-[^a][^s]' && msgfail || msgpass +testwarning --nomsg aptget update -o Acquire::Languages=ast_DE +testfailure grep -q -e 'Translation-[^a][^s]' rootdir/tmp/testsuccess.output rm -rf rootdir/var/lib/apt/lists msgtest 'Download of nothing if none is forced' 'without Index' -aptget update -o Acquire::Languages=none | grep -q -e 'Translation' && msgfail || msgpass +testwarning --nomsg aptget update -o Acquire::Languages=none +testfailure grep -q -e 'Translation' rootdir/tmp/testsuccess.output rm -rf rootdir/var/lib/apt/lists mkdir -p rootdir/var/lib/apt/lists touch rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_i18n_Translation-ast_DE msgtest 'Download of builtin files' 'without Index' -aptget update | grep -q -e 'Translation-ast_DE' && msgpass || msgfail +testwarning --nomsg aptget update +cp rootdir/tmp/testsuccess.output testsuccess.output +testsuccess grep -q -e 'Translation-ast_DE' testsuccess.output rm -rf rootdir/var/lib/apt/lists mkdir -p rootdir/var/lib/apt/lists touch rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_i18n_Translation-ast_DE msgtest 'Download of nothing (even builtin) if none is forced' 'without Index' -aptget update -o Acquire::Languages=none | grep -q -e 'Translation' && msgfail || msgpass +testwarning --nomsg aptget update -o Acquire::Languages=none +testfailure grep -q -e 'Translation' rootdir/tmp/testsuccess.output rm -rf rootdir/var/lib/apt/lists -- cgit v1.2.3-70-g09d2 From 374f8492e6f109e8427816a8f513e5e8feda9049 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 15:40:19 +0100 Subject: allow uninstalled packages to be put on hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dpkg wants to know about a package before it can be put on hold, so we have to at least hint about its existance in the available file it "maintaince" to know about such stuff. The simple thing would probably be to just feed all Packages files into dpkg as well, but what would be the point really? Exactly, so we take a shortcut here and just create dummies in the available file if we need to which isn't going to be that common as usually you are holding packages back and not off. Who would have thought that a simple feature like setting a package on hold requires more than 200 lines of code… at least with the testcase it is now explicitly tested code. --- cmdline/apt-mark.cc | 63 ++++++++++++++++++++++++++++++++---- test/integration/test-apt-mark | 72 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 6 deletions(-) create mode 100755 test/integration/test-apt-mark (limited to 'test') diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 2702dbbd3..860982f9e 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -275,10 +275,64 @@ static bool DoHold(CommandLine &CmdL) } Args.erase(Args.begin() + BaseArgs, Args.end()); - Args.push_back("--set-selections"); + Args.push_back("--merge-avail"); + Args.push_back("-"); Args.push_back(NULL); int external[2] = {-1, -1}; + if (pipe(external) != 0) + return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); + + pid_t dpkgMergeAvail = ExecFork(); + if (dpkgMergeAvail == 0) + { + close(external[1]); + std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); + if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) + _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); + dup2(external[0], STDIN_FILENO); + int const nullfd = open("/dev/null", O_RDONLY); + dup2(nullfd, STDOUT_FILENO); + execvp(Args[0], (char**) &Args[0]); + _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); + _exit(2); + } + + FILE* dpkg = fdopen(external[1], "w"); + for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) + { + if (Pkg->CurrentVer != 0) + continue; + char const * Arch; + if (Pkg->VersionList != 0) + Arch = Pkg.VersionList().Arch(); + else + Arch = Pkg.Arch(); + fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" + "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + } + fclose(dpkg); + + if (dpkgMergeAvail > 0) + { + int Status = 0; + while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + { + if (errno == EINTR) + continue; + _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); + break; + } + if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) + return _error->Error(_("Executing dpkg failed. Are you root?")); + } + + Args.erase(Args.begin() + BaseArgs, Args.end()); + Args.push_back("--set-selections"); + Args.push_back(NULL); + + external[0] = -1; + external[1] = -1; if (pipe(external) != 0) return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --set-selections"); @@ -289,16 +343,13 @@ static bool DoHold(CommandLine &CmdL) std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --set-selections", chrootDir.c_str()); - int const nullfd = open("/dev/null", O_RDONLY); dup2(external[0], STDIN_FILENO); - dup2(nullfd, STDOUT_FILENO); - dup2(nullfd, STDERR_FILENO); execvp(Args[0], (char**) &Args[0]); - _error->WarningE("dpkgGo", "Can't detect if dpkg supports multi-arch!"); + _error->WarningE("dpkgGo", "Can't get dpkg --set-selections running!"); _exit(2); } - FILE* dpkg = fdopen(external[1], "w"); + dpkg = fdopen(external[1], "w"); for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (dpkgMultiArch == false) diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark new file mode 100755 index 000000000..69e0f933d --- /dev/null +++ b/test/integration/test-apt-mark @@ -0,0 +1,72 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'amd64' 'i386' + +insertpackage 'unstable' 'bar' 'amd64,i386' '1' +insertpackage 'unstable' 'uninstalled' 'all' '1' +insertpackage 'unstable' 'uninstalled-native' 'amd64' '1' + +insertinstalledpackage 'foo' 'all' '1' +insertinstalledpackage 'bar' 'amd64' '1' + +setupaptarchive + +# dpkg is "installed" by our test framework +testdpkginstalled dpkg + +testnoautopkg() { + testempty aptmark showauto + testequal 'bar +dpkg +foo' aptmark showmanual + testequal 'bar +foo' aptmark showmanual bar foo uninstalled +} +testmarkonpkgasauto() { + testsuccess aptmark $1 foo + testequal 'foo' aptmark showauto + testequal 'foo' aptmark showauto foo + testequal 'bar +dpkg' aptmark showmanual + testequal 'bar' aptmark showmanual bar + + testsuccess aptmark $2 foo + testnoautopkg +} + +testequal 'E: No packages found' aptmark auto +testequal 'E: No packages found' aptmark manual + +testnoautopkg +testmarkonpkgasauto 'auto' 'manual' +testmarkonpkgasauto 'markauto' 'unmarkauto' + +testnoholdpkg() { + testempty aptmark showhold + testempty aptmark showholds # typical "typo" + testempty aptmark showhold dpkg + testempty aptmark showholds dpkg +} +testmarkonepkgashold() { + testsuccess aptmark hold $1 + testequal "$1" aptmark showhold + testequal "$1" aptmark showholds + testsuccess aptmark unhold $1 + testnoholdpkg +} + +testequal 'E: No packages found' aptmark hold +testequal 'E: No packages found' aptmark unhold + +testnoholdpkg +testmarkonepkgashold 'foo' +testmarkonepkgashold 'bar' +testmarkonepkgashold 'uninstalled' +testmarkonepkgashold 'uninstalled-native' + +testequal 'uninstalled set on hold.' aptmark hold uninstalled +testequal 'uninstalled-native set on hold.' aptmark hold uninstalled-native -- cgit v1.2.3-70-g09d2 From 546dbfc82ad9ff0308b365bca3731a1118b1d251 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 21:26:20 +0100 Subject: disable the lock disabling in the tests We create our own directories here and work without root in them, so we can also test the locking with them as it is how we usually operate. Git-Dch: Ignore --- test/integration/framework | 3 +-- test/integration/test-apt-get-update-unauth-warning | 5 +++-- test/integration/test-apt-update-transactions | 1 + test/integration/test-ubuntu-bug-346386-apt-get-update-paywall | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index 0da9e2aab..d28085ee4 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -236,7 +236,6 @@ setupenvironment() { echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf - echo "Debug::NoLocking \"true\";" >> aptconfig.conf echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf echo "Dir::Bin::Methods \"${METHODSDIR}\";" >> aptconfig.conf # store apt-key were we can access it, even if we run it as a different user @@ -1412,7 +1411,7 @@ aptautotest() { aptautotest_aptget_update() { if ! test -d "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"; then return; fi # all copied files are properly chmodded - for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f); do + for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f ! -name 'lock'); do testfilestats "$file" '%U:%G:%a' '=' "${USER}:${USER}:644" done } diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 0389415c1..80c51152d 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -27,8 +27,9 @@ W: The repository 'file: unstable Release' does not have a Release file. This is E: Use --allow-insecure-repositories to force the update" aptget update --no-allow-insecure-repositories # no package foo -testequal "Listing..." apt list foo -testequal "partial" ls rootdir/var/lib/apt/lists +testequal 'Listing...' apt list foo +testequal 'lock +partial' ls rootdir/var/lib/apt/lists # allow override testequal "Ign file: unstable InRelease diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index fe352c762..bf425a22e 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -49,6 +49,7 @@ testsetup() { msgmsg 'Test with no initial data over' "$1" rm -rf rootdir/var/lib/apt/lists mkdir -m 700 -p rootdir/var/lib/apt/lists/partial + touch rootdir/var/lib/apt/lists/lock if [ "$(id -u)" = '0' ]; then chown _apt:root rootdir/var/lib/apt/lists/partial fi diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall index df2c69cf6..ea516fc12 100755 --- a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall +++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall @@ -40,7 +40,8 @@ msgtest 'Got expected failure message' 'apt-get update' aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0 -testequal 'partial' ls $LISTS +testequal 'lock +partial' ls $LISTS # and again with pre-existing files with "valid data" which should remain for f in Release Release.gpg main_binary-amd64_Packages main_source_Sources; do -- cgit v1.2.3-70-g09d2 From 3f439e2b7126fb82952cd7bc12b8d6cb01352219 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 18 Aug 2013 23:17:05 +0200 Subject: add a simple container for HashStrings APT supports more than just one HashString and even allows to enforce the usage of a specific hash. This class is intended to help with storage and passing around of the HashStrings. The cherry-pick here the un-const-ification of HashType() compared to f4c3850ea335545e297504941dc8c7a8f1c83358. The point of this commit is adding infrastructure for the next one. All by itself, it just adds new symbols. Git-Dch: Ignore --- apt-pkg/contrib/hashes.cc | 118 +++++++++++++++++++++++++++++++++++++----- apt-pkg/contrib/hashes.h | 84 +++++++++++++++++++++++++++++- debian/libapt-pkg4.12.symbols | 8 +++ test/libapt/hashsums_test.cc | 64 ++++++++++++++++++----- 4 files changed, 247 insertions(+), 27 deletions(-) (limited to 'test') diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index 15f83615d..bb11a3fca 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -27,7 +27,7 @@ #include /*}}}*/ -const char* HashString::_SupportedHashes[] = +const char * HashString::_SupportedHashes[] = { "SHA512", "SHA256", "SHA1", "MD5Sum", NULL }; @@ -42,11 +42,16 @@ HashString::HashString(std::string Type, std::string Hash) : Type(Type), Hash(Ha HashString::HashString(std::string StringedHash) /*{{{*/ { - // legacy: md5sum without "MD5Sum:" prefix - if (StringedHash.find(":") == std::string::npos && StringedHash.size() == 32) + if (StringedHash.find(":") == std::string::npos) { - Type = "MD5Sum"; - Hash = StringedHash; + // legacy: md5sum without "MD5Sum:" prefix + if (StringedHash.size() == 32) + { + Type = "MD5Sum"; + Hash = StringedHash; + } + if(_config->FindB("Debug::Hashes",false) == true) + std::clog << "HashString(string): invalid StringedHash " << StringedHash << std::endl; return; } std::string::size_type pos = StringedHash.find(":"); @@ -82,25 +87,25 @@ std::string HashString::GetHashForFile(std::string filename) const /*{{{*/ std::string fileHash; FileFd Fd(filename, FileFd::ReadOnly); - if(Type == "MD5Sum") + if(strcasecmp(Type.c_str(), "MD5Sum") == 0) { MD5Summation MD5; MD5.AddFD(Fd); fileHash = (std::string)MD5.Result(); } - else if (Type == "SHA1") + else if (strcasecmp(Type.c_str(), "SHA1") == 0) { SHA1Summation SHA1; SHA1.AddFD(Fd); fileHash = (std::string)SHA1.Result(); } - else if (Type == "SHA256") + else if (strcasecmp(Type.c_str(), "SHA256") == 0) { SHA256Summation SHA256; SHA256.AddFD(Fd); fileHash = (std::string)SHA256.Result(); } - else if (Type == "SHA512") + else if (strcasecmp(Type.c_str(), "SHA512") == 0) { SHA512Summation SHA512; SHA512.AddFD(Fd); @@ -111,20 +116,105 @@ std::string HashString::GetHashForFile(std::string filename) const /*{{{*/ return fileHash; } /*}}}*/ -const char** HashString::SupportedHashes() +const char** HashString::SupportedHashes() /*{{{*/ { return _SupportedHashes; } - -APT_PURE bool HashString::empty() const + /*}}}*/ +APT_PURE bool HashString::empty() const /*{{{*/ { return (Type.empty() || Hash.empty()); } + /*}}}*/ +std::string HashString::toStr() const /*{{{*/ +{ + return Type + ":" + Hash; +} + /*}}}*/ +APT_PURE bool HashString::operator==(HashString const &other) const /*{{{*/ +{ + return (strcasecmp(Type.c_str(), other.Type.c_str()) == 0 && Hash == other.Hash); +} +APT_PURE bool HashString::operator!=(HashString const &other) const +{ + return !(*this == other); +} + /*}}}*/ + +HashString const * HashStringList::find(char const * const type) const /*{{{*/ +{ + if (type == NULL || type[0] == '\0') + { + std::string forcedType = _config->Find("Acquire::ForceHash", ""); + if (forcedType.empty() == false) + return find(forcedType.c_str()); + for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t) + for (std::vector::const_iterator hs = list.begin(); hs != list.end(); ++hs) + if (strcasecmp(hs->HashType().c_str(), *t) == 0) + return &*hs; + return NULL; + } + for (std::vector::const_iterator hs = list.begin(); hs != list.end(); ++hs) + if (strcasecmp(hs->HashType().c_str(), type) == 0) + return &*hs; + return NULL; +} + /*}}}*/ +bool HashStringList::supported(char const * const type) /*{{{*/ +{ + for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t) + if (strcasecmp(*t, type) == 0) + return true; + return false; +} + /*}}}*/ +bool HashStringList::push_back(const HashString &hashString) /*{{{*/ +{ + if (hashString.HashType().empty() == true || + hashString.HashValue().empty() == true || + supported(hashString.HashType().c_str()) == false) + return false; + + // ensure that each type is added only once + HashString const * const hs = find(hashString.HashType().c_str()); + if (hs != NULL) + return *hs == hashString; -std::string HashString::toStr() const + list.push_back(hashString); + return true; +} + /*}}}*/ +bool HashStringList::VerifyFile(std::string filename) const /*{{{*/ { - return Type + std::string(":") + Hash; + if (list.empty() == true) + return false; + HashString const * const hs = find(NULL); + if (hs == NULL || hs->VerifyFile(filename) == false) + return false; + return true; } + /*}}}*/ +bool HashStringList::operator==(HashStringList const &other) const /*{{{*/ +{ + short matches = 0; + for (const_iterator hs = begin(); hs != end(); ++hs) + { + HashString const * const ohs = other.find(hs->HashType()); + if (ohs == NULL) + continue; + if (*hs != *ohs) + return false; + ++matches; + } + if (matches == 0) + return false; + return true; +} +bool HashStringList::operator!=(HashStringList const &other) const +{ + return !(*this == other); +} + /*}}}*/ // Hashes::AddFD - Add the contents of the FD /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h index 7a62f8a8f..5a4213868 100644 --- a/apt-pkg/contrib/hashes.h +++ b/apt-pkg/contrib/hashes.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -41,7 +42,7 @@ class HashString protected: std::string Type; std::string Hash; - static const char* _SupportedHashes[10]; + static const char * _SupportedHashes[10]; // internal helper std::string GetHashForFile(std::string filename) const; @@ -53,6 +54,8 @@ class HashString // get hash type used std::string HashType() { return Type; }; + std::string HashType() const { return Type; }; + std::string HashValue() const { return Hash; }; // verify the given filename against the currently loaded hash bool VerifyFile(std::string filename) const; @@ -64,11 +67,90 @@ class HashString // helper std::string toStr() const; // convert to str as "type:hash" bool empty() const; + bool operator==(HashString const &other) const; + bool operator!=(HashString const &other) const; // return the list of hashes we support static APT_CONST const char** SupportedHashes(); }; +class HashStringList +{ + public: + /** find best hash if no specific one is requested + * + * @param type of the checksum to return, can be \b NULL + * @return If type is \b NULL (or the empty string) it will + * return the 'best' hash; otherwise the hash which was + * specifically requested. If no hash is found \b NULL will be returned. + */ + HashString const * find(char const * const type) const; + HashString const * find(std::string const &type) const { return find(type.c_str()); } + /** check if the given hash type is supported + * + * @param type to check + * @return true if supported, otherwise false + */ + static APT_PURE bool supported(char const * const type); + /** add the given #HashString to the list + * + * @param hashString to add + * @return true if the hash is added because it is supported and + * not already a different hash of the same type included, otherwise false + */ + bool push_back(const HashString &hashString); + /** @return size of the list of HashStrings */ + size_t size() const { return list.size(); } + + /** take the 'best' hash and verify file with it + * + * @param filename to verify + * @return true if the file matches the hashsum, otherwise false + */ + bool VerifyFile(std::string filename) const; + + /** is the list empty ? + * + * @return \b true if the list is empty, otherwise \b false + */ + bool empty() const { return list.empty(); } + + typedef std::vector::const_iterator const_iterator; + + /** iterator to the first element */ + const_iterator begin() const { return list.begin(); } + + /** iterator to the end element */ + const_iterator end() const { return list.end(); } + + /** start fresh with a clear list */ + void clear() { list.clear(); } + + /** compare two HashStringList for similarity. + * + * Two lists are similar if at least one hashtype is in both lists + * and the hashsum matches. All hashes are checked, if one doesn't + * match false is returned regardless of how many matched before. + */ + bool operator==(HashStringList const &other) const; + bool operator!=(HashStringList const &other) const; + + HashStringList() {} + + // simplifying API-compatibility constructors + HashStringList(std::string const &hash) { + if (hash.empty() == false) + list.push_back(HashString(hash)); + } + HashStringList(char const * const hash) { + if (hash != NULL && hash[0] != '\0') + list.push_back(HashString(hash)); + } + + private: + std::vector list; +}; + class Hashes { public: diff --git a/debian/libapt-pkg4.12.symbols b/debian/libapt-pkg4.12.symbols index 3fa128cff..d89f07bb5 100644 --- a/debian/libapt-pkg4.12.symbols +++ b/debian/libapt-pkg4.12.symbols @@ -1579,6 +1579,14 @@ libapt-pkg.so.4.12 libapt-pkg4.12 #MINVER# (c++)"typeinfo for debTranslationsParser@Base" 1.0.4 (c++)"typeinfo name for debTranslationsParser@Base" 1.0.4 (c++)"vtable for debTranslationsParser@Base" 1.0.4 + (c++)"HashStringList::find(char const*) const@Base" 1.0.9.4 + (c++)"HashStringList::operator==(HashStringList const&) const@Base" 1.0.9.4 + (c++)"HashStringList::operator!=(HashStringList const&) const@Base" 1.0.9.4 + (c++)"HashStringList::push_back(HashString const&)@Base" 1.0.9.4 + (c++)"HashStringList::supported(char const*)@Base" 1.0.9.4 + (c++)"HashStringList::VerifyFile(std::basic_string, std::allocator >) const@Base" 1.0.9.4 + (c++)"HashString::operator==(HashString const&) const@Base" 1.0.9.4 + (c++)"HashString::operator!=(HashString const&) const@Base" 1.0.9.4 ### demangle strangeness - buildd report it as MISSING and as new… (c++)"pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire*, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::vector > const*, indexRecords*)@Base" 0.8.0 ### gcc-4.6 artefacts diff --git a/test/libapt/hashsums_test.cc b/test/libapt/hashsums_test.cc index c06d85e03..ac7d41582 100644 --- a/test/libapt/hashsums_test.cc +++ b/test/libapt/hashsums_test.cc @@ -207,16 +207,56 @@ TEST(HashSumsTest, FileBased) } fd.Close(); - { - HashString sha2("SHA256", sha256.Value()); - EXPECT_TRUE(sha2.VerifyFile(__FILE__)); - } - { - HashString sha2("SHA512", sha512.Value()); - EXPECT_TRUE(sha2.VerifyFile(__FILE__)); - } - { - HashString sha2("SHA256:" + sha256.Value()); - EXPECT_TRUE(sha2.VerifyFile(__FILE__)); - } + HashString sha2file("SHA512", sha512.Value()); + EXPECT_TRUE(sha2file.VerifyFile(__FILE__)); + HashString sha2wrong("SHA512", "00000000000"); + EXPECT_FALSE(sha2wrong.VerifyFile(__FILE__)); + EXPECT_EQ(sha2file, sha2file); + EXPECT_TRUE(sha2file == sha2file); + EXPECT_NE(sha2file, sha2wrong); + EXPECT_TRUE(sha2file != sha2wrong); + + HashString sha2big("SHA256", sha256.Value()); + EXPECT_TRUE(sha2big.VerifyFile(__FILE__)); + HashString sha2small("sha256:" + sha256.Value()); + EXPECT_TRUE(sha2small.VerifyFile(__FILE__)); + EXPECT_EQ(sha2big, sha2small); + EXPECT_TRUE(sha2big == sha2small); + EXPECT_FALSE(sha2big != sha2small); + + HashStringList hashes; + EXPECT_TRUE(hashes.empty()); + EXPECT_TRUE(hashes.push_back(sha2file)); + EXPECT_FALSE(hashes.empty()); + EXPECT_EQ(1, hashes.size()); + + HashStringList wrong; + EXPECT_TRUE(wrong.push_back(sha2wrong)); + EXPECT_NE(wrong, hashes); + EXPECT_FALSE(wrong == hashes); + EXPECT_TRUE(wrong != hashes); + + HashStringList similar; + EXPECT_TRUE(similar.push_back(sha2big)); + EXPECT_NE(similar, hashes); + EXPECT_FALSE(similar == hashes); + EXPECT_TRUE(similar != hashes); + + EXPECT_TRUE(hashes.push_back(sha2big)); + EXPECT_EQ(2, hashes.size()); + EXPECT_TRUE(hashes.push_back(sha2small)); + EXPECT_EQ(2, hashes.size()); + EXPECT_FALSE(hashes.push_back(sha2wrong)); + EXPECT_EQ(2, hashes.size()); + EXPECT_TRUE(hashes.VerifyFile(__FILE__)); + + EXPECT_EQ(similar, hashes); + EXPECT_TRUE(similar == hashes); + EXPECT_FALSE(similar != hashes); + similar.clear(); + EXPECT_TRUE(similar.empty()); + EXPECT_EQ(0, similar.size()); + EXPECT_NE(similar, hashes); + EXPECT_FALSE(similar == hashes); + EXPECT_TRUE(similar != hashes); } -- cgit v1.2.3-70-g09d2 From 3a2b39ee602dd5a98b8fdaee2f1c8e0b13a276e2 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 18 Aug 2013 23:27:24 +0200 Subject: use 'best' hash for source authentication Collect all hashes we can get from the source record and put them into a HashStringList so that 'apt-get source' can use it instead of using always the MD5sum. We therefore also deprecate the MD5 struct member in favor of the list. While at it, the parsing of the Files is enhanced so that records which miss "Files" (aka MD5 checksums) are still searched for other checksums as they include just as much data, just not with a nice and catchy name. This is a cherry-pick of 1262d35 with some dirty tricks to preserve ABI. LP: 1098738 --- apt-pkg/deb/debsrcrecords.cc | 162 +++++++++---- apt-pkg/deb/debsrcrecords.h | 1 + apt-pkg/srcrecords.cc | 31 ++- apt-pkg/srcrecords.h | 21 +- cmdline/apt-get.cc | 28 ++- debian/libapt-pkg4.12.symbols | 2 + .../test-ubuntu-bug-1098738-apt-get-source-md5sum | 260 +++++++++++++++++++++ 7 files changed, 445 insertions(+), 60 deletions(-) create mode 100755 test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum (limited to 'test') diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc index a444cbe4d..49a348dd4 100644 --- a/apt-pkg/deb/debsrcrecords.cc +++ b/apt-pkg/deb/debsrcrecords.cc @@ -118,13 +118,32 @@ bool debSrcRecordParser::BuildDepends(std::vector &List) +bool debSrcRecordParser::Files(std::vector &F) { - List.erase(List.begin(),List.end()); - - string Files = Sect.FindS("Files"); - if (Files.empty() == true) + std::vector F2; + if (Files2(F2) == false) return false; + for (std::vector::const_iterator f2 = F2.begin(); f2 != F2.end(); ++f2) + { + pkgSrcRecords::File2 f; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + f.MD5Hash = f2->MD5Hash; + f.Size = f2->Size; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + f.Path = f2->Path; + f.Type = f2->Type; + F.push_back(f); + } + return true; +} +bool debSrcRecordParser::Files2(std::vector &List) +{ + List.clear(); // Stash the / terminated directory prefix string Base = Sect.FindS("Directory"); @@ -133,51 +152,106 @@ bool debSrcRecordParser::Files(std::vector &List) std::vector const compExts = APT::Configuration::getCompressorExtensions(); - // Iterate over the entire list grabbing each triplet - const char *C = Files.c_str(); - while (*C != 0) - { - pkgSrcRecords::File F; - string Size; - - // Parse each of the elements - if (ParseQuoteWord(C,F.MD5Hash) == false || - ParseQuoteWord(C,Size) == false || - ParseQuoteWord(C,F.Path) == false) - return _error->Error("Error parsing file record"); - - // Parse the size and append the directory - F.Size = atoi(Size.c_str()); - F.Path = Base + F.Path; - - // Try to guess what sort of file it is we are getting. - string::size_type Pos = F.Path.length()-1; - while (1) + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + // derive field from checksum type + std::string checksumField("Checksums-"); + if (strcmp(*type, "MD5Sum") == 0) + checksumField = "Files"; // historic name for MD5 checksums + else + checksumField.append(*type); + + string const Files = Sect.FindS(checksumField.c_str()); + if (Files.empty() == true) + continue; + + // Iterate over the entire list grabbing each triplet + const char *C = Files.c_str(); + while (*C != 0) { - string::size_type Tmp = F.Path.rfind('.',Pos); - if (Tmp == string::npos) - break; - if (F.Type == "tar") { - // source v3 has extension 'debian.tar.*' instead of 'diff.*' - if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") - F.Type = "diff"; - break; - } - F.Type = string(F.Path,Tmp+1,Pos-Tmp); - - if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || - F.Type == "tar") + string hash, size, path; + + // Parse each of the elements + if (ParseQuoteWord(C, hash) == false || + ParseQuoteWord(C, size) == false || + ParseQuoteWord(C, path) == false) + return _error->Error("Error parsing file record in %s of source package %s", checksumField.c_str(), Package().c_str()); + + HashString const hashString(*type, hash); + if (Base.empty() == false) + path = Base + path; + + // look if we have a record for this file already + std::vector::iterator file = List.begin(); + for (; file != List.end(); ++file) + if (file->Path == path) + break; + + // we have it already, store the new hash and be done + if (file != List.end()) { - Pos = Tmp-1; +#if __GNUC__ >= 4 + // set for compatibility only, so warn users not us + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + if (checksumField == "Files") + file->MD5Hash = hash; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + // an error here indicates that we have two different hashes for the same file + if (file->Hashes.push_back(hashString) == false) + return _error->Error("Error parsing checksum in %s of source package %s", checksumField.c_str(), Package().c_str()); continue; } - - break; + + // we haven't seen this file yet + pkgSrcRecords::File2 F; + F.Path = path; + F.FileSize = strtoull(size.c_str(), NULL, 10); + F.Hashes.push_back(hashString); + +#if __GNUC__ >= 4 + // set for compatibility only, so warn users not us + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + F.Size = F.FileSize; + if (checksumField == "Files") + F.MD5Hash = hash; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + + // Try to guess what sort of file it is we are getting. + string::size_type Pos = F.Path.length()-1; + while (1) + { + string::size_type Tmp = F.Path.rfind('.',Pos); + if (Tmp == string::npos) + break; + if (F.Type == "tar") { + // source v3 has extension 'debian.tar.*' instead of 'diff.*' + if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") + F.Type = "diff"; + break; + } + F.Type = string(F.Path,Tmp+1,Pos-Tmp); + + if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || + F.Type == "tar") + { + Pos = Tmp-1; + continue; + } + + break; + } + List.push_back(F); } - - List.push_back(F); } - + return true; } /*}}}*/ diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h index b65d1480b..2a3fc86c9 100644 --- a/apt-pkg/deb/debsrcrecords.h +++ b/apt-pkg/deb/debsrcrecords.h @@ -53,6 +53,7 @@ class debSrcRecordParser : public pkgSrcRecords::Parser return std::string(Start,Stop); }; virtual bool Files(std::vector &F); + bool Files2(std::vector &F); debSrcRecordParser(std::string const &File,pkgIndexFile const *Index) : Parser(Index), Fd(File,FileFd::ReadOnly, FileFd::Extension), Tags(&Fd,102400), diff --git a/apt-pkg/srcrecords.cc b/apt-pkg/srcrecords.cc index 81b1c545d..3175ee75f 100644 --- a/apt-pkg/srcrecords.cc +++ b/apt-pkg/srcrecords.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -147,5 +148,33 @@ const char *pkgSrcRecords::Parser::BuildDepType(unsigned char const &Type) return fields[Type]; } /*}}}*/ +bool pkgSrcRecords::Parser::Files2(std::vector &F2)/*{{{*/ +{ + debSrcRecordParser * const deb = dynamic_cast(this); + if (deb != NULL) + return deb->Files2(F2); - + std::vector F; + if (Files(F) == false) + return false; + for (std::vector::const_iterator f = F.begin(); f != F.end(); ++f) + { + pkgSrcRecords::File2 f2; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + f2.MD5Hash = f->MD5Hash; + f2.Size = f->Size; + f2.Hashes.push_back(HashString("MD5Sum", f->MD5Hash)); + f2.FileSize = f->Size; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + f2.Path = f->Path; + f2.Type = f->Type; + F2.push_back(f2); + } + return true; +} + /*}}}*/ diff --git a/apt-pkg/srcrecords.h b/apt-pkg/srcrecords.h index e000e176a..dde22bd65 100644 --- a/apt-pkg/srcrecords.h +++ b/apt-pkg/srcrecords.h @@ -14,6 +14,7 @@ #define PKGLIB_SRCRECORDS_H #include +#include #include #include @@ -29,15 +30,28 @@ class pkgSrcRecords { public: +#if __GNUC__ >= 4 + // ensure that con- & de-structor don't trigger this warning + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif // Describes a single file struct File { - std::string MD5Hash; - unsigned long Size; + APT_DEPRECATED std::string MD5Hash; + APT_DEPRECATED unsigned long Size; std::string Path; std::string Type; }; - + struct File2 : public File + { + unsigned long long FileSize; + HashStringList Hashes; + }; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + // Abstract parser for each source record class Parser { @@ -77,6 +91,7 @@ class pkgSrcRecords static const char *BuildDepType(unsigned char const &Type) APT_PURE; virtual bool Files(std::vector &F) = 0; + bool Files2(std::vector &F); Parser(const pkgIndexFile *Index) : iIndex(Index) {}; virtual ~Parser() {}; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index cfa79339b..a28537712 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -797,13 +797,13 @@ static bool DoSource(CommandLine &CmdL) } // Back track - vector Lst; - if (Last->Files(Lst) == false) { + vector Lst; + if (Last->Files2(Lst) == false) { return false; } // Load them into the fetcher - for (vector::const_iterator I = Lst.begin(); + for (vector::const_iterator I = Lst.begin(); I != Lst.end(); ++I) { // Try to guess what sort of file it is we are getting. @@ -832,22 +832,26 @@ static bool DoSource(CommandLine &CmdL) queued.insert(Last->Index().ArchiveURI(I->Path)); // check if we have a file with that md5 sum already localy - if(!I->MD5Hash.empty() && FileExists(flNotDir(I->Path))) - { - FileFd Fd(flNotDir(I->Path), FileFd::ReadOnly); - MD5Summation sum; - sum.AddFD(Fd.Fd(), Fd.Size()); - Fd.Close(); - if((string)sum.Result() == I->MD5Hash) + std::string localFile = flNotDir(I->Path); + if (FileExists(localFile) == true) + if(I->Hashes.VerifyFile(localFile) == true) { ioprintf(c1out,_("Skipping already downloaded file '%s'\n"), - flNotDir(I->Path).c_str()); + localFile.c_str()); continue; } + + // see if we have a hash (Acquire::ForceHash is the only way to have none) + HashString const * const hs = I->Hashes.find(NULL); + if (hs == NULL && _config->FindB("APT::Get::AllowUnauthenticated",false) == false) + { + ioprintf(c1out, "Skipping download of file '%s' as requested hashsum is not available for authentication\n", + localFile.c_str()); + continue; } new pkgAcqFile(&Fetcher,Last->Index().ArchiveURI(I->Path), - I->MD5Hash,I->Size, + hs != NULL ? hs->toStr() : "", I->FileSize, Last->Index().SourceInfo(*Last,*I),Src); } } diff --git a/debian/libapt-pkg4.12.symbols b/debian/libapt-pkg4.12.symbols index d89f07bb5..d481e51ed 100644 --- a/debian/libapt-pkg4.12.symbols +++ b/debian/libapt-pkg4.12.symbols @@ -1587,6 +1587,8 @@ libapt-pkg.so.4.12 libapt-pkg4.12 #MINVER# (c++)"HashStringList::VerifyFile(std::basic_string, std::allocator >) const@Base" 1.0.9.4 (c++)"HashString::operator==(HashString const&) const@Base" 1.0.9.4 (c++)"HashString::operator!=(HashString const&) const@Base" 1.0.9.4 + (c++)"pkgSrcRecords::Parser::Files2(std::vector >&)@Base" 1.0.9.4 + (c++)"debSrcRecordParser::Files2(std::vector >&)@Base" 1.0.9.4 ### demangle strangeness - buildd report it as MISSING and as new… (c++)"pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire*, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::vector > const*, indexRecords*)@Base" 0.8.0 ### gcc-4.6 artefacts diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum new file mode 100755 index 000000000..9bdc81264 --- /dev/null +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -0,0 +1,260 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'native' + +cat > aptarchive/Sources < +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.tar.gz + +Package: pkg-sha256-ok +Binary: pkg-sha256-ok +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.tar.gz + +Package: pkg-sha256-bad +Binary: pkg-sha256-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.tar.gz +Checksums-Sha256: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-bad_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-bad_1.0.tar.gz + +Package: pkg-no-md5 +Binary: pkg-no-md5 +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.tar.gz + +Package: pkg-mixed-ok +Binary: pkg-mixed-ok +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-ok_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-ok_1.0.dsc + +Package: pkg-mixed-sha1-bad +Binary: pkg-mixed-sha1-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-mixed-sha1-bad_1.0.dsc +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-sha1-bad_1.0.tar.gz + +Package: pkg-mixed-sha2-bad +Binary: pkg-mixed-sha2-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-sha2-bad_1.0.dsc +Checksums-Sha256: + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-mixed-sha2-bad_1.0.tar.gz + +Package: pkg-md5-disagree +Binary: pkg-md5-disagree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.tar.gz + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-md5-disagree_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-md5-disagree_1.0.tar.gz + +Package: pkg-md5-agree +Binary: pkg-md5-agree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc + +Package: pkg-sha256-disagree +Binary: pkg-sha256-disagree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.tar.gz + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-disagree_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-disagree_1.0.tar.gz +EOF + +# create fetchable files +for x in 'pkg-md5-ok' 'pkg-sha256-ok' 'pkg-sha256-bad' 'pkg-no-md5' \ + 'pkg-mixed-ok' 'pkg-mixed-sha1-bad' 'pkg-mixed-sha2-bad' \ + 'pkg-md5-agree' 'pkg-md5-disagree' 'pkg-sha256-disagree'; do + touch aptarchive/${x}_1.0.dsc aptarchive/${x}_1.0.tar.gz +done + +setupaptarchive +changetowebserver +testsuccess aptget update + +testok() { + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +Download complete and in download only mode" aptget source -d "$@" + msgtest 'Files were successfully downloaded for' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz +} + +testkeep() { + touch ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Skipping already downloaded file '${1}_1.0.dsc' +Skipping already downloaded file '${1}_1.0.tar.gz' +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d "$@" + msgtest 'Files already downloaded are kept for' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz +} + +testmismatch() { + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +E: Failed to fetch http://localhost:8080/${1}_1.0.dsc Hash Sum mismatch + +E: Failed to fetch http://localhost:8080/${1}_1.0.tar.gz Hash Sum mismatch + +E: Failed to fetch some archives." aptget source -d "$@" + msgtest 'Files were not download as they have hashsum mismatches for' "$1" + testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Skipping download of file 'pkg-sha256-bad_1.0.dsc' as requested hashsum is not available for authentication +Skipping download of file 'pkg-sha256-bad_1.0.tar.gz' as requested hashsum is not available for authentication +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d "$@" -o Acquire::ForceHash=ROT26 + msgtest 'Files were not download as hash is unavailable for' "$1" + testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +Download complete and in download only mode" aptget source --allow-unauthenticated -d "$@" -o Acquire::ForceHash=ROT26 + msgtest 'Files were downloaded unauthenticated as user allowed it' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz +} + +testok pkg-md5-ok +testkeep pkg-md5-ok +testok pkg-sha256-ok +testkeep pkg-sha256-ok + +# pkg-sha256-bad has a bad SHA sum, but good MD5 sum. If apt is +# checking the best available hash (as it should), this will trigger +# a hash mismatch. +testmismatch pkg-sha256-bad +testmismatch pkg-sha256-bad +testok pkg-sha256-bad -o Acquire::ForceHash=MD5Sum + +# not having MD5 sum doesn't mean the file doesn't exist at all … +testok pkg-no-md5 +testok pkg-no-md5 -o Acquire::ForceHash=SHA256 +testequal "Reading package lists... +Building dependency tree... +Skipping download of file 'pkg-no-md5_1.0.dsc' as requested hashsum is not available for authentication +Skipping download of file 'pkg-no-md5_1.0.tar.gz' as requested hashsum is not available for authentication +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d pkg-no-md5 -o Acquire::ForceHash=MD5Sum +msgtest 'Files were not download as MD5 is not available for this package' 'pkg-no-md5' +testfailure --nomsg test -e pkg-no-md5_1.0.dsc -a -e pkg-no-md5_1.0.tar.gz + +# deal with cases in which we haven't for all files the same checksum type +# mostly pathologic as this shouldn't happen, but just to be sure +testok pkg-mixed-ok +testequal 'Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (tar) +Get:2 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) +E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc Hash Sum mismatch + +E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha1-bad +msgtest 'Only tar file is downloaded as the dsc has hashsum mismatch' 'pkg-mixed-sha1-bad' +testsuccess --nomsg test ! -e pkg-mixed-sha1-bad_1.0.dsc -a -e pkg-mixed-sha1-bad_1.0.tar.gz +testequal 'Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) +Get:2 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (dsc) +E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz Hash Sum mismatch + +E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha2-bad +msgtest 'Only dsc file is downloaded as the tar has hashsum mismatch' 'pkg-mixed-sha2-bad' +testsuccess --nomsg test -e pkg-mixed-sha2-bad_1.0.dsc -a ! -e pkg-mixed-sha2-bad_1.0.tar.gz + +# it gets even more pathologic: multiple entries for one file, some even disagreeing! +testok pkg-md5-agree +testequal 'Reading package lists... +Building dependency tree... +E: Error parsing checksum in Files of source package pkg-md5-disagree' aptget source -d pkg-md5-disagree +testequal 'Reading package lists... +Building dependency tree... +E: Error parsing checksum in Checksums-SHA256 of source package pkg-sha256-disagree' aptget source -d pkg-sha256-disagree -- cgit v1.2.3-70-g09d2 From c505fa33a6441b451971ce6c636cf2ca4dacdc1d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 28 Sep 2014 01:25:21 +0200 Subject: allow options between command and -- on commandline This used to work before we implemented a stricter commandline parser and e.g. the dd-schroot-cmd command constructs commandlines like this. Reported-By: Helmut Grohne --- apt-pkg/contrib/cmndline.cc | 19 +++++++----- test/libapt/commandline_test.cc | 68 +++++++++++++++++++++++++++++++++++++++++ test/libapt/makefile | 4 +-- 3 files changed, 81 insertions(+), 10 deletions(-) (limited to 'test') diff --git a/apt-pkg/contrib/cmndline.cc b/apt-pkg/contrib/cmndline.cc index 3799c822d..93c1f4664 100644 --- a/apt-pkg/contrib/cmndline.cc +++ b/apt-pkg/contrib/cmndline.cc @@ -47,23 +47,26 @@ CommandLine::~CommandLine() char const * CommandLine::GetCommand(Dispatch const * const Map, unsigned int const argc, char const * const * const argv) { - // if there is a -- on the line there must be the word we search for around it - // as -- marks the end of the options, just not sure if the command can be - // considered an option or not, so accept both + // if there is a -- on the line there must be the word we search for either + // before it (as -- marks the end of the options) or right after it (as we can't + // decide if the command is actually an option, given that in theory, you could + // have parameters named like commands) for (size_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "--") != 0) continue; - ++i; - if (i < argc) + // check if command is before -- + for (size_t k = 1; k < i; ++k) for (size_t j = 0; Map[j].Match != NULL; ++j) - if (strcmp(argv[i], Map[j].Match) == 0) + if (strcmp(argv[k], Map[j].Match) == 0) return Map[j].Match; - i -= 2; - if (i != 0) + // see if the next token after -- is the command + ++i; + if (i < argc) for (size_t j = 0; Map[j].Match != NULL; ++j) if (strcmp(argv[i], Map[j].Match) == 0) return Map[j].Match; + // we found a --, but not a command return NULL; } // no --, so search for the first word matching a command diff --git a/test/libapt/commandline_test.cc b/test/libapt/commandline_test.cc index e403a28c8..627f1b486 100644 --- a/test/libapt/commandline_test.cc +++ b/test/libapt/commandline_test.cc @@ -2,6 +2,7 @@ #include #include +#include #include @@ -85,3 +86,70 @@ TEST(CommandLineTest, BoolParsing) } } + +bool DoVoid(CommandLine &) { return false; } + +TEST(CommandLineTest,GetCommand) +{ + CommandLine::Dispatch Cmds[] = { {"install",&DoVoid}, {"remove", &DoVoid}, {0,0} }; + { + char const * argv[] = { "apt-get", "-t", "unstable", "remove", "-d", "foo" }; + char const * com = CommandLine::GetCommand(Cmds, sizeof(argv)/sizeof(argv[0]), argv); + EXPECT_STREQ("remove", com); + std::vector Args = getCommandArgs("apt-get", com); + ::Configuration c; + CommandLine CmdL(Args.data(), &c); + ASSERT_TRUE(CmdL.Parse(sizeof(argv)/sizeof(argv[0]), argv)); + EXPECT_EQ(c.Find("APT::Default-Release"), "unstable"); + EXPECT_TRUE(c.FindB("APT::Get::Download-Only")); + ASSERT_EQ(2, CmdL.FileSize()); + EXPECT_EQ(std::string(CmdL.FileList[0]), "remove"); + EXPECT_EQ(std::string(CmdL.FileList[1]), "foo"); + } + { + char const * argv[] = {"apt-get", "-t", "unstable", "remove", "--", "-d", "foo" }; + char const * com = CommandLine::GetCommand(Cmds, sizeof(argv)/sizeof(argv[0]), argv); + EXPECT_STREQ("remove", com); + std::vector Args = getCommandArgs("apt-get", com); + ::Configuration c; + CommandLine CmdL(Args.data(), &c); + ASSERT_TRUE(CmdL.Parse(sizeof(argv)/sizeof(argv[0]), argv)); + EXPECT_EQ(c.Find("APT::Default-Release"), "unstable"); + EXPECT_FALSE(c.FindB("APT::Get::Download-Only")); + ASSERT_EQ(3, CmdL.FileSize()); + EXPECT_EQ(std::string(CmdL.FileList[0]), "remove"); + EXPECT_EQ(std::string(CmdL.FileList[1]), "-d"); + EXPECT_EQ(std::string(CmdL.FileList[2]), "foo"); + } + { + char const * argv[] = {"apt-get", "-t", "unstable", "--", "remove", "-d", "foo" }; + char const * com = CommandLine::GetCommand(Cmds, sizeof(argv)/sizeof(argv[0]), argv); + EXPECT_STREQ("remove", com); + std::vector Args = getCommandArgs("apt-get", com); + ::Configuration c; + CommandLine CmdL(Args.data(), &c); + ASSERT_TRUE(CmdL.Parse(sizeof(argv)/sizeof(argv[0]), argv)); + EXPECT_EQ(c.Find("APT::Default-Release"), "unstable"); + EXPECT_FALSE(c.FindB("APT::Get::Download-Only")); + ASSERT_EQ(CmdL.FileSize(), 3); + EXPECT_EQ(std::string(CmdL.FileList[0]), "remove"); + EXPECT_EQ(std::string(CmdL.FileList[1]), "-d"); + EXPECT_EQ(std::string(CmdL.FileList[2]), "foo"); + } + { + char const * argv[] = {"apt-get", "install", "-t", "unstable", "--", "remove", "-d", "foo" }; + char const * com = CommandLine::GetCommand(Cmds, sizeof(argv)/sizeof(argv[0]), argv); + EXPECT_STREQ("install", com); + std::vector Args = getCommandArgs("apt-get", com); + ::Configuration c; + CommandLine CmdL(Args.data(), &c); + ASSERT_TRUE(CmdL.Parse(sizeof(argv)/sizeof(argv[0]), argv)); + EXPECT_EQ(c.Find("APT::Default-Release"), "unstable"); + EXPECT_FALSE(c.FindB("APT::Get::Download-Only")); + ASSERT_EQ(CmdL.FileSize(), 4); + EXPECT_EQ(std::string(CmdL.FileList[0]), "install"); + EXPECT_EQ(std::string(CmdL.FileList[1]), "remove"); + EXPECT_EQ(std::string(CmdL.FileList[2]), "-d"); + EXPECT_EQ(std::string(CmdL.FileList[3]), "foo"); + } +} diff --git a/test/libapt/makefile b/test/libapt/makefile index 69a13fd92..7f23ace46 100644 --- a/test/libapt/makefile +++ b/test/libapt/makefile @@ -14,8 +14,8 @@ test: $(BIN)/gtest$(BASENAME) $(BIN)/gtest$(BASENAME): $(LIB)/gtest.a PROGRAM = gtest${BASENAME} -SLIBS = -lapt-pkg -pthread $(LIB)/gtest.a -LIB_MAKES = apt-pkg/makefile +SLIBS = -lapt-pkg -lapt-private -pthread $(LIB)/gtest.a +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = gtest_runner.cc $(wildcard *-helpers.cc *_test.cc) include $(PROGRAM_H) -- cgit v1.2.3-70-g09d2 From 8cc3535f2cdcfe1301b641dae8dfadf99658c732 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 18 Oct 2014 14:44:41 +0200 Subject: reenable support for -s (and co) in apt-get source The conversion to accept only relevant options for commands has forgotten another one, so adding it again even through the usecase might very well be equally good served by --print-uris. Closes: 742578 --- apt-private/private-cmndline.cc | 2 +- test/integration/test-apt-get-source | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'test') diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc index a4490f5b4..0b5ba5b4f 100644 --- a/apt-private/private-cmndline.cc +++ b/apt-private/private-cmndline.cc @@ -166,7 +166,7 @@ static bool addArgumentsAPTGet(std::vector &Args, char const if (CmdMatches("install", "remove", "purge", "upgrade", "dist-upgrade", "deselect-upgrade", "autoremove", "clean", "autoclean", "check", - "build-dep", "full-upgrade")) + "build-dep", "full-upgrade", "source")) { addArg('s', "simulate", "APT::Get::Simulate", 0); addArg('s', "just-print", "APT::Get::Simulate", 0); diff --git a/test/integration/test-apt-get-source b/test/integration/test-apt-get-source index 33bd980d0..b27cbbe96 100755 --- a/test/integration/test-apt-get-source +++ b/test/integration/test-apt-get-source @@ -82,3 +82,7 @@ testequal "$HEADER Need to get 0 B of source archives. 'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e 'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris -t unstable foo=0.0.1 + +testequal "$HEADER +Need to get 0 B of source archives. +Fetch source foo" aptget source -q -s foo -- cgit v1.2.3-70-g09d2 From 081c9d442a6d39fb9bc419fe3ce697cc791cb844 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 21:38:53 +0100 Subject: various small additional tests and testcases Usually they don't provide a lot in terms of what they test, but they help in covering many lines from strictly anecdotal commands (stats, moo) and error messages, so that stuff which really needs to be tested, but isn't is better visible in coverage reports. Git-Dch: Ignore --- cmdline/apt-cache.cc | 4 +- test/integration/framework | 2 +- test/integration/test-00-commands-have-help | 11 ++ test/integration/test-apt-cache | 124 +++++++++++++++++++++ test/integration/test-apt-cli-search | 3 + test/integration/test-apt-cli-show | 11 ++ test/integration/test-apt-cli-update | 7 ++ test/integration/test-apt-config | 36 ++++++ test/integration/test-apt-helper | 12 +- test/integration/test-apt-mark | 34 ++++-- test/integration/test-apt-update-stale | 6 +- .../test-external-dependency-solver-protocol | 32 +++++- 12 files changed, 263 insertions(+), 19 deletions(-) create mode 100755 test/integration/test-apt-cache create mode 100755 test/integration/test-apt-config (limited to 'test') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 342ad1858..a5024c581 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -116,7 +116,7 @@ static bool ShowUnMet(pkgCache::VerIterator const &V, bool const Important) continue; // Skip conflicts and replaces - if (End.IsNegative() == true) + if (End.IsNegative() == true || End->Type == pkgCache::Dep::Replaces) continue; // Verify the or group @@ -133,7 +133,7 @@ static bool ShowUnMet(pkgCache::VerIterator const &V, bool const Important) break; } delete [] VList; - + if (Start == End) break; ++Start; diff --git a/test/integration/framework b/test/integration/framework index d28085ee4..c9f62c141 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -300,7 +300,7 @@ getarchitecture() { } getarchitectures() { - echo "$(aptconfig dump | grep APT::Architecture | cut -d'"' -f 2 | sed '/^$/ d' | sort | uniq | tr '\n' ' ')" + aptconfig dump --no-empty --format '%v%n' APT::Architecture APT::Architectures | sort -u | tr '\n' ' ' } getarchitecturesfromcommalist() { diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help index ebf8b8cfa..bbd1475eb 100755 --- a/test/integration/test-00-commands-have-help +++ b/test/integration/test-00-commands-have-help @@ -50,3 +50,14 @@ done for CMD in 'apt-dump-solver' 'apt-internal-solver'; do checkoptions "$(echo "$CMD" | tr -d '-')" done + +# in times of need, we all look for super cow to save the day +testsuccess aptget moo +testsuccess aptget moo -q=2 +testsuccess aptget moo moo +testsuccess aptget moo moo -q=2 +testsuccess aptget moo moo --color +testsuccess aptget moo moo moo +testsuccess aptget moo moo moo -q=2 +testsuccess aptget moo moo moo moo +testsuccess aptget moo moo moo moo -q=2 diff --git a/test/integration/test-apt-cache b/test/integration/test-apt-cache new file mode 100755 index 000000000..f47c0e08b --- /dev/null +++ b/test/integration/test-apt-cache @@ -0,0 +1,124 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' 'i386' + +DESCR='Some description + That has multiple lines' +insertpackage 'unstable' 'fancy' 'all' '1' +insertpackage 'unstable,installed' 'foo' 'all' '1' 'Depends: bar +Conflicts: foobar +Recommends: cool (>= 2) | cooler (<< 5)' "$DESCR" +insertpackage 'unstable' 'bar' 'all' '1' 'Depends: bar +Breaks: foo (<< 1) +Replaces: foo (<< 1)' "$DESCR" + +setupaptarchive + +# dpkg is installed by our framework +testdpkginstalled 'dpkg' +testempty aptcache unmet dpkg + +# FIXME: Find some usecase for unmet as it seems kinda useless/broken +#testsuccess aptcache unmet +#testsuccess aptcache unmet foo + +# not too useful to test, but makes coverage green… +testsuccess aptcache stats +cp rootdir/tmp/testsuccess.output stats.output +testsuccess test -s stats.output +testsuccess aptcache xvcg foo +cp rootdir/tmp/testsuccess.output xvcg.output +testsuccess test -s xvcg.output +testsuccess aptcache dotty foo +cp rootdir/tmp/testsuccess.output dotty.output +testsuccess test -s dotty.output +# for this, even the sourcecode says it is useless (expect debugging) +testsuccess aptcache dump +cp rootdir/tmp/testsuccess.output dump.output +testsuccess test -s dump.output + +testequal 'dpkg +bar +fancy +foo' aptcache pkgnames +testequal 'bar' aptcache pkgnames bar +testequal 'fancy +foo' aptcache pkgnames f + +testequal " foo | 1 | file:$(readlink -f .)/aptarchive/ unstable/main amd64 Packages" aptcache madison foo + +### depends + +testequal 'foo + Depends: bar + |Recommends: + Recommends: + Conflicts: + Conflicts: ' aptcache depends foo +testequal 'foo + Depends: bar + Recommends: + Conflicts: + Conflicts: ' aptcache depends foo -o APT::Cache::ShowOnlyFirstOr=1 +testequal 'foo + Depends: bar + |Recommends: (>= 2) + Recommends: (<< 5) + Conflicts: + Conflicts: ' aptcache depends foo -o APT::Cache::ShowVersion=1 +testequal 'foo + Depends: bar + Conflicts: + Conflicts: ' aptcache depends foo --no-recommends +testequal 'foo + Depends: bar' aptcache depends foo --important +testequal 'foo + Conflicts: + Conflicts: ' aptcache depends foo --important --no-depends --conflicts +testequal 'foo + Depends: bar + |Recommends: + Recommends: + Conflicts: + Conflicts: +bar + Depends: bar + Breaks: foo + Breaks: + Replaces: foo + Replaces: + + + + +' aptcache depends foo --recurse +testequal 'foo + Depends: bar +bar + Depends: bar + Replaces: foo + Replaces: +' aptcache depends foo --recurse --important --replaces + +## rdpends + +testequal 'foo +Reverse Depends: + bar + bar' aptcache rdepends foo +testequal 'foo +Reverse Depends: + Replaces: bar + Breaks: bar' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 +testequal 'foo +Reverse Depends: + Replaces: bar (<< 1) + Breaks: bar (<< 1)' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 -o APT::Cache::ShowVersion=1 +testequal 'foo +Reverse Depends: + Breaks: bar (<< 1)' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 -o APT::Cache::ShowVersion=1 --important --breaks diff --git a/test/integration/test-apt-cli-search b/test/integration/test-apt-cli-search index 8f009d57c..1a28ba4da 100755 --- a/test/integration/test-apt-cli-search +++ b/test/integration/test-apt-cli-search @@ -25,6 +25,9 @@ setupaptarchive APTARCHIVE=$(readlink -f ./aptarchive) +testequal 'E: You must give at least one search pattern' aptcache search +testequal 'E: You must give at least one search pattern' apt search + # with OP progress testequal "Sorting... Full Text Search... diff --git a/test/integration/test-apt-cli-show b/test/integration/test-apt-cli-show index 4c8e134d6..930e591e0 100755 --- a/test/integration/test-apt-cli-show +++ b/test/integration/test-apt-cli-show @@ -36,3 +36,14 @@ APT-Sources: file:$APTARCHIVE/ unstable/main i386 Packages Description: Some description That has multiple lines " apt show foo + +# this is the default, but disabled by the testcases +testsuccess apt show foo -o Apt::Cmd::Disable-Script-Warning=0 +cp rootdir/tmp/testsuccess.output aptshow.output +testsuccess grep '^WARNING: ' aptshow.output + +if [ "$(id -u)" != '0' ]; then + testsuccess apt install foo -s -o APT::Get::Show-User-Simulation-Note=1 + cp rootdir/tmp/testsuccess.output aptshow.output + testsuccess grep '^NOTE: ' aptshow.output +fi diff --git a/test/integration/test-apt-cli-update b/test/integration/test-apt-cli-update index 987bb9adb..83cc94b93 100755 --- a/test/integration/test-apt-cli-update +++ b/test/integration/test-apt-cli-update @@ -8,10 +8,17 @@ setupenvironment configarchitecture "i386" insertpackage 'unstable' 'foo' 'all' '2.0' +cp rootdir/var/lib/dpkg/status dpkg.status insertinstalledpackage 'foo' 'all' '1.0' setupaptarchive --no-update APTARCHIVE=$(readlink -f ./aptarchive) +testequal 'E: The update command takes no arguments' apt update -q arguments + testequal "1 package can be upgraded. Run 'apt list --upgradable' to see it." apt update -q + +cp dpkg.status rootdir/var/lib/dpkg/status +insertinstalledpackage 'foo' 'all' '2.0' +testequal 'All packages are up to date.' apt update -q diff --git a/test/integration/test-apt-config b/test/integration/test-apt-config new file mode 100755 index 000000000..2f2ff9d38 --- /dev/null +++ b/test/integration/test-apt-config @@ -0,0 +1,36 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' + +testsuccess aptconfig dump +testequal 'APT::Architecture "amd64";' aptconfig dump APT::Architecture +testempty aptconfig dump config::which::does::not::exist + +testequal 'APT::Architectures ""; +APT::Architectures:: "amd64";' aptconfig dump APT::Architectures +testequal 'APT::Architectures:: "amd64";' aptconfig dump --no-empty APT::Architectures +testequal 'amd64' aptconfig dump --no-empty --format='%v%n' APT::Architectures + +testempty aptconfig shell +testequal 'E: Arguments not in pairs' aptconfig shell APT::Architecture +testempty aptconfig shell APT::Architecture ARCH # incorrect order +testequal "ARCH='amd64'" aptconfig shell ARCH APT::Architecture + +ROOTDIR="$(readlink -f rootdir)" +testequal "CONFIG='apt.conf'" aptconfig shell CONFIG Dir::Etc::main +testequal "CONFIG='${ROOTDIR}/etc/apt/apt.conf'" aptconfig shell CONFIG Dir::Etc::main/f +testequal "CONFIG='etc/apt/'" aptconfig shell CONFIG Dir::Etc +testequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/ # old style +testequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/d + +testempty aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles +export DEB_BUILD_PROFILES='nodoc stage1' +testequal 'nodoc +stage1' aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles +unset DEB_BUILD_PROFILES +testempty aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index 06e497ff7..3c1d393a6 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -53,7 +53,7 @@ echo "http://some-proxy" EOF chmod 755 apt-proxy-detect echo "Acquire::http::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect - + testequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com @@ -64,13 +64,15 @@ echo "https://https-proxy" EOF chmod 755 apt-proxy-detect echo "Acquire::https::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect - - testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com - - + testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com } test_apt_helper_download test_apt_helper_detect_proxy +# test failure modes +testequal 'E: Invalid operation download' apthelper download +testequal 'E: Must specify at least one pair url/filename' apthelper download-file +testequal 'E: Must specify at least one pair url/filename' apthelper download-file http://example.org/ +testequal 'E: Need one URL as argument' apthelper auto-detect-proxy diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark index 69e0f933d..0f62a12b4 100755 --- a/test/integration/test-apt-mark +++ b/test/integration/test-apt-mark @@ -20,21 +20,31 @@ testdpkginstalled dpkg testnoautopkg() { testempty aptmark showauto + testempty aptcache showauto testequal 'bar dpkg foo' aptmark showmanual testequal 'bar foo' aptmark showmanual bar foo uninstalled } -testmarkonpkgasauto() { - testsuccess aptmark $1 foo +testfooisauto() { testequal 'foo' aptmark showauto + testequal 'foo' aptcache showauto testequal 'foo' aptmark showauto foo + testequal 'foo' aptcache showauto foo testequal 'bar dpkg' aptmark showmanual testequal 'bar' aptmark showmanual bar +} +testmarkonpkgasauto() { + testsuccess $1 $2 foo + testfooisauto + testsuccess $1 $2 foo + testfooisauto - testsuccess aptmark $2 foo + testsuccess $1 $3 foo + testnoautopkg + testsuccess $1 $3 foo testnoautopkg } @@ -42,8 +52,9 @@ testequal 'E: No packages found' aptmark auto testequal 'E: No packages found' aptmark manual testnoautopkg -testmarkonpkgasauto 'auto' 'manual' -testmarkonpkgasauto 'markauto' 'unmarkauto' +testmarkonpkgasauto 'aptmark' 'auto' 'manual' +testmarkonpkgasauto 'aptmark' 'markauto' 'unmarkauto' +testmarkonpkgasauto 'aptget' 'markauto' 'unmarkauto' testnoholdpkg() { testempty aptmark showhold @@ -51,10 +62,19 @@ testnoholdpkg() { testempty aptmark showhold dpkg testempty aptmark showholds dpkg } -testmarkonepkgashold() { - testsuccess aptmark hold $1 +testpkgonhold() { testequal "$1" aptmark showhold testequal "$1" aptmark showholds + testequal "$1" aptmark showhold $1 + testequal "$1" aptmark showholds $1 +} +testmarkonepkgashold() { + testsuccess aptmark hold $1 + testpkgonhold $1 + testsuccess aptmark hold $1 + testpkgonhold $1 + testsuccess aptmark unhold $1 + testnoholdpkg testsuccess aptmark unhold $1 testnoholdpkg } diff --git a/test/integration/test-apt-update-stale b/test/integration/test-apt-update-stale index 52f94591f..277aa5b09 100755 --- a/test/integration/test-apt-update-stale +++ b/test/integration/test-apt-update-stale @@ -14,9 +14,11 @@ configarchitecture "i386" insertpackage 'unstable' 'foo' 'all' '1.0' -setupaptarchive +setupaptarchive --no-update changetowebserver -aptget update -qq + +echo "Acquire::Languages \"none\";" > rootdir/etc/apt/apt.conf.d/00nolanguages +testsuccess aptget update listcurrentlistsdirectory > lists.before # insert new version diff --git a/test/integration/test-external-dependency-solver-protocol b/test/integration/test-external-dependency-solver-protocol index 07d2441b6..fd68578c5 100755 --- a/test/integration/test-external-dependency-solver-protocol +++ b/test/integration/test-external-dependency-solver-protocol @@ -12,7 +12,10 @@ insertinstalledpackage 'stuff' 'all' '1' insertpackage 'unstable' 'cool' 'all' '2' 'Multi-Arch: foreign' insertpackage 'unstable' 'stuff' 'all' '2' 'Multi-Arch: foreign' insertpackage 'unstable' 'coolstuff' 'i386,amd64' '2' 'Depends: cool, stuff' -insertpackage 'unstable' 'awesome' 'all' '2' 'Multi-Arch: foreign' +insertpackage 'unstable' 'awesome' 'all' '2' 'Multi-Arch: foreign +Conflicts: badstuff' +insertpackage 'unstable' 'badstuff' 'all' '2' 'Multi-Arch: foreign +Conflicts: awesome' insertpackage 'unstable' 'awesomecoolstuff' 'i386' '2' 'Depends: coolstuff, awesome' insertpackage 'experimental' 'cool' 'all' '3' 'Multi-Arch: foreign' @@ -44,6 +47,14 @@ The following NEW packages will be installed: Inst coolstuff (3 experimental [amd64]) Conf coolstuff (3 experimental [amd64])' aptget install --solver apt coolstuff -s +testequal 'Reading package lists... +Building dependency tree... +Execute external solver... +The following packages will be REMOVED: + cool* +0 upgraded, 0 newly installed, 1 to remove and 1 not upgraded. +Purg cool [1]' aptget purge --solver apt cool -s + testsuccess aptget install awesomecoolstuff:i386 -s testsuccess aptget install --solver apt awesomecoolstuff:i386 -s @@ -57,9 +68,13 @@ testsuccess aptget dist-upgrade -s --solver apt testsuccess aptget upgrade -s testsuccess aptget upgrade -s --solver apt +testfailure aptget install awesome badstuff -s +testfailure aptget install awesome badstuff -s --solver apt +testsuccess grep 'ERR_UNSOLVABLE' rootdir/tmp/testfailure.output + configarchitecture 'armel' msgtest 'Test direct calling is okay for' 'apt-internal-solver' -cat /tmp/dump.edsp | aptinternalsolver > solver.result 2>&1 || true +cat /tmp/dump.edsp | aptinternalsolver -q=0 > solver.result 2>&1 || true if [ "$(tail -n2 solver.result | head -n1 )" = "Message: Done" ]; then msgpass else @@ -69,3 +84,16 @@ fi rm -f /tmp/dump.edsp testfailure aptget install --solver apt awesomecoolstuff:i386 -s + +testsuccess aptinternalsolver scenario +testequal 'Package: stuff +Source: stuff +Architecture: all +Version: 1 +Installed: yes +APT-ID: 2 +Priority: optional +Section: other +APT-Pin: 100 +APT-Candidate: yes +' aptinternalsolver scenario stuff -- cgit v1.2.3-70-g09d2 From 8fe964f148344b8a55252fe52b6292a4ab86ea98 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 14 Nov 2014 18:01:09 +0100 Subject: create our cache and lib directory always with mode 755 We autocreate for a while now the last two directories in /var/lib/apt/lists (similar for /var/cache/apt/archives) which is very nice for systems having any of those on tmpfs or other non-persistent storage. This also means though that this creation is effected by the default umask, so for people with aggressive umasks like 027 the directories will be created with 750, which means all non-root users are left out, which is usually exactly what we want then this umask is set, but the cache and lib directories contain public knowledge. There isn't any need to protect them from viewers and they render apt completely useless if not readable. --- apt-pkg/acquire.cc | 7 +++++-- apt-pkg/cdrom.cc | 7 +++++-- test/integration/framework | 24 ++++++++++++++++++++++-- test/integration/test-apt-cdrom | 1 - test/integration/test-apt-update-transactions | 6 +----- 5 files changed, 33 insertions(+), 12 deletions(-) (limited to 'test') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 07c4646f5..0c815c005 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -92,8 +92,11 @@ void pkgAcquire::Initialize() static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent) { std::string const partial = parent + "partial"; - if (CreateAPTDirectoryIfNeeded(grand, partial) == false && - CreateAPTDirectoryIfNeeded(parent, partial) == false) + mode_t const mode = umask(S_IWGRP | S_IWOTH); + bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false && + CreateAPTDirectoryIfNeeded(parent, partial) == false); + umask(mode); + if (creation_fail == true) return false; std::string const SandboxUser = _config->Find("APT::Sandbox::User"); diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index b97f7b036..aaa7b82e0 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -822,8 +822,11 @@ bool pkgCdrom::Add(pkgCdromStatus *log) /*{{{*/ // check for existence and possibly create state directory for copying string const listDir = _config->FindDir("Dir::State::lists"); string const partialListDir = listDir + "partial/"; - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && - CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false) + mode_t const mode = umask(S_IWGRP | S_IWOTH); + bool const creation_fail = (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && + CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false); + umask(mode); + if (creation_fail == true) { UnmountCDROM(CDROM, NULL); return _error->Errno("cdrom", _("List directory %spartial is missing."), listDir.c_str()); diff --git a/test/integration/framework b/test/integration/framework index c9f62c141..191e205ce 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -203,7 +203,7 @@ setupenvironment() { mkdir rootdir aptarchive keys cd rootdir mkdir -p etc/apt/apt.conf.d etc/apt/sources.list.d etc/apt/trusted.gpg.d etc/apt/preferences.d - mkdir -p usr/bin var/cache var/lib/apt var/log tmp + mkdir -p usr/bin var/cache var/lib var/log tmp mkdir -p var/lib/dpkg/info var/lib/dpkg/updates var/lib/dpkg/triggers touch var/lib/dpkg/available mkdir -p usr/lib/apt @@ -1341,7 +1341,7 @@ testfilestats() { msgpass else echo >&2 - ls -l >&2 "$1" + ls -ld >&2 "$1" echo -n >&2 "stat(1) reports for $2: " stat --format "$2" "$1" msgfail @@ -1386,6 +1386,24 @@ listcurrentlistsdirectory() { done } +### convinience hacks ### +mkdir() { + # creating some directories by hand is a tedious task, so make it look simple + if [ "$*" = '-p rootdir/var/lib/apt/lists' ] || [ "$*" = "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" ] || + [ "$*" = '-p rootdir/var/lib/apt/lists/partial' ] || [ "$*" = "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" ]; then + # only the last directory created by mkdir is effected by the -m ! + command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" + command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" + command mkdir -m 700 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" + touch "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/lock" + if [ "$(id -u)" = '0' ]; then + chown _apt:root "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" + fi + else + command mkdir "$@" + fi +} + ### The following tests are run by most test methods automatically to check ### general things about commands executed without writing the test every time. @@ -1410,6 +1428,8 @@ aptautotest() { aptautotest_aptget_update() { if ! test -d "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"; then return; fi + testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" '%U:%G:%a' '=' "${USER}:${USER}:755" + testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" '%U:%G:%a' '=' "${USER}:${USER}:755" # all copied files are properly chmodded for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f ! -name 'lock'); do testfilestats "$file" '%U:%G:%a' '=' "${USER}:${USER}:644" diff --git a/test/integration/test-apt-cdrom b/test/integration/test-apt-cdrom index 3a33219fe..7da645e83 100755 --- a/test/integration/test-apt-cdrom +++ b/test/integration/test-apt-cdrom @@ -125,7 +125,6 @@ $CDROM_POST" aptcdromlog add msgtest 'Test for the german description translation of' 'testing' aptcache show testing -o Acquire::Languages=de | grep -q '^Description-de: ' && msgpass || msgfail rm -rf rootdir/var/lib/apt/lists -mkdir -p rootdir/var/lib/apt/lists/partial testequal "$CDROM_PRE Found 2 package indexes, 1 source indexes, 1 translation indexes and 1 signatures $CDROM_POST" aptcdromlog add diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index bf425a22e..b325733ac 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -48,11 +48,7 @@ testrun() { testsetup() { msgmsg 'Test with no initial data over' "$1" rm -rf rootdir/var/lib/apt/lists - mkdir -m 700 -p rootdir/var/lib/apt/lists/partial - touch rootdir/var/lib/apt/lists/lock - if [ "$(id -u)" = '0' ]; then - chown _apt:root rootdir/var/lib/apt/lists/partial - fi + mkdir -p rootdir/var/lib/apt/lists/partial listcurrentlistsdirectory > listsdir.lst testrun 'listsdir.lst' -- cgit v1.2.3-70-g09d2 From 4bb006d1ddcf1807474067dcbef9fb0bb5def0ac Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 17 Nov 2014 22:54:29 +0100 Subject: fix file ownership tests to work on kfreebsd While on linux files are created in /tmp with $USER:$USER, on my kfreebsd testmachine they are created with $USER:root, so we pull some strings here to make it work on both. --- test/integration/framework | 14 +++++++++++--- test/integration/test-apt-cdrom | 2 +- test/integration/test-apt-get-changelog | 4 ++-- test/integration/test-apt-get-source-authenticated | 1 - test/integration/test-authentication-basic | 4 ++-- 5 files changed, 16 insertions(+), 9 deletions(-) (limited to 'test') diff --git a/test/integration/framework b/test/integration/framework index 191e205ce..51c7b8cae 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -272,6 +272,14 @@ setupenvironment() { echo "Apt::Cmd::Disable-Script-Warning \"1\";" > rootdir/etc/apt/apt.conf.d/apt-binary configcompression '.' 'gz' #'bz2' 'lzma' 'xz' + # create some files in /tmp and look at user/group to get what this means + TEST_DEFAULT_USER="$USER" + if [ "$(uname)" = 'GNU/kFreeBSD' ]; then + TEST_DEFAULT_GROUP='root' + else + TEST_DEFAULT_GROUP="$USER" + fi + # Acquire::AllowInsecureRepositories=false is not yet the default # but we want it to be the default soon configallowinsecurerepositories "false"; @@ -1428,11 +1436,11 @@ aptautotest() { aptautotest_aptget_update() { if ! test -d "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"; then return; fi - testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" '%U:%G:%a' '=' "${USER}:${USER}:755" - testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" '%U:%G:%a' '=' "${USER}:${USER}:755" + testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755" + testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755" # all copied files are properly chmodded for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f ! -name 'lock'); do - testfilestats "$file" '%U:%G:%a' '=' "${USER}:${USER}:644" + testfilestats "$file" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" done } aptautotest_apt_update() { aptautotest_aptget_update "$@"; } diff --git a/test/integration/test-apt-cdrom b/test/integration/test-apt-cdrom index 7da645e83..7f4b3c257 100755 --- a/test/integration/test-apt-cdrom +++ b/test/integration/test-apt-cdrom @@ -146,5 +146,5 @@ testcdromusage msgmsg 'Check that nothing touched our' 'CD-ROM' for file in $(find rootdir/media/cdrom-unmounted/dists); do - testfilestats "$file" '%U:%G:%a' '=' "${USER}:${USER}:555" + testfilestats "$file" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:555" done diff --git a/test/integration/test-apt-get-changelog b/test/integration/test-apt-get-changelog index 648dccf40..01f2bd393 100755 --- a/test/integration/test-apt-get-changelog +++ b/test/integration/test-apt-get-changelog @@ -33,7 +33,7 @@ testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/apt_ testsuccess aptget changelog apt -d testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0/changelog)" -testfilestats 'apt.changelog' '%U:%G:%a' '=' "${USER}:${USER}:644" +testfilestats 'apt.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" rm -f apt.changelog ../aptarchive/pool/apt_1.0/changelog testequal "$(cat ../aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \ @@ -41,7 +41,7 @@ testequal "$(cat ../aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \ testsuccess aptget changelog apt -d testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0.changelog)" -testfilestats 'apt.changelog' '%U:%G:%a' '=' "${USER}:${USER}:644" +testfilestats 'apt.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" rm -f apt.changelog ../aptarchive/pool/apt_1.0.changelog testequal 'E: changelog download failed' aptget changelog apt -qq -d -o APT::Changelogs::Server='http://not-on-the-main-server:8080/' diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated index f68c32386..685bc566b 100755 --- a/test/integration/test-apt-get-source-authenticated +++ b/test/integration/test-apt-get-source-authenticated @@ -31,4 +31,3 @@ testfailure test -e foo_2.0.dsc # allow overriding the warning testsuccess aptget source --allow-unauthenticated foo -o Debug::pkgAcquire::Worker=1 testsuccess test -s foo_2.0.dsc -a -L foo_2.0.dsc -testaccessrights 'foo_2.0.dsc' '777' diff --git a/test/integration/test-authentication-basic b/test/integration/test-authentication-basic index 21b024970..7e74726be 100755 --- a/test/integration/test-authentication-basic +++ b/test/integration/test-authentication-basic @@ -24,7 +24,7 @@ testauthfailure() { testauthsuccess() { testsuccess apthelper download-file "${1}/bash" ./downloaded/bash testfileequal ./downloaded/bash "$(cat aptarchive/bash)" - testfilestats ./downloaded/bash '%U:%G:%a' '=' "${USER}:${USER}:644" + testfilestats ./downloaded/bash '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" rm -f ./downloaded/bash # lets see if got/retains acceptable permissions @@ -32,7 +32,7 @@ testauthsuccess() { if [ "$(id -u)" = '0' ]; then testfilestats "$AUTHCONF" '%U:%G:%a' '=' "_apt:root:600" else - testfilestats "$AUTHCONF" '%U:%G:%a' '=' "${USER}:${USER}:600" + testfilestats "$AUTHCONF" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:600" fi fi -- cgit v1.2.3-70-g09d2 From 0c78757010a17173bdc79b818133697e7dcbf6a4 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 17 Nov 2014 15:06:35 +0100 Subject: close leaking slave fd after setting up pty magic The fd moves out of scope here anyway, so we should close it properly instead of leaking it which will tickle down to dpkg maintainer scripts. Closes: 767774 --- apt-pkg/deb/dpkgpm.cc | 6 +- test/integration/framework | 71 ++++++++++++++++++---- test/integration/test-failing-maintainer-scripts | 46 +------------- .../test-no-fds-leaked-to-maintainer-scripts | 40 ++++++++++++ 4 files changed, 103 insertions(+), 60 deletions(-) create mode 100755 test/integration/test-no-fds-leaked-to-maintainer-scripts (limited to 'test') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 3f6039e0d..dd03eac2c 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1158,8 +1158,7 @@ void pkgDPkgPM::SetupSlavePtyMagic() int const slaveFd = open(d->slave, O_RDWR); if (slaveFd == -1) _error->FatalE("open", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); - - if (ioctl(slaveFd, TIOCSCTTY, 0) < 0) + else if (ioctl(slaveFd, TIOCSCTTY, 0) < 0) _error->FatalE("ioctl", "Setting TIOCSCTTY for slave fd %d failed!", slaveFd); else { @@ -1170,6 +1169,9 @@ void pkgDPkgPM::SetupSlavePtyMagic() if (tcsetattr(0, TCSANOW, &d->tt) < 0) _error->FatalE("tcsetattr", "Setting in Setup via TCSANOW for slave fd %d failed!", slaveFd); } + + if (slaveFd != -1) + close(slaveFd); } void pkgDPkgPM::StopPtyMagic() { diff --git a/test/integration/framework b/test/integration/framework index 51c7b8cae..ff059f59e 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -358,6 +358,54 @@ configdpkg() { fi } +configdpkgnoopchroot() { + # create a library to noop chroot() and rewrite maintainer script executions + # via execvp() as used by dpkg as we don't want our rootdir to be a fullblown + # chroot directory dpkg could chroot into to execute the maintainer scripts + msgtest 'Building library to preload to make maintainerscript work in' 'dpkg' + cat << EOF > noopchroot.c +#define _GNU_SOURCE +#include +#include +#include +#include + +static char * chrootdir = NULL; + +int chroot(const char *path) { + printf("WARNING: CHROOTing to %s was ignored!\n", path); + free(chrootdir); + chrootdir = strdup(path); + return 0; +} +int execvp(const char *file, char *const argv[]) { + static int (*func_execvp) (const char *, char * const []) = NULL; + if (func_execvp == NULL) + func_execvp = (int (*) (const char *, char * const [])) dlsym(RTLD_NEXT, "execvp"); + if (chrootdir == NULL || strncmp(file, "/var/lib/dpkg/", strlen("/var/lib/dpkg/")) != 0) + return func_execvp(file, argv); + printf("REWRITE execvp call %s into %s\n", file, chrootdir); + char newfile[strlen(chrootdir) + strlen(file)]; + strcpy(newfile, chrootdir); + strcat(newfile, file); + return func_execvp(newfile, argv); +} +EOF + testsuccess --nomsg gcc -fPIC -shared -o noopchroot.so noopchroot.c -ldl + + mkdir -p "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/" + DPKG="${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg" + echo "#!/bin/sh +if [ -n \"\$LD_PRELOAD\" ]; then + export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so \${LD_PRELOAD}\" +else + export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so\" +fi +dpkg \"\$@\"" > $DPKG + chmod +x $DPKG + sed -ie "s|^DPKG::options:: \"dpkg\";\$|DPKG::options:: \"$DPKG\";|" aptconfig.conf +} + configallowinsecurerepositories() { echo "Acquire::AllowInsecureRepositories \"$1\";" > rootdir/etc/apt/apt.conf.d/allow-insecure-repositories.conf @@ -480,7 +528,7 @@ buildsimplenativepackage() { fi local BUILDDIR=${TMPWORKINGDIRECTORY}/incoming/${NAME}-${VERSION} - msgninfo "Build package ${NAME} in ${VERSION} for ${RELEASE} in ${DISTSECTION}… " + msgtest "Build source package in version ${VERSION} for ${RELEASE} in ${DISTSECTION}" "$NAME" mkdir -p $BUILDDIR/debian/source echo "* most suckless software product ever" > ${BUILDDIR}/FEATURES echo "#!/bin/sh @@ -512,7 +560,10 @@ Package: $NAME" >> ${BUILDDIR}/debian/control echo "Description: $DESCRIPTION" >> ${BUILDDIR}/debian/control echo '3.0 (native)' > ${BUILDDIR}/debian/source/format - (cd ${BUILDDIR}/..; dpkg-source -b ${NAME}-${VERSION} 2>&1) | sed -n 's#^dpkg-source: info: building [^ ]\+ in ##p' \ + cd ${BUILDDIR}/.. + testsuccess --nomsg dpkg-source -b ${NAME}-${VERSION} + cd - >/dev/null + sed -n 's#^dpkg-source: info: building [^ ]\+ in ##p' ${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output \ | while read SRC; do echo "pool/${SRC}" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.srclist # if expr match "${SRC}" '.*\.dsc' >/dev/null 2>&1; then @@ -524,6 +575,7 @@ Package: $NAME" >> ${BUILDDIR}/debian/control done for arch in $(getarchitecturesfromcommalist "$ARCH"); do + msgtest "Build binary package for ${RELEASE} in ${SECTION}" "$NAME" rm -rf ${BUILDDIR}/debian/tmp mkdir -p ${BUILDDIR}/debian/tmp/DEBIAN ${BUILDDIR}/debian/tmp/usr/share/doc/${NAME} ${BUILDDIR}/debian/tmp/usr/bin cp ${BUILDDIR}/debian/copyright ${BUILDDIR}/debian/changelog ${BUILDDIR}/FEATURES ${BUILDDIR}/debian/tmp/usr/share/doc/${NAME} @@ -537,11 +589,7 @@ Package: $NAME" >> ${BUILDDIR}/debian/control local LOG="${BUILDDIR}/../${NAME}_${VERSION}_${arch}.dpkg-deb.log" # ensure the right permissions as dpkg-deb ensists chmod 755 ${BUILDDIR}/debian/tmp/DEBIAN - if ! dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/.. >$LOG 2>&1; then - cat $LOG - false - fi - rm $LOG + testsuccess --nomsg dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/.. echo "pool/${NAME}_${VERSION}_${arch}.deb" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.pkglist done @@ -559,15 +607,13 @@ buildpackage() { local ARCH=$(getarchitecture $4) local PKGNAME="$(echo "$BUILDDIR" | grep -o '[^/]*$')" local BUILDLOG="$(readlink -f "${BUILDDIR}/../${PKGNAME}_${RELEASE}_${SECTION}.dpkg-bp.log")" - msgninfo "Build package ${PKGNAME} for ${RELEASE} in ${SECTION}… " + msgtest "Build package for ${RELEASE} in ${SECTION}" "$PKGNAME" cd $BUILDDIR if [ "$ARCH" = "all" ]; then ARCH="$(dpkg-architecture -qDEB_HOST_ARCH 2> /dev/null)" fi - if ! dpkg-buildpackage -uc -us -a$ARCH >$BUILDLOG 2>&1 ; then - cat $BUILDLOG - false - fi + testsuccess --nomsg dpkg-buildpackage -uc -us -a$ARCH + cp ${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output $BUILDLOG local PKGS="$(grep '^dpkg-deb: building package' $BUILDLOG | cut -d'/' -f 2 | sed -e "s#'\.##")" local SRCS="$(grep '^dpkg-source: info: building' $BUILDLOG | grep -o '[a-z0-9._+~-]*$')" cd - > /dev/null @@ -577,7 +623,6 @@ buildpackage() { for SRC in $SRCS; do echo "pool/${SRC}" >> ${TMPWORKINGDIRECTORY}/incoming/${RELEASE}.${SECTION}.srclist done - msgdone "info" } buildaptarchive() { diff --git a/test/integration/test-failing-maintainer-scripts b/test/integration/test-failing-maintainer-scripts index 3dd7d643e..953506aa5 100755 --- a/test/integration/test-failing-maintainer-scripts +++ b/test/integration/test-failing-maintainer-scripts @@ -6,6 +6,7 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture 'native' +configdpkgnoopchroot # create a bunch of failures createfailure() { @@ -25,51 +26,6 @@ createfailure 'postrm' setupaptarchive -# create a library to noop chroot() and rewrite maintainer script executions -# via execvp() as used by dpkg as we don't want our rootdir to be a fullblown -# chroot directory dpkg could chroot into to execute the maintainer scripts -cat << EOF > noopchroot.c -#define _GNU_SOURCE -#include -#include -#include -#include - -static char * chrootdir = NULL; - -int chroot(const char *path) { - printf("WARNING: CHROOTing to %s was ignored!\n", path); - free(chrootdir); - chrootdir = strdup(path); - return 0; -} -int execvp(const char *file, char *const argv[]) { - static int (*func_execvp) (const char *, char * const []) = NULL; - if (func_execvp == NULL) - func_execvp = (int (*) (const char *, char * const [])) dlsym(RTLD_NEXT, "execvp"); - if (chrootdir == NULL || strncmp(file, "/var/lib/dpkg/", strlen("/var/lib/dpkg/")) != 0) - return func_execvp(file, argv); - printf("REWRITE execvp call %s into %s\n", file, chrootdir); - char newfile[strlen(chrootdir) + strlen(file)]; - strcpy(newfile, chrootdir); - strcat(newfile, file); - return func_execvp(newfile, argv); -} -EOF -testsuccess gcc -fPIC -shared -o noopchroot.so noopchroot.c -ldl - -mkdir -p "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/" -DPKG="${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg" -echo "#!/bin/sh -if [ -n \"\$LD_PRELOAD\" ]; then - export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so \${LD_PRELOAD}\" -else - export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so\" -fi -dpkg \"\$@\"" > $DPKG -chmod +x $DPKG -sed -ie "s|^DPKG::options:: \"dpkg\";\$|DPKG::options:: \"$DPKG\";|" aptconfig.conf - # setup some pre- and post- invokes to check the output isn't garbled later APTHOOK="${TMPWORKINGDIRECTORY}/rootdir/usr/bin/apthook" echo '#!/bin/sh diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts new file mode 100755 index 000000000..6ed120090 --- /dev/null +++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts @@ -0,0 +1,40 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'native' +configdpkgnoopchroot + +setupsimplenativepackage "fdleaks" 'native' '1.0' 'unstable' +BUILDDIR="incoming/fdleaks-1.0" +for script in 'preinst' 'postinst' 'prerm' 'postrm'; do + echo '#!/bin/sh +ls -l /proc/self/fd/' > ${BUILDDIR}/debian/$script +done +buildpackage "$BUILDDIR" 'unstable' 'main' 'native' +rm -rf "$BUILDDIR" + +setupaptarchive + +testsuccess aptget install -y fdleaks +msgtest 'Check if fds were not' 'leaked' +if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '8' ]; then + msgpass +else + echo + cat rootdir/tmp/testsuccess.output + msgfail +fi + +testsuccess aptget purge -y fdleaks +msgtest 'Check if fds were not' 'leaked' +if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '12' ]; then + msgpass +else + echo + cat rootdir/tmp/testsuccess.output + msgfail +fi -- cgit v1.2.3-70-g09d2 From 150bdc9ca5d656f9fba94d37c5f4f183b02bd746 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 18 Nov 2014 00:59:39 +0100 Subject: fix PTY interaction on linux and kfreebsd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We run dpkg on its own pty, so we can log its output and have our own output around it (like the progress bar), while also allowing debconf and configfile prompts to happen. In commit 223ae57d468fdcac451209a095047a07a5698212 we changed to constantly reopening the slave for kfreebsd. This has the sideeffect though that in some cases slave and master will lose their connection on linux, so that no output is passed along anymore. We fix this by having always an fd referencing the slave open (linux), but we don't use it (kfreebsd). Failing to get our PTY up and running has many (bad) consequences including (not limited to, nor all at ones or in any case) garbled ouput, no output, no logging, a (partial) mixture of the previous items, … This commit is therefore also reshuffling quiet a bit of the creation code to get especially the output part up and running on linux and the logging for kfreebsd. Note that the testcase tries to cover some cases, but this is an interactivity issue so only interactive usage can really be a good test. Closes: 765687 --- apt-pkg/deb/dpkgpm.cc | 99 ++++++++++++---------- .../test-no-fds-leaked-to-maintainer-scripts | 40 ++++++++- 2 files changed, 92 insertions(+), 47 deletions(-) (limited to 'test') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index dd03eac2c..593875071 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -72,7 +72,8 @@ class pkgDPkgPMPrivate public: pkgDPkgPMPrivate() : stdin_is_dev_null(false), dpkgbuf_pos(0), term_out(NULL), history_out(NULL), - progress(NULL), master(-1), slave(NULL) + progress(NULL), tt_is_valid(false), master(-1), + slave(NULL), protect_slave_from_dying(-1) { dpkgbuf[0] = '\0'; } @@ -90,8 +91,10 @@ public: // pty stuff struct termios tt; + bool tt_is_valid; int master; char * slave; + int protect_slave_from_dying; // signals sigset_t sigmask; @@ -1069,47 +1072,40 @@ void pkgDPkgPM::StartPtyMagic() } _error->PushToStack(); - // if tcgetattr for both stdin/stdout returns 0 (no error) - // we do the pty magic - if (tcgetattr(STDOUT_FILENO, &d->tt) == 0 && - tcgetattr(STDIN_FILENO, &d->tt) == 0) + + d->master = posix_openpt(O_RDWR | O_NOCTTY); + if (d->master == -1) + _error->Errno("posix_openpt", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); + else if (unlockpt(d->master) == -1) + _error->Errno("unlockpt", "Unlocking the slave of master fd %d failed!", d->master); + else { - d->master = posix_openpt(O_RDWR | O_NOCTTY); - if (d->master == -1) - _error->Errno("posix_openpt", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); - else if (unlockpt(d->master) == -1) - { - _error->Errno("unlockpt", "Unlocking the slave of master fd %d failed!", d->master); - close(d->master); - d->master = -1; - } + char const * const slave_name = ptsname(d->master); + if (slave_name == NULL) + _error->Errno("ptsname", "Getting name for slave of master fd %d failed!", d->master); else { - char const * const slave_name = ptsname(d->master); - if (slave_name == NULL) + d->slave = strdup(slave_name); + if (d->slave == NULL) + _error->Errno("strdup", "Copying name %s for slave of master fd %d failed!", slave_name, d->master); + else if (grantpt(d->master) == -1) + _error->Errno("grantpt", "Granting access to slave %s based on master fd %d failed!", slave_name, d->master); + else if (tcgetattr(STDIN_FILENO, &d->tt) == 0) { - _error->Errno("unlockpt", "Getting name for slave of master fd %d failed!", d->master); - close(d->master); - d->master = -1; - } - else - { - d->slave = strdup(slave_name); - if (d->slave == NULL) + d->tt_is_valid = true; + struct termios raw_tt; + // copy window size of stdout if its a 'good' terminal + if (tcgetattr(STDOUT_FILENO, &raw_tt) == 0) { - _error->Errno("strdup", "Copying name %s for slave of master fd %d failed!", slave_name, d->master); - close(d->master); - d->master = -1; + struct winsize win; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) < 0) + _error->Errno("ioctl", "Getting TIOCGWINSZ from stdout failed!"); + if (ioctl(d->master, TIOCSWINSZ, &win) < 0) + _error->Errno("ioctl", "Setting TIOCSWINSZ for master fd %d failed!", d->master); } - struct winsize win; - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) < 0) - _error->Errno("ioctl", "Getting TIOCGWINSZ from stdout failed!"); - if (ioctl(d->master, TIOCSWINSZ, &win) < 0) - _error->Errno("ioctl", "Setting TIOCSWINSZ for master fd %d failed!", d->master); if (tcsetattr(d->master, TCSANOW, &d->tt) == -1) _error->Errno("tcsetattr", "Setting in Start via TCSANOW for master fd %d failed!", d->master); - struct termios raw_tt; raw_tt = d->tt; cfmakeraw(&raw_tt); raw_tt.c_lflag &= ~ECHO; @@ -1121,18 +1117,22 @@ void pkgDPkgPM::StartPtyMagic() sigaddset(&d->sigmask, SIGTTOU); sigprocmask(SIG_BLOCK,&d->sigmask, &d->original_sigmask); if (tcsetattr(STDIN_FILENO, TCSAFLUSH, &raw_tt) == -1) - _error->Errno("tcsetattr", "Setting in Start via TCSAFLUSH for stdout failed!"); + _error->Errno("tcsetattr", "Setting in Start via TCSAFLUSH for stdin failed!"); sigprocmask(SIG_SETMASK, &d->original_sigmask, NULL); + + } + if (d->slave != NULL) + { + /* on linux, closing (and later reopening) all references to the slave + makes the slave a death end, so we open it here to have one open all + the time. We could use this fd in SetupSlavePtyMagic() for linux, but + on kfreebsd we get an incorrect ("step like") output then while it has + no problem with closing all references… so to avoid platform specific + code here we combine both and be happy once more */ + d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC); } } } - else - { - // complain only if stdout is either a terminal (but still failed) or is an invalid - // descriptor otherwise we would complain about redirection to e.g. /dev/null as well. - if (isatty(STDOUT_FILENO) == 1 || errno == EBADF) - _error->Errno("tcgetattr", _("Can not write log (%s)"), _("Is stdout a terminal?")); - } if (_error->PendingError() == true) { @@ -1141,17 +1141,23 @@ void pkgDPkgPM::StartPtyMagic() close(d->master); d->master = -1; } + if (d->slave != NULL) + { + free(d->slave); + d->slave = NULL; + } _error->DumpErrors(std::cerr); } _error->RevertToStack(); } void pkgDPkgPM::SetupSlavePtyMagic() { - if(d->master == -1) + if(d->master == -1 || d->slave == NULL) return; if (close(d->master) == -1) _error->FatalE("close", "Closing master %d in child failed!", d->master); + d->master = -1; if (setsid() == -1) _error->FatalE("setsid", "Starting a new session for child failed!"); @@ -1166,7 +1172,7 @@ void pkgDPkgPM::SetupSlavePtyMagic() if (dup2(slaveFd, i) == -1) _error->FatalE("dup2", "Dupping %d to %d in child failed!", slaveFd, i); - if (tcsetattr(0, TCSANOW, &d->tt) < 0) + if (d->tt_is_valid == true && tcsetattr(STDIN_FILENO, TCSANOW, &d->tt) < 0) _error->FatalE("tcsetattr", "Setting in Setup via TCSANOW for slave fd %d failed!", slaveFd); } @@ -1178,9 +1184,14 @@ void pkgDPkgPM::StopPtyMagic() if (d->slave != NULL) free(d->slave); d->slave = NULL; + if (d->protect_slave_from_dying != -1) + { + close(d->protect_slave_from_dying); + d->protect_slave_from_dying = -1; + } if(d->master >= 0) { - if (tcsetattr(0, TCSAFLUSH, &d->tt) == -1) + if (d->tt_is_valid == true && tcsetattr(STDIN_FILENO, TCSAFLUSH, &d->tt) == -1) _error->FatalE("tcsetattr", "Setting in Stop via TCSAFLUSH for stdin failed!"); close(d->master); d->master = -1; diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts index 6ed120090..3c6457cab 100755 --- a/test/integration/test-no-fds-leaked-to-maintainer-scripts +++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts @@ -8,7 +8,7 @@ setupenvironment configarchitecture 'native' configdpkgnoopchroot -setupsimplenativepackage "fdleaks" 'native' '1.0' 'unstable' +setupsimplenativepackage "fdleaks" 'all' '1.0' 'unstable' BUILDDIR="incoming/fdleaks-1.0" for script in 'preinst' 'postinst' 'prerm' 'postrm'; do echo '#!/bin/sh @@ -19,7 +19,8 @@ rm -rf "$BUILDDIR" setupaptarchive -testsuccess aptget install -y fdleaks +rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log +testsuccess aptget install -y fdleaks -qq < /dev/null msgtest 'Check if fds were not' 'leaked' if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '8' ]; then msgpass @@ -29,7 +30,23 @@ else msgfail fi -testsuccess aptget purge -y fdleaks +cp rootdir/tmp/testsuccess.output terminal.output +tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log +testfileequal 'terminal.log' "$(cat terminal.output)" + +testequal 'startup archives unpack +install fdleaks:all 1.0 +status half-installed fdleaks:all 1.0 +status unpacked fdleaks:all 1.0 +status unpacked fdleaks:all 1.0 +startup packages configure +configure fdleaks:all 1.0 +status unpacked fdleaks:all 1.0 +status half-configured fdleaks:all 1.0 +status installed fdleaks:all 1.0' cut -f 3- -d' ' rootdir/var/log/dpkg.log + +rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log +testsuccess aptget purge -y fdleaks -qq msgtest 'Check if fds were not' 'leaked' if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '12' ]; then msgpass @@ -38,3 +55,20 @@ else cat rootdir/tmp/testsuccess.output msgfail fi +cp rootdir/tmp/testsuccess.output terminal.output +tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log +testfileequal 'terminal.log' "$(cat terminal.output)" + +testequal 'startup packages purge +status installed fdleaks:all 1.0 +remove fdleaks:all 1.0 +status half-configured fdleaks:all 1.0 +status half-installed fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +purge fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status not-installed fdleaks:all ' cut -f 3- -d' ' rootdir/var/log/dpkg.log -- cgit v1.2.3-70-g09d2 From b8ef7ef414600109b931fc8eebb1dfafdfe390cd Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 18 Nov 2014 13:06:53 +0100 Subject: fix test to support non-multiarch dpkg versions On travis we work with a pre-multiarch version of dpkg, so the output is slightly different in regards to package names. Git-Dch: Ignore --- .../test-no-fds-leaked-to-maintainer-scripts | 51 ++++++++++++---------- 1 file changed, 28 insertions(+), 23 deletions(-) (limited to 'test') diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts index 3c6457cab..428db46ef 100755 --- a/test/integration/test-no-fds-leaked-to-maintainer-scripts +++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts @@ -5,7 +5,7 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture 'native' +configarchitecture 'amd64' 'i386' configdpkgnoopchroot setupsimplenativepackage "fdleaks" 'all' '1.0' 'unstable' @@ -17,6 +17,11 @@ done buildpackage "$BUILDDIR" 'unstable' 'main' 'native' rm -rf "$BUILDDIR" +PKGNAME='fdleaks:all' +if ! dpkg-checkbuilddeps -d 'dpkg (>= 1.16.2)' /dev/null; then + PKGNAME='fdleaks' +fi + setupaptarchive rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log @@ -34,16 +39,16 @@ cp rootdir/tmp/testsuccess.output terminal.output tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log testfileequal 'terminal.log' "$(cat terminal.output)" -testequal 'startup archives unpack -install fdleaks:all 1.0 -status half-installed fdleaks:all 1.0 -status unpacked fdleaks:all 1.0 -status unpacked fdleaks:all 1.0 +testequal "startup archives unpack +install $PKGNAME 1.0 +status half-installed $PKGNAME 1.0 +status unpacked $PKGNAME 1.0 +status unpacked $PKGNAME 1.0 startup packages configure -configure fdleaks:all 1.0 -status unpacked fdleaks:all 1.0 -status half-configured fdleaks:all 1.0 -status installed fdleaks:all 1.0' cut -f 3- -d' ' rootdir/var/log/dpkg.log +configure $PKGNAME 1.0 +status unpacked $PKGNAME 1.0 +status half-configured $PKGNAME 1.0 +status installed $PKGNAME 1.0" cut -f 3- -d' ' rootdir/var/log/dpkg.log rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log testsuccess aptget purge -y fdleaks -qq @@ -59,16 +64,16 @@ cp rootdir/tmp/testsuccess.output terminal.output tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log testfileequal 'terminal.log' "$(cat terminal.output)" -testequal 'startup packages purge -status installed fdleaks:all 1.0 -remove fdleaks:all 1.0 -status half-configured fdleaks:all 1.0 -status half-installed fdleaks:all 1.0 -status config-files fdleaks:all 1.0 -purge fdleaks:all 1.0 -status config-files fdleaks:all 1.0 -status config-files fdleaks:all 1.0 -status config-files fdleaks:all 1.0 -status config-files fdleaks:all 1.0 -status config-files fdleaks:all 1.0 -status not-installed fdleaks:all ' cut -f 3- -d' ' rootdir/var/log/dpkg.log +testequal "startup packages purge +status installed $PKGNAME 1.0 +remove $PKGNAME 1.0 +status half-configured $PKGNAME 1.0 +status half-installed $PKGNAME 1.0 +status config-files $PKGNAME 1.0 +purge $PKGNAME 1.0 +status config-files $PKGNAME 1.0 +status config-files $PKGNAME 1.0 +status config-files $PKGNAME 1.0 +status config-files $PKGNAME 1.0 +status config-files $PKGNAME 1.0 +status not-installed $PKGNAME " cut -f 3- -d' ' rootdir/var/log/dpkg.log -- cgit v1.2.3-70-g09d2 From 1d838084dc775c0a4184edb4f3b9138903ac27fb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 18 Nov 2014 13:41:18 +0100 Subject: use dpkg --merge-avail only if needed in apt-mark Only "recent" versions of dpkg support stdin for merge instead of a file, so as a quick fix we delay calling it until we really need it which fixes most of the problem already. Checking for a specific dpkg version here is deemed too much work, just like using a temporary file here and depends a too high requirement for this minor usecase. After all, it didn't work at all before, so we break nobody here and can fix it if someone complains (with a patch). --- cmdline/apt-mark.cc | 100 ++++++++++++++++++++++------------------- test/integration/test-apt-mark | 9 ++++ 2 files changed, 64 insertions(+), 45 deletions(-) (limited to 'test') diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 860982f9e..8974397c9 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -274,65 +274,75 @@ static bool DoHold(CommandLine &CmdL) return true; } - Args.erase(Args.begin() + BaseArgs, Args.end()); - Args.push_back("--merge-avail"); - Args.push_back("-"); - Args.push_back(NULL); - - int external[2] = {-1, -1}; - if (pipe(external) != 0) - return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); - - pid_t dpkgMergeAvail = ExecFork(); - if (dpkgMergeAvail == 0) - { - close(external[1]); - std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); - if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) - _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); - dup2(external[0], STDIN_FILENO); - int const nullfd = open("/dev/null", O_RDONLY); - dup2(nullfd, STDOUT_FILENO); - execvp(Args[0], (char**) &Args[0]); - _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); - _exit(2); - } - - FILE* dpkg = fdopen(external[1], "w"); + APT::PackageList keepoffset; for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (Pkg->CurrentVer != 0) continue; - char const * Arch; - if (Pkg->VersionList != 0) - Arch = Pkg.VersionList().Arch(); - else - Arch = Pkg.Arch(); - fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" - "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + keepoffset.insert(Pkg); } - fclose(dpkg); - if (dpkgMergeAvail > 0) + if (keepoffset.empty() == false) { - int Status = 0; - while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + Args.erase(Args.begin() + BaseArgs, Args.end()); + Args.push_back("--merge-avail"); + // FIXME: supported only since 1.17.7 in dpkg + Args.push_back("-"); + Args.push_back(NULL); + + int external[2] = {-1, -1}; + if (pipe(external) != 0) + return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); + + pid_t dpkgMergeAvail = ExecFork(); + if (dpkgMergeAvail == 0) { - if (errno == EINTR) - continue; - _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); - break; + close(external[1]); + std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); + if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) + _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); + dup2(external[0], STDIN_FILENO); + int const nullfd = open("/dev/null", O_RDONLY); + dup2(nullfd, STDOUT_FILENO); + execvp(Args[0], (char**) &Args[0]); + _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); + _exit(2); + } + + FILE* dpkg = fdopen(external[1], "w"); + for (APT::PackageList::iterator Pkg = keepoffset.begin(); Pkg != keepoffset.end(); ++Pkg) + { + char const * Arch; + if (Pkg->VersionList != 0) + Arch = Pkg.VersionList().Arch(); + else + Arch = Pkg.Arch(); + fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" + "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + } + fclose(dpkg); + keepoffset.clear(); + + if (dpkgMergeAvail > 0) + { + int Status = 0; + while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + { + if (errno == EINTR) + continue; + _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); + break; + } + if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) + return _error->Error(_("Executing dpkg failed. Are you root?")); } - if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) - return _error->Error(_("Executing dpkg failed. Are you root?")); } Args.erase(Args.begin() + BaseArgs, Args.end()); Args.push_back("--set-selections"); Args.push_back(NULL); - external[0] = -1; - external[1] = -1; + int external[2] = {-1, -1}; if (pipe(external) != 0) return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --set-selections"); @@ -349,7 +359,7 @@ static bool DoHold(CommandLine &CmdL) _exit(2); } - dpkg = fdopen(external[1], "w"); + FILE* dpkg = fdopen(external[1], "w"); for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (dpkgMultiArch == false) diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark index 0f62a12b4..5a3ae4b2f 100755 --- a/test/integration/test-apt-mark +++ b/test/integration/test-apt-mark @@ -85,6 +85,15 @@ testequal 'E: No packages found' aptmark unhold testnoholdpkg testmarkonepkgashold 'foo' testmarkonepkgashold 'bar' + +msgtest 'dpkg supports --merge-avail via' 'stdin' +if dpkg --merge-avail - < /dev/null >/dev/null 2>&1; then + msgpass +else + msgskip 'dpkg version too old' + exit 0 +fi + testmarkonepkgashold 'uninstalled' testmarkonepkgashold 'uninstalled-native' -- cgit v1.2.3-70-g09d2 From 9fc0b435593839de47098212f0ae5f15b6263099 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 17 Nov 2014 15:06:35 +0100 Subject: close leaking slave fd after setting up pty magic The fd moves out of scope here anyway, so we should close it properly instead of leaking it which will tickle down to dpkg maintainer scripts. Closes: 767774 --- apt-pkg/deb/dpkgpm.cc | 6 +- test/integration/framework | 78 ++++++++++++++++++---- test/integration/test-failing-maintainer-scripts | 46 +------------ .../test-no-fds-leaked-to-maintainer-scripts | 40 +++++++++++ 4 files changed, 109 insertions(+), 61 deletions(-) create mode 100755 test/integration/test-no-fds-leaked-to-maintainer-scripts (limited to 'test') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 7bbf18cba..e1f1ec680 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1160,8 +1160,7 @@ void pkgDPkgPM::SetupSlavePtyMagic() int const slaveFd = open(d->slave, O_RDWR); if (slaveFd == -1) _error->FatalE("open", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); - - if (ioctl(slaveFd, TIOCSCTTY, 0) < 0) + else if (ioctl(slaveFd, TIOCSCTTY, 0) < 0) _error->FatalE("ioctl", "Setting TIOCSCTTY for slave fd %d failed!", slaveFd); else { @@ -1172,6 +1171,9 @@ void pkgDPkgPM::SetupSlavePtyMagic() if (tcsetattr(0, TCSANOW, &d->tt) < 0) _error->FatalE("tcsetattr", "Setting in Setup via TCSANOW for slave fd %d failed!", slaveFd); } + + if (slaveFd != -1) + close(slaveFd); } void pkgDPkgPM::StopPtyMagic() { diff --git a/test/integration/framework b/test/integration/framework index 7923e23d9..df1942ff9 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -325,6 +325,59 @@ configdpkg() { fi } +configdpkgnoopchroot() { + # create a library to noop chroot() and rewrite maintainer script executions + # via execvp() as used by dpkg as we don't want our rootdir to be a fullblown + # chroot directory dpkg could chroot into to execute the maintainer scripts + msgtest 'Building library to preload to make maintainerscript work in' 'dpkg' + cat << EOF > noopchroot.c +#define _GNU_SOURCE +#include +#include +#include +#include + +static char * chrootdir = NULL; + +int chroot(const char *path) { + printf("WARNING: CHROOTing to %s was ignored!\n", path); + free(chrootdir); + chrootdir = strdup(path); + return 0; +} +int execvp(const char *file, char *const argv[]) { + static int (*func_execvp) (const char *, char * const []) = NULL; + if (func_execvp == NULL) + func_execvp = (int (*) (const char *, char * const [])) dlsym(RTLD_NEXT, "execvp"); + if (chrootdir == NULL || strncmp(file, "/var/lib/dpkg/", strlen("/var/lib/dpkg/")) != 0) + return func_execvp(file, argv); + printf("REWRITE execvp call %s into %s\n", file, chrootdir); + char newfile[strlen(chrootdir) + strlen(file)]; + strcpy(newfile, chrootdir); + strcat(newfile, file); + return func_execvp(newfile, argv); +} +EOF + testsuccess --nomsg gcc -fPIC -shared -o noopchroot.so noopchroot.c -ldl + + mkdir -p "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/" + DPKG="${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg" + echo "#!/bin/sh +if [ -n \"\$LD_PRELOAD\" ]; then + export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so \${LD_PRELOAD}\" +else + export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so\" +fi +dpkg \"\$@\"" > $DPKG + chmod +x $DPKG + sed -ie "s|^DPKG::options:: \"dpkg\";\$|DPKG::options:: \"$DPKG\";|" aptconfig.conf +} + +configallowinsecurerepositories() { + echo "Acquire::AllowInsecureRepositories \"$1\";" > rootdir/etc/apt/apt.conf.d/allow-insecure-repositories.conf + +} + configcompression() { while [ -n "$1" ]; do case "$1" in @@ -442,7 +495,7 @@ buildsimplenativepackage() { fi local BUILDDIR=${TMPWORKINGDIRECTORY}/incoming/${NAME}-${VERSION} - msgninfo "Build package ${NAME} in ${VERSION} for ${RELEASE} in ${DISTSECTION}… " + msgtest "Build source package in version ${VERSION} for ${RELEASE} in ${DISTSECTION}" "$NAME" mkdir -p $BUILDDIR/debian/source echo "* most suckless software product ever" > ${BUILDDIR}/FEATURES echo "#!/bin/sh @@ -474,7 +527,10 @@ Package: $NAME" >> ${BUILDDIR}/debian/control echo "Description: $DESCRIPTION" >> ${BUILDDIR}/debian/control echo '3.0 (native)' > ${BUILDDIR}/debian/source/format - (cd ${BUILDDIR}/..; dpkg-source -b ${NAME}-${VERSION} 2>&1) | sed -n 's#^dpkg-source: info: building [^ ]\+ in ##p' \ + cd ${BUILDDIR}/.. + testsuccess --nomsg dpkg-source -b ${NAME}-${VERSION} + cd - >/dev/null + sed -n 's#^dpkg-source: info: building [^ ]\+ in ##p' ${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output \ | while read SRC; do echo "pool/${SRC}" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.srclist # if expr match "${SRC}" '.*\.dsc' >/dev/null 2>&1; then @@ -486,6 +542,7 @@ Package: $NAME" >> ${BUILDDIR}/debian/control done for arch in $(getarchitecturesfromcommalist "$ARCH"); do + msgtest "Build binary package for ${RELEASE} in ${SECTION}" "$NAME" rm -rf ${BUILDDIR}/debian/tmp mkdir -p ${BUILDDIR}/debian/tmp/DEBIAN ${BUILDDIR}/debian/tmp/usr/share/doc/${NAME} ${BUILDDIR}/debian/tmp/usr/bin cp ${BUILDDIR}/debian/copyright ${BUILDDIR}/debian/changelog ${BUILDDIR}/FEATURES ${BUILDDIR}/debian/tmp/usr/share/doc/${NAME} @@ -499,11 +556,7 @@ Package: $NAME" >> ${BUILDDIR}/debian/control local LOG="${BUILDDIR}/../${NAME}_${VERSION}_${arch}.dpkg-deb.log" # ensure the right permissions as dpkg-deb ensists chmod 755 ${BUILDDIR}/debian/tmp/DEBIAN - if ! dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/.. >$LOG 2>&1; then - cat $LOG - false - fi - rm $LOG + testsuccess --nomsg dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/.. echo "pool/${NAME}_${VERSION}_${arch}.deb" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.pkglist done @@ -521,15 +574,13 @@ buildpackage() { local ARCH=$(getarchitecture $4) local PKGNAME="$(echo "$BUILDDIR" | grep -o '[^/]*$')" local BUILDLOG="$(readlink -f "${BUILDDIR}/../${PKGNAME}_${RELEASE}_${SECTION}.dpkg-bp.log")" - msgninfo "Build package ${PKGNAME} for ${RELEASE} in ${SECTION}… " + msgtest "Build package for ${RELEASE} in ${SECTION}" "$PKGNAME" cd $BUILDDIR if [ "$ARCH" = "all" ]; then ARCH="$(dpkg-architecture -qDEB_HOST_ARCH 2> /dev/null)" fi - if ! dpkg-buildpackage -uc -us -a$ARCH >$BUILDLOG 2>&1 ; then - cat $BUILDLOG - false - fi + testsuccess --nomsg dpkg-buildpackage -uc -us -a$ARCH + cp ${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output $BUILDLOG local PKGS="$(grep '^dpkg-deb: building package' $BUILDLOG | cut -d'/' -f 2 | sed -e "s#'\.##")" local SRCS="$(grep '^dpkg-source: info: building' $BUILDLOG | grep -o '[a-z0-9._+~-]*$')" cd - > /dev/null @@ -539,7 +590,6 @@ buildpackage() { for SRC in $SRCS; do echo "pool/${SRC}" >> ${TMPWORKINGDIRECTORY}/incoming/${RELEASE}.${SECTION}.srclist done - msgdone "info" } buildaptarchive() { @@ -1072,7 +1122,7 @@ testequal() { if [ -n "$MSG" ]; then msgtest "$MSG" "$*" fi - $* 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail + "$@" 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail } testequalor2() { diff --git a/test/integration/test-failing-maintainer-scripts b/test/integration/test-failing-maintainer-scripts index 3dd7d643e..953506aa5 100755 --- a/test/integration/test-failing-maintainer-scripts +++ b/test/integration/test-failing-maintainer-scripts @@ -6,6 +6,7 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture 'native' +configdpkgnoopchroot # create a bunch of failures createfailure() { @@ -25,51 +26,6 @@ createfailure 'postrm' setupaptarchive -# create a library to noop chroot() and rewrite maintainer script executions -# via execvp() as used by dpkg as we don't want our rootdir to be a fullblown -# chroot directory dpkg could chroot into to execute the maintainer scripts -cat << EOF > noopchroot.c -#define _GNU_SOURCE -#include -#include -#include -#include - -static char * chrootdir = NULL; - -int chroot(const char *path) { - printf("WARNING: CHROOTing to %s was ignored!\n", path); - free(chrootdir); - chrootdir = strdup(path); - return 0; -} -int execvp(const char *file, char *const argv[]) { - static int (*func_execvp) (const char *, char * const []) = NULL; - if (func_execvp == NULL) - func_execvp = (int (*) (const char *, char * const [])) dlsym(RTLD_NEXT, "execvp"); - if (chrootdir == NULL || strncmp(file, "/var/lib/dpkg/", strlen("/var/lib/dpkg/")) != 0) - return func_execvp(file, argv); - printf("REWRITE execvp call %s into %s\n", file, chrootdir); - char newfile[strlen(chrootdir) + strlen(file)]; - strcpy(newfile, chrootdir); - strcat(newfile, file); - return func_execvp(newfile, argv); -} -EOF -testsuccess gcc -fPIC -shared -o noopchroot.so noopchroot.c -ldl - -mkdir -p "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/" -DPKG="${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg" -echo "#!/bin/sh -if [ -n \"\$LD_PRELOAD\" ]; then - export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so \${LD_PRELOAD}\" -else - export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so\" -fi -dpkg \"\$@\"" > $DPKG -chmod +x $DPKG -sed -ie "s|^DPKG::options:: \"dpkg\";\$|DPKG::options:: \"$DPKG\";|" aptconfig.conf - # setup some pre- and post- invokes to check the output isn't garbled later APTHOOK="${TMPWORKINGDIRECTORY}/rootdir/usr/bin/apthook" echo '#!/bin/sh diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts new file mode 100755 index 000000000..6ed120090 --- /dev/null +++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts @@ -0,0 +1,40 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'native' +configdpkgnoopchroot + +setupsimplenativepackage "fdleaks" 'native' '1.0' 'unstable' +BUILDDIR="incoming/fdleaks-1.0" +for script in 'preinst' 'postinst' 'prerm' 'postrm'; do + echo '#!/bin/sh +ls -l /proc/self/fd/' > ${BUILDDIR}/debian/$script +done +buildpackage "$BUILDDIR" 'unstable' 'main' 'native' +rm -rf "$BUILDDIR" + +setupaptarchive + +testsuccess aptget install -y fdleaks +msgtest 'Check if fds were not' 'leaked' +if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '8' ]; then + msgpass +else + echo + cat rootdir/tmp/testsuccess.output + msgfail +fi + +testsuccess aptget purge -y fdleaks +msgtest 'Check if fds were not' 'leaked' +if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '12' ]; then + msgpass +else + echo + cat rootdir/tmp/testsuccess.output + msgfail +fi -- cgit v1.2.3-70-g09d2 From 299aea924ccef428219ed6f1a026c122678429e6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 18 Nov 2014 00:59:39 +0100 Subject: fix PTY interaction on linux and kfreebsd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We run dpkg on its own pty, so we can log its output and have our own output around it (like the progress bar), while also allowing debconf and configfile prompts to happen. In commit 223ae57d468fdcac451209a095047a07a5698212 we changed to constantly reopening the slave for kfreebsd. This has the sideeffect though that in some cases slave and master will lose their connection on linux, so that no output is passed along anymore. We fix this by having always an fd referencing the slave open (linux), but we don't use it (kfreebsd). Failing to get our PTY up and running has many (bad) consequences including (not limited to, nor all at ones or in any case) garbled ouput, no output, no logging, a (partial) mixture of the previous items, … This commit is therefore also reshuffling quiet a bit of the creation code to get especially the output part up and running on linux and the logging for kfreebsd. Note that the testcase tries to cover some cases, but this is an interactivity issue so only interactive usage can really be a good test. Closes: 765687 --- apt-pkg/deb/dpkgpm.cc | 99 ++++++++++++---------- .../test-no-fds-leaked-to-maintainer-scripts | 40 ++++++++- 2 files changed, 92 insertions(+), 47 deletions(-) (limited to 'test') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index e1f1ec680..e36a52c3e 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -72,7 +72,8 @@ class pkgDPkgPMPrivate public: pkgDPkgPMPrivate() : stdin_is_dev_null(false), dpkgbuf_pos(0), term_out(NULL), history_out(NULL), - progress(NULL), master(-1), slave(NULL) + progress(NULL), tt_is_valid(false), master(-1), + slave(NULL), protect_slave_from_dying(-1) { dpkgbuf[0] = '\0'; } @@ -90,8 +91,10 @@ public: // pty stuff struct termios tt; + bool tt_is_valid; int master; char * slave; + int protect_slave_from_dying; // signals sigset_t sigmask; @@ -1071,47 +1074,40 @@ void pkgDPkgPM::StartPtyMagic() } _error->PushToStack(); - // if tcgetattr for both stdin/stdout returns 0 (no error) - // we do the pty magic - if (tcgetattr(STDOUT_FILENO, &d->tt) == 0 && - tcgetattr(STDIN_FILENO, &d->tt) == 0) + + d->master = posix_openpt(O_RDWR | O_NOCTTY); + if (d->master == -1) + _error->Errno("posix_openpt", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); + else if (unlockpt(d->master) == -1) + _error->Errno("unlockpt", "Unlocking the slave of master fd %d failed!", d->master); + else { - d->master = posix_openpt(O_RDWR | O_NOCTTY); - if (d->master == -1) - _error->Errno("posix_openpt", _("Can not write log (%s)"), _("Is /dev/pts mounted?")); - else if (unlockpt(d->master) == -1) - { - _error->Errno("unlockpt", "Unlocking the slave of master fd %d failed!", d->master); - close(d->master); - d->master = -1; - } + char const * const slave_name = ptsname(d->master); + if (slave_name == NULL) + _error->Errno("ptsname", "Getting name for slave of master fd %d failed!", d->master); else { - char const * const slave_name = ptsname(d->master); - if (slave_name == NULL) + d->slave = strdup(slave_name); + if (d->slave == NULL) + _error->Errno("strdup", "Copying name %s for slave of master fd %d failed!", slave_name, d->master); + else if (grantpt(d->master) == -1) + _error->Errno("grantpt", "Granting access to slave %s based on master fd %d failed!", slave_name, d->master); + else if (tcgetattr(STDIN_FILENO, &d->tt) == 0) { - _error->Errno("unlockpt", "Getting name for slave of master fd %d failed!", d->master); - close(d->master); - d->master = -1; - } - else - { - d->slave = strdup(slave_name); - if (d->slave == NULL) + d->tt_is_valid = true; + struct termios raw_tt; + // copy window size of stdout if its a 'good' terminal + if (tcgetattr(STDOUT_FILENO, &raw_tt) == 0) { - _error->Errno("strdup", "Copying name %s for slave of master fd %d failed!", slave_name, d->master); - close(d->master); - d->master = -1; + struct winsize win; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) < 0) + _error->Errno("ioctl", "Getting TIOCGWINSZ from stdout failed!"); + if (ioctl(d->master, TIOCSWINSZ, &win) < 0) + _error->Errno("ioctl", "Setting TIOCSWINSZ for master fd %d failed!", d->master); } - struct winsize win; - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &win) < 0) - _error->Errno("ioctl", "Getting TIOCGWINSZ from stdout failed!"); - if (ioctl(d->master, TIOCSWINSZ, &win) < 0) - _error->Errno("ioctl", "Setting TIOCSWINSZ for master fd %d failed!", d->master); if (tcsetattr(d->master, TCSANOW, &d->tt) == -1) _error->Errno("tcsetattr", "Setting in Start via TCSANOW for master fd %d failed!", d->master); - struct termios raw_tt; raw_tt = d->tt; cfmakeraw(&raw_tt); raw_tt.c_lflag &= ~ECHO; @@ -1123,18 +1119,22 @@ void pkgDPkgPM::StartPtyMagic() sigaddset(&d->sigmask, SIGTTOU); sigprocmask(SIG_BLOCK,&d->sigmask, &d->original_sigmask); if (tcsetattr(STDIN_FILENO, TCSAFLUSH, &raw_tt) == -1) - _error->Errno("tcsetattr", "Setting in Start via TCSAFLUSH for stdout failed!"); + _error->Errno("tcsetattr", "Setting in Start via TCSAFLUSH for stdin failed!"); sigprocmask(SIG_SETMASK, &d->original_sigmask, NULL); + + } + if (d->slave != NULL) + { + /* on linux, closing (and later reopening) all references to the slave + makes the slave a death end, so we open it here to have one open all + the time. We could use this fd in SetupSlavePtyMagic() for linux, but + on kfreebsd we get an incorrect ("step like") output then while it has + no problem with closing all references… so to avoid platform specific + code here we combine both and be happy once more */ + d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC); } } } - else - { - // complain only if stdout is either a terminal (but still failed) or is an invalid - // descriptor otherwise we would complain about redirection to e.g. /dev/null as well. - if (isatty(STDOUT_FILENO) == 1 || errno == EBADF) - _error->Errno("tcgetattr", _("Can not write log (%s)"), _("Is stdout a terminal?")); - } if (_error->PendingError() == true) { @@ -1143,17 +1143,23 @@ void pkgDPkgPM::StartPtyMagic() close(d->master); d->master = -1; } + if (d->slave != NULL) + { + free(d->slave); + d->slave = NULL; + } _error->DumpErrors(std::cerr); } _error->RevertToStack(); } void pkgDPkgPM::SetupSlavePtyMagic() { - if(d->master == -1) + if(d->master == -1 || d->slave == NULL) return; if (close(d->master) == -1) _error->FatalE("close", "Closing master %d in child failed!", d->master); + d->master = -1; if (setsid() == -1) _error->FatalE("setsid", "Starting a new session for child failed!"); @@ -1168,7 +1174,7 @@ void pkgDPkgPM::SetupSlavePtyMagic() if (dup2(slaveFd, i) == -1) _error->FatalE("dup2", "Dupping %d to %d in child failed!", slaveFd, i); - if (tcsetattr(0, TCSANOW, &d->tt) < 0) + if (d->tt_is_valid == true && tcsetattr(STDIN_FILENO, TCSANOW, &d->tt) < 0) _error->FatalE("tcsetattr", "Setting in Setup via TCSANOW for slave fd %d failed!", slaveFd); } @@ -1180,9 +1186,14 @@ void pkgDPkgPM::StopPtyMagic() if (d->slave != NULL) free(d->slave); d->slave = NULL; + if (d->protect_slave_from_dying != -1) + { + close(d->protect_slave_from_dying); + d->protect_slave_from_dying = -1; + } if(d->master >= 0) { - if (tcsetattr(0, TCSAFLUSH, &d->tt) == -1) + if (d->tt_is_valid == true && tcsetattr(STDIN_FILENO, TCSAFLUSH, &d->tt) == -1) _error->FatalE("tcsetattr", "Setting in Stop via TCSAFLUSH for stdin failed!"); close(d->master); d->master = -1; diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts index 6ed120090..3c6457cab 100755 --- a/test/integration/test-no-fds-leaked-to-maintainer-scripts +++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts @@ -8,7 +8,7 @@ setupenvironment configarchitecture 'native' configdpkgnoopchroot -setupsimplenativepackage "fdleaks" 'native' '1.0' 'unstable' +setupsimplenativepackage "fdleaks" 'all' '1.0' 'unstable' BUILDDIR="incoming/fdleaks-1.0" for script in 'preinst' 'postinst' 'prerm' 'postrm'; do echo '#!/bin/sh @@ -19,7 +19,8 @@ rm -rf "$BUILDDIR" setupaptarchive -testsuccess aptget install -y fdleaks +rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log +testsuccess aptget install -y fdleaks -qq < /dev/null msgtest 'Check if fds were not' 'leaked' if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '8' ]; then msgpass @@ -29,7 +30,23 @@ else msgfail fi -testsuccess aptget purge -y fdleaks +cp rootdir/tmp/testsuccess.output terminal.output +tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log +testfileequal 'terminal.log' "$(cat terminal.output)" + +testequal 'startup archives unpack +install fdleaks:all 1.0 +status half-installed fdleaks:all 1.0 +status unpacked fdleaks:all 1.0 +status unpacked fdleaks:all 1.0 +startup packages configure +configure fdleaks:all 1.0 +status unpacked fdleaks:all 1.0 +status half-configured fdleaks:all 1.0 +status installed fdleaks:all 1.0' cut -f 3- -d' ' rootdir/var/log/dpkg.log + +rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log +testsuccess aptget purge -y fdleaks -qq msgtest 'Check if fds were not' 'leaked' if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '12' ]; then msgpass @@ -38,3 +55,20 @@ else cat rootdir/tmp/testsuccess.output msgfail fi +cp rootdir/tmp/testsuccess.output terminal.output +tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log +testfileequal 'terminal.log' "$(cat terminal.output)" + +testequal 'startup packages purge +status installed fdleaks:all 1.0 +remove fdleaks:all 1.0 +status half-configured fdleaks:all 1.0 +status half-installed fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +purge fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status config-files fdleaks:all 1.0 +status not-installed fdleaks:all ' cut -f 3- -d' ' rootdir/var/log/dpkg.log -- cgit v1.2.3-70-g09d2 From 69d8b8537af1dd52db5f2e0e785bdce3e52fdf8d Mon Sep 17 00:00:00 2001 From: James McCoy Date: Fri, 28 Nov 2014 14:21:06 +0100 Subject: support long keyids in "apt-key del" instead of ignoring them apt-key given a long keyid reports just "OK" all the time, but doesn't delete the mentioned key as it doesn't find the key. Note: In debian/experimental this was closed with 29f1b977100aeb6d6ebd38923eeb7a623e264ffe which just added the testcase as the rewrite of apt-key had fixed this as well. Closes: 754436 --- cmdline/apt-key.in | 2 +- test/integration/test-apt-key | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'test') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 0774cf4b7..b4e071000 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -180,7 +180,7 @@ update() { remove_key_from_keyring() { local GPG="$GPG_CMD --keyring $1" # check if the key is in this keyring: the key id is in the 5 column at the end - if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+$2:"; then + if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then return fi if [ ! -w "$1" ]; then diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index 68b3f9710..47230cb55 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -105,3 +105,9 @@ testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ + +msgtest 'Test key removal with' '8 byte key ID' +cleanplate +cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg +testsuccess --nomsg aptkey --fakeroot del 5A90D141DBAC8DAE +testempty aptkey list -- cgit v1.2.3-70-g09d2 From 4e6a7e260eb713318b1b5019a72073050242ac3c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 25 Nov 2014 10:50:58 +0100 Subject: properly handle already reinstall pkgs in ordering The bugreport itself describes the case of the ordering code detecting a loop where none is present, but the testcase finds also cases in which there is actually a loop and we fail to realize it. --reinstall can be considered an interactive command through and it usually doesn't encounter such "hard" problems (= looping essentials), so this is less serious than it sounds at first. Closes: 770291 --- apt-pkg/packagemanager.cc | 46 +++++++++++--- test/integration/test-bug-770291-reinstall | 98 ++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 10 deletions(-) create mode 100755 test/integration/test-bug-770291-reinstall (limited to 'test') diff --git a/apt-pkg/packagemanager.cc b/apt-pkg/packagemanager.cc index ba48c53cc..d137dc75a 100644 --- a/apt-pkg/packagemanager.cc +++ b/apt-pkg/packagemanager.cc @@ -400,7 +400,8 @@ bool pkgPackageManager::SmartConfigure(PkgIterator Pkg, int const Depth) // Check if the current version of the package is available and will satisfy this dependency if (DepPkg.CurrentVer() == Ver && List->IsNow(DepPkg) == true && List->IsFlag(DepPkg,pkgOrderList::Removed) == false && - DepPkg.State() == PkgIterator::NeedsNothing) + DepPkg.State() == PkgIterator::NeedsNothing && + (Cache[DepPkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) { Bad = false; break; @@ -413,8 +414,13 @@ bool pkgPackageManager::SmartConfigure(PkgIterator Pkg, int const Depth) if (PkgLoop == true) { if (Debug) - std::clog << OutputInDepth(Depth) << "Package " << Pkg << " loops in SmartConfigure" << std::endl; - Bad = false; + std::clog << OutputInDepth(Depth) << "Package " << Pkg << " loops in SmartConfigure"; + if (List->IsFlag(DepPkg,pkgOrderList::UnPacked)) + Bad = false; + else if (Debug) + std::clog << ", but it isn't unpacked yet"; + if (Debug) + std::clog << std::endl; } } @@ -426,7 +432,7 @@ bool pkgPackageManager::SmartConfigure(PkgIterator Pkg, int const Depth) if (Bad == false) { if (Debug) - std::clog << OutputInDepth(Depth) << "Found ok dep " << D.TargetPkg() << std::endl; + std::clog << OutputInDepth(Depth) << "Found ok dep " << Start.TargetPkg() << std::endl; continue; } @@ -444,7 +450,8 @@ bool pkgPackageManager::SmartConfigure(PkgIterator Pkg, int const Depth) // Check if the current version of the package is available and will satisfy this dependency if (DepPkg.CurrentVer() == Ver && List->IsNow(DepPkg) == true && List->IsFlag(DepPkg,pkgOrderList::Removed) == false && - DepPkg.State() == PkgIterator::NeedsNothing) + DepPkg.State() == PkgIterator::NeedsNothing && + (Cache[DepPkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) continue; // Check if the version that is going to be installed will satisfy the dependency @@ -454,8 +461,13 @@ bool pkgPackageManager::SmartConfigure(PkgIterator Pkg, int const Depth) if (PkgLoop == true) { if (Debug) - std::clog << OutputInDepth(Depth) << "Package " << Pkg << " loops in SmartConfigure" << std::endl; - Bad = false; + std::clog << OutputInDepth(Depth) << "Package " << Pkg << " loops in SmartConfigure"; + if (List->IsFlag(DepPkg,pkgOrderList::UnPacked)) + Bad = false; + else if (Debug) + std::clog << ", but it isn't unpacked yet"; + if (Debug) + std::clog << std::endl; } else { @@ -722,7 +734,8 @@ bool pkgPackageManager::SmartUnPack(PkgIterator Pkg, bool const Immediate, int c // See if the current version is ok if (Pkg.CurrentVer() == Ver && List->IsNow(Pkg) == true && - Pkg.State() == PkgIterator::NeedsNothing) + Pkg.State() == PkgIterator::NeedsNothing && + (Cache[Pkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) { Bad = false; if (Debug) @@ -744,8 +757,11 @@ bool pkgPackageManager::SmartUnPack(PkgIterator Pkg, bool const Immediate, int c PkgIterator DepPkg = Ver.ParentPkg(); // Not the install version - if (Cache[DepPkg].InstallVer != *I || - (Cache[DepPkg].Keep() == true && DepPkg.State() == PkgIterator::NeedsNothing)) + if (Cache[DepPkg].InstallVer != *I) + continue; + + if (Cache[DepPkg].Keep() == true && DepPkg.State() == PkgIterator::NeedsNothing && + (Cache[DepPkg].iFlags & pkgDepCache::ReInstall) != pkgDepCache::ReInstall) continue; if (List->IsFlag(DepPkg,pkgOrderList::Configured)) @@ -757,6 +773,16 @@ bool pkgPackageManager::SmartUnPack(PkgIterator Pkg, bool const Immediate, int c // check if it needs unpack or if if configure is enough if (List->IsFlag(DepPkg,pkgOrderList::UnPacked) == false) { + // two packages pre-depending on each other can't be handled sanely + if (List->IsFlag(DepPkg,pkgOrderList::Loop) && PkgLoop) + { + // this isn't an error as there is potential for something else to satisfy it + // (like a provides or an or-group member) + if (Debug) + clog << OutputInDepth(Depth) << "Unpack loop detected between " << DepPkg.FullName() << " and " << Pkg.FullName() << endl; + continue; + } + if (Debug) clog << OutputInDepth(Depth) << "Trying to SmartUnpack " << DepPkg.FullName() << endl; if (NonLoopingSmart(UNPACK_IMMEDIATE, Pkg, DepPkg, Depth, PkgLoop, &Bad, &Changed) == false) diff --git a/test/integration/test-bug-770291-reinstall b/test/integration/test-bug-770291-reinstall new file mode 100755 index 000000000..ea1f57ede --- /dev/null +++ b/test/integration/test-bug-770291-reinstall @@ -0,0 +1,98 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'i386' + +insertpackage 'unstable,installed' 'libc6' 'i386' '1' +insertpackage 'unstable,installed' 'libselinux1' 'i386' '1' + +cp rootdir/var/lib/dpkg/status dpkg.status + +insertpackage 'unstable,installed' 'init' 'i386' '1' 'Depends: systemd-sysv +Essential: yes' +insertpackage 'unstable,installed' 'systemd-sysv' 'i386' '215-5+b1' 'Depends: systemd (= 215-5+b1) +Pre-Depends: systemd' +# fun fact: we need these two pre-depends to get systemd ordered before systemd-sysv as +# many pre-depends mean: do early (as they are a pain, so get them out of the way early) +insertpackage 'unstable,installed' 'systemd' 'i386' '215-5+b1' 'Pre-Depends: libc6, libselinux1' + +# depends loop +insertpackage 'unstable,installed' 'dependsA' 'i386' '1' 'Depends: dependsB +Essential: yes' +insertpackage 'unstable,installed' 'dependsB' 'i386' '1' 'Depends: dependsA +Essential: yes' + +# pre-depends loop +insertpackage 'unstable,installed' 'predependsA' 'i386' '1' 'Pre-Depends: predependsB +Essential: yes' +insertpackage 'unstable,installed' 'predependsB' 'i386' '1' 'Pre-Depends: predependsA +Essential: yes' + +# pre-depends-to-depends loop +insertpackage 'unstable,installed' 'predependsdependsA' 'i386' '1' 'Pre-Depends: predependsdependsB +Essential: yes' +insertpackage 'unstable,installed' 'predependsdependsB' 'i386' '1' 'Depends: predependsdependsA +Essential: yes' + +setupaptarchive + +testequal 'Reading package lists... +Building dependency tree... +0 upgraded, 0 newly installed, 2 reinstalled, 0 to remove and 0 not upgraded. +Inst systemd [215-5+b1] (215-5+b1 unstable [i386]) +Conf systemd (215-5+b1 unstable [i386]) +Inst systemd-sysv [215-5+b1] (215-5+b1 unstable [i386]) +Conf systemd-sysv (215-5+b1 unstable [i386])' aptget install --reinstall systemd systemd-sysv -s + +testequal 'Reading package lists... +Building dependency tree... +0 upgraded, 0 newly installed, 2 reinstalled, 0 to remove and 0 not upgraded. +Inst dependsA [1] (1 unstable [i386]) +Inst dependsB [1] (1 unstable [i386]) +Conf dependsB (1 unstable [i386]) +Conf dependsA (1 unstable [i386])' aptget install --reinstall dependsA dependsB -s + +# there is a chance dpkg can actually do these, BUT this depends on the maintainerscripts (not) present +# which is very very risky to depend on (and apt doesn't know about that anyhow). +testfailure aptget install --reinstall predependsA predependsB -s -o Debug::pkgPackageManager=1 +testequal "E: Couldn't configure predependsA:i386, probably a dependency cycle." tail -n1 rootdir/tmp/testfailure.output + +# FIXME: the error message is a catch all here, not like the one above +testfailure aptget install --reinstall predependsdependsA predependsdependsB -s -o Debug::pkgPackageManager=1 +testequal "E: Could not configure 'predependsdependsB:i386'. " tail -n1 rootdir/tmp/testfailure.output + + +msgmsg 'While we are at it, lets try these loops without reinstall as well' +cp dpkg.status rootdir/var/lib/dpkg/status + +testequal 'Reading package lists... +Building dependency tree... +The following NEW packages will be installed: + systemd systemd-sysv +0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded. +Inst systemd (215-5+b1 unstable [i386]) +Conf systemd (215-5+b1 unstable [i386]) +Inst systemd-sysv (215-5+b1 unstable [i386]) +Conf systemd-sysv (215-5+b1 unstable [i386])' aptget install systemd systemd-sysv -s + +testequal 'Reading package lists... +Building dependency tree... +The following NEW packages will be installed: + dependsA dependsB +0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded. +Inst dependsA (1 unstable [i386]) [] +Inst dependsB (1 unstable [i386]) +Conf dependsB (1 unstable [i386]) +Conf dependsA (1 unstable [i386])' aptget install dependsA dependsB -s + +# there is a chance dpkg can actually do these, BUT this depends on the maintainerscripts (not) present +# which is very very risky to depend on (and apt doesn't know about that anyhow). +testfailure aptget install predependsA predependsB -s -o Debug::pkgPackageManager=1 +testequal "E: Couldn't configure predependsA:i386, probably a dependency cycle." tail -n1 rootdir/tmp/testfailure.output + +# FIXME: the error message is a catch all here, not like the one above +testfailure aptget install predependsdependsA predependsdependsB -s -o Debug::pkgPackageManager=1 +testequal "E: Could not configure 'predependsdependsB:i386'. " tail -n1 rootdir/tmp/testfailure.output -- cgit v1.2.3-70-g09d2 From 016bea8214e1826b289025f03890f70a5805db87 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 25 Nov 2014 12:10:15 +0100 Subject: correct architecture detection for 'rc' packages for purge We were already considering these cases, but the code was flawed, so that packages changing architectures are incorrectly handled and hence the wrong architecture is used to call dpkg with, so that dpkg says the package isn't installed (which it isn't for the requested architecture). Closes: 770898 --- apt-pkg/deb/dpkgpm.cc | 16 ++---- test/integration/framework | 24 ++++++--- .../test-ubuntu-bug-761175-remove-purge | 62 ++++++++++++++-------- 3 files changed, 63 insertions(+), 39 deletions(-) (limited to 'test') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 593875071..3fbda0552 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -201,18 +201,10 @@ pkgCache::VerIterator FindNowVersion(const pkgCache::PkgIterator &Pkg) { pkgCache::VerIterator Ver; for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver) - { - pkgCache::VerFileIterator Vf = Ver.FileList(); - pkgCache::PkgFileIterator F = Vf.File(); - for (F = Vf.File(); F.end() == false; ++F) - { - if (F && F.Archive()) - { - if (strcmp(F.Archive(), "now")) - return Ver; - } - } - } + for (pkgCache::VerFileIterator Vf = Ver.FileList(); Vf.end() == false; ++Vf) + for (pkgCache::PkgFileIterator F = Vf.File(); F.end() == false; ++F) + if (F->Archive != 0 && strcmp(F.Archive(), "now") == 0) + return Ver; return Ver; } /*}}}*/ diff --git a/test/integration/framework b/test/integration/framework index ff059f59e..298f4c9b2 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -635,12 +635,8 @@ buildaptarchive() { createaptftparchiveconfig() { local COMPRESSORS="$(cut -d' ' -f 1 ${TMPWORKINGDIRECTORY}/rootdir/etc/testcase-compressor.conf | tr '\n' ' ')" - COMPRESSORS="${COMPRESSORS%* }" - local ARCHS="$(find pool/ -name '*.deb' | grep -oE '_[a-z0-9-]+\.deb$' | sort | uniq | sed -e '/^_all.deb$/ d' -e 's#^_\([a-z0-9-]*\)\.deb$#\1#' | tr '\n' ' ')" - if [ -z "$ARCHS" ]; then - # the pool is empty, so we will operate on faked packages - let us use the configured archs - ARCHS="$(getarchitectures)" - fi + local COMPRESSORS="${COMPRESSORS%* }" + local ARCHS="$(getarchitectures)" echo -n 'Dir { ArchiveDir "' >> ftparchive.conf echo -n $(readlink -f .) >> ftparchive.conf @@ -1489,3 +1485,19 @@ aptautotest_aptget_update() { done } aptautotest_apt_update() { aptautotest_aptget_update "$@"; } + +testaptautotestnodpkgwarning() { + local TESTCALL="$1" + while [ -n "$2" ]; do + if [ "$2" = '-s' ]; then return; fi + shift + done + testfailure grep '^dpkg: warning:.*ignor.*' "${TMPWORKINGDIRECTORY}/rootdir/tmp-before/${TESTCALL}.output" +} + +aptautotest_aptget_install() { testaptautotestnodpkgwarning "$@"; } +aptautotest_aptget_remove() { testaptautotestnodpkgwarning "$@"; } +aptautotest_aptget_purge() { testaptautotestnodpkgwarning "$@"; } +aptautotest_apt_install() { testaptautotestnodpkgwarning "$@"; } +aptautotest_apt_remove() { testaptautotestnodpkgwarning "$@"; } +aptautotest_apt_purge() { testaptautotestnodpkgwarning "$@"; } diff --git a/test/integration/test-ubuntu-bug-761175-remove-purge b/test/integration/test-ubuntu-bug-761175-remove-purge index 14648e9b8..0b5a91246 100755 --- a/test/integration/test-ubuntu-bug-761175-remove-purge +++ b/test/integration/test-ubuntu-bug-761175-remove-purge @@ -4,33 +4,53 @@ set -e TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture 'native' - -setupsimplenativepackage 'compiz-core' 'native' '1.0' 'unstable' -BUILDDIR='incoming/compiz-core-1.0' -mkdir -p ${BUILDDIR}/debian/compiz-core/etc -echo 'foo=bar;' > ${BUILDDIR}/compiz.conf -echo 'compiz.conf /etc/compiz.conf' >> ${BUILDDIR}/debian/install -buildpackage "$BUILDDIR" 'unstable' 'main' 'native' -rm -rf "$BUILDDIR" +configarchitecture 'amd64' 'i386' + +buildcompizpkg() { + setupsimplenativepackage "compiz-core-$1" "$2" "$3" "$4" + BUILDDIR="incoming/compiz-core-$1-$3" + mkdir -p ${BUILDDIR}/debian/compiz-core/etc + echo 'foo=bar;' > ${BUILDDIR}/compiz.conf + echo 'compiz.conf /etc/compiz.conf' >> ${BUILDDIR}/debian/install + buildpackage "$BUILDDIR" "$4" 'main' "$2" + rm -rf "$BUILDDIR" +} +buildcompizpkg 'native' 'all' '1.0' 'stable' +buildcompizpkg 'all' 'native' '1.0' 'stable' +buildcompizpkg 'native' 'native' '2.0' 'unstable' +buildcompizpkg 'all' 'all' '2.0' 'unstable' setupaptarchive +runtests() { + testdpkgnotinstalled compiz-core-$1 + testsuccess aptget install compiz-core-$1 -t "${2:-unstable}" + testdpkginstalled compiz-core-$1 -testdpkgnotinstalled compiz-core -testsuccess aptget install compiz-core -testdpkginstalled compiz-core - -testsuccess aptget remove compiz-core -y -testdpkgnotinstalled compiz-core - -msgtest 'Check that conffiles are still around for' 'compiz-core' -dpkg -l compiz-core | grep -q '^rc' && msgpass || msgfail + testsuccess aptget remove compiz-core-$1 -y + testdpkgnotinstalled compiz-core-$1 + testdpkgstatus 'rc' '1' "compiz-core-$1" -testequal 'Reading package lists... + testequal "Reading package lists... Building dependency tree... Reading state information... The following packages will be REMOVED: - compiz-core* + compiz-core-$1* 0 upgraded, 0 newly installed, 1 to remove and 0 not upgraded. -Purg compiz-core' aptget purge compiz-core -s +Purg compiz-core-$1" aptget purge compiz-core-$1 -s + testsuccess aptget purge compiz-core-$1 -y + testequal "dpkg-query: no packages found matching compiz-core-$1" dpkg -l compiz-core-$1 +} + +msgmsg 'Test in multi arch environment' +runtests 'native' +runtests 'all' +runtests 'native' 'stable' +runtests 'all' 'stable' + +msgmsg 'Test in single arch environment' +configarchitecture 'amd64' +runtests 'native' +runtests 'all' +runtests 'native' 'stable' +runtests 'all' 'stable' -- cgit v1.2.3-70-g09d2 From ecb777ddb4d13bb7a18bbf2ebb8e2c810dcaeb72 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 18 Nov 2014 19:53:56 +0100 Subject: always run 'dpkg --configure -a' at the end of our dpkg callings dpkg checks now for dependencies before running triggers, so that packages can now end up in trigger states (especially those we are not touching at all with our calls) after apt is done running. The solution to this is trivial: Just tell dpkg to configure everything after we have (supposely) configured everything already. In the worst case this means dpkg will have to run a bunch of triggers, usually it will just do nothing though. The code to make this happen was already available, so we just flip a config option here to cause it to be run. This way we can keep pretending that triggers are an implementation detail of dpkg. --triggers-only would supposely work as well, but --configure is more robust in regards to future changes to dpkg and something we will hopefully make use of in future versions anyway (as it was planed at the time this and related options were implemented). Closes: 769609 --- apt-pkg/deb/dpkgpm.cc | 9 ++- test/integration/framework | 25 ++++--- test/integration/test-apt-progress-fd | 67 ++++++++++--------- test/integration/test-apt-progress-fd-deb822 | 18 +++-- test/integration/test-apt-progress-fd-error | 2 +- ...est-bug-769609-triggers-still-pending-after-run | 78 ++++++++++++++++++++++ .../test-no-fds-leaked-to-maintainer-scripts | 6 +- 7 files changed, 149 insertions(+), 56 deletions(-) create mode 100755 test/integration/test-bug-769609-triggers-still-pending-after-run (limited to 'test') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 3fbda0552..79120f6f5 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1039,6 +1039,12 @@ void pkgDPkgPM::BuildPackagesProgressMap() PackagesTotal++; } } + /* one extra: We don't want the progress bar to reach 100%, especially not + if we call dpkg --configure --pending and process a bunch of triggers + while showing 100%. Also, spindown takes a while, so never reaching 100% + is way more correct than reaching 100% while still doing stuff even if + doing it this way is slightly bending the rules */ + ++PackagesTotal; } /*}}}*/ bool pkgDPkgPM::Go(int StatusFd) @@ -1266,9 +1272,8 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) // support subpressing of triggers processing for special // cases like d-i that runs the triggers handling manually - bool const SmartConf = (_config->Find("PackageManager::Configure", "all") != "all"); bool const TriggersPending = _config->FindB("DPkg::TriggersPending", false); - if (_config->FindB("DPkg::ConfigurePending", SmartConf) == true) + if (_config->FindB("DPkg::ConfigurePending", true) == true) List.push_back(Item(Item::ConfigurePending, PkgIterator())); // for the progress diff --git a/test/integration/framework b/test/integration/framework index 298f4c9b2..930ab9367 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1237,10 +1237,13 @@ testnopackage() { fi } -testdpkginstalled() { - msgtest "Test for correctly installed package(s) with" "dpkg -l $*" - local PKGS="$(dpkg -l "$@" 2>/dev/null | grep '^i' | wc -l)" - if [ "$PKGS" != $# ]; then +testdpkgstatus() { + local STATE="$1" + local NR="$2" + shift 2 + msgtest "Test that $NR package(s) are in state $STATE with" "dpkg -l $*" + local PKGS="$(dpkg -l "$@" 2>/dev/null | grep "^${STATE}" | wc -l)" + if [ "$PKGS" != $NR ]; then echo >&2 $PKGS dpkg -l "$@" | grep '^[a-z]' >&2 msgfail @@ -1249,16 +1252,12 @@ testdpkginstalled() { fi } +testdpkginstalled() { + testdpkgstatus 'ii' "$#" "$@" +} + testdpkgnotinstalled() { - msgtest "Test for correctly not-installed package(s) with" "dpkg -l $*" - local PKGS="$(dpkg -l "$@" 2> /dev/null | grep '^i' | wc -l)" - if [ "$PKGS" != 0 ]; then - echo - dpkg -l "$@" | grep '^[a-z]' >&2 - msgfail - else - msgpass - fi + testdpkgstatus 'ii' '0' "$@" } testmarkedauto() { diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd index af022f582..90e6ef7e4 100755 --- a/test/integration/test-apt-progress-fd +++ b/test/integration/test-apt-progress-fd @@ -19,13 +19,14 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1 dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) -pmstatus:testing:20:Preparing testing (amd64) -pmstatus:testing:40:Unpacking testing (amd64) -pmstatus:testing:60:Preparing to configure testing (amd64) -pmstatus:dpkg-exec:60:Running dpkg -pmstatus:testing:60:Configuring testing (amd64) -pmstatus:testing:80:Configuring testing (amd64) -pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log +pmstatus:testing:16.6667:Preparing testing (amd64) +pmstatus:testing:33.3333:Unpacking testing (amd64) +pmstatus:testing:50:Preparing to configure testing (amd64) +pmstatus:dpkg-exec:50:Running dpkg +pmstatus:testing:50:Configuring testing (amd64) +pmstatus:testing:66.6667:Configuring testing (amd64) +pmstatus:testing:83.3333:Installed testing (amd64) +pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log # upgrade exec 3> apt-progress.log @@ -34,13 +35,14 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1 dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) -pmstatus:testing:20:Preparing testing (amd64) -pmstatus:testing:40:Unpacking testing (amd64) -pmstatus:testing:60:Preparing to configure testing (amd64) -pmstatus:dpkg-exec:60:Running dpkg -pmstatus:testing:60:Configuring testing (amd64) -pmstatus:testing:80:Configuring testing (amd64) -pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log +pmstatus:testing:16.6667:Preparing testing (amd64) +pmstatus:testing:33.3333:Unpacking testing (amd64) +pmstatus:testing:50:Preparing to configure testing (amd64) +pmstatus:dpkg-exec:50:Running dpkg +pmstatus:testing:50:Configuring testing (amd64) +pmstatus:testing:66.6667:Configuring testing (amd64) +pmstatus:testing:83.3333:Installed testing (amd64) +pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log # reinstall exec 3> apt-progress.log @@ -49,22 +51,24 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1 dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) -pmstatus:testing:20:Preparing testing (amd64) -pmstatus:testing:40:Unpacking testing (amd64) -pmstatus:testing:60:Preparing to configure testing (amd64) -pmstatus:dpkg-exec:60:Running dpkg -pmstatus:testing:60:Configuring testing (amd64) -pmstatus:testing:80:Configuring testing (amd64) -pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log +pmstatus:testing:16.6667:Preparing testing (amd64) +pmstatus:testing:33.3333:Unpacking testing (amd64) +pmstatus:testing:50:Preparing to configure testing (amd64) +pmstatus:dpkg-exec:50:Running dpkg +pmstatus:testing:50:Configuring testing (amd64) +pmstatus:testing:66.6667:Configuring testing (amd64) +pmstatus:testing:83.3333:Installed testing (amd64) +pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log # and remove exec 3> apt-progress.log testsuccess aptget remove testing -y -o APT::Status-Fd=3 testequal "pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Removing testing (amd64) -pmstatus:testing:33.3333:Preparing for removal of testing (amd64) -pmstatus:testing:66.6667:Removing testing (amd64) -pmstatus:testing:100:Removed testing (amd64)" cat apt-progress.log +pmstatus:testing:25:Preparing for removal of testing (amd64) +pmstatus:testing:50:Removing testing (amd64) +pmstatus:testing:75:Removed testing (amd64) +pmstatus:dpkg-exec:75:Running dpkg" cat apt-progress.log # install non-native and ensure we get proper progress info exec 3> apt-progress.log @@ -75,12 +79,13 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1 dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing2:0:Installing testing2 (i386) -pmstatus:testing2:20:Preparing testing2 (i386) -pmstatus:testing2:40:Unpacking testing2 (i386) -pmstatus:testing2:60:Preparing to configure testing2 (i386) -pmstatus:dpkg-exec:60:Running dpkg -pmstatus:testing2:60:Configuring testing2 (i386) -pmstatus:testing2:80:Configuring testing2 (i386) -pmstatus:testing2:100:Installed testing2 (i386)" cat apt-progress.log +pmstatus:testing2:16.6667:Preparing testing2 (i386) +pmstatus:testing2:33.3333:Unpacking testing2 (i386) +pmstatus:testing2:50:Preparing to configure testing2 (i386) +pmstatus:dpkg-exec:50:Running dpkg +pmstatus:testing2:50:Configuring testing2 (i386) +pmstatus:testing2:66.6667:Configuring testing2 (i386) +pmstatus:testing2:83.3333:Installed testing2 (i386) +pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log rm -f apt-progress*.log diff --git a/test/integration/test-apt-progress-fd-deb822 b/test/integration/test-apt-progress-fd-deb822 index 9d227942d..badc985e4 100755 --- a/test/integration/test-apt-progress-fd-deb822 +++ b/test/integration/test-apt-progress-fd-deb822 @@ -27,37 +27,41 @@ Message: Installing testing (amd64) Status: progress Package: testing:amd64 -Percent: 20 +Percent: 16.6667 Message: Preparing testing (amd64) Status: progress Package: testing:amd64 -Percent: 40 +Percent: 33.3333 Message: Unpacking testing (amd64) Status: progress Package: testing:amd64 -Percent: 60 +Percent: 50 Message: Preparing to configure testing (amd64) Status: progress -Percent: 60 +Percent: 50 Message: Running dpkg Status: progress Package: testing:amd64 -Percent: 60 +Percent: 50 Message: Configuring testing (amd64) Status: progress Package: testing:amd64 -Percent: 80 +Percent: 66.6667 Message: Configuring testing (amd64) Status: progress Package: testing:amd64 -Percent: 100 +Percent: 83.3333 Message: Installed testing (amd64) + +Status: progress +Percent: 83.3333 +Message: Running dpkg " cat apt-progress.log diff --git a/test/integration/test-apt-progress-fd-error b/test/integration/test-apt-progress-fd-error index a47095b9b..632300765 100755 --- a/test/integration/test-apt-progress-fd-error +++ b/test/integration/test-apt-progress-fd-error @@ -18,7 +18,7 @@ setupaptarchive exec 3> apt-progress.log testfailure aptget install foo1 foo2 -y -o APT::Status-Fd=3 msgtest "Ensure correct error message" -if grep -q "aptarchive/pool/foo2_0.8.15_amd64.deb:40:trying to overwrite '/usr/bin/file-conflict', which is also in package foo1 0.8.15" apt-progress.log; then +if grep -q "aptarchive/pool/foo2_0.8.15_amd64.deb:36.3636:trying to overwrite '/usr/bin/file-conflict', which is also in package foo1 0.8.15" apt-progress.log; then msgpass else cat apt-progress.log diff --git a/test/integration/test-bug-769609-triggers-still-pending-after-run b/test/integration/test-bug-769609-triggers-still-pending-after-run new file mode 100755 index 000000000..0588b793f --- /dev/null +++ b/test/integration/test-bug-769609-triggers-still-pending-after-run @@ -0,0 +1,78 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' + +msgtest 'Check if installed dpkg supports' 'noawait trigger' +if dpkg-checkbuilddeps -d 'dpkg (>= 1.16.1)' /dev/null; then + msgpass +else + msgskip 'dpkg version too old' + exit 0 +fi +configdpkgnoopchroot + +buildtriggerpackages() { + local TYPE="$1" + setupsimplenativepackage "triggerable-$TYPE" 'all' '1.0' 'unstable' "Depends: trigdepends-$TYPE" + BUILDDIR="incoming/triggerable-${TYPE}-1.0" + cat >${BUILDDIR}/debian/postinst < ${BUILDDIR}/debian/triggers + buildpackage "$BUILDDIR" 'unstable' 'main' 'native' + rm -rf "$BUILDDIR" + buildsimplenativepackage "trigdepends-$TYPE" 'all' '1.0' 'unstable' +} + +# FIXME: implement test with activate-style triggers +#buildtriggerpackages 'interest' +buildtriggerpackages 'interest-noawait' + +buildsimplenativepackage "trigstuff" 'all' '1.0' 'unstable' + + +setupaptarchive + +runtests() { + local TYPE="$1" + msgmsg 'Working with trigger type' "$TYPE" + testsuccess aptget install triggerable-$TYPE -y + cp rootdir/tmp/testsuccess.output terminal.output + testsuccess grep '^REWRITE ' terminal.output + testdpkginstalled triggerable-$TYPE trigdepends-$TYPE + + testsuccess aptget install trigdepends-$TYPE -y --reinstall + cp rootdir/tmp/testsuccess.output terminal.output + testsuccess grep '^REWRITE ' terminal.output + testsuccess grep ' root root ' terminal.output + testdpkginstalled triggerable-$TYPE trigdepends-$TYPE + + testsuccess aptget install trigstuff -y + cp rootdir/tmp/testsuccess.output terminal.output + testsuccess grep '^REWRITE ' terminal.output + testsuccess grep ' root root ' terminal.output + testdpkginstalled triggerable-$TYPE trigdepends-$TYPE trigstuff + + testsuccess aptget purge trigstuff -y + cp rootdir/tmp/testsuccess.output terminal.output + testsuccess grep '^REWRITE ' terminal.output + testsuccess grep ' root root ' terminal.output + testdpkginstalled triggerable-$TYPE trigdepends-$TYPE + testdpkgnotinstalled trigstuff + + testsuccess aptget purge trigdepends-$TYPE -y + cp rootdir/tmp/testsuccess.output terminal.output + testfailure grep '^REWRITE ' terminal.output + testfailure grep ' root root ' terminal.output + testdpkgnotinstalled triggerable-$TYPE trigdepends-$TYPE +} +#runtests 'interest' +runtests 'interest-noawait' diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts index 428db46ef..cde987bd6 100755 --- a/test/integration/test-no-fds-leaked-to-maintainer-scripts +++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts @@ -48,7 +48,8 @@ startup packages configure configure $PKGNAME 1.0 status unpacked $PKGNAME 1.0 status half-configured $PKGNAME 1.0 -status installed $PKGNAME 1.0" cut -f 3- -d' ' rootdir/var/log/dpkg.log +status installed $PKGNAME 1.0 +startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log testsuccess aptget purge -y fdleaks -qq @@ -76,4 +77,5 @@ status config-files $PKGNAME 1.0 status config-files $PKGNAME 1.0 status config-files $PKGNAME 1.0 status config-files $PKGNAME 1.0 -status not-installed $PKGNAME " cut -f 3- -d' ' rootdir/var/log/dpkg.log +status not-installed $PKGNAME +startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log -- cgit v1.2.3-70-g09d2 From ed793a19ec00b83254029509bc516e3ba911c75a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 29 Nov 2014 17:59:52 +0100 Subject: dispose http(s) 416 error page as non-content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real webservers (like apache) actually send an error page with a 416 response, but our client didn't expect it leaving the page on the socket to be parsed as response for the next request (http) or as file content (https), which isn't what we want at all… Symptom is a "Bad header line" as html usually doesn't parse that well to an http-header. This manifests itself e.g. if we have a complete file (or larger) in partial/ which isn't discarded by If-Range as the server doesn't support it (or it is just newer, think: mirror rotation). It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a, which removed the filesize - 1 trick, but this had its own problems… To properly test this our webserver gains the ability to reply with transfer-encoding: chunked as most real webservers will use it to send the dynamically generated error pages. Closes: 768797 --- cmdline/apt-helper.cc | 31 +++-- methods/http.cc | 2 + methods/https.cc | 15 ++- methods/server.cc | 26 ++-- methods/server.h | 5 +- test/integration/framework | 4 +- test/integration/test-partial-file-support | 62 ++++++++- test/interactive-helper/aptwebserver.cc | 193 ++++++++++++++++++----------- 8 files changed, 233 insertions(+), 105 deletions(-) (limited to 'test') diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index 1b832f165..a05ae90a2 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -51,22 +51,33 @@ static bool DoDownloadFile(CommandLine &CmdL) AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); pkgAcquire Fetcher(&Stat); - std::string download_uri = CmdL.FileList[1]; - std::string targetfile = CmdL.FileList[2]; - std::string hash; - if (CmdL.FileSize() > 3) - hash = CmdL.FileList[3]; - // we use download_uri as descr and targetfile as short-descr - new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, - "dest-dir-ignored", targetfile); + size_t fileind = 0; + std::vector targetfiles; + while (fileind + 2 <= CmdL.FileSize()) + { + std::string download_uri = CmdL.FileList[fileind + 1]; + std::string targetfile = CmdL.FileList[fileind + 2]; + std::string hash; + if (CmdL.FileSize() > fileind + 3) + hash = CmdL.FileList[fileind + 3]; + // we use download_uri as descr and targetfile as short-descr + new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, + "dest-dir-ignored", targetfile); + targetfiles.push_back(targetfile); + fileind += 3; + } // Disable drop-privs if "_apt" can not write to the target dir CheckDropPrivsMustBeDisabled(Fetcher); bool Failed = false; - if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true || - FileExists(targetfile) == false) + if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true) return _error->Error(_("Download Failed")); + if (targetfiles.empty() == false) + for (std::vector::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f) + if (FileExists(*f) == false) + return _error->Error(_("Download Failed")); + return true; } diff --git a/methods/http.cc b/methods/http.cc index a5de13511..ad1347d36 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -444,6 +444,8 @@ bool HttpServerState::RunData(FileFd * const File) loss of the connection means we are done */ if (Encoding == Closes) In.Limit(-1); + else if (JunkSize != 0) + In.Limit(JunkSize); else In.Limit(Size - StartPos); diff --git a/methods/https.cc b/methods/https.cc index 366148e19..23b3a10d4 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -69,6 +69,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { me->Server->Result = 200; me->Server->StartPos = me->Server->Size; + // the actual size is not important for https as curl will deal with it + // by itself and e.g. doesn't bother us with transport-encoding… + me->Server->JunkSize = std::numeric_limits::max(); } else me->Server->StartPos = 0; @@ -86,19 +89,25 @@ size_t HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) { HttpsMethod *me = (HttpsMethod *)userp; + size_t buffer_size = size * nmemb; + // we don't need to count the junk here, just drop anything we get as + // we don't always know how long it would be, e.g. in chunked encoding. + if (me->Server->JunkSize != 0) + return buffer_size; if (me->Res.Size == 0) me->URIStart(me->Res); - if(me->File->Write(buffer, size*nmemb) != true) + if(me->File->Write(buffer, buffer_size) != true) return false; if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) { me->SetFailReason("MaximumSizeExceeded"); - return _error->Error("Writing more data than expected (%llu > %llu)", + _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->MaximumSize); + return 0; } - return size*nmemb; + return buffer_size; } int diff --git a/methods/server.cc b/methods/server.cc index c4689ff12..9b3d39cf2 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, Minor = 0; Result = 0; Size = 0; + JunkSize = 0; StartPos = 0; Encoding = Closes; HaveContent = false; @@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line) Encoding = Stream; HaveContent = true; - // The length is already set from the Content-Range header - if (StartPos != 0) - return true; + unsigned long long * SizePtr = &Size; + if (Result == 416) + SizePtr = &JunkSize; - Size = strtoull(Val.c_str(), NULL, 10); - if (Size >= std::numeric_limits::max()) + *SizePtr = strtoull(Val.c_str(), NULL, 10); + if (*SizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (Size == 0) + else if (*SizePtr == 0) HaveContent = false; return true; } @@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line) // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) - { - StartPos = 1; // ignore Content-Length, it would override Size - HaveContent = false; - } + ; // we got the expected filesize which is all we wanted else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); if ((unsigned long long)StartPos > Size) @@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res) if ((unsigned long long)SBuf.st_size == Server->Size) { // the file is completely downloaded, but was not moved + if (Server->HaveContent == true) + { + // Send to error page to dev/null + FileFd DevNull("/dev/null",FileFd::WriteExists); + Server->RunData(&DevNull); + } + Server->HaveContent = false; Server->StartPos = Server->Size; Server->Result = 200; - Server->HaveContent = false; } else if (unlink(Queue->DestFile.c_str()) == 0) { diff --git a/methods/server.h b/methods/server.h index 7d5198478..b974ec89a 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,7 +34,8 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; + unsigned long long Size; // size of the usable content (aka: the file) + unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; time_t Date; bool HaveContent; @@ -73,7 +74,7 @@ struct ServerState RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; diff --git a/test/integration/framework b/test/integration/framework index 930ab9367..f7f69f5d5 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1119,8 +1119,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom } downloadfile() { - local PROTO="$(echo "$1" | cut -d':' -f 1 )" - apthelper -o Debug::Acquire::${PROTO}=1 \ + local PROTO="${1%%:*}" + apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \ download-file "$1" "$2" 2>&1 || true # only if the file exists the download was successful if [ -r "$2" ]; then diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support index 98b2f242a..b6b305d25 100755 --- a/test/integration/test-partial-file-support +++ b/test/integration/test-partial-file-support @@ -24,13 +24,25 @@ testdownloadfile() { else msgpass fi - cat "$DOWNLOADLOG" | while read field hash; do + sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log + testsuccess test -s receivedhashes.log + local HASHES_OK=0 + local HASHES_BAD=0 + while read field hash; do local EXPECTED case "$field" in 'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";; + 'Checksum-FileSize-Hash:') + #filesize is too weak to check for != + if [ "$4" = '=' ]; then + EXPECTED="$(stat -c '%s' "$TESTFILE")" + else + continue + fi + ;; *) continue;; esac if [ "$4" = '=' ]; then @@ -40,15 +52,41 @@ testdownloadfile() { fi if [ "$EXPECTED" "$4" "$hash" ]; then msgpass + HASHES_OK=$((HASHES_OK+1)); else - cat >&2 "$DOWNLOADLOG" msgfail "expected: $EXPECTED ; got: $hash" + HASHES_BAD=$((HASHES_BAD+1)); fi - done + done < receivedhashes.log + msgtest 'At least one good hash and no bad ones' + if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then + cat >&2 "$DOWNLOADLOG" + msgfail + else + msgpass + fi } TESTFILE='aptarchive/testfile' cp -a ${TESTDIR}/framework $TESTFILE +cp -a ${TESTDIR}/framework "${TESTFILE}2" + +followuprequest() { + local DOWN='./downloaded/testfile' + + copysource $TESTFILE 1M $DOWN + testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' + testwebserverlaststatuscode '416' "$DOWNLOADLOG" + + copysource $TESTFILE 1M $DOWN + copysource "${TESTFILE}2" 20 "${DOWN}2" + msgtest 'Testing download of files with' 'completely downloaded file + partial file' + testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \ + download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2" + testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output' + testsuccess diff -u "$TESTFILE" "${DOWN}" + testsuccess diff -u "${DOWN}" "${DOWN}2" +} testrun() { webserverconfig 'aptwebserver::support::range' 'true' @@ -66,9 +104,11 @@ testrun() { testdownloadfile 'invalid partial data' "${1}/testfile" "$DOWN" '!=' testwebserverlaststatuscode '206' "$DOWNLOADLOG" - copysource $TESTFILE 1M $DOWN - testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' - testwebserverlaststatuscode '416' "$DOWNLOADLOG" + webserverconfig 'aptwebserver::closeOnError' 'false' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'true' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'false' copysource /dev/zero 1M $DOWN testdownloadfile 'too-big partial file' "${1}/testfile" "$DOWN" '=' @@ -86,8 +126,18 @@ testrun() { testwebserverlaststatuscode '200' "$DOWNLOADLOG" } +msgmsg 'http: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'http://localhost:8080' +msgmsg 'http: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'http://localhost:8080' changetohttpswebserver +msgmsg 'https: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'https://localhost:4433' +msgmsg 'https: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'https://localhost:4433' diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 00004a524..ca6f88c58 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -19,6 +19,8 @@ #include #include #include + +#include #include #include #include @@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode) /*{{{*/ return NULL; } /*}}}*/ +static bool chunkedTransferEncoding(std::list const &headers) { + if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end()) + return true; + if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true) + return true; + return false; +} static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/ { - std::ostringstream contentlength; - contentlength << "Content-Length: " << data.FileSize(); - headers.push_back(contentlength.str()); - + if (chunkedTransferEncoding(headers) == false) + { + std::ostringstream contentlength; + contentlength << "Content-Length: " << data.FileSize(); + headers.push_back(contentlength.str()); + } std::string lastmodified("Last-Modified: "); lastmodified.append(TimeRFC1123(data.ModificationTime())); headers.push_back(lastmodified); @@ -92,9 +103,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/ /*}}}*/ static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/ { - std::ostringstream contentlength; - contentlength << "Content-Length: " << data.size(); - headers.push_back(contentlength.str()); + if (chunkedTransferEncoding(headers) == false) + { + std::ostringstream contentlength; + contentlength << "Content-Length: " << data.size(); + headers.push_back(contentlength.str()); + } } /*}}}*/ static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/ @@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl; bool Success = true; for (std::list::const_iterator h = headers.begin(); @@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list const &headers, FileFd &data)/*{{{*/ { bool Success = true; + bool const chunked = chunkedTransferEncoding(headers); char buffer[500]; unsigned long long actual = 0; while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true) { if (actual == 0) break; - Success &= FileFd::Write(client, buffer, actual); + + if (chunked == true) + { + std::string size; + strprintf(size, "%llX\r\n", actual); + Success &= FileFd::Write(client, size.c_str(), size.size()); + Success &= FileFd::Write(client, buffer, actual); + Success &= FileFd::Write(client, "\r\n", strlen("\r\n")); + } + else + Success &= FileFd::Write(client, buffer, actual); + } + if (chunked == true) + { + char const * const finish = "0\r\n\r\n"; + Success &= FileFd::Write(client, finish, strlen(finish)); } if (Success == false) - std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl; + std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl; return Success; } /*}}}*/ -static bool sendData(int const client, std::string const &data) /*{{{*/ +static bool sendData(int const client, std::list const &headers, std::string const &data)/*{{{*/ { - if (FileFd::Write(client, data.c_str(), data.size()) == false) + if (chunkedTransferEncoding(headers) == true) + { + unsigned long long const ullsize = data.length(); + std::string size; + strprintf(size, "%llX\r\n", ullsize); + char const * const finish = "\r\n0\r\n\r\n"; + if (FileFd::Write(client, size.c_str(), size.length()) == false || + FileFd::Write(client, data.c_str(), ullsize) == false || + FileFd::Write(client, finish, strlen(finish)) == false) + { + std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl; + return false; + } + } + else if (FileFd::Write(client, data.c_str(), data.size()) == false) { std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl; return false; @@ -157,33 +204,38 @@ static bool sendData(int const client, std::string const &data) /*{{{*/ } /*}}}*/ static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/ - bool content, std::string const &error = "", std::list headers = std::list()) + bool const content, std::string const &error, std::list &headers) { std::string response(""); response.append(httpcodeToStr(httpcode)).append(""); response.append("

").append(httpcodeToStr(httpcode)).append("

"); if (httpcode != 200) - { - if (error.empty() == false) - response.append("

Error: ").append(error).append("

"); - response.append("This error is a result of the request: 
");
-   }
+      response.append("
Error: ");
+   else
+      response.append("
Success: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("
This error is a result of the request: 
");
    else
-   {
-      if (error.empty() == false)
-	 response.append("
Success: ").append(error).append("
");
       response.append("The successfully executed operation was requested by: 
");
-   }
    response.append(request).append("
");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -220,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -262,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list &headers)
 {
-   std::list headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -310,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string ¶ms, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -332,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -344,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -359,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -370,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
 
@@ -382,9 +433,9 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       if (authConf.empty() != auth.empty())
       {
 	 if (auth.empty())
-	    sendError(client, 407, request, sendContent, "Proxy requires authentication");
+	    sendError(client, 407, request, sendContent, "Proxy requires authentication", headers);
 	 else
-	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it");
+	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it", headers);
 	return false;
       }
       if (authConf.empty() == false)
@@ -395,7 +446,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 	    auth.erase(0, strlen(basic));
 	    if (auth != authConf)
 	    {
-	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match");
+	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match", headers);
 	       return false;
 	    }
 	 }
@@ -410,7 +461,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos && APT::String::Startswith(filename, "/_config/") == false)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -421,9 +472,9 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       if (authConf.empty() != auth.empty())
       {
 	 if (auth.empty())
-	    sendError(client, 401, request, sendContent, "Server requires authentication");
+	    sendError(client, 401, request, sendContent, "Server requires authentication", headers);
 	 else
-	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it");
+	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it", headers);
 	 return false;
       }
       if (authConf.empty() == false)
@@ -434,13 +485,12 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 	    auth.erase(0, strlen(basic));
 	    if (auth != authConf)
 	    {
-	       sendError(client, 401, request, sendContent, "Authentication doesn't match");
+	       sendError(client, 401, request, sendContent, "Authentication doesn't match", headers);
 	       return false;
 	    }
 	 }
 	 else
 	 {
-	    std::list headers;
 	    headers.push_back("WWW-Authenticate: Basic");
 	    sendError(client, 401, request, sendContent, "Unsupported Authentication Scheme", headers);
 	    return false;
@@ -463,7 +513,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -499,7 +550,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector parts, std::list &headers)
 {
    size_t const pcount = parts.size();
    for (size_t i = 0; i < pcount; ++i)
@@ -507,40 +559,38 @@ static bool handleOnTheFlyReconfiguration(int const client, std::string const &r
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -549,18 +599,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -569,7 +623,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -601,7 +655,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -620,7 +674,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -676,17 +730,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -695,22 +748,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From c6bc9735cf1486d40d85bba90cfc3aaa6537a9c0 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Wed, 10 Dec 2014 22:26:59 +0100
Subject: do not make PTY slave the controlling terminal
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If we have no controlling terminal opening a terminal will make this
terminal our controller, which is a serious problem if this happens to
be the pseudo terminal we created to run dpkg in as we will close this
terminal at the end hanging ourself up in the process…

The offending open is the one we do to have at least one slave fd open
all the time, but for good measure, we apply the flag also to the slave
fd opening in the child process as we set the controlling terminal
explicitely here.

This is a regression from 150bdc9ca5d656f9fba94d37c5f4f183b02bd746 with
the slight twist that this usecase was silently broken before in that it
wasn't logging the output in term.log (as a pseudo terminal wasn't
created).

Closes: 772641
---
 apt-pkg/deb/dpkgpm.cc                              |  4 +-
 .../test-no-fds-leaked-to-maintainer-scripts       | 60 +++++++++++++---------
 2 files changed, 38 insertions(+), 26 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index 79120f6f5..8a8214c19 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -1127,7 +1127,7 @@ void pkgDPkgPM::StartPtyMagic()
 	       on kfreebsd we get an incorrect ("step like") output then while it has
 	       no problem with closing all references… so to avoid platform specific
 	       code here we combine both and be happy once more */
-	    d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC);
+	    d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC | O_NOCTTY);
 	 }
       }
    }
@@ -1159,7 +1159,7 @@ void pkgDPkgPM::SetupSlavePtyMagic()
    if (setsid() == -1)
       _error->FatalE("setsid", "Starting a new session for child failed!");
 
-   int const slaveFd = open(d->slave, O_RDWR);
+   int const slaveFd = open(d->slave, O_RDWR | O_NOCTTY);
    if (slaveFd == -1)
       _error->FatalE("open", _("Can not write log (%s)"), _("Is /dev/pts mounted?"));
    else if (ioctl(slaveFd, TIOCSCTTY, 0) < 0)
diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts
index cde987bd6..a7d556b87 100755
--- a/test/integration/test-no-fds-leaked-to-maintainer-scripts
+++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts
@@ -26,20 +26,25 @@ setupaptarchive
 
 rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
 testsuccess aptget install -y fdleaks -qq < /dev/null
-msgtest 'Check if fds were not' 'leaked'
-if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '8' ]; then
-	msgpass
-else
-	echo
-	cat rootdir/tmp/testsuccess.output
-	msgfail
-fi
 
-cp rootdir/tmp/testsuccess.output terminal.output
-tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
-testfileequal 'terminal.log' "$(cat terminal.output)"
+checkfdleak() {
+	msgtest 'Check if fds were not' 'leaked'
+	if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = "$1" ]; then
+		msgpass
+	else
+		echo
+		cat rootdir/tmp/testsuccess.output
+		msgfail
+	fi
+}
+checkinstall() {
+	checkfdleak 8
+
+	cp rootdir/tmp/testsuccess.output terminal.output
+	tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
+	testfileequal 'terminal.log' "$(cat terminal.output)"
 
-testequal "startup archives unpack
+	testequal "startup archives unpack
 install $PKGNAME  1.0
 status half-installed $PKGNAME 1.0
 status unpacked $PKGNAME 1.0
@@ -50,22 +55,19 @@ status unpacked $PKGNAME 1.0
 status half-configured $PKGNAME 1.0
 status installed $PKGNAME 1.0
 startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log
+}
+checkinstall
 
 rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
 testsuccess aptget purge -y fdleaks -qq
-msgtest 'Check if fds were not' 'leaked'
-if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '12' ]; then
-	msgpass
-else
-	echo
-	cat rootdir/tmp/testsuccess.output
-	msgfail
-fi
-cp rootdir/tmp/testsuccess.output terminal.output
-tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
-testfileequal 'terminal.log' "$(cat terminal.output)"
+checkpurge() {
+	checkfdleak 12
+
+	cp rootdir/tmp/testsuccess.output terminal.output
+	tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
+	testfileequal 'terminal.log' "$(cat terminal.output)"
 
-testequal "startup packages purge
+	testequal "startup packages purge
 status installed $PKGNAME 1.0
 remove $PKGNAME 1.0 
 status half-configured $PKGNAME 1.0
@@ -79,3 +81,13 @@ status config-files $PKGNAME 1.0
 status config-files $PKGNAME 1.0
 status not-installed $PKGNAME 
 startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log
+}
+checkpurge
+
+rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
+testsuccess runapt command setsid -w "${BUILDDIRECTORY}/apt-get" install -y fdleaks -qq < /dev/null
+checkinstall
+
+rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
+testsuccess runapt command setsid -w "${BUILDDIRECTORY}/apt-get" purge -y fdleaks -qq
+checkpurge
-- 
cgit v1.2.3-70-g09d2


From 92e8c1ff287ab829de825e00cdf94744e699ff97 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 29 Nov 2014 17:59:52 +0100
Subject: dispose http(s) 416 error page as non-content
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Real webservers (like apache) actually send an error page with a 416
response, but our client didn't expect it leaving the page on the socket
to be parsed as response for the next request (http) or as file content
(https), which isn't what we want at all… Symptom is a "Bad header line"
as html usually doesn't parse that well to an http-header.

This manifests itself e.g. if we have a complete file (or larger) in
partial/ which isn't discarded by If-Range as the server doesn't support
it (or it is just newer, think: mirror rotation).
It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a,
which removed the filesize - 1 trick, but this had its own problems…

To properly test this our webserver gains the ability to reply with
transfer-encoding: chunked as most real webservers will use it to send
the dynamically generated error pages.

(The tests and their binary helpers had to be slightly modified to
apply, but the patch to fix the issue itself is unchanged.)

Closes: 768797
---
 cmdline/apt-helper.cc                      |  35 ++++--
 methods/http.cc                            |   2 +
 methods/https.cc                           |  12 +-
 methods/server.cc                          |  26 +++--
 methods/server.h                           |   5 +-
 test/integration/framework                 |   6 +-
 test/integration/test-apt-helper           |  24 ++--
 test/integration/test-partial-file-support |  62 +++++++++-
 test/interactive-helper/aptwebserver.cc    | 181 ++++++++++++++++++-----------
 9 files changed, 241 insertions(+), 112 deletions(-)

(limited to 'test')

diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index dd43ea1bc..63f70983c 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,23 +48,34 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
-
    pkgAcquire Fetcher;
    AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
    Fetcher.Setup(&Stat);
-   std::string download_uri = CmdL.FileList[1];
-   std::string targetfile = CmdL.FileList[2];
-   std::string hash;
-   if (CmdL.FileSize() > 3)
-      hash = CmdL.FileList[3];
-   // we use download_uri as descr and targetfile as short-descr
-   new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, 
-                  "dest-dir-ignored", targetfile);
-   Fetcher.Run();
+
+   size_t fileind = 0;
+   std::vector targetfiles;
+   while (fileind + 2 <= CmdL.FileSize())
+   {
+      std::string download_uri = CmdL.FileList[fileind + 1];
+      std::string targetfile = CmdL.FileList[fileind + 2];
+      std::string hash;
+      if (CmdL.FileSize() > fileind + 3)
+	 hash = CmdL.FileList[fileind + 3];
+      // we use download_uri as descr and targetfile as short-descr
+      new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile,
+	    "dest-dir-ignored", targetfile);
+      targetfiles.push_back(targetfile);
+      fileind += 3;
+   }
+
    bool Failed = false;
-   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true ||
-	 FileExists(targetfile) == false)
+   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true)
       return _error->Error(_("Download Failed"));
+   if (targetfiles.empty() == false)
+      for (std::vector::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f)
+	 if (FileExists(*f) == false)
+	    return _error->Error(_("Download Failed"));
+
    return true;
 }
 
diff --git a/methods/http.cc b/methods/http.cc
index f2a4a4db6..1b996db98 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -440,6 +440,8 @@ bool HttpServerState::RunData(FileFd * const File)
          loss of the connection means we are done */
       if (Encoding == Closes)
 	 In.Limit(-1);
+      else if (JunkSize != 0)
+	 In.Limit(JunkSize);
       else
 	 In.Limit(Size - StartPos);
       
diff --git a/methods/https.cc b/methods/https.cc
index 0499af0c5..65a744e2a 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -59,6 +59,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       {
          me->Server->Result = 200;
 	 me->Server->StartPos = me->Server->Size;
+	 // the actual size is not important for https as curl will deal with it
+	 // by itself and e.g. doesn't bother us with transport-encoding…
+	 me->Server->JunkSize = std::numeric_limits::max();
       }
       else
 	 me->Server->StartPos = 0;
@@ -76,13 +79,18 @@ size_t
 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    HttpsMethod *me = (HttpsMethod *)userp;
+   size_t buffer_size = size * nmemb;
+   // we don't need to count the junk here, just drop anything we get as
+   // we don't always know how long it would be, e.g. in chunked encoding.
+   if (me->Server->JunkSize != 0)
+      return buffer_size;
 
    if (me->Res.Size == 0)
       me->URIStart(me->Res);
-   if(me->File->Write(buffer, size*nmemb) != true)
+   if(me->File->Write(buffer, buffer_size) != true)
       return false;
 
-   return size*nmemb;
+   return buffer_size;
 }
 
 int
diff --git a/methods/server.cc b/methods/server.cc
index 92d94e638..cb0341d5f 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
    Minor = 0; 
    Result = 0; 
    Size = 0; 
+   JunkSize = 0;
    StartPos = 0;
    Encoding = Closes;
    HaveContent = false;
@@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line)
 	 Encoding = Stream;
       HaveContent = true;
 
-      // The length is already set from the Content-Range header
-      if (StartPos != 0)
-	 return true;
+      unsigned long long * SizePtr = &Size;
+      if (Result == 416)
+	 SizePtr = &JunkSize;
 
-      Size = strtoull(Val.c_str(), NULL, 10);
-      if (Size >= std::numeric_limits::max())
+      *SizePtr = strtoull(Val.c_str(), NULL, 10);
+      if (*SizePtr >= std::numeric_limits::max())
 	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
-      else if (Size == 0)
+      else if (*SizePtr == 0)
 	 HaveContent = false;
       return true;
    }
@@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line)
 
       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
-      {
-	 StartPos = 1; // ignore Content-Length, it would override Size
-	 HaveContent = false;
-      }
+	 ; // we got the expected filesize which is all we wanted
       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
@@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
 	 if ((unsigned long long)SBuf.st_size == Server->Size)
 	 {
 	    // the file is completely downloaded, but was not moved
+	    if (Server->HaveContent == true)
+	    {
+	       // Send to error page to dev/null
+	       FileFd DevNull("/dev/null",FileFd::WriteExists);
+	       Server->RunData(&DevNull);
+	    }
+	    Server->HaveContent = false;
 	    Server->StartPos = Server->Size;
 	    Server->Result = 200;
-	    Server->HaveContent = false;
 	 }
 	 else if (unlink(Queue->DestFile.c_str()) == 0)
 	 {
diff --git a/methods/server.h b/methods/server.h
index f5e68d902..1b81e3549 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,7 +34,8 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
-   unsigned long long Size;
+   unsigned long long Size; // size of the usable content (aka: the file)
+   unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
    time_t Date;
    bool HaveContent;
@@ -71,7 +72,7 @@ struct ServerState
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
-   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
+   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
 		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true;};
    virtual bool WriteResponse(std::string const &Data) = 0;
diff --git a/test/integration/framework b/test/integration/framework
index df1942ff9..ac482a7a0 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1064,8 +1064,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 }
 
 downloadfile() {
-	local PROTO="$(echo "$1" | cut -d':' -f 1 )"
-	apthelper -o Debug::Acquire::${PROTO}=1 \
+	local PROTO="${1%%:*}"
+	apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
 		download-file "$1" "$2" 2>&1 || true
 	# only if the file exists the download was successful
 	if [ -e "$2" ]; then
@@ -1221,7 +1221,7 @@ testsuccess() {
 		msgtest 'Test for successful execution of' "$*"
 	fi
 	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
-	if $@ >${OUTPUT} 2>&1; then
+	if "$@" >${OUTPUT} 2>&1; then
 		msgpass
 	else
 		echo >&2
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index c749224ca..31e471677 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -10,34 +10,36 @@ configarchitecture "i386"
 changetohttpswebserver
 
 test_apt_helper_download() {
-    echo "foo" > aptarchive/foo
+    echo 'foo' > aptarchive/foo
+    echo 'bar' > aptarchive/foo2
 
     msgtest 'apt-file download-file md5sum'
-    apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00
     testfileequal foo2 'foo'
 
     msgtest 'apt-file download-file sha1'
-    apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
     testfileequal foo1 'foo'
 
     msgtest 'apt-file download-file sha256'
-    apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c
     testfileequal foo3 'foo'
 
     msgtest 'apt-file download-file no-hash'
-    apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo4
     testfileequal foo4 'foo'
     
     msgtest 'apt-file download-file wrong hash'
-    if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then
-        msgpass
-    else
-        msgfail
-    fi
-    testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
+    testfailure --nomsg apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc
+    testfileequal rootdir/tmp/testfailure.output 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
 
 E: Download Failed'
     testfileequal foo5.FAILED 'foo'
+
+    msgtest 'apt-file download-file md5sum sha1'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo6 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 http://localhost:8080/foo2 foo7 SHA1:e242ed3bffccdf271b7fbaf34ed72d089537b42f
+    testfileequal foo6 'foo'
+    testfileequal foo7 'bar'
 }
 
 test_apt_helper_detect_proxy() {
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 5ab326def..160d451b6 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -24,13 +24,25 @@ testdownloadfile() {
 	else
 		msgpass
 	fi
-	cat "$DOWNLOADLOG" | while read field hash; do
+	sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log
+	testsuccess test -s receivedhashes.log
+	local HASHES_OK=0
+	local HASHES_BAD=0
+	while read field hash; do
 		local EXPECTED
 		case "$field" in
 		'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";;
+		'Checksum-FileSize-Hash:')
+			#filesize is too weak to check for !=
+			if [ "$4" = '=' ]; then
+				EXPECTED="$(stat -c '%s' "$TESTFILE")"
+			else
+				continue
+			fi
+			;;
 		*) continue;;
 		esac
 		if [ "$4" = '=' ]; then
@@ -40,15 +52,41 @@ testdownloadfile() {
 		fi
 		if [ "$EXPECTED" "$4" "$hash" ]; then
 			msgpass
+			HASHES_OK=$((HASHES_OK+1));
 		else
-			cat >&2 "$DOWNLOADLOG"
 			msgfail "expected: $EXPECTED ; got: $hash"
+			HASHES_BAD=$((HASHES_BAD+1));
 		fi
-	done
+	done < receivedhashes.log
+	msgtest 'At least one good hash and no bad ones'
+	if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then
+		cat >&2 "$DOWNLOADLOG"
+		msgfail
+	else
+		msgpass
+	fi
 }
 
 TESTFILE='aptarchive/testfile'
 cp -a ${TESTDIR}/framework $TESTFILE
+cp -a ${TESTDIR}/framework "${TESTFILE}2"
+
+followuprequest() {
+	local DOWN='./testfile'
+
+	copysource $TESTFILE 1M $DOWN
+	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
+	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+
+	copysource $TESTFILE 1M $DOWN
+	copysource "${TESTFILE}2" 20 "${DOWN}2"
+	msgtest 'Testing download of files with' 'completely downloaded file + partial file'
+	testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \
+		download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2"
+	testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output'
+	testsuccess diff -u "$TESTFILE" "${DOWN}"
+	testsuccess diff -u "${DOWN}" "${DOWN}2"
+}
 
 testrun() {
 	webserverconfig 'aptwebserver::support::range' 'true'
@@ -65,9 +103,11 @@ testrun() {
 	testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
 	testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 
-	copysource $TESTFILE 1M ./testfile
-	testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '='
-	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'true'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
 
 	copysource /dev/zero 1M ./testfile
 	testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='
@@ -85,8 +125,18 @@ testrun() {
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 
+msgmsg 'http: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'http://localhost:8080'
+msgmsg 'http: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'http://localhost:8080'
 
 changetohttpswebserver
 
+msgmsg 'https: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'https://localhost:4433'
+msgmsg 'https: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'https://localhost:4433'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 34476e1af..cd52da692 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -19,6 +19,8 @@
 #include 
 #include 
 #include 
+
+#include 
 #include 
 #include 
 #include 
@@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode)		/*{{{*/
    return NULL;
 }
 									/*}}}*/
+static bool chunkedTransferEncoding(std::list const &headers) {
+   if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end())
+      return true;
+   if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true)
+      return true;
+   return false;
+}
 static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.FileSize();
-   headers.push_back(contentlength.str());
-
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.FileSize();
+      headers.push_back(contentlength.str());
+   }
    std::string lastmodified("Last-Modified: ");
    lastmodified.append(TimeRFC1123(data.ModificationTime()));
    headers.push_back(lastmodified);
@@ -92,9 +103,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
 									/*}}}*/
 static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.size();
-   headers.push_back(contentlength.str());
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.size();
+      headers.push_back(contentlength.str());
+   }
 }
 									/*}}}*/
 static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/
@@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl;
    bool Success = true;
    for (std::list::const_iterator h = headers.begin();
@@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list const &headers, FileFd &data)/*{{{*/
 {
    bool Success = true;
+   bool const chunked = chunkedTransferEncoding(headers);
    char buffer[500];
    unsigned long long actual = 0;
    while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true)
    {
       if (actual == 0)
 	 break;
-      Success &= FileFd::Write(client, buffer, actual);
+
+      if (chunked == true)
+      {
+	 std::string size;
+	 strprintf(size, "%llX\r\n", actual);
+	 Success &= FileFd::Write(client, size.c_str(), size.size());
+	 Success &= FileFd::Write(client, buffer, actual);
+	 Success &= FileFd::Write(client, "\r\n", strlen("\r\n"));
+      }
+      else
+	 Success &= FileFd::Write(client, buffer, actual);
+   }
+   if (chunked == true)
+   {
+      char const * const finish = "0\r\n\r\n";
+      Success &= FileFd::Write(client, finish, strlen(finish));
    }
    if (Success == false)
-      std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl;
+      std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl;
    return Success;
 }
 									/*}}}*/
-static bool sendData(int const client, std::string const &data)		/*{{{*/
+static bool sendData(int const client, std::list const &headers, std::string const &data)/*{{{*/
 {
-   if (FileFd::Write(client, data.c_str(), data.size()) == false)
+   if (chunkedTransferEncoding(headers) == true)
+   {
+      unsigned long long const ullsize = data.length();
+      std::string size;
+      strprintf(size, "%llX\r\n", ullsize);
+      char const * const finish = "\r\n0\r\n\r\n";
+      if (FileFd::Write(client, size.c_str(), size.length()) == false ||
+	    FileFd::Write(client, data.c_str(), ullsize) == false ||
+	    FileFd::Write(client, finish, strlen(finish)) == false)
+      {
+	 std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl;
+	 return false;
+      }
+   }
+   else if (FileFd::Write(client, data.c_str(), data.size()) == false)
    {
       std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl;
       return false;
@@ -157,34 +204,38 @@ static bool sendData(int const client, std::string const &data)		/*{{{*/
 }
 									/*}}}*/
 static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   std::list headers;
    std::string response("");
    response.append(httpcodeToStr(httpcode)).append("");
    response.append("
").append(httpcodeToStr(httpcode)).append("
");
    if (httpcode != 200)
-   {
-      if (error.empty() == false)
-	 response.append("
Error: ").append(error).append("
");
-      response.append("This error is a result of the request: 
");
-   }
+      response.append("
Error: ");
+   else
+      response.append("
Success: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("
This error is a result of the request: 
");
    else
-   {
-      if (error.empty() == false)
-	 response.append("
Success: ").append(error).append("
");
       response.append("The successfully executed operation was requested by: 
");
-   }
    response.append(request).append("
");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -221,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -263,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list &headers)
 {
-   std::list headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -311,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string ¶ms, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -333,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -345,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -360,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -371,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
       // strip the host from the request to make it an absolute path
@@ -379,7 +429,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -398,7 +448,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -434,46 +485,45 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector const &parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector parts, std::list &headers)
 {
    size_t const pcount = parts.size();
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -482,18 +532,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -502,7 +556,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -534,7 +588,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -553,7 +607,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -609,17 +663,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -628,22 +681,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From e18f6133b254db9e1dc7b202366b067b15a68123 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Wed, 10 Dec 2014 22:26:59 +0100
Subject: do not make PTY slave the controlling terminal
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If we have no controlling terminal opening a terminal will make this
terminal our controller, which is a serious problem if this happens to
be the pseudo terminal we created to run dpkg in as we will close this
terminal at the end hanging ourself up in the process…

The offending open is the one we do to have at least one slave fd open
all the time, but for good measure, we apply the flag also to the slave
fd opening in the child process as we set the controlling terminal
explicitely here.

This is a regression from 150bdc9ca5d656f9fba94d37c5f4f183b02bd746 with
the slight twist that this usecase was silently broken before in that it
wasn't logging the output in term.log (as a pseudo terminal wasn't
created).

Closes: 772641
---
 apt-pkg/deb/dpkgpm.cc                              |   4 +-
 test/integration/framework                         |   2 +-
 .../test-no-fds-leaked-to-maintainer-scripts       | 109 +++++++++++++--------
 3 files changed, 70 insertions(+), 45 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index e36a52c3e..93a007d14 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -1131,7 +1131,7 @@ void pkgDPkgPM::StartPtyMagic()
 	       on kfreebsd we get an incorrect ("step like") output then while it has
 	       no problem with closing all references… so to avoid platform specific
 	       code here we combine both and be happy once more */
-	    d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC);
+	    d->protect_slave_from_dying = open(d->slave, O_RDWR | O_CLOEXEC | O_NOCTTY);
 	 }
       }
    }
@@ -1163,7 +1163,7 @@ void pkgDPkgPM::SetupSlavePtyMagic()
    if (setsid() == -1)
       _error->FatalE("setsid", "Starting a new session for child failed!");
 
-   int const slaveFd = open(d->slave, O_RDWR);
+   int const slaveFd = open(d->slave, O_RDWR | O_NOCTTY);
    if (slaveFd == -1)
       _error->FatalE("open", _("Can not write log (%s)"), _("Is /dev/pts mounted?"));
    else if (ioctl(slaveFd, TIOCSCTTY, 0) < 0)
diff --git a/test/integration/framework b/test/integration/framework
index ac482a7a0..9e183057f 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -102,7 +102,7 @@ runapt() {
 	local CMD="$1"
 	shift
 	case $CMD in
-	sh|aptitude|*/*) ;;
+	sh|aptitude|*/*|command) ;;
 	*) CMD="${BUILDDIRECTORY}/$CMD";;
 	esac
 	MALLOC_PERTURB_=21 MALLOC_CHECK_=2 APT_CONFIG="$(getaptconfig)" LD_LIBRARY_PATH=${BUILDDIRECTORY} $CMD "$@"
diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts
index 3c6457cab..6eb033055 100755
--- a/test/integration/test-no-fds-leaked-to-maintainer-scripts
+++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts
@@ -5,7 +5,7 @@ TESTDIR=$(readlink -f $(dirname $0))
 . $TESTDIR/framework
 
 setupenvironment
-configarchitecture 'native'
+configarchitecture 'amd64' 'i386'
 configdpkgnoopchroot
 
 setupsimplenativepackage "fdleaks" 'all' '1.0' 'unstable'
@@ -17,58 +17,83 @@ done
 buildpackage "$BUILDDIR" 'unstable' 'main' 'native'
 rm -rf "$BUILDDIR"
 
+PKGNAME='fdleaks:all'
+if ! dpkg-checkbuilddeps -d 'dpkg (>= 1.16.2)' /dev/null >/dev/null 2>&1; then
+	PKGNAME='fdleaks'
+fi
+
 setupaptarchive
 
 rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
 testsuccess aptget install -y fdleaks -qq < /dev/null
-msgtest 'Check if fds were not' 'leaked'
-if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '8' ]; then
-	msgpass
-else
-	echo
-	cat rootdir/tmp/testsuccess.output
-	msgfail
-fi
 
-cp rootdir/tmp/testsuccess.output terminal.output
-tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
-testfileequal 'terminal.log' "$(cat terminal.output)"
+checkfdleak() {
+	msgtest 'Check if fds were not' 'leaked'
+	if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = "$1" ]; then
+		msgpass
+	else
+		echo
+		cat rootdir/tmp/testsuccess.output
+		msgfail
+	fi
+}
+checkinstall() {
+	checkfdleak 8
+
+	cp rootdir/tmp/testsuccess.output terminal.output
+	tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
+	testfileequal 'terminal.log' "$(cat terminal.output)"
 
-testequal 'startup archives unpack
-install fdleaks:all  1.0
-status half-installed fdleaks:all 1.0
-status unpacked fdleaks:all 1.0
-status unpacked fdleaks:all 1.0
+	testequal "startup archives unpack
+install $PKGNAME  1.0
+status half-installed $PKGNAME 1.0
+status unpacked $PKGNAME 1.0
+status unpacked $PKGNAME 1.0
 startup packages configure
-configure fdleaks:all 1.0 
-status unpacked fdleaks:all 1.0
-status half-configured fdleaks:all 1.0
-status installed fdleaks:all 1.0' cut -f 3- -d' ' rootdir/var/log/dpkg.log
+configure $PKGNAME 1.0 
+status unpacked $PKGNAME 1.0
+status half-configured $PKGNAME 1.0
+status installed $PKGNAME 1.0" cut -f 3- -d' ' rootdir/var/log/dpkg.log
+}
+checkinstall
 
 rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
 testsuccess aptget purge -y fdleaks -qq
-msgtest 'Check if fds were not' 'leaked'
-if [ "$(grep 'root root' rootdir/tmp/testsuccess.output | wc -l)" = '12' ]; then
+checkpurge() {
+	checkfdleak 12
+
+	cp rootdir/tmp/testsuccess.output terminal.output
+	tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
+	testfileequal 'terminal.log' "$(cat terminal.output)"
+
+	testequal "startup packages purge
+status installed $PKGNAME 1.0
+remove $PKGNAME 1.0 
+status half-configured $PKGNAME 1.0
+status half-installed $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+purge $PKGNAME 1.0 
+status config-files $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+status not-installed $PKGNAME " cut -f 3- -d' ' rootdir/var/log/dpkg.log
+}
+checkpurge
+
+msgtest 'setsid provided is new enough to support' '-w'
+if dpkg-checkbuilddeps -d 'util-linux (>= 2.24.2-1)' /dev/null >/dev/null 2>&1; then
 	msgpass
 else
-	echo
-	cat rootdir/tmp/testsuccess.output
-	msgfail
+	msgskip "$(command dpkg -l util-linux)"
+	exit
 fi
-cp rootdir/tmp/testsuccess.output terminal.output
-tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
-testfileequal 'terminal.log' "$(cat terminal.output)"
 
-testequal 'startup packages purge
-status installed fdleaks:all 1.0
-remove fdleaks:all 1.0 
-status half-configured fdleaks:all 1.0
-status half-installed fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-purge fdleaks:all 1.0 
-status config-files fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-status config-files fdleaks:all 1.0
-status not-installed fdleaks:all ' cut -f 3- -d' ' rootdir/var/log/dpkg.log
+rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
+testsuccess runapt command setsid -w "${BUILDDIRECTORY}/apt-get" install -y fdleaks -qq < /dev/null
+checkinstall
+
+rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
+testsuccess runapt command setsid -w "${BUILDDIRECTORY}/apt-get" purge -y fdleaks -qq
+checkpurge
-- 
cgit v1.2.3-70-g09d2


From a2a75ff4516f7609f4c55b42270abb8d08943c60 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 18 Nov 2014 19:53:56 +0100
Subject: always run 'dpkg --configure -a' at the end of our dpkg callings

dpkg checks now for dependencies before running triggers, so that
packages can now end up in trigger states (especially those we are not
touching at all with our calls) after apt is done running.

The solution to this is trivial: Just tell dpkg to configure everything
after we have (supposely) configured everything already. In the worst
case this means dpkg will have to run a bunch of triggers, usually it
will just do nothing though.

The code to make this happen was already available, so we just flip a
config option here to cause it to be run. This way we can keep
pretending that triggers are an implementation detail of dpkg.
--triggers-only would supposely work as well, but --configure is more
robust in regards to future changes to dpkg and something we will
hopefully make use of in future versions anyway (as it was planed at the
time this and related options were implemented).

Note that dpkg currently has a workaround implemented to allow upgrades
to jessie to be clean, so that the test works before and after. Also
note that test (compared to the one in the bug) drops the await test as
its is considered a loop by dpkg now.

Closes: 769609
---
 apt-pkg/deb/dpkgpm.cc                              |  9 ++-
 test/integration/framework                         | 25 ++++----
 test/integration/test-apt-progress-fd              | 67 ++++++++++---------
 test/integration/test-apt-progress-fd-deb822       | 18 ++++--
 test/integration/test-apt-progress-fd-error        |  2 +-
 ...est-bug-769609-triggers-still-pending-after-run | 75 ++++++++++++++++++++++
 .../test-no-fds-leaked-to-maintainer-scripts       |  6 +-
 7 files changed, 146 insertions(+), 56 deletions(-)
 create mode 100755 test/integration/test-bug-769609-triggers-still-pending-after-run

(limited to 'test')

diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index 93a007d14..d54b7b50f 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -1047,6 +1047,12 @@ void pkgDPkgPM::BuildPackagesProgressMap()
 	 PackagesTotal++;
       }
    }
+   /* one extra: We don't want the progress bar to reach 100%, especially not
+      if we call dpkg --configure --pending and process a bunch of triggers
+      while showing 100%. Also, spindown takes a while, so never reaching 100%
+      is way more correct than reaching 100% while still doing stuff even if
+      doing it this way is slightly bending the rules */
+   ++PackagesTotal;
 }
                                                                         /*}}}*/
 #if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR < 13)
@@ -1280,9 +1286,8 @@ bool pkgDPkgPM::GoNoABIBreak(APT::Progress::PackageManager *progress)
 
    // support subpressing of triggers processing for special
    // cases like d-i that runs the triggers handling manually
-   bool const SmartConf = (_config->Find("PackageManager::Configure", "all") != "all");
    bool const TriggersPending = _config->FindB("DPkg::TriggersPending", false);
-   if (_config->FindB("DPkg::ConfigurePending", SmartConf) == true)
+   if (_config->FindB("DPkg::ConfigurePending", true) == true)
       List.push_back(Item(Item::ConfigurePending, PkgIterator()));
 
    // for the progress
diff --git a/test/integration/framework b/test/integration/framework
index 9e183057f..c9445065b 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1178,10 +1178,13 @@ testnopackage() {
 	fi
 }
 
-testdpkginstalled() {
-	msgtest "Test for correctly installed package(s) with" "dpkg -l $*"
-	local PKGS="$(dpkg -l "$@" 2>/dev/null | grep '^i' | wc -l)"
-	if [ "$PKGS" != $# ]; then
+testdpkgstatus() {
+	local STATE="$1"
+	local NR="$2"
+	shift 2
+	msgtest "Test that $NR package(s) are in state $STATE with" "dpkg -l $*"
+	local PKGS="$(dpkg -l "$@" 2>/dev/null | grep "^${STATE}" | wc -l)"
+	if [ "$PKGS" != $NR ]; then
 		echo >&2 $PKGS
 		dpkg -l "$@" | grep '^[a-z]' >&2
 		msgfail
@@ -1190,16 +1193,12 @@ testdpkginstalled() {
 	fi
 }
 
+testdpkginstalled() {
+	testdpkgstatus 'ii' "$#" "$@"
+}
+
 testdpkgnotinstalled() {
-	msgtest "Test for correctly not-installed package(s) with" "dpkg -l $*"
-	local PKGS="$(dpkg -l "$@" 2> /dev/null | grep '^i' | wc -l)"
-	if [ "$PKGS" != 0 ]; then
-		echo
-		dpkg -l "$@" | grep '^[a-z]' >&2
-		msgfail
-	else
-		msgpass
-	fi
+	testdpkgstatus 'ii' '0' "$@"
 }
 
 testmarkedauto() {
diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd
index d72e7e72d..68cc0439c 100755
--- a/test/integration/test-apt-progress-fd
+++ b/test/integration/test-apt-progress-fd
@@ -19,13 +19,14 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:0:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
-pmstatus:testing:20:Preparing testing (amd64)
-pmstatus:testing:40:Unpacking testing (amd64)
-pmstatus:testing:60:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:60:Running dpkg
-pmstatus:testing:60:Configuring testing (amd64)
-pmstatus:testing:80:Configuring testing (amd64)
-pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log
+pmstatus:testing:16.6667:Preparing testing (amd64)
+pmstatus:testing:33.3333:Unpacking testing (amd64)
+pmstatus:testing:50:Preparing to configure testing (amd64)
+pmstatus:dpkg-exec:50:Running dpkg
+pmstatus:testing:50:Configuring testing (amd64)
+pmstatus:testing:66.6667:Configuring testing (amd64)
+pmstatus:testing:83.3333:Installed testing (amd64)
+pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
 
 # upgrade
 exec 3> apt-progress.log
@@ -34,13 +35,14 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:0:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
-pmstatus:testing:20:Preparing testing (amd64)
-pmstatus:testing:40:Unpacking testing (amd64)
-pmstatus:testing:60:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:60:Running dpkg
-pmstatus:testing:60:Configuring testing (amd64)
-pmstatus:testing:80:Configuring testing (amd64)
-pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log
+pmstatus:testing:16.6667:Preparing testing (amd64)
+pmstatus:testing:33.3333:Unpacking testing (amd64)
+pmstatus:testing:50:Preparing to configure testing (amd64)
+pmstatus:dpkg-exec:50:Running dpkg
+pmstatus:testing:50:Configuring testing (amd64)
+pmstatus:testing:66.6667:Configuring testing (amd64)
+pmstatus:testing:83.3333:Installed testing (amd64)
+pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
 
 # reinstall
 exec 3> apt-progress.log
@@ -49,22 +51,24 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:0:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
-pmstatus:testing:20:Preparing testing (amd64)
-pmstatus:testing:40:Unpacking testing (amd64)
-pmstatus:testing:60:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:60:Running dpkg
-pmstatus:testing:60:Configuring testing (amd64)
-pmstatus:testing:80:Configuring testing (amd64)
-pmstatus:testing:100:Installed testing (amd64)" cat apt-progress.log
+pmstatus:testing:16.6667:Preparing testing (amd64)
+pmstatus:testing:33.3333:Unpacking testing (amd64)
+pmstatus:testing:50:Preparing to configure testing (amd64)
+pmstatus:dpkg-exec:50:Running dpkg
+pmstatus:testing:50:Configuring testing (amd64)
+pmstatus:testing:66.6667:Configuring testing (amd64)
+pmstatus:testing:83.3333:Installed testing (amd64)
+pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
 
 # and remove
 exec 3> apt-progress.log
 testsuccess aptget remove testing -y -o APT::Status-Fd=3
 testequal "pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Removing testing (amd64)
-pmstatus:testing:33.3333:Preparing for removal of testing (amd64)
-pmstatus:testing:66.6667:Removing testing (amd64)
-pmstatus:testing:100:Removed testing (amd64)" cat apt-progress.log
+pmstatus:testing:25:Preparing for removal of testing (amd64)
+pmstatus:testing:50:Removing testing (amd64)
+pmstatus:testing:75:Removed testing (amd64)
+pmstatus:dpkg-exec:75:Running dpkg" cat apt-progress.log
 
 # install non-native and ensure we get proper progress info
 exec 3> apt-progress.log
@@ -75,12 +79,13 @@ testequal "dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:0:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)
-pmstatus:testing2:20:Preparing testing2 (i386)
-pmstatus:testing2:40:Unpacking testing2 (i386)
-pmstatus:testing2:60:Preparing to configure testing2 (i386)
-pmstatus:dpkg-exec:60:Running dpkg
-pmstatus:testing2:60:Configuring testing2 (i386)
-pmstatus:testing2:80:Configuring testing2 (i386)
-pmstatus:testing2:100:Installed testing2 (i386)" cat apt-progress.log
+pmstatus:testing2:16.6667:Preparing testing2 (i386)
+pmstatus:testing2:33.3333:Unpacking testing2 (i386)
+pmstatus:testing2:50:Preparing to configure testing2 (i386)
+pmstatus:dpkg-exec:50:Running dpkg
+pmstatus:testing2:50:Configuring testing2 (i386)
+pmstatus:testing2:66.6667:Configuring testing2 (i386)
+pmstatus:testing2:83.3333:Installed testing2 (i386)
+pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
 
 rm -f apt-progress*.log
diff --git a/test/integration/test-apt-progress-fd-deb822 b/test/integration/test-apt-progress-fd-deb822
index 9d227942d..badc985e4 100755
--- a/test/integration/test-apt-progress-fd-deb822
+++ b/test/integration/test-apt-progress-fd-deb822
@@ -27,37 +27,41 @@ Message: Installing testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 20
+Percent: 16.6667
 Message: Preparing testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 40
+Percent: 33.3333
 Message: Unpacking testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 60
+Percent: 50
 Message: Preparing to configure testing (amd64)
 
 Status: progress
-Percent: 60
+Percent: 50
 Message: Running dpkg
 
 Status: progress
 Package: testing:amd64
-Percent: 60
+Percent: 50
 Message: Configuring testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 80
+Percent: 66.6667
 Message: Configuring testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 100
+Percent: 83.3333
 Message: Installed testing (amd64)
+
+Status: progress
+Percent: 83.3333
+Message: Running dpkg
 " cat apt-progress.log
 
 
diff --git a/test/integration/test-apt-progress-fd-error b/test/integration/test-apt-progress-fd-error
index a47095b9b..632300765 100755
--- a/test/integration/test-apt-progress-fd-error
+++ b/test/integration/test-apt-progress-fd-error
@@ -18,7 +18,7 @@ setupaptarchive
 exec 3> apt-progress.log
 testfailure aptget install foo1 foo2 -y -o APT::Status-Fd=3
 msgtest "Ensure correct error message"
-if grep -q "aptarchive/pool/foo2_0.8.15_amd64.deb:40:trying to overwrite '/usr/bin/file-conflict', which is also in package foo1 0.8.15" apt-progress.log; then
+if grep -q "aptarchive/pool/foo2_0.8.15_amd64.deb:36.3636:trying to overwrite '/usr/bin/file-conflict', which is also in package foo1 0.8.15" apt-progress.log; then
 	msgpass
 else
 	cat apt-progress.log
diff --git a/test/integration/test-bug-769609-triggers-still-pending-after-run b/test/integration/test-bug-769609-triggers-still-pending-after-run
new file mode 100755
index 000000000..146fa766b
--- /dev/null
+++ b/test/integration/test-bug-769609-triggers-still-pending-after-run
@@ -0,0 +1,75 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'amd64'
+
+msgtest 'Check if installed dpkg supports' 'noawait trigger'
+if dpkg-checkbuilddeps -d 'dpkg (>= 1.16.1)' /dev/null; then
+	msgpass
+else
+	msgskip 'dpkg version too old'
+	exit 0
+fi
+configdpkgnoopchroot
+
+buildtriggerpackages() {
+	local TYPE="$1"
+	setupsimplenativepackage "triggerable-$TYPE" 'all' '1.0' 'unstable' "Depends: trigdepends-$TYPE"
+	BUILDDIR="incoming/triggerable-${TYPE}-1.0"
+	cat >${BUILDDIR}/debian/postinst < ${BUILDDIR}/debian/triggers
+	buildpackage "$BUILDDIR" 'unstable' 'main' 'native'
+	rm -rf "$BUILDDIR"
+	buildsimplenativepackage "trigdepends-$TYPE" 'all' '1.0' 'unstable'
+}
+
+#buildtriggerpackages 'interest'
+buildtriggerpackages 'interest-noawait'
+buildsimplenativepackage "trigstuff" 'all' '1.0' 'unstable'
+
+setupaptarchive
+
+runtests() {
+	local TYPE="$1"
+	msgmsg 'Working with trigger type' "$TYPE"
+	testsuccess aptget install triggerable-$TYPE -y
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testsuccess grep '^REWRITE ' terminal.output
+	testdpkginstalled triggerable-$TYPE trigdepends-$TYPE
+
+	testsuccess aptget install trigdepends-$TYPE -y --reinstall
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testsuccess grep '^REWRITE ' terminal.output
+	testsuccess grep ' root root ' terminal.output
+	testdpkginstalled triggerable-$TYPE trigdepends-$TYPE
+
+	testsuccess aptget install trigstuff -y
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testsuccess grep '^REWRITE ' terminal.output
+	testsuccess grep ' root root ' terminal.output
+	testdpkginstalled triggerable-$TYPE trigdepends-$TYPE trigstuff
+
+	testsuccess aptget purge trigstuff -y
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testsuccess grep '^REWRITE ' terminal.output
+	testsuccess grep ' root root ' terminal.output
+	testdpkginstalled triggerable-$TYPE trigdepends-$TYPE
+	testdpkgnotinstalled trigstuff
+
+	testsuccess aptget purge trigdepends-$TYPE -y
+	cp rootdir/tmp/testsuccess.output terminal.output
+	testfailure grep '^REWRITE ' terminal.output
+	testfailure grep ' root root ' terminal.output
+	testdpkgnotinstalled triggerable-$TYPE trigdepends-$TYPE
+}
+#runtests 'interest'
+runtests 'interest-noawait'
diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts
index 6eb033055..7d0c1c6c1 100755
--- a/test/integration/test-no-fds-leaked-to-maintainer-scripts
+++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts
@@ -53,7 +53,8 @@ startup packages configure
 configure $PKGNAME 1.0 
 status unpacked $PKGNAME 1.0
 status half-configured $PKGNAME 1.0
-status installed $PKGNAME 1.0" cut -f 3- -d' ' rootdir/var/log/dpkg.log
+status installed $PKGNAME 1.0
+startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log
 }
 checkinstall
 
@@ -78,7 +79,8 @@ status config-files $PKGNAME 1.0
 status config-files $PKGNAME 1.0
 status config-files $PKGNAME 1.0
 status config-files $PKGNAME 1.0
-status not-installed $PKGNAME " cut -f 3- -d' ' rootdir/var/log/dpkg.log
+status not-installed $PKGNAME 
+startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log
 }
 checkpurge
 
-- 
cgit v1.2.3-70-g09d2


From 748a2177dcf8ff72bca90f5c7d516559ddd67352 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 22 Dec 2014 23:14:08 +0100
Subject: pass-through stdin fd instead of content if not a terminal

Commit 299aea924ccef428219ed6f1a026c122678429e6 fixes the problem of
not logging terminal in case stdin & stdout are not a terminal. The
problem is that we are then trying to pass-through stdin content by
reading from the apt-process stdin and writing it to the stdin of the
child (dpkg), which works great for users who can control themselves,
but pipes and co are a bit less forgiving causing us to pass everything
to the first child process, which if the sending part of the pipe is
e.g. 'yes' we will never see the end of it (as the pipe is full at some
point and further writing blocks).

There is a simple solution for that of course: If stdin isn't a terminal,
we us the apt-process stdin as stdin for the child directly (We don't do
this if it is a terminal to be able to save the typed input in the log).

Closes: 773061
---
 apt-pkg/deb/dpkgpm.cc                              | 16 ++++++++++++----
 .../test-no-fds-leaked-to-maintainer-scripts       | 22 ++++++++++++++++++++--
 2 files changed, 32 insertions(+), 6 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index d54b7b50f..e23ca466d 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -73,7 +73,8 @@ public:
    pkgDPkgPMPrivate() : stdin_is_dev_null(false), dpkgbuf_pos(0),
 			term_out(NULL), history_out(NULL),
 			progress(NULL), tt_is_valid(false), master(-1),
-			slave(NULL), protect_slave_from_dying(-1)
+			slave(NULL), protect_slave_from_dying(-1),
+			direct_stdin(false)
    {
       dpkgbuf[0] = '\0';
    }
@@ -100,6 +101,7 @@ public:
    sigset_t sigmask;
    sigset_t original_sigmask;
 
+   bool direct_stdin;
 };
 
 namespace
@@ -1079,6 +1081,9 @@ void pkgDPkgPM::StartPtyMagic()
       return;
    }
 
+   if (isatty(STDIN_FILENO) == 0)
+      d->direct_stdin = true;
+
    _error->PushToStack();
 
    d->master = posix_openpt(O_RDWR | O_NOCTTY);
@@ -1176,7 +1181,10 @@ void pkgDPkgPM::SetupSlavePtyMagic()
       _error->FatalE("ioctl", "Setting TIOCSCTTY for slave fd %d failed!", slaveFd);
    else
    {
-      for (unsigned short i = 0; i < 3; ++i)
+      unsigned short i = 0;
+      if (d->direct_stdin == true)
+	 ++i;
+      for (; i < 3; ++i)
 	 if (dup2(slaveFd, i) == -1)
 	    _error->FatalE("dup2", "Dupping %d to %d in child failed!", slaveFd, i);
 
@@ -1596,8 +1604,8 @@ bool pkgDPkgPM::GoNoABIBreak(APT::Progress::PackageManager *progress)
 
 	 // wait for input or output here
 	 FD_ZERO(&rfds);
-	 if (d->master >= 0 && !d->stdin_is_dev_null)
-	    FD_SET(0, &rfds); 
+	 if (d->master >= 0 && d->direct_stdin == false && d->stdin_is_dev_null == false)
+	    FD_SET(STDIN_FILENO, &rfds);
 	 FD_SET(_dpkgin, &rfds);
 	 if(d->master >= 0)
 	    FD_SET(d->master, &rfds);
diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts
index 7d0c1c6c1..41c057042 100755
--- a/test/integration/test-no-fds-leaked-to-maintainer-scripts
+++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts
@@ -11,8 +11,14 @@ configdpkgnoopchroot
 setupsimplenativepackage "fdleaks" 'all' '1.0' 'unstable'
 BUILDDIR="incoming/fdleaks-1.0"
 for script in 'preinst' 'postinst' 'prerm' 'postrm'; do
-	echo '#!/bin/sh
-ls -l /proc/self/fd/' > ${BUILDDIR}/debian/$script
+	cat > ${BUILDDIR}/debian/$script << EOF
+#!/bin/sh
+if [ -e "$(pwd)/rootdir/tmp/read_stdin" ]; then
+	read line;
+	echo "STDIN: -\$line-"
+fi
+ls -l /proc/self/fd/
+EOF
 done
 buildpackage "$BUILDDIR" 'unstable' 'main' 'native'
 rm -rf "$BUILDDIR"
@@ -99,3 +105,15 @@ checkinstall
 rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
 testsuccess runapt command setsid -w "${BUILDDIRECTORY}/apt-get" purge -y fdleaks -qq
 checkpurge
+
+touch rootdir/tmp/read_stdin
+
+rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
+for i in $(seq 1 10); do echo "$i"; done | testsuccess aptget install -y fdleaks -qq
+checkinstall
+testequal '2' grep -c '^STDIN: ' rootdir/var/log/apt/term.log
+
+rm -f rootdir/var/log/dpkg.log rootdir/var/log/apt/term.log
+yes '' | testsuccess runapt command setsid -w "${BUILDDIRECTORY}/apt-get" purge -y fdleaks -qq
+checkpurge
+testequal '3' grep -c '^STDIN: ' rootdir/var/log/apt/term.log
-- 
cgit v1.2.3-70-g09d2


From d13f2ef5dd2cf41d7abd7f309a9e8965a77d2a63 Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Tue, 6 Jan 2015 10:54:24 +0100
Subject: Add regression test for the previous commit

The issue was that https.cc never called URIStart(), one way to
detect this is that no download progress is generated without
this call. The test now checks for this and as a side-effect will
also ensure that we do not break download progress reporting and
Acquire::{http,https}::Dl-Limit accidently.
---
 test/integration/test-apt-download-progress | 43 +++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100755 test/integration/test-apt-download-progress

(limited to 'test')

diff --git a/test/integration/test-apt-download-progress b/test/integration/test-apt-download-progress
new file mode 100755
index 000000000..0a9020bec
--- /dev/null
+++ b/test/integration/test-apt-download-progress
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# ensure downloading sends progress as a regression test for commit 9127d7ae
+#
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+changetohttpswebserver
+
+assertprogress() {
+    T="$1"
+    testsuccess grep "dlstatus:1:0:Retrieving file 1 of 1" "$T"
+    if ! egrep -q "dlstatus:1:[0-9]{1,2}\.(.*):Retrieving file 1 of 1" "$T"; then
+        cat "$T"
+        msgfail "Failed to detect download progress"
+    fi
+    testsuccess grep "dlstatus:1:100:Retrieving file 1 of 1" "$T"
+    #cat $T
+}
+
+# we need to ensure the file is reasonable big so that apt has a chance to
+# actually report progress - but not too big to ensure its not delaying the
+# test too much
+TESTFILE=testfile.big
+testsuccess dd if=/dev/zero of=./aptarchive/$TESTFILE bs=800k count=1 
+
+msgtest 'download progress works via' 'http'
+printf '\n'
+exec 3> apt-progress.log
+testsuccess apthelper download-file "http://localhost:8080/$TESTFILE" http-$TESTFILE -o APT::Status-Fd=3 -o Acquire::http::Dl-Limit=800
+assertprogress apt-progress.log
+
+msgtest 'download progress works via' 'https'
+printf '\n'
+exec 3> apt-progress.log
+testsuccess apthelper download-file "https://localhost:4433/$TESTFILE" https-$TESTFILE -o APT::Status-Fd=3 -o Acquire::https::Dl-Limit=800
+assertprogress apt-progress.log
+
+# cleanup
+rm -f apt-progress*.log
-- 
cgit v1.2.3-70-g09d2


From 31be38d205406d4c756684e20b93d62c4701e091 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 9 Jan 2015 01:03:31 +0100
Subject: 128 KiB DSC files ought to be enough for everyone

Your mileage may vary, but don't worry: There is more than one way to
do it, but our one size fits all is not a bigger hammer, but an entire
roundhouse kick! So brace yourself for the tl;dr: The limit is gone.*

Beware: This fixes also the problem that a double newline is
unconditionally added 'later' which is an overcommitment in case
the dsc filesize is limit-2 <= x <= limit.

* limited to numbers fitting into an unsigned long long.

Closes: 774893
---
 ftparchive/cachedb.cc                            |  6 ++--
 ftparchive/sources.cc                            | 41 ++++++++++++++++--------
 ftparchive/sources.h                             | 26 ++++++---------
 ftparchive/writer.cc                             | 12 ++-----
 test/integration/framework                       |  2 +-
 test/integration/test-apt-ftparchive-src-cachedb |  4 ---
 6 files changed, 42 insertions(+), 49 deletions(-)

(limited to 'test')

diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc
index 0901492f7..c73a64fb7 100644
--- a/ftparchive/cachedb.cc
+++ b/ftparchive/cachedb.cc
@@ -328,12 +328,12 @@ bool CacheDB::LoadSource()
    if (Dsc.Read(FileName) == false)
       return false;
 
-   if (Dsc.Data == 0)
+   if (Dsc.Length == 0)
       return _error->Error(_("Failed to read .dsc"));
-   
+
    // Write back the control information
    InitQuerySource();
-   if (Put(Dsc.Data, Dsc.Length) == true)
+   if (Put(Dsc.Data.c_str(), Dsc.Length) == true)
       CurStat.Flags |= FlSource;
 
    return true;
diff --git a/ftparchive/sources.cc b/ftparchive/sources.cc
index d0878a70a..ab976b490 100644
--- a/ftparchive/sources.cc
+++ b/ftparchive/sources.cc
@@ -1,5 +1,5 @@
 #include 
-#include 
+#include 
 
 // for memcpy
 #include 
@@ -9,17 +9,19 @@
 
 #include "sources.h"
 
-bool DscExtract::TakeDsc(const void *newData, unsigned long newSize)
+bool DscExtract::TakeDsc(const void *newData, unsigned long long newSize)
 {
-   if(newSize > maxSize)
-     return _error->Error("DSC data is too large %lu!", newSize);
-
    if (newSize == 0)
    {
+      // adding two newlines 'off record' for pkgTagSection.Scan() calls
+      Data = "\n\n";
       Length = 0;
       return true;
    }
-   memcpy(Data, newData, newSize);
+
+   Data = std::string((const char*)newData, newSize);
+   // adding two newlines 'off record' for pkgTagSection.Scan() calls
+   Data.append("\n\n");
    Length = newSize;
 
    return true;
@@ -27,20 +29,31 @@ bool DscExtract::TakeDsc(const void *newData, unsigned long newSize)
 
 bool DscExtract::Read(std::string FileName)
 {
+   Data.clear();
+   Length = 0;
+
    FileFd F;
    if (OpenMaybeClearSignedFile(FileName, F) == false)
       return false;
-   
-   unsigned long long const FSize = F.FileSize();
-   if(FSize > maxSize)
-     return _error->Error("DSC file '%s' is too large!",FileName.c_str());
-
-   if (F.Read(Data, FSize) == false)
-      return false;
-   Length = FSize;
 
    IsClearSigned = (FileName != F.Name());
 
+   std::ostringstream data;
+   char buffer[1024];
+   do {
+      unsigned long long actual = 0;
+      if (F.Read(buffer, sizeof(buffer)-1, &actual) == false)
+	 return _error->Errno("read", "Failed to read dsc file %s", FileName.c_str());
+      if (actual == 0)
+	 break;
+      Length += actual;
+      buffer[actual] = '\0';
+      data << buffer;
+   } while(true);
+
+   // adding two newlines 'off record' for pkgTagSection.Scan() calls
+   data << "\n\n";
+   Data = data.str();
    return true;
 }
 
diff --git a/ftparchive/sources.h b/ftparchive/sources.h
index 91e0b1376..a125ec6a4 100644
--- a/ftparchive/sources.h
+++ b/ftparchive/sources.h
@@ -3,29 +3,21 @@
 
 #include 
 
-class DscExtract 
+#include 
+
+class DscExtract
 {
  public:
-   //FIXME: do we really need to enforce a maximum size of the dsc file?
-   static const int maxSize = 128*1024;
-
-   char *Data;
+   std::string Data;
    pkgTagSection Section;
-   unsigned long Length;
+   unsigned long long Length;
    bool IsClearSigned;
 
-   bool TakeDsc(const void *Data, unsigned long Size);
+   bool TakeDsc(const void *Data, unsigned long long Size);
    bool Read(std::string FileName);
-   
-   DscExtract() : Data(0), Length(0) {
-     Data = new char[maxSize];
-   };
-   ~DscExtract() { 
-      if(Data != NULL) {
-         delete [] Data;
-         Data = NULL;
-      } 
-   };
+
+   DscExtract() : Length(0), IsClearSigned(false) {};
+   ~DscExtract() {};
 };
 
 
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index 7c1c9cc03..0f6cc177b 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -634,18 +634,10 @@ bool SourcesWriter::DoPackage(string FileName)
    // the "db cursor"
    Db.Finish();
 
-   // read stuff
-   char *Start = Db.Dsc.Data;
-   char *BlkEnd = Db.Dsc.Data + Db.Dsc.Length;
-
-   // Add extra \n to the end, just in case (as in clearsigned they are missing)
-   *BlkEnd++ = '\n';
-   *BlkEnd++ = '\n';
-
    pkgTagSection Tags;
-   if (Tags.Scan(Start,BlkEnd - Start) == false)
+   if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
       return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
-   
+
    if (Tags.Exists("Source") == false)
       return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
    Tags.Trim();
diff --git a/test/integration/framework b/test/integration/framework
index c9445065b..70ad381e9 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -780,7 +780,7 @@ buildaptarchivefromincoming() {
 	[ -e ftparchive.conf ] || createaptftparchiveconfig
 	[ -e dists ] || buildaptftparchivedirectorystructure
 	msgninfo "\tGenerate Packages, Sources and Contents files… "
-	aptftparchive -qq generate ftparchive.conf
+	testsuccess aptftparchive generate ftparchive.conf
 	cd - > /dev/null
 	msgdone "info"
 	generatereleasefiles
diff --git a/test/integration/test-apt-ftparchive-src-cachedb b/test/integration/test-apt-ftparchive-src-cachedb
index adcca6217..0ac4d558f 100755
--- a/test/integration/test-apt-ftparchive-src-cachedb
+++ b/test/integration/test-apt-ftparchive-src-cachedb
@@ -180,10 +180,6 @@ testequal "
 E: Could not find a Source entry in the DSC 'aptarchive/pool/invalid/invalid_1.0.dsc'" aptftparchive sources aptarchive/pool/invalid
 rm -f aptarchive/pool/invalid/invalid_1.0.dsc
 
-dd if=/dev/zero of="aptarchive/pool/invalid/toobig_1.0.dsc" bs=1k count=129 2>/dev/null
-testequal "
-E: DSC file 'aptarchive/pool/invalid/toobig_1.0.dsc' is too large!" aptftparchive sources aptarchive/pool/invalid
-
 # ensure clean works
 rm -f aptarchive/pool/main/*
 aptftparchive clean apt-ftparchive.conf -o Debug::APT::FTPArchive::Clean=1 > clean-out.txt 2>&1 
-- 
cgit v1.2.3-70-g09d2


From 77b6f202e1629b7794a03b6522d636ff1436d074 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 10 Jan 2015 12:31:18 +0100
Subject: award points for positive dependencies again

Commit 9ec748ff103840c4c65471ca00d3b72984131ce4 from Feb 23 last year
adds a version check after 8daf68e366fa9fa2794ae667f51562663856237c
added 8 days earlier negative points for breaks/conflicts with the
intended that only dependencies which are satisfied propagate points
(aka: old conflicts do not).

The implementation was needlessly complex and flawed through preventing
positive dependencies from gaining points like they did before these
commits making library transitions harder instead of simpler. It worked
out anyhow most of the time out of pure 'luck' (and other ways of
gaining points) or got miss attributed to being a temporary hick-up.

Closes: 774924
---
 apt-pkg/algorithms.cc                              |  2 +-
 .../test-allow-scores-for-all-dependency-types     | 28 ++++++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

(limited to 'test')

diff --git a/apt-pkg/algorithms.cc b/apt-pkg/algorithms.cc
index 608ec7fce..b83831053 100644
--- a/apt-pkg/algorithms.cc
+++ b/apt-pkg/algorithms.cc
@@ -468,7 +468,7 @@ void pkgProblemResolver::MakeScores()
 	 if (D->Version != 0)
 	 {
 	    pkgCache::VerIterator const IV = Cache[T].InstVerIter(Cache);
-	    if (IV.end() == true || D.IsSatisfied(IV) != D.IsNegative())
+	    if (IV.end() == true || D.IsSatisfied(IV) == false)
 	       continue;
 	 }
 	 Scores[T->ID] += DepMap[D->Type];
diff --git a/test/integration/test-allow-scores-for-all-dependency-types b/test/integration/test-allow-scores-for-all-dependency-types
index a5c98f3d6..d60cb8daf 100755
--- a/test/integration/test-allow-scores-for-all-dependency-types
+++ b/test/integration/test-allow-scores-for-all-dependency-types
@@ -32,6 +32,11 @@ insertpackage 'multipleyes' 'foo' 'amd64' '2.2' 'Conflicts: bar (<= 3)'
 # having foo multiple times as conflict is a non-advisable hack in general
 insertpackage 'multipleyes' 'bar' 'amd64' '2.2' 'Conflicts: foo (<= 3), foo (<= 3)'
 
+#774924 - slightly simplified
+insertpackage 'jessie' 'login' 'amd64' '2' 'Pre-Depends: libaudit1 (>= 0)'
+insertpackage 'jessie' 'libaudit1' 'amd64' '2' 'Depends: libaudit-common (>= 0)'
+insertpackage 'jessie' 'libaudit-common' 'amd64' '2' 'Breaks: libaudit0, libaudit1 (<< 2)'
+
 cp rootdir/var/lib/dpkg/status rootdir/var/lib/dpkg/status-backup
 setupaptarchive
 
@@ -142,3 +147,26 @@ Inst foo [1] (2 versioned [amd64])
 Inst baz (2 versioned [amd64])
 Conf foo (2 versioned [amd64])
 Conf baz (2 versioned [amd64])' aptget install baz -st versioned
+
+# recreating the exact situation is hard, so we pull tricks to get the score
+cp -f rootdir/var/lib/dpkg/status-backup rootdir/var/lib/dpkg/status
+insertinstalledpackage 'gdm3' 'amd64' '1' 'Depends: libaudit0, libaudit0'
+insertinstalledpackage 'login' 'amd64' '1' 'Essential: yes'
+insertinstalledpackage 'libaudit0' 'amd64' '1'
+testequal 'Reading package lists...
+Building dependency tree...
+The following packages will be REMOVED:
+  gdm3 libaudit0
+The following NEW packages will be installed:
+  libaudit-common libaudit1
+The following packages will be upgraded:
+  login
+1 upgraded, 2 newly installed, 2 to remove and 0 not upgraded.
+Remv gdm3 [1]
+Remv libaudit0 [1]
+Inst libaudit-common (2 jessie [amd64])
+Conf libaudit-common (2 jessie [amd64])
+Inst libaudit1 (2 jessie [amd64])
+Conf libaudit1 (2 jessie [amd64])
+Inst login [1] (2 jessie [amd64])
+Conf login (2 jessie [amd64])' aptget dist-upgrade -st jessie
-- 
cgit v1.2.3-70-g09d2


From b0be0e09cfbbcb033eb0b92eaf17ac31a6b9f423 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 22 Dec 2014 12:20:28 +0100
Subject: rework dpkg-wrapping in test framework

We are able to run maintainer scripts if needed before, but we need to
set ADMINDIR to be able to call dpkg tools like dpkg-trigger inside of
them.

Git-Dch: Ignore
---
 test/integration/framework | 44 +++++++++++++++++++++++++-------------------
 1 file changed, 25 insertions(+), 19 deletions(-)

(limited to 'test')

diff --git a/test/integration/framework b/test/integration/framework
index f7f69f5d5..2c794d1f3 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -131,7 +131,7 @@ aptinternalsolver() { runapt "${APTINTERNALSOLVER}" "$@"; }
 aptdumpsolver() { runapt "${APTDUMPSOLVER}" "$@"; }
 
 dpkg() {
-	command dpkg --root=${TMPWORKINGDIRECTORY}/rootdir --force-not-root --force-bad-path --log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log "$@"
+	"${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg" "$@"
 }
 dpkgcheckbuilddeps() {
 	command dpkg-checkbuilddeps --admindir=${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg "$@"
@@ -247,15 +247,28 @@ setupenvironment() {
 	else
 		echo "Dir::Bin::apt-key \"${BUILDDIRECTORY}/apt-key\";" >> aptconfig.conf
 	fi
-	echo "Dir::Bin::dpkg \"fakeroot\";" >> aptconfig.conf
-	echo "DPKG::options:: \"dpkg\";" >> aptconfig.conf
-	echo "DPKG::options:: \"--root=${TMPWORKINGDIRECTORY}/rootdir\";" >> aptconfig.conf
-	echo "DPKG::options:: \"--force-not-root\";" >> aptconfig.conf
-	echo "DPKG::options:: \"--force-bad-path\";" >> aptconfig.conf
+
+	cat > "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg" < rootdir/etc/apt/apt.conf.d/99dpkg
+
 	if ! command dpkg --assert-multi-arch >/dev/null 2>&1; then
 		echo "DPKG::options:: \"--force-architecture\";" >> aptconfig.conf # Added to test multiarch before dpkg is ready for it…
 	fi
-	echo "DPKG::options:: \"--log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log\";" >> aptconfig.conf
 	echo 'quiet::NoUpdate "true";' >> aptconfig.conf
 	echo 'quiet::NoStatistic "true";' >> aptconfig.conf
 	# too distracting for users, but helpful to detect changes
@@ -388,22 +401,15 @@ int execvp(const char *file, char *const argv[]) {
 	char newfile[strlen(chrootdir) + strlen(file)];
 	strcpy(newfile, chrootdir);
 	strcat(newfile, file);
+	char const * const baseadmindir = "/var/lib/dpkg";
+	char admindir[strlen(chrootdir) + strlen(baseadmindir)];
+	strcpy(admindir, chrootdir);
+	strcat(admindir, baseadmindir);
+	setenv("DPKG_ADMINDIR", admindir, 1);
 	return func_execvp(newfile, argv);
 }
 EOF
 	testsuccess --nomsg gcc -fPIC -shared -o noopchroot.so noopchroot.c -ldl
-
-	mkdir -p "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/"
-	DPKG="${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg"
-	echo "#!/bin/sh
-if [ -n \"\$LD_PRELOAD\" ]; then
-	export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so \${LD_PRELOAD}\"
-else
-	export LD_PRELOAD=\"${TMPWORKINGDIRECTORY}/noopchroot.so\"
-fi
-dpkg \"\$@\"" > $DPKG
-	chmod +x $DPKG
-	sed -ie "s|^DPKG::options:: \"dpkg\";\$|DPKG::options:: \"$DPKG\";|" aptconfig.conf
 }
 
 configallowinsecurerepositories() {
-- 
cgit v1.2.3-70-g09d2


From 905fba60a046646a26a56b4c5d4a5dc7d5906f0d Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 9 Mar 2015 01:54:46 +0100
Subject: derive more of https from http method

Bug #778375 uncovered that https wasn't properly integrated in the class
family tree of http as it was supposed to be leading to a NULL pointer
dereference. Fixing this 'properly' was deemed to much diff for
practically no gain that late in the release, so commit
0c2dc43d4fe1d026650b5e2920a021557f9534a6 just fixed the synptom, while
this commit here is fixing the cause plus adding a test.
---
 methods/http.cc                                    |  2 -
 methods/https.cc                                   | 37 +++++----
 methods/https.h                                    | 16 ++--
 methods/server.cc                                  |  7 +-
 methods/server.h                                   |  3 +-
 .../test-bug-778375-server-has-no-reason-phrase    | 40 ++++++++++
 test/interactive-helper/aptwebserver.cc            | 88 +++++++++++-----------
 7 files changed, 124 insertions(+), 69 deletions(-)
 create mode 100755 test/integration/test-bug-778375-server-has-no-reason-phrase

(limited to 'test')

diff --git a/methods/http.cc b/methods/http.cc
index ad1347d36..021b284d0 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -772,8 +772,6 @@ bool HttpMethod::Configuration(string Message)
    if (ServerMethod::Configuration(Message) == false)
       return false;
 
-   DropPrivsOrDie();
-
    AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 				  PipelineDepth);
diff --git a/methods/https.cc b/methods/https.cc
index 37a8ff5fd..32de42e4b 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -37,16 +37,6 @@
 									/*}}}*/
 using namespace std;
 
-bool HttpsMethod::Configuration(std::string Message)
-{
-   if (pkgAcqMethod::Configuration(Message) == false)
-      return false;
-
-   DropPrivsOrDie();
-
-   return true;
-}
-
 size_t
 HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
 {
@@ -131,7 +121,7 @@ HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/,
 }
 
 // HttpsServerState::HttpsServerState - Constructor			/*{{{*/
-HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * /*Owner*/) : ServerState(Srv, NULL)
+HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner)
 {
    TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
    ReceivedData = false;
@@ -335,13 +325,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout);
 
    // set redirect options and default to 10 redirects
-   bool const AllowRedirect = _config->FindB("Acquire::https::AllowRedirect",
-	_config->FindB("Acquire::http::AllowRedirect",true));
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect);
    curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10);
 
    // debug
-   if(_config->FindB("Debug::Acquire::https", false))
+   if (Debug == true)
       curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
 
    // error handling
@@ -378,7 +366,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
 
    // go for it - if the file exists, append on it
    File = new FileFd(Itm->DestFile, FileFd::WriteAny);
-   Server = new HttpsServerState(Itm->Uri, this);
+   Server = CreateServerState(Itm->Uri);
 
    // keep apt updated
    Res.Filename = Itm->DestFile;
@@ -473,6 +461,25 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
 
    return true;
 }
+									/*}}}*/
+// HttpsMethod::Configuration - Handle a configuration message		/*{{{*/
+bool HttpsMethod::Configuration(string Message)
+{
+   if (ServerMethod::Configuration(Message) == false)
+      return false;
+
+   AllowRedirect = _config->FindB("Acquire::https::AllowRedirect",
+	_config->FindB("Acquire::http::AllowRedirect", true));
+   Debug = _config->FindB("Debug::Acquire::https",false);
+
+   return true;
+}
+									/*}}}*/
+ServerState * HttpsMethod::CreateServerState(URI uri)			/*{{{*/
+{
+   return new HttpsServerState(uri, this);
+}
+									/*}}}*/
 
 int main()
 {
diff --git a/methods/https.h b/methods/https.h
index 6917a6ff6..433a84680 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -50,17 +50,14 @@ class HttpsServerState : public ServerState
 
    HttpsServerState(URI Srv, HttpsMethod *Owner);
    virtual ~HttpsServerState() {Close();};
-
-   bool ReceivedData;
 };
 
-class HttpsMethod : public pkgAcqMethod
+class HttpsMethod : public ServerMethod
 {
    // minimum speed in bytes/se that triggers download timeout handling
    static const int DL_MIN_SPEED = 10;
 
    virtual bool Fetch(FetchItem *);
-   virtual bool Configuration(std::string Message);
 
    static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp);
    static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp);
@@ -69,12 +66,19 @@ class HttpsMethod : public pkgAcqMethod
    void SetupProxy();
    CURL *curl;
    FetchResult Res;
-   HttpsServerState *Server;
+   ServerState *Server;
+
+   // Used by ServerMethods unused by https
+   virtual void SendReq(FetchItem *) { exit(42); }
+   virtual void RotateDNS() { exit(42); }
 
    public:
    FileFd *File;
 
-   HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), Server(NULL), File(NULL)
+   virtual bool Configuration(std::string Message);
+   virtual ServerState * CreateServerState(URI uri);
+
+   HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL)
    {
       curl = curl_easy_init();
    };
diff --git a/methods/server.cc b/methods/server.cc
index c17f27f73..91ec824d1 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -238,7 +238,12 @@ ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOu
 
 bool ServerMethod::Configuration(string Message)			/*{{{*/
 {
-   return pkgAcqMethod::Configuration(Message);
+   if (pkgAcqMethod::Configuration(Message) == false)
+      return false;
+
+   DropPrivsOrDie();
+
+   return true;
 }
 									/*}}}*/
 
diff --git a/methods/server.h b/methods/server.h
index b974ec89a..3b232dcac 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -37,6 +37,7 @@ struct ServerState
    unsigned long long Size; // size of the usable content (aka: the file)
    unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
+   bool ReceivedData;
    time_t Date;
    bool HaveContent;
    enum {Chunked,Stream,Closes} Encoding;
@@ -75,7 +76,7 @@ struct ServerState
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
    virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
-		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
+		 StartPos = 0; ReceivedData = false; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;};
    virtual bool WriteResponse(std::string const &Data) = 0;
 
diff --git a/test/integration/test-bug-778375-server-has-no-reason-phrase b/test/integration/test-bug-778375-server-has-no-reason-phrase
new file mode 100755
index 000000000..23481ef88
--- /dev/null
+++ b/test/integration/test-bug-778375-server-has-no-reason-phrase
@@ -0,0 +1,40 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'native'
+
+echo 'found' > aptarchive/working
+changetohttpswebserver  -o 'aptwebserver::redirect::replace::/redirectme/=/' \
+	-o 'aptwebserver::httpcode::200=200' -o 'aptwebserver::httpcode::404=404' \
+	-o 'aptwebserver::httpcode::301=301'
+
+testdownload() {
+	rm -f downfile
+	msgtest "download of a $1 via" "${3%%:*}"
+	$2 --nomsg downloadfile "$3" downfile
+
+	cp rootdir/tmp/testsuccess.output download.log
+	#looking for "HTTP server doesn't give Reason-Phrase for 200"
+	testsuccess grep 'give Reason-Phrase for' download.log
+
+	if [ "$2" = 'testsuccess' ]; then
+		testfileequal downfile 'found'
+	else
+		testfailure test -e downfile
+	fi
+}
+
+runtest() {
+	testdownload 'file works' 'testsuccess' "$1/working"
+	testdownload 'file via redirect works' 'testsuccess' "$1/redirectme/working"
+
+	testdownload 'non-existent file fails' 'testfailure' "$1/failing"
+	testdownload 'non-existent file via redirect fails' 'testfailure' "$1/redirectme/failing"
+}
+
+runtest 'http://localhost:8080'
+runtest 'https://localhost:4433'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 3403bbdd2..644629a33 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -27,58 +27,58 @@
 #include 
 #include 
 
-static char const * httpcodeToStr(int const httpcode)		/*{{{*/
+static std::string httpcodeToStr(int const httpcode)			/*{{{*/
 {
    switch (httpcode)
    {
       // Informational 1xx
-      case 100: return "100 Continue";
-      case 101: return "101 Switching Protocols";
+      case 100: return _config->Find("aptwebserver::httpcode::100", "100 Continue");
+      case 101: return _config->Find("aptwebserver::httpcode::101", "101 Switching Protocols");
       // Successful 2xx
-      case 200: return "200 OK";
-      case 201: return "201 Created";
-      case 202: return "202 Accepted";
-      case 203: return "203 Non-Authoritative Information";
-      case 204: return "204 No Content";
-      case 205: return "205 Reset Content";
-      case 206: return "206 Partial Content";
+      case 200: return _config->Find("aptwebserver::httpcode::200", "200 OK");
+      case 201: return _config->Find("aptwebserver::httpcode::201", "201 Created");
+      case 202: return _config->Find("aptwebserver::httpcode::202", "202 Accepted");
+      case 203: return _config->Find("aptwebserver::httpcode::203", "203 Non-Authoritative Information");
+      case 204: return _config->Find("aptwebserver::httpcode::204", "204 No Content");
+      case 205: return _config->Find("aptwebserver::httpcode::205", "205 Reset Content");
+      case 206: return _config->Find("aptwebserver::httpcode::206", "206 Partial Content");
       // Redirections 3xx
-      case 300: return "300 Multiple Choices";
-      case 301: return "301 Moved Permanently";
-      case 302: return "302 Found";
-      case 303: return "303 See Other";
-      case 304: return "304 Not Modified";
-      case 305: return "304 Use Proxy";
-      case 307: return "307 Temporary Redirect";
+      case 300: return _config->Find("aptwebserver::httpcode::300", "300 Multiple Choices");
+      case 301: return _config->Find("aptwebserver::httpcode::301", "301 Moved Permanently");
+      case 302: return _config->Find("aptwebserver::httpcode::302", "302 Found");
+      case 303: return _config->Find("aptwebserver::httpcode::303", "303 See Other");
+      case 304: return _config->Find("aptwebserver::httpcode::304", "304 Not Modified");
+      case 305: return _config->Find("aptwebserver::httpcode::305", "305 Use Proxy");
+      case 307: return _config->Find("aptwebserver::httpcode::307", "307 Temporary Redirect");
       // Client errors 4xx
-      case 400: return "400 Bad Request";
-      case 401: return "401 Unauthorized";
-      case 402: return "402 Payment Required";
-      case 403: return "403 Forbidden";
-      case 404: return "404 Not Found";
-      case 405: return "405 Method Not Allowed";
-      case 406: return "406 Not Acceptable";
-      case 407: return "407 Proxy Authentication Required";
-      case 408: return "408 Request Time-out";
-      case 409: return "409 Conflict";
-      case 410: return "410 Gone";
-      case 411: return "411 Length Required";
-      case 412: return "412 Precondition Failed";
-      case 413: return "413 Request Entity Too Large";
-      case 414: return "414 Request-URI Too Large";
-      case 415: return "415 Unsupported Media Type";
-      case 416: return "416 Requested range not satisfiable";
-      case 417: return "417 Expectation Failed";
-      case 418: return "418 I'm a teapot";
+      case 400: return _config->Find("aptwebserver::httpcode::400", "400 Bad Request");
+      case 401: return _config->Find("aptwebserver::httpcode::401", "401 Unauthorized");
+      case 402: return _config->Find("aptwebserver::httpcode::402", "402 Payment Required");
+      case 403: return _config->Find("aptwebserver::httpcode::403", "403 Forbidden");
+      case 404: return _config->Find("aptwebserver::httpcode::404", "404 Not Found");
+      case 405: return _config->Find("aptwebserver::httpcode::405", "405 Method Not Allowed");
+      case 406: return _config->Find("aptwebserver::httpcode::406", "406 Not Acceptable");
+      case 407: return _config->Find("aptwebserver::httpcode::407", "407 Proxy Authentication Required");
+      case 408: return _config->Find("aptwebserver::httpcode::408", "408 Request Time-out");
+      case 409: return _config->Find("aptwebserver::httpcode::409", "409 Conflict");
+      case 410: return _config->Find("aptwebserver::httpcode::410", "410 Gone");
+      case 411: return _config->Find("aptwebserver::httpcode::411", "411 Length Required");
+      case 412: return _config->Find("aptwebserver::httpcode::412", "412 Precondition Failed");
+      case 413: return _config->Find("aptwebserver::httpcode::413", "413 Request Entity Too Large");
+      case 414: return _config->Find("aptwebserver::httpcode::414", "414 Request-URI Too Large");
+      case 415: return _config->Find("aptwebserver::httpcode::415", "415 Unsupported Media Type");
+      case 416: return _config->Find("aptwebserver::httpcode::416", "416 Requested range not satisfiable");
+      case 417: return _config->Find("aptwebserver::httpcode::417", "417 Expectation Failed");
+      case 418: return _config->Find("aptwebserver::httpcode::418", "418 I'm a teapot");
       // Server error 5xx
-      case 500: return "500 Internal Server Error";
-      case 501: return "501 Not Implemented";
-      case 502: return "502 Bad Gateway";
-      case 503: return "503 Service Unavailable";
-      case 504: return "504 Gateway Time-out";
-      case 505: return "505 HTTP Version not supported";
-   }
-   return NULL;
+      case 500: return _config->Find("aptwebserver::httpcode::500", "500 Internal Server Error");
+      case 501: return _config->Find("aptwebserver::httpcode::501", "501 Not Implemented");
+      case 502: return _config->Find("aptwebserver::httpcode::502", "502 Bad Gateway");
+      case 503: return _config->Find("aptwebserver::httpcode::503", "503 Service Unavailable");
+      case 504: return _config->Find("aptwebserver::httpcode::504", "504 Gateway Time-out");
+      case 505: return _config->Find("aptwebserver::httpcode::505", "505 HTTP Version not supported");
+   }
+   return "";
 }
 									/*}}}*/
 static bool chunkedTransferEncoding(std::list const &headers) {
-- 
cgit v1.2.3-70-g09d2


From 25b86db159fbc3c043628e285c0c1ef24dec2c6e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 10 Mar 2015 00:59:44 +0100
Subject: test exitcode as well as string equality

We use test{success,failure} now all over the place in the framework, so
its only consequencial to do this in the situations in which we test for
a specific output as well.

Git-Dch: Ignore
---
 test/integration/framework                         | 24 +++++-
 .../test-allow-scores-for-all-dependency-types     | 22 +++---
 .../test-apt-bug-718329-support-data.tar           | 34 ---------
 test/integration/test-apt-by-hash-update           |  4 +-
 test/integration/test-apt-cache                    | 32 ++++----
 test/integration/test-apt-cdrom                    | 18 ++---
 test/integration/test-apt-cli-list                 | 21 +++---
 test/integration/test-apt-cli-search               | 26 +++----
 test/integration/test-apt-cli-show                 |  2 +-
 test/integration/test-apt-cli-update               |  6 +-
 test/integration/test-apt-cli-upgrade              |  6 +-
 test/integration/test-apt-config                   | 24 +++---
 test/integration/test-apt-extracttemplates         |  2 +-
 test/integration/test-apt-ftparchive-cachedb       | 12 +--
 .../test-apt-ftparchive-cachedb-lp1274466          |  2 +-
 test/integration/test-apt-ftparchive-src-cachedb   |  2 +-
 test/integration/test-apt-get-autoremove           |  6 +-
 test/integration/test-apt-get-build-dep            |  6 +-
 test/integration/test-apt-get-changelog            |  4 +-
 test/integration/test-apt-get-install-deb          |  2 +-
 test/integration/test-apt-get-source               | 18 ++---
 test/integration/test-apt-get-source-arch          | 16 ++--
 test/integration/test-apt-get-source-multisources  |  4 +-
 .../integration/test-apt-get-update-unauth-warning |  8 +-
 test/integration/test-apt-get-upgrade              |  6 +-
 test/integration/test-apt-helper                   | 14 ++--
 test/integration/test-apt-key                      |  4 +-
 test/integration/test-apt-key-net-update           |  4 +-
 test/integration/test-apt-mark                     | 36 ++++-----
 test/integration/test-apt-progress-fd              | 20 ++---
 test/integration/test-apt-progress-fd-deb822       |  5 +-
 .../test-apt-translation-has-no-packages           |  4 +-
 test/integration/test-apt-update-expected-size     |  2 +-
 test/integration/test-apt-update-nofallback        | 12 +--
 test/integration/test-apt-update-rollback          | 20 ++---
 test/integration/test-apt-update-stale             |  2 +-
 .../test-architecture-specification-parsing        | 20 ++---
 test/integration/test-authentication-basic         |  2 +-
 .../test-bug-470115-new-and-tighten-recommends     | 20 ++---
 .../test-bug-507998-dist-upgrade-recommends        |  2 +-
 .../test-bug-543966-downgrade-below-1000-pin       | 10 +--
 ...est-bug-549968-install-depends-of-not-installed |  2 +-
 .../test-bug-590041-prefer-non-virtual-packages    |  8 +-
 ...g-590438-broken-provides-thanks-to-remove-order |  2 +-
 test/integration/test-bug-591882-conkeror          |  4 +-
 .../integration/test-bug-593360-modifiers-in-names | 16 ++--
 .../test-bug-596498-trusted-unsigned-repo          | 12 +--
 ...test-bug-598669-install-postfix-gets-exim-heavy |  2 +-
 test/integration/test-bug-601961-install-info      |  6 +-
 test/integration/test-bug-602412-dequote-redirect  |  2 +-
 .../integration/test-bug-604222-new-and-autoremove | 10 +--
 .../test-bug-605394-versioned-or-groups            |  2 +-
 test/integration/test-bug-611729-mark-as-manual    |  6 +-
 .../test-bug-612099-multiarch-conflicts            | 30 ++++----
 test/integration/test-bug-612557-garbage-upgrade   |  6 +-
 .../test-bug-613420-new-garbage-dependency         |  2 +-
 .../test-bug-618848-always-respect-user-requests   |  2 +-
 .../test-bug-624218-Translation-file-handling      | 17 ++---
 .../test-bug-632221-cross-dependency-satisfaction  | 30 ++++----
 ...test-bug-64141-install-dependencies-for-on-hold |  4 +-
 .../test-bug-657695-resolver-breaks-on-virtuals    |  2 +-
 .../test-bug-661537-build-profiles-support         |  8 +-
 .../test-bug-675449-essential-are-protected        | 10 +--
 .../test-bug-679371-apt-get-autoclean-multiarch    |  2 +-
 .../test-bug-680041-apt-mark-holds-correctly       | 20 ++---
 .../test-bug-683786-build-dep-on-virtual-packages  | 18 ++---
 .../test-bug-686346-package-missing-architecture   | 10 +--
 .../test-bug-689582-100-char-long-path-names       |  2 +-
 .../test-bug-691453-apt-cache-search-multi-pattern | 12 +--
 .../test-bug-709560-set-candidate-release          |  2 +-
 .../test-bug-712435-missing-descriptions           | 18 ++---
 .../test-bug-717891-abolute-uris-for-proxies       |  2 +-
 .../test-bug-718329-support-data.tar-uncompressed  | 31 ++++++++
 ...st-bug-719263-print-uris-removes-authentication |  2 +-
 test/integration/test-bug-720597-build-dep-purge   |  4 +-
 .../test-bug-722207-print-uris-even-if-very-quiet  | 14 ++--
 .../test-bug-723586-any-stripped-in-single-arch    | 10 +--
 test/integration/test-bug-728500-tempdir           |  2 +-
 test/integration/test-bug-732746-preferences       |  2 +-
 .../test-bug-735967-lib32-to-i386-unavailable      |  8 +-
 test/integration/test-bug-738785-switch-protocol   |  4 +-
 .../test-bug-745036-new-foreign-invalidates-cache  |  2 +-
 .../test-bug-745046-candidate-propagation-fails    |  4 +-
 test/integration/test-bug-753297-upgradable        |  4 +-
 .../test-bug-758153-versioned-provides-support     | 88 +++++++++++++++++++---
 test/integration/test-bug-770291-reinstall         |  8 +-
 test/integration/test-bug-multiarch-upgrade        |  2 +-
 .../test-cachecontainer-architecture-specification | 32 ++++----
 test/integration/test-compressed-indexes           | 26 +++----
 test/integration/test-conflicts-loop               |  2 +-
 .../integration/test-conflicts-real-multiarch-same |  6 +-
 .../test-cve-2013-1051-InRelease-parsing           |  8 +-
 test/integration/test-essential-force-loopbreak    |  2 +-
 .../test-external-dependency-solver-protocol       |  8 +-
 test/integration/test-handling-broken-orgroups     | 14 ++--
 .../test-ignore-provides-if-versioned-breaks       | 18 ++---
 .../test-ignore-provides-if-versioned-conflicts    | 18 ++---
 .../test-implicit-conflicts-real-not-virtual       |  8 +-
 test/integration/test-kernel-helper-autoremove     |  6 +-
 test/integration/test-multiarch-foreign            | 24 +++---
 .../test-ordering-ignore-not-matching-breaks       |  8 +-
 test/integration/test-package-reinstallation       |  2 +-
 test/integration/test-pdiff-usage                  | 12 +--
 test/integration/test-pin-non-existent-package     | 12 +--
 test/integration/test-policy-pinning               |  4 +-
 .../test-prefer-higher-priority-providers          | 14 ++--
 ...prefer-native-architecture-over-higher-priority |  2 +-
 ...prevent-markinstall-multiarch-same-versionscrew |  6 +-
 test/integration/test-provides-gone-with-upgrade   |  2 +-
 test/integration/test-release-candidate-switching  | 32 ++++----
 test/integration/test-releasefile-verification     | 30 ++++----
 .../test-resolve-by-keep-new-recommends            |  2 +-
 test/integration/test-sourceslist-trusted-options  |  4 +-
 .../test-specific-architecture-dependencies        | 30 ++++----
 .../test-suggest-installed-multiarch-silbing       | 16 ++--
 .../test-ubuntu-bug-1098738-apt-get-source-md5sum  | 20 ++---
 ...u-bug-1130419-prefer-installed-ma-same-siblings | 12 +--
 ...t-ubuntu-bug-1304403-obsolete-priority-standard |  4 +-
 test/integration/test-ubuntu-bug-614993            |  2 +-
 .../test-ubuntu-bug-761175-remove-purge            |  4 +-
 ...st-ubuntu-bug-784473-InRelease-one-message-only |  2 +-
 .../test-ubuntu-bug-802901-multiarch-early-remove  |  2 +-
 .../test-ubuntu-bug-806274-install-suggests        |  8 +-
 .../test-ubuntu-bug-859188-multiarch-reinstall     |  8 +-
 ...ubuntu-bug-985852-pre-depends-or-group-ordering |  2 +-
 .../test-unpack-different-version-unpacked         | 14 ++--
 test/integration/test-xorg-break-providers         |  6 +-
 127 files changed, 725 insertions(+), 645 deletions(-)
 delete mode 100755 test/integration/test-apt-bug-718329-support-data.tar
 create mode 100755 test/integration/test-bug-718329-support-data.tar-uncompressed

(limited to 'test')

diff --git a/test/integration/framework b/test/integration/framework
index 5f13df1c0..ec23e41e6 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1338,7 +1338,7 @@ testwarning() {
 	else
 		msgtest 'Test for successful execution with warnings of' "$*"
 	fi
-	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
+	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testwarning.output"
 	if "$@" >${OUTPUT} 2>&1; then
 		if expr match "$1" '^apt.*' >/dev/null; then
 			if grep -q -E ' runtime error: ' "$OUTPUT"; then
@@ -1388,6 +1388,26 @@ testfailure() {
 	aptautotest 'testfailure' "$@"
 }
 
+testsuccessequal() {
+	local CMP="$1"
+	shift
+	testsuccess "$@"
+	testfileequal "${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output" "$CMP"
+}
+testwarningequal() {
+	local CMP="$1"
+	shift
+	testwarning "$@"
+	testfileequal "${TMPWORKINGDIRECTORY}/rootdir/tmp/testwarning.output" "$CMP"
+}
+testfailureequal() {
+	local CMP="$1"
+	shift
+	testfailure "$@"
+	testfileequal "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output" "$CMP"
+}
+
+
 testfilestats() {
 	msgtest "Test that file $1 has $2 $3" "$4"
 	if [ "$4" "$3" "$(stat --format "$2" "$1")" ]; then
@@ -1493,7 +1513,7 @@ aptautotest_apt_update() { aptautotest_aptget_update "$@"; }
 testaptautotestnodpkgwarning() {
 	local TESTCALL="$1"
 	while [ -n "$2" ]; do
-		if [ "$2" = '-s' ]; then return; fi
+		if expr match "$2" '^-[a-z]*s' >/dev/null 2>&1; then return; fi
 		shift
 	done
 	testfailure grep '^dpkg: warning:.*ignor.*' "${TMPWORKINGDIRECTORY}/rootdir/tmp-before/${TESTCALL}.output"
diff --git a/test/integration/test-allow-scores-for-all-dependency-types b/test/integration/test-allow-scores-for-all-dependency-types
index e1d805ce9..56cfc9a69 100755
--- a/test/integration/test-allow-scores-for-all-dependency-types
+++ b/test/integration/test-allow-scores-for-all-dependency-types
@@ -42,7 +42,7 @@ setupaptarchive
 
 insertinstalledpackage 'libdb-dev' 'amd64' '5.1.7' 'Depends: libdb5.1-dev'
 insertinstalledpackage 'libdb5.1-dev' 'amd64' '5.1.29-7'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
@@ -57,7 +57,7 @@ Inst libdb-dev [5.1.7] (5.3.0 unversioned [amd64]) []
 Inst libdb5.3-dev (5.3.28-3 unversioned [amd64])
 Conf libdb5.3-dev (5.3.28-3 unversioned [amd64])
 Conf libdb-dev (5.3.0 unversioned [amd64])' aptget dist-upgrade -st unversioned
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
@@ -76,25 +76,25 @@ Conf libdb-dev (5.3.0 versioned [amd64])' aptget dist-upgrade -st versioned
 cp -f rootdir/var/lib/dpkg/status-backup rootdir/var/lib/dpkg/status
 insertinstalledpackage 'foo' 'amd64' '1'
 insertinstalledpackage 'bar' 'amd64' '1'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages have been kept back:
   bar foo
 0 upgraded, 0 newly installed, 0 to remove and 2 not upgraded.' aptget dist-upgrade -st unversioned
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages have been kept back:
   bar foo
 0 upgraded, 0 newly installed, 0 to remove and 2 not upgraded.' aptget dist-upgrade -st versioned
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages have been kept back:
   bar foo
 0 upgraded, 0 newly installed, 0 to remove and 2 not upgraded.' aptget dist-upgrade -st multipleno
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
@@ -106,14 +106,14 @@ Remv foo [1]
 Inst bar [1] (2.2 multipleyes [amd64])
 Conf bar (2.2 multipleyes [amd64])' aptget dist-upgrade -st multipleyes
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   baz
 0 upgraded, 1 newly installed, 0 to remove and 2 not upgraded.
 Inst baz (2 unversioned [amd64])
 Conf baz (2 unversioned [amd64])' aptget install baz -st unversioned
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -130,14 +130,14 @@ Inst baz (2 versioned [amd64])
 Conf foo (2 versioned [amd64])
 Conf baz (2 versioned [amd64])' aptget install baz -st versioned
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   baz
 0 upgraded, 1 newly installed, 0 to remove and 2 not upgraded.
 Inst baz (2 unversioned [amd64])
 Conf baz (2 unversioned [amd64])' aptget install baz -st unversioned
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -159,7 +159,7 @@ cp -f rootdir/var/lib/dpkg/status-backup rootdir/var/lib/dpkg/status
 insertinstalledpackage 'gdm3' 'amd64' '1' 'Depends: libaudit0, libaudit0'
 insertinstalledpackage 'login' 'amd64' '1' 'Essential: yes'
 insertinstalledpackage 'libaudit0' 'amd64' '1'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
diff --git a/test/integration/test-apt-bug-718329-support-data.tar b/test/integration/test-apt-bug-718329-support-data.tar
deleted file mode 100755
index 5cfb31917..000000000
--- a/test/integration/test-apt-bug-718329-support-data.tar
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/sh
-set -e
-
-test_process_package_with_compression() {
-    COMPRESSOR="$1"
-    DATA_TAR="$2"
-
-    msgtest "Testing apt-ftparchive with compression type: $COMPRESSOR"
-
-    buildsimplenativepackage 'pkg' 'all' '1.0' '' '' 'some descr' '' '' '' "$COMPRESSOR"
-    testequal "debian-binary
-control.tar.gz
-$DATA_TAR" ar t incoming/pkg_1.0_all.deb
-
-    testequal "Package: pkg" echo "$(aptftparchive packages incoming/|grep ^Package)"
-
-    testequal "usr/bin/pkg-all						    pkg
-usr/share/doc/pkg/FEATURES				    pkg
-usr/share/doc/pkg/changelog				    pkg
-usr/share/doc/pkg/copyright				    pkg" aptftparchive contents incoming/
-
-    rm -rf incoming/*
-}
-
-TESTDIR=$(readlink -f $(dirname $0))
-. $TESTDIR/framework
-
-setupenvironment
-test_process_package_with_compression "gzip" "data.tar.gz"
-test_process_package_with_compression "none" "data.tar"
-test_process_package_with_compression "xz" "data.tar.xz"
-
-
-
diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update
index d9d0b146f..8300c532c 100755
--- a/test/integration/test-apt-by-hash-update
+++ b/test/integration/test-apt-by-hash-update
@@ -29,7 +29,7 @@ mkdir -p aptarchive/dists/unstable/main/source/by-hash/SHA512
 testfailure aptget upate
 
 # ensure we do not know about "foo"
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 E: Unable to locate package foo" aptget install -q -s foo
 
@@ -37,7 +37,7 @@ E: Unable to locate package foo" aptget install -q -s foo
 testsuccess aptget update -o APT::Acquire::By-Hash=1 -o Acquire::Languages=none
 
 # ensure it works
-testequal "Inst foo (1.0 unstable [all])
+testsuccessequal "Inst foo (1.0 unstable [all])
 Conf foo (1.0 unstable [all])" aptget install -qq -s foo
 
 # add magic string to Release file ...
diff --git a/test/integration/test-apt-cache b/test/integration/test-apt-cache
index f47c0e08b..f3db8024f 100755
--- a/test/integration/test-apt-cache
+++ b/test/integration/test-apt-cache
@@ -42,45 +42,45 @@ testsuccess aptcache dump
 cp rootdir/tmp/testsuccess.output dump.output
 testsuccess test -s dump.output
 
-testequal 'dpkg
+testsuccessequal 'dpkg
 bar
 fancy
 foo' aptcache pkgnames
-testequal 'bar' aptcache pkgnames bar
-testequal 'fancy
+testsuccessequal 'bar' aptcache pkgnames bar
+testsuccessequal 'fancy
 foo' aptcache pkgnames f
 
-testequal "       foo |          1 | file:$(readlink -f .)/aptarchive/ unstable/main amd64 Packages" aptcache madison foo
+testsuccessequal "       foo |          1 | file:$(readlink -f .)/aptarchive/ unstable/main amd64 Packages" aptcache madison foo
 
 ### depends
 
-testequal 'foo
+testsuccessequal 'foo
   Depends: bar
  |Recommends: 
   Recommends: 
   Conflicts: 
   Conflicts: ' aptcache depends foo
-testequal 'foo
+testsuccessequal 'foo
   Depends: bar
   Recommends: 
   Conflicts: 
   Conflicts: ' aptcache depends foo -o APT::Cache::ShowOnlyFirstOr=1
-testequal 'foo
+testsuccessequal 'foo
   Depends: bar
  |Recommends:  (>= 2)
   Recommends:  (<< 5)
   Conflicts: 
   Conflicts: ' aptcache depends foo -o APT::Cache::ShowVersion=1
-testequal 'foo
+testsuccessequal 'foo
   Depends: bar
   Conflicts: 
   Conflicts: ' aptcache depends foo --no-recommends
-testequal 'foo
+testsuccessequal 'foo
   Depends: bar' aptcache depends foo --important
-testequal 'foo
+testsuccessequal 'foo
   Conflicts: 
   Conflicts: ' aptcache depends foo --important --no-depends --conflicts
-testequal 'foo
+testsuccessequal 'foo
   Depends: bar
  |Recommends: 
   Recommends: 
@@ -97,7 +97,7 @@ bar
 
 
 ' aptcache depends foo --recurse
-testequal 'foo
+testsuccessequal 'foo
   Depends: bar
 bar
   Depends: bar
@@ -107,18 +107,18 @@ bar
 
 ## rdpends
 
-testequal 'foo
+testsuccessequal 'foo
 Reverse Depends:
   bar
   bar' aptcache rdepends foo
-testequal 'foo
+testsuccessequal 'foo
 Reverse Depends:
   Replaces: bar
   Breaks: bar' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1
-testequal 'foo
+testsuccessequal 'foo
 Reverse Depends:
   Replaces: bar (<< 1)
   Breaks: bar (<< 1)' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 -o APT::Cache::ShowVersion=1
-testequal 'foo
+testsuccessequal 'foo
 Reverse Depends:
   Breaks: bar (<< 1)' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 -o APT::Cache::ShowVersion=1 --important --breaks
diff --git a/test/integration/test-apt-cdrom b/test/integration/test-apt-cdrom
index 7f4b3c257..9906795ca 100755
--- a/test/integration/test-apt-cdrom
+++ b/test/integration/test-apt-cdrom
@@ -49,12 +49,12 @@ deb-src cdrom:[Debian APT Testdisk 0.8.15]/ stable main
 Unmounting CD-ROM...
 Repeat this process for the rest of the CDs in your set."
 
-testequal "$CDROM_PRE
+testsuccessequal "$CDROM_PRE
 Found 2 package indexes, 1 source indexes, 1 translation indexes and 1 signatures
 Found label 'Debian APT Testdisk 0.8.15'
 $CDROM_POST" aptcdromlog add
 
-testequal "Using CD-ROM mount point $(readlink -f ./rootdir/media)/cdrom/
+testsuccessequal "Using CD-ROM mount point $(readlink -f ./rootdir/media)/cdrom/
 Mounting CD-ROM...
 Stored label: Debian APT Testdisk 0.8.15
 Unmounting CD-ROM..." aptcdromlog ident
@@ -63,13 +63,13 @@ Unmounting CD-ROM..." aptcdromlog ident
 ident="$(LC_ALL=C aptcdrom ident 2>&1 )"
 CD_ID="$(echo "$ident" | grep "^Identifying" | head -n1 | cut -d" " -f2 | tr --delete '[]')"
 CD_LABEL="$(echo "$ident" | grep "^Stored label:" | head -n1 | sed "s/^[^:]*: //")"
-testequal "CD::${CD_ID} \"${CD_LABEL}\";
-CD::${CD_ID}::Label \"${CD_LABEL}\";" cat rootdir/var/lib/apt/cdroms.list
+testfileequal rootdir/var/lib/apt/cdroms.list "CD::${CD_ID} \"${CD_LABEL}\";
+CD::${CD_ID}::Label \"${CD_LABEL}\";"
 
 testcdromusage() {
 	touch rootdir/var/lib/apt/extended_states
 
-	testequal 'Reading package lists...
+	testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following NEW packages will be installed:
@@ -84,7 +84,7 @@ Conf testing (0.8.15 stable [amd64])' aptget install testing -s
 	testsuccess aptget purge testing -y
 	testdpkgnotinstalled testing
 
-	testequal 'Reading package lists...
+	testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following NEW packages will be installed:
@@ -114,18 +114,18 @@ Conf testing:i386 (0.8.15 stable [i386])' aptget install testing:i386 -s
 testcdromusage
 
 # check Idempotence of apt-cdrom (and disabling of Translation dropping)
-testequal "$CDROM_PRE
+testsuccessequal "$CDROM_PRE
 Found 2 package indexes, 1 source indexes, 2 translation indexes and 1 signatures
 $CDROM_POST" aptcdromlog add -o APT::CDROM::DropTranslation=0
 
 # take Translations from previous runs as needed
-testequal "$CDROM_PRE
+testsuccessequal "$CDROM_PRE
 Found 2 package indexes, 1 source indexes, 2 translation indexes and 1 signatures
 $CDROM_POST" aptcdromlog add
 msgtest 'Test for the german description translation of' 'testing'
 aptcache show testing -o Acquire::Languages=de | grep -q '^Description-de: ' && msgpass || msgfail
 rm -rf rootdir/var/lib/apt/lists
-testequal "$CDROM_PRE
+testsuccessequal "$CDROM_PRE
 Found 2 package indexes, 1 source indexes, 1 translation indexes and 1 signatures
 $CDROM_POST" aptcdromlog add
 msgtest 'Test for the english description translation of' 'testing'
diff --git a/test/integration/test-apt-cli-list b/test/integration/test-apt-cli-list
index 1487afd55..d3c44e126 100755
--- a/test/integration/test-apt-cli-list
+++ b/test/integration/test-apt-cli-list
@@ -27,50 +27,49 @@ setupaptarchive
 
 APTARCHIVE=$(readlink -f ./aptarchive)
 
-testequal "Listing...
+testsuccessequal "Listing...
 bar/now 1.0 i386 [installed,local]
 baz/unstable 2.0 all [upgradable from: 0.1]
 foo/unstable 1.0 all
 foobar/unstable 2.0 i386 [upgradable from: 1.0]" apt list
 
-testequal "Listing...
+testsuccessequal "Listing...
 foo/unstable 1.0 all
 foobar/unstable 2.0 i386 [upgradable from: 1.0]" apt list "foo*"
 
-testequal "Listing...
+testsuccessequal "Listing...
 baz/unstable 2.0 all [upgradable from: 0.1]
 foobar/unstable 2.0 i386 [upgradable from: 1.0]" apt list --upgradable
 
 # FIXME: hm, hm - does it make sense to have this different? shouldn't
 #        we use "installed,upgradable" consitently?
-testequal "Listing...
+testsuccessequal "Listing...
 bar/now 1.0 i386 [installed,local]
 baz/now 0.1 all [installed,upgradable to: 2.0]
 foobar/now 1.0 i386 [installed,upgradable to: 2.0]" apt list --installed
 
-testequal "Listing...
+testsuccessequal "Listing...
 bar/now 1.0 i386 [installed,local]
 
 foobar/unstable 2.0 i386 [upgradable from: 1.0]
 foobar/now 1.0 i386 [installed,upgradable to: 2.0]
 " apt list bar foobar --all-versions
 
-testequal "Listing...
+testsuccessequal "Listing...
 bar/now 1.0 i386 [installed,local]
   an autogenerated dummy bar=1.0/installed
 " apt list bar --verbose
 
 # test for dpkg ^rc state
 insertinstalledpackage 'conf-only' 'i386' '1.0' '' '' 'deinstall ok config-files'
-testequal "Listing...
+testsuccessequal "Listing...
 conf-only/now 1.0 i386 [residual-config]" apt list conf-only
 
 # ensure that the users learns about multiple versions too
-testequal "Listing...
+testsuccessequal "Listing...
 baz/unstable 2.0 all [upgradable from: 0.1]
 N: There are 2 additional versions. Please use the '-a' switch to see them." apt list baz -o quiet=0
 
 # test format strings for machine parseable output
-apt list -qq bar baz  -o APT::Cmd::use-format=true  -o APT::Cmd::format="\${Package} - \${installed:Version} - \${candidate:Version}" > output.txt
-testequal "bar - 1.0 - 1.0
-baz - 0.1 - 2.0" cat output.txt
+testsuccessequal 'bar - 1.0 - 1.0
+baz - 0.1 - 2.0' apt list -qq bar baz  -o APT::Cmd::use-format=true  -o APT::Cmd::format="\${Package} - \${installed:Version} - \${candidate:Version}"
diff --git a/test/integration/test-apt-cli-search b/test/integration/test-apt-cli-search
index 1a28ba4da..e86661dcb 100755
--- a/test/integration/test-apt-cli-search
+++ b/test/integration/test-apt-cli-search
@@ -25,51 +25,51 @@ setupaptarchive
 
 APTARCHIVE=$(readlink -f ./aptarchive)
 
-testequal 'E: You must give at least one search pattern' aptcache search
-testequal 'E: You must give at least one search pattern' apt search
+testfailureequal 'E: You must give at least one search pattern' aptcache search
+testfailureequal 'E: You must give at least one search pattern' apt search
 
 # with OP progress 
-testequal "Sorting...
+testsuccessequal "Sorting...
 Full Text Search...
 foo/unstable 1.0 all
   $DESCR
 " apt search xxyyzz
 
 # without op progress
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
 " apt search -qq xxyyzz
 testempty apt search -qq --names-only xxyyzz
 
 # search name
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
 " apt search -qq foo
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
 " apt search -qq --names-only foo
 
 # search with multiple words is a AND search
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
 " apt search -qq aabbcc xxyyzz
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
 " apt search -qq 'a+b+c+' 'i*xxy{0,2}zz'
 
 # search is not case-sensitive by default
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
 " apt search -qq uppercase
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
 " apt search -qq 'up[pP]erc[Aa]se'
 
 # search is done in the long description
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
 " apt search -qq 'long description'
-testequal "foo/unstable 1.0 all
+testsuccessequal "foo/unstable 1.0 all
   $DESCR
   Long description of stuff and such, with lines
   .
@@ -77,7 +77,7 @@ testequal "foo/unstable 1.0 all
 " apt search --full -qq 'long description'
 
 # output is sorted and search word finds both package
-testequal "bar/testing 2.0 i386
+testsuccessequal "bar/testing 2.0 i386
   $DESCR2
 
 foo/unstable 1.0 all
diff --git a/test/integration/test-apt-cli-show b/test/integration/test-apt-cli-show
index 930e591e0..5604620fd 100755
--- a/test/integration/test-apt-cli-show
+++ b/test/integration/test-apt-cli-show
@@ -24,7 +24,7 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 
 # note that we do not display Description-md5 with the "apt" cmd
 # and also show some additional fields that are calculated
-testequal "Package: foo
+testsuccessequal "Package: foo
 Priority: optional
 Section: other
 Installed-Size: 43.0 kB
diff --git a/test/integration/test-apt-cli-update b/test/integration/test-apt-cli-update
index 83cc94b93..d68ab25e4 100755
--- a/test/integration/test-apt-cli-update
+++ b/test/integration/test-apt-cli-update
@@ -15,10 +15,10 @@ setupaptarchive --no-update
 
 APTARCHIVE=$(readlink -f ./aptarchive)
 
-testequal 'E: The update command takes no arguments' apt update -q arguments
+testfailureequal 'E: The update command takes no arguments' apt update -q arguments
 
-testequal "1 package can be upgraded. Run 'apt list --upgradable' to see it." apt update -q
+testsuccessequal "1 package can be upgraded. Run 'apt list --upgradable' to see it." apt update -q
 
 cp dpkg.status rootdir/var/lib/dpkg/status
 insertinstalledpackage 'foo' 'all' '2.0'
-testequal 'All packages are up to date.' apt update -q
+testsuccessequal 'All packages are up to date.' apt update -q
diff --git a/test/integration/test-apt-cli-upgrade b/test/integration/test-apt-cli-upgrade
index b6ee2270b..54f2ecd11 100755
--- a/test/integration/test-apt-cli-upgrade
+++ b/test/integration/test-apt-cli-upgrade
@@ -23,7 +23,7 @@ setupaptarchive
 APTARCHIVE=$(readlink -f ./aptarchive)
 
 # default is to allow new dependencies
-testequal "The following NEW packages will be installed:
+testsuccessequal "The following NEW packages will be installed:
   foo-new-dependency
 The following packages will be upgraded:
   foo
@@ -33,7 +33,7 @@ Inst foo [1.0] (2.0 unstable [all])
 Conf foo-new-dependency (2.0 unstable [all])
 Conf foo (2.0 unstable [all])" apt upgrade -qq -s
 
-# ensure 
-testequal "The following packages have been kept back:
+# ensure the 'old' way works as well
+testsuccessequal "The following packages have been kept back:
   foo
 0 upgraded, 0 newly installed, 0 to remove and 1 not upgraded." apt upgrade -qq -s --no-new-pkgs
diff --git a/test/integration/test-apt-config b/test/integration/test-apt-config
index 2f2ff9d38..2eea9a0f8 100755
--- a/test/integration/test-apt-config
+++ b/test/integration/test-apt-config
@@ -8,29 +8,29 @@ setupenvironment
 configarchitecture 'amd64'
 
 testsuccess aptconfig dump
-testequal 'APT::Architecture "amd64";' aptconfig dump APT::Architecture
+testsuccessequal 'APT::Architecture "amd64";' aptconfig dump APT::Architecture
 testempty aptconfig dump config::which::does::not::exist
 
-testequal 'APT::Architectures "";
+testsuccessequal 'APT::Architectures "";
 APT::Architectures:: "amd64";' aptconfig dump APT::Architectures
-testequal 'APT::Architectures:: "amd64";' aptconfig dump --no-empty APT::Architectures
-testequal 'amd64' aptconfig dump --no-empty --format='%v%n' APT::Architectures
+testsuccessequal 'APT::Architectures:: "amd64";' aptconfig dump --no-empty APT::Architectures
+testsuccessequal 'amd64' aptconfig dump --no-empty --format='%v%n' APT::Architectures
 
 testempty aptconfig shell
-testequal 'E: Arguments not in pairs' aptconfig shell APT::Architecture
+testfailureequal 'E: Arguments not in pairs' aptconfig shell APT::Architecture
 testempty aptconfig shell APT::Architecture ARCH # incorrect order
-testequal "ARCH='amd64'" aptconfig shell ARCH APT::Architecture
+testsuccessequal "ARCH='amd64'" aptconfig shell ARCH APT::Architecture
 
 ROOTDIR="$(readlink -f rootdir)"
-testequal "CONFIG='apt.conf'" aptconfig shell CONFIG Dir::Etc::main
-testequal "CONFIG='${ROOTDIR}/etc/apt/apt.conf'" aptconfig shell CONFIG Dir::Etc::main/f
-testequal "CONFIG='etc/apt/'" aptconfig shell CONFIG Dir::Etc
-testequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/ # old style
-testequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/d
+testsuccessequal "CONFIG='apt.conf'" aptconfig shell CONFIG Dir::Etc::main
+testsuccessequal "CONFIG='${ROOTDIR}/etc/apt/apt.conf'" aptconfig shell CONFIG Dir::Etc::main/f
+testsuccessequal "CONFIG='etc/apt/'" aptconfig shell CONFIG Dir::Etc
+testsuccessequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/ # old style
+testsuccessequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/d
 
 testempty aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles
 export DEB_BUILD_PROFILES='nodoc stage1'
-testequal 'nodoc
+testsuccessequal 'nodoc
 stage1' aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles
 unset DEB_BUILD_PROFILES
 testempty aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles
diff --git a/test/integration/test-apt-extracttemplates b/test/integration/test-apt-extracttemplates
index 276862464..5dadc4933 100755
--- a/test/integration/test-apt-extracttemplates
+++ b/test/integration/test-apt-extracttemplates
@@ -35,7 +35,7 @@ Description: Some bar var
 	OUT='rootdir/tmp/testsuccess.output'
 	testequal "$1" cut -f1 -d' ' $OUT
 	if [ -n "$2" ]; then
-		testequal '' cut -f2 -d' ' $OUT
+		testequal '' cut -s -f2 -d' ' $OUT
 	else
 		testequal '1.0' cut -f2 -d' ' $OUT
 	fi
diff --git a/test/integration/test-apt-ftparchive-cachedb b/test/integration/test-apt-ftparchive-cachedb
index 866e5a469..3454ee36a 100755
--- a/test/integration/test-apt-ftparchive-cachedb
+++ b/test/integration/test-apt-ftparchive-cachedb
@@ -9,14 +9,14 @@ $(dpkg-deb -I ./aptarchive/pool/main/foo_1_i386.deb | grep 'Installed-Size:' | s
 Maintainer: Joe Sixpack 
 Architecture: i386
 Version: 1
-Filename: pool/main/foo_1_i386.deb" head -n8 ./aptarchive/dists/test/main/binary-i386/Packages 
+Filename: pool/main/foo_1_i386.deb" head -n8 ./aptarchive/dists/test/main/binary-i386/Packages
 }
 
 ensure_correct_contents_file() {
-    testequal "usr/bin/foo-i386					    others/foo
+    testfileequal ./aptarchive/dists/test/Contents-i386 "usr/bin/foo-i386					    others/foo
 usr/share/doc/foo/FEATURES				    others/foo
 usr/share/doc/foo/changelog				    others/foo
-usr/share/doc/foo/copyright				    others/foo" cat ./aptarchive/dists/test/Contents-i386
+usr/share/doc/foo/copyright				    others/foo"
 }
 
 #
@@ -99,8 +99,8 @@ testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 0
 
 # and clean
 rm -rf aptarchive/pool/main/*
-testequal "packages-main-i386.db" aptftparchive clean ftparchive.conf -q=0
+testsuccessequal "packages-main-i386.db" aptftparchive clean ftparchive.conf -q=0
 testsuccess aptftparchive clean ftparchive.conf -q=0 -o Debug::APT::FTPArchive::Clean=1
 cp rootdir/tmp/testsuccess.output clean-out.txt
-testequal "0	Number of unique keys in the tree" grep unique clean-out.txt
-testequal "packages-main-i386.db" grep packages-main-i386.db clean-out.txt
+testsuccessequal "0	Number of unique keys in the tree" grep unique clean-out.txt
+testsuccessequal "packages-main-i386.db" grep packages-main-i386.db clean-out.txt
diff --git a/test/integration/test-apt-ftparchive-cachedb-lp1274466 b/test/integration/test-apt-ftparchive-cachedb-lp1274466
index 579ae33a6..1f86e367f 100755
--- a/test/integration/test-apt-ftparchive-cachedb-lp1274466
+++ b/test/integration/test-apt-ftparchive-cachedb-lp1274466
@@ -26,7 +26,7 @@ testsuccess grep 7da58ff901a40ecf42a730dc33198b182e9ba9ec98799fc2c2b6fabeeee40cc
 testfailure grep 7da58ff901a40ecf42a730dc33198b182e9ba9ec98799fc2c2b6fabeeee40cc12a0e7cadb4b66764235c56e1009dbfe8a9a566fb1eedf47a992d1fff2cc3332c old-format.dump
 
 # regression test for corruption with previous generation of cachedb
-testequal "Package: foo
+testsuccessequal "Package: foo
 Priority: optional
 Section: others
 Installed-Size: 29
diff --git a/test/integration/test-apt-ftparchive-src-cachedb b/test/integration/test-apt-ftparchive-src-cachedb
index 28321e3c5..2a361ecc0 100755
--- a/test/integration/test-apt-ftparchive-src-cachedb
+++ b/test/integration/test-apt-ftparchive-src-cachedb
@@ -2,7 +2,7 @@
 set -e
 
 assert_correct_sources_file() {
-    testequal "Package: bar
+    testsuccessequal "Package: bar
 Architecture: all
 Version: 1.0
 Binary: bar
diff --git a/test/integration/test-apt-get-autoremove b/test/integration/test-apt-get-autoremove
index acde4b096..a0e4d3c24 100755
--- a/test/integration/test-apt-get-autoremove
+++ b/test/integration/test-apt-get-autoremove
@@ -20,7 +20,7 @@ testdpkgnotinstalled 'debhelper'
 testdpkginstalled 'po-debconf' 'unrelated'
 
 echo 'APT::NeverAutoRemove { "^debc.*nf$"; };' > rootdir/etc/apt/apt.conf.d/00autoremove
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
@@ -55,7 +55,7 @@ testdpkginstalled 'unrelated' 'debhelper' 'po-debconf'
 testsuccess aptmark auto debhelper
 
 testmarkedauto 'debhelper' 'po-debconf'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
@@ -65,7 +65,7 @@ Remv debhelper [8.0.0]
 Remv po-debconf [1.0.16]' aptget autoremove -s
 
 testsuccess aptmark hold debhelper
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.' aptget autoremove -s
diff --git a/test/integration/test-apt-get-build-dep b/test/integration/test-apt-get-build-dep
index 87ec6e54d..f7af5b782 100755
--- a/test/integration/test-apt-get-build-dep
+++ b/test/integration/test-apt-get-build-dep
@@ -32,7 +32,7 @@ Files:
  1e806d32233af87437258d86b1561f57 2036 2vcard_0.5-3.diff.gz
 EOF
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Note, using file '2vcard_0.5-3.dsc' to get the build dependencies
 The following NEW packages will be installed:
@@ -75,7 +75,7 @@ z2UAn1oXgTai6opwhVfkxrlmJ+iRxzuc
 -----END PGP SIGNATURE-----
 EOF
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Note, using file '2vcard_0.5-3.dsc' to get the build dependencies
 The following NEW packages will be installed:
@@ -117,7 +117,7 @@ Description: install packages using the apt protocol - common data
 
 EOF
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Note, using directory './foo-1.0' to get the build dependencies
 The following NEW packages will be installed:
diff --git a/test/integration/test-apt-get-changelog b/test/integration/test-apt-get-changelog
index 01f2bd393..7e81c71b6 100755
--- a/test/integration/test-apt-get-changelog
+++ b/test/integration/test-apt-get-changelog
@@ -21,9 +21,9 @@ chmod -R -w rootdir/var/cache/apt/archives
 
 echo 'Apt::Changelogs::Server "http://localhost:8080/";' > rootdir/etc/apt/apt.conf.d/changelog.conf
 
-testequal "'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
+testsuccessequal "'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
 
-testequal "'http://localhost:8080/pool/apt_1.0/changelog'
+testsuccessequal "'http://localhost:8080/pool/apt_1.0/changelog'
 'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt apt --print-uris
 
 cd downloaded
diff --git a/test/integration/test-apt-get-install-deb b/test/integration/test-apt-get-install-deb
index f2e5229cd..0f34692fe 100755
--- a/test/integration/test-apt-get-install-deb
+++ b/test/integration/test-apt-get-install-deb
@@ -8,7 +8,7 @@ setupenvironment
 configarchitecture "i386"
 
 # regression test for #754904
-testequal 'E: Unable to locate package /dev/null' aptget install -qq /dev/null
+testfailureequal 'E: Unable to locate package /dev/null' aptget install -qq /dev/null
 
 # and ensure we fail for invalid debs
 cat > foo.deb < rootdir/etc/apt/apt.conf.d/02proxy-detect
 
-    testequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com
+    testsuccessequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com
 
 
     # https auto detect proxy script
@@ -66,14 +66,14 @@ EOF
     chmod 755 apt-proxy-detect
     echo "Acquire::https::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect
 
-    testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com
+    testsuccessequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com
 }
 
 test_apt_helper_download
 test_apt_helper_detect_proxy
 
 # test failure modes
-testequal 'E: Invalid operation download' apthelper download
-testequal 'E: Must specify at least one pair url/filename' apthelper download-file
-testequal 'E: Must specify at least one pair url/filename' apthelper download-file http://example.org/
-testequal 'E: Need one URL as argument' apthelper auto-detect-proxy
+testfailureequal 'E: Invalid operation download' apthelper download
+testfailureequal 'E: Must specify at least one pair url/filename' apthelper download-file
+testfailureequal 'E: Must specify at least one pair url/filename' apthelper download-file http://example.org/
+testfailureequal 'E: Need one URL as argument' apthelper auto-detect-proxy
diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key
index b6b7b7909..989fe658c 100755
--- a/test/integration/test-apt-key
+++ b/test/integration/test-apt-key
@@ -17,7 +17,7 @@ testaptkeys() {
 	if ! aptkey list | grep '^pub' > aptkey.list; then
 		echo -n > aptkey.list
 	fi
-	testequal "$1" cat ./aptkey.list
+	testfileequal './aptkey.list' "$1"
 }
 
 echo 'APT::Key::ArchiveKeyring "./keys/joesixpack.pub";
@@ -35,7 +35,7 @@ testrun() {
 
 	testaptkeys 'pub   2048R/DBAC8DAE 2010-08-18'
 
-	testequal 'gpg: key DBAC8DAE: "Joe Sixpack (APT Testcases Dummy) " not changed
+	testsuccessequal 'gpg: key DBAC8DAE: "Joe Sixpack (APT Testcases Dummy) " not changed
 gpg: Total number processed: 1
 gpg:              unchanged: 1' aptkey --fakeroot update
 
diff --git a/test/integration/test-apt-key-net-update b/test/integration/test-apt-key-net-update
index b3c118555..2a0823bec 100755
--- a/test/integration/test-apt-key-net-update
+++ b/test/integration/test-apt-key-net-update
@@ -23,7 +23,7 @@ echo 'APT::Key::ArchiveKeyringURI "http://localhost:8080/ubuntu/project/test-arc
 echo 'APT::Key::Net-Update-Enabled "1";' >> ./aptconfig.conf
 
 # test against the "real" webserver
-testequal 'Checking for new archive signing keys now
+testsuccessequal 'Checking for new archive signing keys now
 gpg: key F68C85A3: public key "Test Automatic Archive Signing Key " imported
 gpg: Total number processed: 1
 gpg:               imported: 1  (RSA: 1)' aptkey --fakeroot net-update
@@ -40,7 +40,7 @@ echo 'APT::Key::ArchiveKeyringURI "http://localhost:8080/ubuntu/project/marvinpa
 echo 'APT::Key::Net-Update-Enabled "1";' >> ./aptconfig.conf
 
 # test against the "real" webserver
-testequal "Checking for new archive signing keys now
+testsuccessequal "Checking for new archive signing keys now
 Key 'DE66AECA9151AFA1877EC31DE8525D47528144E2' not added. It is not signed with a master key" aptkey --fakeroot net-update
 
 aptkey list | grep '^pub' > aptkey.list
diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark
index 5a3ae4b2f..9b68945f9 100755
--- a/test/integration/test-apt-mark
+++ b/test/integration/test-apt-mark
@@ -21,20 +21,20 @@ testdpkginstalled dpkg
 testnoautopkg() {
 	testempty aptmark showauto
 	testempty aptcache showauto
-	testequal 'bar
+	testsuccessequal 'bar
 dpkg
 foo' aptmark showmanual
-	testequal 'bar
+	testsuccessequal 'bar
 foo' aptmark showmanual bar foo uninstalled
 }
 testfooisauto() {
-	testequal 'foo' aptmark showauto
-	testequal 'foo' aptcache showauto
-	testequal 'foo' aptmark showauto foo
-	testequal 'foo' aptcache showauto foo
-	testequal 'bar
+	testsuccessequal 'foo' aptmark showauto
+	testsuccessequal 'foo' aptcache showauto
+	testsuccessequal 'foo' aptmark showauto foo
+	testsuccessequal 'foo' aptcache showauto foo
+	testsuccessequal 'bar
 dpkg' aptmark showmanual
-	testequal 'bar' aptmark showmanual bar
+	testsuccessequal 'bar' aptmark showmanual bar
 }
 testmarkonpkgasauto() {
 	testsuccess $1 $2 foo
@@ -48,8 +48,8 @@ testmarkonpkgasauto() {
 	testnoautopkg
 }
 
-testequal 'E: No packages found' aptmark auto
-testequal 'E: No packages found' aptmark manual
+testfailureequal 'E: No packages found' aptmark auto
+testfailureequal 'E: No packages found' aptmark manual
 
 testnoautopkg
 testmarkonpkgasauto 'aptmark' 'auto' 'manual'
@@ -63,10 +63,10 @@ testnoholdpkg() {
 	testempty aptmark showholds dpkg
 }
 testpkgonhold() {
-	testequal "$1" aptmark showhold
-	testequal "$1" aptmark showholds
-	testequal "$1" aptmark showhold $1
-	testequal "$1" aptmark showholds $1
+	testsuccessequal "$1" aptmark showhold
+	testsuccessequal "$1" aptmark showholds
+	testsuccessequal "$1" aptmark showhold $1
+	testsuccessequal "$1" aptmark showholds $1
 }
 testmarkonepkgashold() {
 	testsuccess aptmark hold $1
@@ -79,8 +79,8 @@ testmarkonepkgashold() {
 	testnoholdpkg
 }
 
-testequal 'E: No packages found' aptmark hold
-testequal 'E: No packages found' aptmark unhold
+testfailureequal 'E: No packages found' aptmark hold
+testfailureequal 'E: No packages found' aptmark unhold
 
 testnoholdpkg
 testmarkonepkgashold 'foo'
@@ -97,5 +97,5 @@ fi
 testmarkonepkgashold 'uninstalled'
 testmarkonepkgashold 'uninstalled-native'
 
-testequal 'uninstalled set on hold.' aptmark hold uninstalled
-testequal 'uninstalled-native set on hold.' aptmark hold uninstalled-native
+testsuccessequal 'uninstalled set on hold.' aptmark hold uninstalled
+testsuccessequal 'uninstalled-native set on hold.' aptmark hold uninstalled-native
diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd
index 90e6ef7e4..0c11aba7e 100755
--- a/test/integration/test-apt-progress-fd
+++ b/test/integration/test-apt-progress-fd
@@ -15,7 +15,7 @@ setupaptarchive
 # install native
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3
-testequal "dlstatus:1:0:Retrieving file 1 of 1
+testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -26,12 +26,12 @@ pmstatus:dpkg-exec:50:Running dpkg
 pmstatus:testing:50:Configuring testing (amd64)
 pmstatus:testing:66.6667:Configuring testing (amd64)
 pmstatus:testing:83.3333:Installed testing (amd64)
-pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
+pmstatus:dpkg-exec:83.3333:Running dpkg'
 
 # upgrade
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3
-testequal "dlstatus:1:0:Retrieving file 1 of 1
+testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -42,12 +42,12 @@ pmstatus:dpkg-exec:50:Running dpkg
 pmstatus:testing:50:Configuring testing (amd64)
 pmstatus:testing:66.6667:Configuring testing (amd64)
 pmstatus:testing:83.3333:Installed testing (amd64)
-pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
+pmstatus:dpkg-exec:83.3333:Running dpkg'
 
 # reinstall
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3
-testequal "dlstatus:1:0:Retrieving file 1 of 1
+testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -58,24 +58,24 @@ pmstatus:dpkg-exec:50:Running dpkg
 pmstatus:testing:50:Configuring testing (amd64)
 pmstatus:testing:66.6667:Configuring testing (amd64)
 pmstatus:testing:83.3333:Installed testing (amd64)
-pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
+pmstatus:dpkg-exec:83.3333:Running dpkg'
 
 # and remove
 exec 3> apt-progress.log
 testsuccess aptget remove testing -y -o APT::Status-Fd=3
-testequal "pmstatus:dpkg-exec:0:Running dpkg
+testfileequal './apt-progress.log' 'pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Removing testing (amd64)
 pmstatus:testing:25:Preparing for removal of testing (amd64)
 pmstatus:testing:50:Removing testing (amd64)
 pmstatus:testing:75:Removed testing (amd64)
-pmstatus:dpkg-exec:75:Running dpkg" cat apt-progress.log
+pmstatus:dpkg-exec:75:Running dpkg'
 
 # install non-native and ensure we get proper progress info
 exec 3> apt-progress.log
 testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3
 
 # and compare
-testequal "dlstatus:1:0:Retrieving file 1 of 1
+testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)
@@ -86,6 +86,6 @@ pmstatus:dpkg-exec:50:Running dpkg
 pmstatus:testing2:50:Configuring testing2 (i386)
 pmstatus:testing2:66.6667:Configuring testing2 (i386)
 pmstatus:testing2:83.3333:Installed testing2 (i386)
-pmstatus:dpkg-exec:83.3333:Running dpkg" cat apt-progress.log
+pmstatus:dpkg-exec:83.3333:Running dpkg'
 
 rm -f apt-progress*.log
diff --git a/test/integration/test-apt-progress-fd-deb822 b/test/integration/test-apt-progress-fd-deb822
index badc985e4..ca7f14cb9 100755
--- a/test/integration/test-apt-progress-fd-deb822
+++ b/test/integration/test-apt-progress-fd-deb822
@@ -16,7 +16,7 @@ setupaptarchive
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-deb822-Fd=3
 
-testequal "Status: progress
+testfileequal './apt-progress.log' 'Status: progress
 Percent: 0
 Message: Running dpkg
 
@@ -62,7 +62,6 @@ Message: Installed testing (amd64)
 Status: progress
 Percent: 83.3333
 Message: Running dpkg
-" cat apt-progress.log
-
+'
 
 rm -f apt-progress*.log
diff --git a/test/integration/test-apt-translation-has-no-packages b/test/integration/test-apt-translation-has-no-packages
index bb2353a33..440fd30cf 100755
--- a/test/integration/test-apt-translation-has-no-packages
+++ b/test/integration/test-apt-translation-has-no-packages
@@ -17,7 +17,7 @@ configarchitecture "amd64"
 
 if [ ! -x ${BUILDDIRECTORY}/apt ]; then
     msgmsg "No ${BUILDDIRECTORY}/apt"
-    msgskip 
+    msgskip
     exit 0
 fi
 
@@ -33,7 +33,7 @@ cp $APTARCHIVE/dists/unstable/main/binary-amd64/Packages \
 
 # ensure that there is no Version for the package foo generated out of
 # the corrupted Translation-en file
-testequal "foo:
+testsuccessequal "foo:
   Installed: (none)
   Candidate: 1.0
   Version table:
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 045217a77..9711c293a 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -39,6 +39,6 @@ find aptarchive -name 'Packages*' | while read pkg; do
 done
 NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
 rm -f rootdir/var/lib/apt/lists/localhost*
-testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages  Writing more data than expected ($NEW_SIZE > $SIZE)
+testfailureequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages  Writing more data than expected ($NEW_SIZE > $SIZE)
 
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback
index e82a976a6..71576de81 100755
--- a/test/integration/test-apt-update-nofallback
+++ b/test/integration/test-apt-update-nofallback
@@ -31,14 +31,14 @@ EOF
 
 assert_update_is_refused_and_last_good_state_used()
 {
-    testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
+    testfailureequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
 
     assert_repo_is_intact
 }
 
 assert_repo_is_intact()
 {
-    testequal "foo/unstable 2.0 all" apt list -q
+    testsuccessequal "foo/unstable 2.0 all" apt list -q
     testsuccess aptget install -y -s foo
     testfailure aptget install -y evil
     testsuccess aptget source foo --print-uris
@@ -97,7 +97,7 @@ test_from_inrelease_to_unsigned_with_override()
     testwarning aptget update --allow-insecure-repositories \
         -o Acquire::AllowDowngradeToInsecureRepositories=1
     # but that the individual packages are still considered untrusted
-    testequal "WARNING: The following packages cannot be authenticated!
+    testfailureequal "WARNING: The following packages cannot be authenticated!
   evil
 E: There are problems and -y was used without --force-yes" aptget install -qq -y evil
 }
@@ -149,7 +149,7 @@ test_subvert_inrelease()
     # replace InRelease with something else
     mv $APTARCHIVE/dists/unstable/Release $APTARCHIVE/dists/unstable/InRelease
 
-    testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  Does not start with a cleartext signature
+    testfailureequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  Does not start with a cleartext signature
 
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
 
@@ -169,7 +169,7 @@ test_inrelease_to_invalid_inrelease()
     sed -i 's/Codename.*/Codename: evil!'/ $APTARCHIVE/dists/unstable/InRelease
     inject_evil_package
 
-    testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
+    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
 W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
@@ -193,7 +193,7 @@ test_release_gpg_to_invalid_release_release_gpg()
     echo "Some evil data" >>  $APTARCHIVE/dists/unstable/Release
     inject_evil_package
 
-    testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release.gpg: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
+    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release.gpg: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
 W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg  The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback
index 9efc194a0..f4500b69d 100755
--- a/test/integration/test-apt-update-rollback
+++ b/test/integration/test-apt-update-rollback
@@ -38,7 +38,7 @@ start_with_good_inrelease() {
     create_fresh_archive
     testsuccess aptget update
     listcurrentlistsdirectory > lists.before
-    testequal "old/unstable 1.0 all" apt list -q
+    testsuccessequal "old/unstable 1.0 all" apt list -q
 }
 
 test_inrelease_to_new_inrelease() {
@@ -47,7 +47,7 @@ test_inrelease_to_new_inrelease() {
 
     add_new_package '+1hour'
     testsuccess aptget update -o Debug::Acquire::Transaction=1
-    testequal "new/unstable 1.0 all
+    testsuccessequal "new/unstable 1.0 all
 old/unstable 1.0 all" apt list -q
 }
 
@@ -60,12 +60,12 @@ test_inrelease_to_broken_hash_reverts_all() {
     break_repository_sources_index '+1hour'
 
     # test the error condition
-    testequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources  Hash Sum mismatch
+    testfailureequal "W: Failed to fetch file:${APTARCHIVE}/dists/unstable/main/source/Sources  Hash Sum mismatch
 
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
     # ensure that the Packages file is also rolled back
     testfileequal lists.before "$(listcurrentlistsdirectory)"
-    testequal "E: Unable to locate package new" aptget install new -s -qq
+    testfailureequal "E: Unable to locate package new" aptget install new -s -qq
 }
 
 test_inrelease_to_valid_release() {
@@ -78,7 +78,7 @@ test_inrelease_to_valid_release() {
     rm $APTARCHIVE/dists/unstable/Release.gpg
 
     # update fails
-    testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
+    testfailureequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
 
     # test that security downgrade was not successful
     testfileequal lists.before "$(listcurrentlistsdirectory)"
@@ -101,7 +101,7 @@ test_inrelease_to_release_reverts_all() {
     break_repository_sources_index '+1hour'
 
     # ensure error
-    testequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1
+    testfailureequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1
 
     # ensure that the Packages file is also rolled back
     testfileequal lists.before "$(listcurrentlistsdirectory)"
@@ -119,7 +119,7 @@ test_unauthenticated_to_invalid_inrelease() {
 
     testwarning aptget update --allow-insecure-repositories
     listcurrentlistsdirectory > lists.before
-    testequal "WARNING: The following packages cannot be authenticated!
+    testfailureequal "WARNING: The following packages cannot be authenticated!
   old
 E: There are problems and -y was used without --force-yes" aptget install -qq -y old
 
@@ -127,13 +127,13 @@ E: There are problems and -y was used without --force-yes" aptget install -qq -y
     add_new_package '+1hour'
     break_repository_sources_index '+1hour'
 
-    testequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources  Hash Sum mismatch
+    testfailureequal "W: Failed to fetch file:$APTARCHIVE/dists/unstable/main/source/Sources  Hash Sum mismatch
 
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
 
     testfileequal lists.before "$(listcurrentlistsdirectory)"
     testfailure ls rootdir/var/lib/apt/lists/*_InRelease
-    testequal "WARNING: The following packages cannot be authenticated!
+    testfailureequal "WARNING: The following packages cannot be authenticated!
   old
 E: There are problems and -y was used without --force-yes" aptget install -qq -y old
 }
@@ -144,7 +144,7 @@ test_inrelease_to_unauth_inrelease() {
 
     signreleasefiles 'Marvin Paranoid'
 
-    testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
+    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
 
 W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease  The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
 
diff --git a/test/integration/test-apt-update-stale b/test/integration/test-apt-update-stale
index 277aa5b09..05154641a 100755
--- a/test/integration/test-apt-update-stale
+++ b/test/integration/test-apt-update-stale
@@ -39,7 +39,7 @@ cp -p aptarchive/dists/unstable/main/binary-i386/saved/Packages* \
      aptarchive/dists/unstable/main/binary-i386/
 
 # ensure this raises an error
-testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages  Hash Sum mismatch
+testfailureequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages  Hash Sum mismatch
 
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
 testfileequal lists.before "$(listcurrentlistsdirectory)"
diff --git a/test/integration/test-architecture-specification-parsing b/test/integration/test-architecture-specification-parsing
index d1f6011de..f5a5b123e 100755
--- a/test/integration/test-architecture-specification-parsing
+++ b/test/integration/test-architecture-specification-parsing
@@ -26,7 +26,7 @@ insertinstalledpackage 'build-essential' 'all' '11.5' 'Multi-Arch: foreign'
 
 setupaptarchive
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -38,7 +38,7 @@ Inst pkg-arch-foo (1.0 stable [${NATIVE}])
 Conf foo (1.0 stable [${NATIVE}])
 Conf pkg-arch-foo (1.0 stable [${NATIVE}])" aptget install pkg-arch-foo -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   pkg-arch-no-foo
@@ -46,7 +46,7 @@ The following NEW packages will be installed:
 Inst pkg-arch-no-foo (1.0 stable [${NATIVE}])
 Conf pkg-arch-no-foo (1.0 stable [${NATIVE}])" aptget install pkg-arch-no-foo -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -58,7 +58,7 @@ Inst pkg-arch-foo-unrelated-no (1.0 stable [${NATIVE}])
 Conf foo (1.0 stable [${NATIVE}])
 Conf pkg-arch-foo-unrelated-no (1.0 stable [${NATIVE}])" aptget install pkg-arch-foo-unrelated-no -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -70,7 +70,7 @@ Inst pkg-arch-foo-unrelated-no2 (1.0 stable [${NATIVE}])
 Conf foo (1.0 stable [${NATIVE}])
 Conf pkg-arch-foo-unrelated-no2 (1.0 stable [${NATIVE}])" aptget install pkg-arch-foo-unrelated-no2 -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo
@@ -78,11 +78,11 @@ The following NEW packages will be installed:
 Inst foo (1.0 stable [${NATIVE}])
 Conf foo (1.0 stable [${NATIVE}])" aptget build-dep pkg-arch-foo -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.' aptget build-dep pkg-arch-no-foo -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo
@@ -90,7 +90,7 @@ The following NEW packages will be installed:
 Inst foo (1.0 stable [${NATIVE}])
 Conf foo (1.0 stable [${NATIVE}])" aptget build-dep pkg-arch-foo-unrelated-no -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo
@@ -98,13 +98,13 @@ The following NEW packages will be installed:
 Inst foo (1.0 stable [${NATIVE}])
 Conf foo (1.0 stable [${NATIVE}])" aptget build-dep pkg-arch-foo-unrelated-no2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.' aptget build-dep no-depends -s
 
 # this is not really testing APT - more that dpkg is in line with us
 configarchitecture 'amd64' 'armel'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   no-depends:armel
diff --git a/test/integration/test-authentication-basic b/test/integration/test-authentication-basic
index 7e74726be..3a6897b59 100755
--- a/test/integration/test-authentication-basic
+++ b/test/integration/test-authentication-basic
@@ -38,7 +38,7 @@ testauthsuccess() {
 
 	rm -rf rootdir/var/lib/apt/lists
 	testsuccess aptget update
-	testequal 'Reading package lists...
+	testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo
diff --git a/test/integration/test-bug-470115-new-and-tighten-recommends b/test/integration/test-bug-470115-new-and-tighten-recommends
index 6bc22ea7b..0970e2f23 100755
--- a/test/integration/test-bug-470115-new-and-tighten-recommends
+++ b/test/integration/test-bug-470115-new-and-tighten-recommends
@@ -47,7 +47,7 @@ insertpackage 'unstable' 'now-satisfiable' 'all' '2' 'Recommends: cool (>= 2)'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   cool
@@ -59,7 +59,7 @@ Inst tighten-cool [1] (2 unstable [all])
 Conf cool (2 unstable [all])
 Conf tighten-cool (2 unstable [all])' aptget install tighten-cool -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   stuff
@@ -71,7 +71,7 @@ Inst tighten-coolorstuff [1] (2 unstable [all])
 Conf stuff (2 unstable [all])
 Conf tighten-coolorstuff (2 unstable [all])' aptget install tighten-coolorstuff -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   cool
@@ -83,7 +83,7 @@ Inst tighten-coolorstuff2 [1] (2 unstable [all])
 Conf cool (2 unstable [all])
 Conf tighten-coolorstuff2 (2 unstable [all])' aptget install tighten-coolorstuff2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   cool
@@ -95,7 +95,7 @@ Inst newrec-cool [1] (2 unstable [all])
 Conf cool (2 unstable [all])
 Conf newrec-cool (2 unstable [all])' aptget install newrec-cool -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   super
@@ -109,7 +109,7 @@ Inst super (2 unstable [all])
 Conf newrec-super (2 unstable [all])
 Conf super (2 unstable [all])' aptget install newrec-super -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   cool
@@ -121,7 +121,7 @@ Inst newrec-coolorstuff [1] (2 unstable [all])
 Conf cool (2 unstable [all])
 Conf newrec-coolorstuff (2 unstable [all])' aptget install newrec-coolorstuff -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   stuff
@@ -133,7 +133,7 @@ Inst stuff [1] (2 unstable [all])
 Conf cool-gone (2 unstable [all])
 Conf stuff (2 unstable [all])' aptget install cool-gone -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   super
@@ -149,7 +149,7 @@ Conf super-overtake (2 unstable [all])' aptget install super-overtake -s
 
 # if super would be in front, we would get a new here as it is new and
 # the first option in an or-group should be the preferred one…
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   cool
@@ -163,7 +163,7 @@ Conf upgrade-over-new (2 unstable [all])' aptget install upgrade-over-new -s
 
 # the recommends wasn't used before so while we could do it now,
 # the user doesn't seem to need it so avoid upgrading it
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be upgraded:
   now-satisfiable
diff --git a/test/integration/test-bug-507998-dist-upgrade-recommends b/test/integration/test-bug-507998-dist-upgrade-recommends
index f3b4e04fb..70c6fb496 100755
--- a/test/integration/test-bug-507998-dist-upgrade-recommends
+++ b/test/integration/test-bug-507998-dist-upgrade-recommends
@@ -14,7 +14,7 @@ insertpackage 'unstable' 'wireshark' 'amd64' '1.2.1-2' 'Depends: wireshark-commo
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be upgraded:
diff --git a/test/integration/test-bug-543966-downgrade-below-1000-pin b/test/integration/test-bug-543966-downgrade-below-1000-pin
index f602bea95..d37539b9f 100755
--- a/test/integration/test-bug-543966-downgrade-below-1000-pin
+++ b/test/integration/test-bug-543966-downgrade-below-1000-pin
@@ -15,7 +15,7 @@ setupaptarchive
 STATUS=$(readlink -f rootdir/var/lib/dpkg/status)
 APTARCHIVE="$(readlink -f aptarchive)/"
 
-testequal "base-files:
+testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0-1
   Version table:
@@ -28,7 +28,7 @@ echo 'Package: base-files
 Pin: release a=unstable
 Pin-Priority: 99' > rootdir/etc/apt/preferences
 
-testequal "base-files:
+testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0-1
   Package pin: 5.0.0
@@ -42,7 +42,7 @@ echo 'Package: base-files
 Pin: release a=unstable
 Pin-Priority: 100' > rootdir/etc/apt/preferences
 
-testequal "base-files:
+testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0-1
   Package pin: 5.0.0
@@ -56,7 +56,7 @@ echo 'Package: base-files
 Pin: release a=unstable
 Pin-Priority: 999' > rootdir/etc/apt/preferences
 
-testequal "base-files:
+testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0-1
   Package pin: 5.0.0
@@ -70,7 +70,7 @@ echo 'Package: base-files
 Pin: release a=unstable
 Pin-Priority: 1000' > rootdir/etc/apt/preferences
 
-testequal "base-files:
+testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0
   Package pin: 5.0.0
diff --git a/test/integration/test-bug-549968-install-depends-of-not-installed b/test/integration/test-bug-549968-install-depends-of-not-installed
index 8c434b3ce..3ff4807de 100755
--- a/test/integration/test-bug-549968-install-depends-of-not-installed
+++ b/test/integration/test-bug-549968-install-depends-of-not-installed
@@ -14,7 +14,7 @@ setupaptarchive
 
 # We check the Markers here as the autoremove nuker will also
 # prevent it, but to late - its better to fail earlier
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
   MarkInstall coolstuff [ i386 ] < none -> 1.0 > ( other ) FU=1
     Ignore MarkInstall of extracoolstuff [ i386 ] < none -> 1.0 > ( other ) as its mode (Keep) is protected
diff --git a/test/integration/test-bug-590041-prefer-non-virtual-packages b/test/integration/test-bug-590041-prefer-non-virtual-packages
index 0ce4c1413..3bd7d436e 100755
--- a/test/integration/test-bug-590041-prefer-non-virtual-packages
+++ b/test/integration/test-bug-590041-prefer-non-virtual-packages
@@ -46,8 +46,8 @@ EOF
 setupaptarchive
 
 testshowvirtual libc6:i386
-testequal "$pkglibc6" aptcache show libc6:armel
-testequal "$pkglibc6" aptcache show libc6
-testequal "$pkglibdb1" aptcache show libdb1:i386
+testsuccessequal "$pkglibc6" aptcache show libc6:armel
+testsuccessequal "$pkglibc6" aptcache show libc6
+testsuccessequal "$pkglibdb1" aptcache show libdb1:i386
 testnopackage libdb1:armel
-testequal "$pkglibdb1" aptcache show libdb1
+testsuccessequal "$pkglibdb1" aptcache show libdb1
diff --git a/test/integration/test-bug-590438-broken-provides-thanks-to-remove-order b/test/integration/test-bug-590438-broken-provides-thanks-to-remove-order
index 645e86d7d..37426ec11 100755
--- a/test/integration/test-bug-590438-broken-provides-thanks-to-remove-order
+++ b/test/integration/test-bug-590438-broken-provides-thanks-to-remove-order
@@ -29,7 +29,7 @@ predependsgawk() {
 	echo "$pkgbasefile
 Pre-Depends: $1
 " >> rootdir/var/lib/dpkg/status
-	testequal "Inst gawk (1:3.1.7.dfsg-5 localhost [i386])
+	testsuccessequal "Inst gawk (1:3.1.7.dfsg-5 localhost [i386])
 Conf gawk (1:3.1.7.dfsg-5 localhost [i386])
 Remv mawk [1.3.3-15]" aptget install gawk mawk- -sqq -o PreDepends=$(echo "$1" | sed 's/ //g')
 }
diff --git a/test/integration/test-bug-591882-conkeror b/test/integration/test-bug-591882-conkeror
index 891ddb8b7..b71d4d5fd 100755
--- a/test/integration/test-bug-591882-conkeror
+++ b/test/integration/test-bug-591882-conkeror
@@ -73,5 +73,5 @@ After this operation, 36.0 MB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation."
 
 # Test that the old behavior can be restored with the option
-testequal "$UPGRADEFAIL" aptget dist-upgrade --trivial-only -o pkgProblemResolver::FixByInstall=0
-testequal "$UPGRADESUCCESS" aptget dist-upgrade --trivial-only #-o pkgProblemResolver::FixByInstall=0
+testfailureequal "$UPGRADEFAIL" aptget dist-upgrade --trivial-only -o pkgProblemResolver::FixByInstall=0
+testfailureequal "$UPGRADESUCCESS" aptget dist-upgrade --trivial-only #-o pkgProblemResolver::FixByInstall=0
diff --git a/test/integration/test-bug-593360-modifiers-in-names b/test/integration/test-bug-593360-modifiers-in-names
index 74826cbdb..57a24683e 100755
--- a/test/integration/test-bug-593360-modifiers-in-names
+++ b/test/integration/test-bug-593360-modifiers-in-names
@@ -7,7 +7,7 @@ setupenvironment
 configarchitecture "i386"
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   g++
@@ -15,12 +15,12 @@ The following NEW packages will be installed:
 Inst g++ (4:4.4.5-1 localhost [i386])
 Conf g++ (4:4.4.5-1 localhost [i386])' aptget install g++ -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'g++' is not installed, so not removed
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget remove g++ -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   g++
@@ -28,7 +28,7 @@ The following NEW packages will be installed:
 Inst g++ (4:4.4.5-1 localhost [i386])
 Conf g++ (4:4.4.5-1 localhost [i386])' aptget install g+++ -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   g++
@@ -36,7 +36,7 @@ The following NEW packages will be installed:
 Inst g++ (4:4.4.5-1 localhost [i386])
 Conf g++ (4:4.4.5-1 localhost [i386])' aptget purge g+++ -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   apt
@@ -44,7 +44,7 @@ The following NEW packages will be installed:
 Inst apt (0.8.8 localhost [all])
 Conf apt (0.8.8 localhost [all])' aptget install apt -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   apt+
@@ -52,7 +52,7 @@ The following NEW packages will be installed:
 Inst apt+ (0.8.8 localhost [all])
 Conf apt+ (0.8.8 localhost [all])' aptget install apt+ -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   apt+
@@ -60,7 +60,7 @@ The following NEW packages will be installed:
 Inst apt+ (0.8.8 localhost [all])
 Conf apt+ (0.8.8 localhost [all])' aptget install apt++ -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   apt+
diff --git a/test/integration/test-bug-596498-trusted-unsigned-repo b/test/integration/test-bug-596498-trusted-unsigned-repo
index 1e5e75b0e..4eb77b9a4 100755
--- a/test/integration/test-bug-596498-trusted-unsigned-repo
+++ b/test/integration/test-bug-596498-trusted-unsigned-repo
@@ -18,16 +18,16 @@ aptgetupdate() {
 PKGTEXT="$(aptget install cool --assume-no -d | head -n 7)"
 DEBFILE='rootdir/etc/apt/sources.list.d/apt-test-unstable-deb.list'
 
-testequal "$PKGTEXT
+testsuccessequal "$PKGTEXT
 Download complete and in download only mode" aptget install cool --assume-no -d
 
-testequal "$PKGTEXT
+testsuccessequal "$PKGTEXT
 Download complete and in download only mode" aptget install cool --assume-no -d --allow-unauthenticated
 
 sed -i -e 's#deb#deb [trusted=no]#' $DEBFILE
 aptgetupdate 'testsuccess'
 
-testequal "$PKGTEXT
+testfailureequal "$PKGTEXT
 WARNING: The following packages cannot be authenticated!
   cool
 Install these packages without verification? [y/N] N
@@ -37,13 +37,13 @@ find aptarchive/ \( -name 'Release.gpg' -o -name 'InRelease' \) -delete
 sed -i -e 's#deb \[trusted=no\]#deb#' $DEBFILE
 aptgetupdate
 
-testequal "$PKGTEXT
+testfailureequal "$PKGTEXT
 WARNING: The following packages cannot be authenticated!
   cool
 Install these packages without verification? [y/N] N
 E: Some packages could not be authenticated" aptget install cool --assume-no -d
 
-testequal "$PKGTEXT
+testsuccessequal "$PKGTEXT
 WARNING: The following packages cannot be authenticated!
   cool
 Authentication warning overridden.
@@ -52,5 +52,5 @@ Download complete and in download only mode" aptget install cool --assume-no -d
 sed -i -e 's#deb#deb [trusted=yes]#' $DEBFILE
 aptgetupdate
 
-testequal "$PKGTEXT
+testsuccessequal "$PKGTEXT
 Download complete and in download only mode" aptget install cool --assume-no -d
diff --git a/test/integration/test-bug-598669-install-postfix-gets-exim-heavy b/test/integration/test-bug-598669-install-postfix-gets-exim-heavy
index c3a77f346..2ec1d3d1c 100755
--- a/test/integration/test-bug-598669-install-postfix-gets-exim-heavy
+++ b/test/integration/test-bug-598669-install-postfix-gets-exim-heavy
@@ -7,7 +7,7 @@ setupenvironment
 configarchitecture "i386"
 setupaptarchive
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   exim4 exim4-daemon-light
diff --git a/test/integration/test-bug-601961-install-info b/test/integration/test-bug-601961-install-info
index 914910597..806d3f547 100755
--- a/test/integration/test-bug-601961-install-info
+++ b/test/integration/test-bug-601961-install-info
@@ -7,7 +7,7 @@ setupenvironment
 configarchitecture "i386"
 setupaptarchive
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   findutils
@@ -18,7 +18,7 @@ This should NOT be done unless you know exactly what you are doing!
 After this operation, 1745 kB disk space will be freed.
 E: Trivial Only specified but this is not a trivial operation.' aptget remove findutils --trivial-only
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   install-info
@@ -26,7 +26,7 @@ The following packages will be REMOVED:
 After this operation, 262 kB disk space will be freed.
 E: Trivial Only specified but this is not a trivial operation.' aptget remove install-info --trivial-only
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   essentialpkg findutils
diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect
index 6393f0c27..d3573a79a 100755
--- a/test/integration/test-bug-602412-dequote-redirect
+++ b/test/integration/test-bug-602412-dequote-redirect
@@ -20,7 +20,7 @@ testrun() {
 	testsuccess --nomsg aptget update
 
 	# check that I-M-S header is kept in redirections
-	testequal "Hit $1 unstable InRelease
+	testsuccessequal "Hit $1 unstable InRelease
 Hit $1 unstable/main Sources
 Hit $1 unstable/main amd64 Packages
 Hit $1 unstable/main Translation-en
diff --git a/test/integration/test-bug-604222-new-and-autoremove b/test/integration/test-bug-604222-new-and-autoremove
index b29347f64..52992680b 100755
--- a/test/integration/test-bug-604222-new-and-autoremove
+++ b/test/integration/test-bug-604222-new-and-autoremove
@@ -11,7 +11,7 @@ touch rootdir/var/lib/apt/extended_states
 testsuccess aptmark markauto 'libvtk5.4'
 testmarkedauto 'libvtk5.4'
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 The following package was automatically installed and is no longer required:
@@ -23,7 +23,7 @@ The following NEW packages will be installed:
 Inst libavcodec52 (4:0.5.2-6 localhost [i386])
 Conf libavcodec52 (4:0.5.2-6 localhost [i386])" aptget install libavcodec52 -s
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 The following package was automatically installed and is no longer required:
@@ -61,6 +61,6 @@ Need to get 0 B/6304 kB of archives.
 After this operation, 17.3 MB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation.'
 
-testequal "$CONFLICTING" aptget install dummy-archive --trivial-only -o Debug::pkgDepCache::Marker=1 -o APT::Get::HideAutoRemove=0
-testequal "$CONFLICTING" aptget install dummy-archive --trivial-only -o Debug::pkgDepCache::Marker=1 -o APT::Get::HideAutoRemove=1
-testequal "$CONFLICTING" aptget install dummy-archive --trivial-only -o Debug::pkgDepCache::Marker=1 -o APT::Get::HideAutoRemove=small
+testfailureequal "$CONFLICTING" aptget install dummy-archive --trivial-only -o Debug::pkgDepCache::Marker=1 -o APT::Get::HideAutoRemove=0
+testfailureequal "$CONFLICTING" aptget install dummy-archive --trivial-only -o Debug::pkgDepCache::Marker=1 -o APT::Get::HideAutoRemove=1
+testfailureequal "$CONFLICTING" aptget install dummy-archive --trivial-only -o Debug::pkgDepCache::Marker=1 -o APT::Get::HideAutoRemove=small
diff --git a/test/integration/test-bug-605394-versioned-or-groups b/test/integration/test-bug-605394-versioned-or-groups
index bb72d59e3..f938ba311 100755
--- a/test/integration/test-bug-605394-versioned-or-groups
+++ b/test/integration/test-bug-605394-versioned-or-groups
@@ -7,7 +7,7 @@ setupenvironment
 configarchitecture "i386"
 setupaptarchive
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be upgraded:
diff --git a/test/integration/test-bug-611729-mark-as-manual b/test/integration/test-bug-611729-mark-as-manual
index e3d454f97..a7bde393b 100755
--- a/test/integration/test-bug-611729-mark-as-manual
+++ b/test/integration/test-bug-611729-mark-as-manual
@@ -34,14 +34,14 @@ testdpkgnotinstalled a
 testdpkginstalled b c
 testmarkedauto 'b'
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 b is already the newest version.
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.' aptget install b --only-upgrade
 testmarkedauto 'b'
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 b is already the newest version.
@@ -59,7 +59,7 @@ sed -i rootdir/var/log/apt/history.log -e '/^Commandline: / d' -e '/^Start-Date:
 testfileequal 'rootdir/var/log/apt/history.log' '
 Reinstall: b:i386 (1.0)'
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 b is already the newest version.
diff --git a/test/integration/test-bug-612099-multiarch-conflicts b/test/integration/test-bug-612099-multiarch-conflicts
index c32600037..401b521a5 100755
--- a/test/integration/test-bug-612099-multiarch-conflicts
+++ b/test/integration/test-bug-612099-multiarch-conflicts
@@ -17,7 +17,7 @@ setupaptarchive
 
 testsuccess aptget install libc6:i386 -t stable -y
 testdpkginstalled libc6:i386
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
@@ -29,7 +29,7 @@ Remv libc6 [1.0]
 Inst libc6:amd64 (1.0 stable [amd64])
 Conf libc6:amd64 (1.0 stable [amd64])' aptget install libc6:amd64 -s -t stable
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following NEW packages will be installed:
@@ -38,7 +38,7 @@ The following NEW packages will be installed:
 Inst foobar (1.0 stable [i386])
 Conf foobar (1.0 stable [i386])' aptget install foobar -st stable
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following extra packages will be installed:
@@ -54,7 +54,7 @@ Inst foobar:amd64 (1.0 stable [amd64])
 Conf libc6:amd64 (1.0 stable [amd64])
 Conf foobar:amd64 (1.0 stable [amd64])' aptget install foobar:amd64 -st stable
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following NEW packages will be installed:
@@ -67,7 +67,7 @@ Inst foobar (1.0 stable [i386])
 Conf libc6 (2.0 testing [all])
 Conf foobar (1.0 stable [i386])' aptget install foobar/stable libc6 -st testing
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 Calculating upgrade...
@@ -84,7 +84,7 @@ testsuccess aptget purge libc6 -y
 testsuccess aptget install libc6:i386 -y
 testdpkginstalled libc6:all
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following NEW packages will be installed:
@@ -93,7 +93,7 @@ The following NEW packages will be installed:
 Inst foobar (1.0 stable [i386])
 Conf foobar (1.0 stable [i386])' aptget install foobar/stable -st testing
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following extra packages will be installed:
@@ -110,7 +110,7 @@ Conf libc6:amd64 (1.0 stable [amd64])
 Conf foobar:amd64 (1.0 stable [amd64])' aptget install foobar:amd64/stable -st testing
 
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 Selected version '1.0' (stable [i386]) for 'libc6'
@@ -133,7 +133,7 @@ setupaptarchive
 testsuccess aptget install libc6-same:i386 -t stable -y
 testdpkginstalled libc6-same:i386
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following NEW packages will be installed:
@@ -142,7 +142,7 @@ The following NEW packages will be installed:
 Inst foobar-same (1.0 stable [i386])
 Conf foobar-same (1.0 stable [i386])' aptget install foobar-same -st stable
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following extra packages will be installed:
@@ -155,7 +155,7 @@ Inst foobar-same:amd64 (1.0 stable [amd64])
 Conf libc6-same:amd64 (1.0 stable [amd64])
 Conf foobar-same:amd64 (1.0 stable [amd64])' aptget install foobar-same:amd64 -st stable
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following NEW packages will be installed:
@@ -166,7 +166,7 @@ Conf libc6-same:amd64 (1.0 stable [amd64])' aptget install libc6-same:amd64 -s -
 
 # FIXME: We should test installing libc6-same:amd64 here, but dpkg doesn't allow it currently
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 Calculating upgrade...
@@ -184,7 +184,7 @@ testsuccess aptget install libc6-same:i386 -y
 testdpkginstalled libc6-same:all
 
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 Selected version '1.0' (stable [i386]) for 'libc6-same'
@@ -194,7 +194,7 @@ The following packages will be DOWNGRADED:
 Inst libc6-same [2.0] (1.0 stable [i386])
 Conf libc6-same (1.0 stable [i386])" aptget install libc6-same/stable -s -q=0
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following NEW packages will be installed:
@@ -203,7 +203,7 @@ The following NEW packages will be installed:
 Inst foobar-same (1.0 stable [i386])
 Conf foobar-same (1.0 stable [i386])' aptget install foobar-same/stable -st testing
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following extra packages will be installed:
diff --git a/test/integration/test-bug-612557-garbage-upgrade b/test/integration/test-bug-612557-garbage-upgrade
index 910b3b149..552330d81 100755
--- a/test/integration/test-bug-612557-garbage-upgrade
+++ b/test/integration/test-bug-612557-garbage-upgrade
@@ -17,7 +17,7 @@ testsuccess aptmark markauto python-uno openoffice.org-common
 #aptmark unmarkauto openoffice.org-emailmerge
 testmarkedauto python-uno openoffice.org-common
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following extra packages will be installed:
@@ -35,7 +35,7 @@ E: Trivial Only specified but this is not a trivial operation.' aptget --trivial
 testsuccess aptmark markauto openoffice.org-emailmerge
 testmarkedauto python-uno openoffice.org-common openoffice.org-emailmerge
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following extra packages will be installed:
@@ -50,7 +50,7 @@ The following packages will be upgraded:
 After this operation, 53.2 MB disk space will be freed.
 E: Trivial Only specified but this is not a trivial operation.' aptget --trivial-only install python-uno
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
diff --git a/test/integration/test-bug-613420-new-garbage-dependency b/test/integration/test-bug-613420-new-garbage-dependency
index 9d9f1096a..5839f8798 100755
--- a/test/integration/test-bug-613420-new-garbage-dependency
+++ b/test/integration/test-bug-613420-new-garbage-dependency
@@ -18,7 +18,7 @@ touch rootdir/var/lib/apt/extended_states
 testsuccess aptmark markauto openoffice.org-officebean
 testmarkedauto openoffice.org-officebean
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages were automatically installed and are no longer required:
diff --git a/test/integration/test-bug-618848-always-respect-user-requests b/test/integration/test-bug-618848-always-respect-user-requests
index 1ebadf280..a7ffee6c1 100755
--- a/test/integration/test-bug-618848-always-respect-user-requests
+++ b/test/integration/test-bug-618848-always-respect-user-requests
@@ -13,7 +13,7 @@ insertpackage 'unstable' 'exim4-daemon-heavy' 'all' '1.0' 'Depends: libdb4.8'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
   MarkDelete libdb4.8 [ i386 ] < 1.0 > ( other ) FU=1
   MarkDelete exim4-daemon-light [ i386 ] < 1.0 > ( other ) FU=0
diff --git a/test/integration/test-bug-624218-Translation-file-handling b/test/integration/test-bug-624218-Translation-file-handling
index 652386892..3987abff1 100755
--- a/test/integration/test-bug-624218-Translation-file-handling
+++ b/test/integration/test-bug-624218-Translation-file-handling
@@ -65,29 +65,27 @@ configallowinsecurerepositories "true";
 
 msgtest 'Download of en as forced language' 'without Index'
 testwarning --nomsg aptget update -o Acquire::Languages=en
-cp rootdir/tmp/testsuccess.output testsuccess.output
-testsuccess grep -q -e 'Translation-en ' testsuccess.output
+testsuccess grep -q -e 'Translation-en ' rootdir/tmp/testwarning.output
 rm -rf rootdir/var/lib/apt/lists
 
 msgtest 'Download of nothing else in forced language' 'without Index'
 testwarning --nomsg aptget update -o Acquire::Languages=en
-testfailure grep -q -e 'Translation-[^e][^n] ' rootdir/tmp/testsuccess.output
+testfailure grep -q -e 'Translation-[^e][^n] ' rootdir/tmp/testwarning.output
 rm -rf rootdir/var/lib/apt/lists
 
 msgtest 'Download of ast_DE as forced language' 'without Index'
 testwarning --nomsg aptget update -o Acquire::Languages=ast_DE
-cp rootdir/tmp/testsuccess.output testsuccess.output
-testsuccess grep -q -e 'Translation-ast_DE$' testsuccess.output
+testsuccess grep -q -e 'Translation-ast_DE$' rootdir/tmp/testwarning.output
 rm -rf rootdir/var/lib/apt/lists
 
 msgtest 'Download of nothing else in forced language' 'without Index'
 testwarning --nomsg aptget update -o Acquire::Languages=ast_DE
-testfailure grep -q -e 'Translation-[^a][^s]' rootdir/tmp/testsuccess.output
+testfailure grep -q -e 'Translation-[^a][^s]' rootdir/tmp/testwarning.output
 rm -rf rootdir/var/lib/apt/lists
 
 msgtest 'Download of nothing if none is forced' 'without Index'
 testwarning --nomsg aptget update -o Acquire::Languages=none
-testfailure grep -q -e 'Translation' rootdir/tmp/testsuccess.output
+testfailure grep -q -e 'Translation' rootdir/tmp/testwarning.output
 rm -rf rootdir/var/lib/apt/lists
 
 mkdir -p rootdir/var/lib/apt/lists
@@ -95,8 +93,7 @@ touch rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_i18n_Translat
 
 msgtest 'Download of builtin files' 'without Index'
 testwarning --nomsg aptget update
-cp rootdir/tmp/testsuccess.output testsuccess.output
-testsuccess grep -q -e 'Translation-ast_DE' testsuccess.output
+testsuccess grep -q -e 'Translation-ast_DE' rootdir/tmp/testwarning.output
 rm -rf rootdir/var/lib/apt/lists
 
 mkdir -p rootdir/var/lib/apt/lists
@@ -104,5 +101,5 @@ touch rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_i18n_Translat
 
 msgtest 'Download of nothing (even builtin) if none is forced' 'without Index'
 testwarning --nomsg aptget update -o Acquire::Languages=none
-testfailure grep -q -e 'Translation' rootdir/tmp/testsuccess.output
+testfailure grep -q -e 'Translation' rootdir/tmp/testwarning.output
 rm -rf rootdir/var/lib/apt/lists
diff --git a/test/integration/test-bug-632221-cross-dependency-satisfaction b/test/integration/test-bug-632221-cross-dependency-satisfaction
index d90a103c9..563821173 100755
--- a/test/integration/test-bug-632221-cross-dependency-satisfaction
+++ b/test/integration/test-bug-632221-cross-dependency-satisfaction
@@ -35,17 +35,17 @@ insertsource 'unstable' 'source-specific-armel' 'armel' '1' 'Build-Depends: spec
 
 setupaptarchive
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 E: Build-Depends dependency for forbidden-none can't be satisfied because amdboot:any is not allowed on 'Multi-Arch: none' packages" aptget build-dep forbidden-none -s -a armel
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 E: Build-Depends dependency for forbidden-same can't be satisfied because libc6:any is not allowed on 'Multi-Arch: same' packages" aptget build-dep forbidden-same -s -a armel
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 E: Build-Depends dependency for forbidden-foreign can't be satisfied because doxygen:any is not allowed on 'Multi-Arch: foreign' packages" aptget build-dep forbidden-foreign -s -a armel
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libc6 specific
@@ -54,7 +54,7 @@ Inst libc6 (1.0 unstable [amd64])
 Inst specific (1.0 unstable [amd64])
 Conf libc6 (1.0 unstable [amd64])
 Conf specific (1.0 unstable [amd64])' aptget build-dep source-specific-amd64 -s
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libc6 specific
@@ -64,7 +64,7 @@ Inst specific (1.0 unstable [amd64])
 Conf libc6 (1.0 unstable [amd64])
 Conf specific (1.0 unstable [amd64])' aptget build-dep source-specific-amd64 -s -a armel
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libc6:armel specific:armel
@@ -73,7 +73,7 @@ Inst libc6:armel (1.0 unstable [armel])
 Inst specific:armel (1.0 unstable [armel])
 Conf libc6:armel (1.0 unstable [armel])
 Conf specific:armel (1.0 unstable [armel])' aptget build-dep source-specific-armel -s
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libc6:armel specific:armel
@@ -83,7 +83,7 @@ Inst specific:armel (1.0 unstable [armel])
 Conf libc6:armel (1.0 unstable [armel])
 Conf specific:armel (1.0 unstable [armel])' aptget build-dep source-specific-armel -s -a armel
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   amdboot cool doxygen foreigner libc6 libc6-dev libfwibble-dev libfwibble1
@@ -108,7 +108,7 @@ Conf libfwibble1 (1.0 unstable [amd64])
 Conf libfwibble-dev (1.0 unstable [amd64])
 Conf linux-stuff (1.0 unstable [amd64])' aptget build-dep apt -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   amdboot arm-stuff:armel cool doxygen foreigner libc6 libc6:armel libc6-dev
@@ -139,7 +139,7 @@ Conf libfwibble-dev:armel (1.0 unstable [armel])' aptget build-dep apt -s -a arm
 
 configarchitecture 'armel' 'amd64'
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   amdboot:amd64 arm-stuff cool doxygen foreigner libc6 libc6-dev
@@ -164,7 +164,7 @@ Conf libc6-dev (1.0 unstable [armel])
 Conf libfwibble1 (1.0 unstable [armel])
 Conf libfwibble-dev (1.0 unstable [armel])' aptget build-dep apt -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   amdboot:amd64 cool doxygen foreigner libc6:amd64 libc6 libc6-dev:amd64
@@ -198,7 +198,7 @@ configarchitecture 'amd64' 'armel'
 insertinstalledpackage 'cool' 'amd64' '0.5'
 insertinstalledpackage 'foreigner' 'armel' '0.5'
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   amdboot doxygen libc6 libc6-dev libfwibble-dev libfwibble1 linux-stuff
@@ -218,7 +218,7 @@ Conf libfwibble1 (1.0 unstable [amd64])
 Conf libfwibble-dev (1.0 unstable [amd64])
 Conf linux-stuff (1.0 unstable [amd64])' aptget build-dep apt -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   amdboot arm-stuff:armel doxygen libc6 libc6:armel libc6-dev libc6-dev:armel
@@ -246,7 +246,7 @@ Conf libfwibble-dev:armel (1.0 unstable [armel])' aptget build-dep apt -s -a arm
 configarchitecture 'armel' 'amd64'
 
 # cool 0.5 is not M-A: allowed, so amd64 is not acceptable
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   cool:amd64
@@ -272,7 +272,7 @@ Conf libc6-dev (1.0 unstable [armel])
 Conf libfwibble1 (1.0 unstable [armel])
 Conf libfwibble-dev (1.0 unstable [armel])' aptget build-dep apt -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   amdboot:amd64 doxygen libc6:amd64 libc6 libc6-dev:amd64 libc6-dev
diff --git a/test/integration/test-bug-64141-install-dependencies-for-on-hold b/test/integration/test-bug-64141-install-dependencies-for-on-hold
index 9e6c223a8..ff8fa4523 100755
--- a/test/integration/test-bug-64141-install-dependencies-for-on-hold
+++ b/test/integration/test-bug-64141-install-dependencies-for-on-hold
@@ -19,7 +19,7 @@ insertpackage 'unstable' 'libdb4.8' 'native' '4.8.30-3'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
@@ -34,7 +34,7 @@ E: Trivial Only specified but this is not a trivial operation.' aptget dist-upgr
 
 testsuccess aptmark hold apt
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages have been kept back:
diff --git a/test/integration/test-bug-657695-resolver-breaks-on-virtuals b/test/integration/test-bug-657695-resolver-breaks-on-virtuals
index 1b92a04fe..14c90b3b9 100755
--- a/test/integration/test-bug-657695-resolver-breaks-on-virtuals
+++ b/test/integration/test-bug-657695-resolver-breaks-on-virtuals
@@ -16,7 +16,7 @@ insertpackage 'unstable' 'xserver-xorg-core' 'amd64' '2:1.11.3-0ubuntu9' 'Breaks
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
diff --git a/test/integration/test-bug-661537-build-profiles-support b/test/integration/test-bug-661537-build-profiles-support
index 6c850fdf9..7e7a74b03 100755
--- a/test/integration/test-bug-661537-build-profiles-support
+++ b/test/integration/test-bug-661537-build-profiles-support
@@ -24,7 +24,7 @@ insertsource 'unstable' 'spec-7' 'any' '1' 'Build-Depends: foo  = 1)'
 insertinstalledpackage 'pkgz' 'none' '1'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
@@ -111,7 +111,7 @@ The following packages will be REMOVED:
 0 upgraded, 0 newly installed, 2 to remove and 0 not upgraded.
 Purg pkgy:none [1]
 Purg pkgx:none [1]' aptget purge pkgx -s
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
diff --git a/test/integration/test-bug-689582-100-char-long-path-names b/test/integration/test-bug-689582-100-char-long-path-names
index 1b4b172b6..58ece1d5a 100755
--- a/test/integration/test-bug-689582-100-char-long-path-names
+++ b/test/integration/test-bug-689582-100-char-long-path-names
@@ -28,7 +28,7 @@ ar cr ../testpkg.deb debian-binary control.tar.gz data.tar.gz
 
 cd - > /dev/null
 
-testequal '000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000102 testpkg
+testsuccessequal '000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000102 testpkg
 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000101 testpkg
 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100 testpkg
 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000099 testpkg
diff --git a/test/integration/test-bug-691453-apt-cache-search-multi-pattern b/test/integration/test-bug-691453-apt-cache-search-multi-pattern
index 0367892fc..15586b726 100755
--- a/test/integration/test-bug-691453-apt-cache-search-multi-pattern
+++ b/test/integration/test-bug-691453-apt-cache-search-multi-pattern
@@ -21,13 +21,13 @@ foo - tool best used with bar
 bar - tool best used with foo
 baz - alternative tool best used with foo'
 
-testequal "$FOOBAR" aptcache search foo
-testequal "$FOOBAR" aptcache search bar
-testequal "$FOOBAR" aptcache search foo bar
+testsuccessequal "$FOOBAR" aptcache search foo
+testsuccessequal "$FOOBAR" aptcache search bar
+testsuccessequal "$FOOBAR" aptcache search foo bar
 
-testequal 'foobar - funky tool
+testsuccessequal 'foobar - funky tool
 foo - tool best used with bar' aptcache search -n foo
-testequal 'foobar - funky tool
+testsuccessequal 'foobar - funky tool
 bar - tool best used with foo
 baz - alternative tool best used with foo' aptcache search -n bar
-testequal 'foobar - funky tool' aptcache search -n foo bar
+testsuccessequal 'foobar - funky tool' aptcache search -n foo bar
diff --git a/test/integration/test-bug-709560-set-candidate-release b/test/integration/test-bug-709560-set-candidate-release
index 48dc5c382..ab41d8f2a 100755
--- a/test/integration/test-bug-709560-set-candidate-release
+++ b/test/integration/test-bug-709560-set-candidate-release
@@ -21,7 +21,7 @@ EOF
 
 setupaptarchive
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Selected version '2.0' (experimental [all]) for 'foo'
 Selected version '2.1' (experimental [all]) for 'foo-dep' because of 'foo'
diff --git a/test/integration/test-bug-712435-missing-descriptions b/test/integration/test-bug-712435-missing-descriptions
index 7a3518745..726134326 100755
--- a/test/integration/test-bug-712435-missing-descriptions
+++ b/test/integration/test-bug-712435-missing-descriptions
@@ -81,43 +81,43 @@ Description-md5: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" > aptarchive/Packages
 
 setupaptarchive
 
-testequal "Package: apt-normal
+testsuccessequal "Package: apt-normal
 $PACKAGESTANZA
 $DESCRIPTION
 Description-md5: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 " aptcache show apt-normal
 
 for variant in 'below' 'middle' 'top'; do
-	testequal "Package: apt-both-$variant
+	testsuccessequal "Package: apt-both-$variant
 $PACKAGESTANZA
 $TRANSDESCRIPTION
 Description-md5: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
 " aptcache show apt-both-$variant
 done
 
-testequal "Package: apt-trans
+testsuccessequal "Package: apt-trans
 $PACKAGESTANZA
 $TRANSDESCRIPTION
 Description-md5: cccccccccccccccccccccccccccccccc
 " aptcache show apt-trans
 
-testequal "Package: apt-md5
+testsuccessequal "Package: apt-md5
 $PACKAGESTANZA
 Description-md5: dddddddddddddddddddddddddddddddd
 " aptcache show apt-md5
 
-testequal "Package: apt-none
+testsuccessequal "Package: apt-none
 $PACKAGESTANZA
 " aptcache show apt-none
 
-testequal "Package: apt-intermixed
+testsuccessequal "Package: apt-intermixed
 $PACKAGESTANZA
 $DESCRIPTION
 Description-md5: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
 X-Some-Flag: yes
 " aptcache show apt-intermixed
 
-testequal "Package: apt-intermixed2
+testsuccessequal "Package: apt-intermixed2
 $PACKAGESTANZA
 $TRANSDESCRIPTION
 Description-md5: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
@@ -126,7 +126,7 @@ X-Foo-Flag: Something with a Description
 X-Bar-Flag: no
 " aptcache show apt-intermixed2
 
-testequal "Package: apt-intermixed3
+testsuccessequal "Package: apt-intermixed3
 $PACKAGESTANZA
 $TRANSDESCRIPTION
 Description-md5: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
@@ -152,7 +152,7 @@ Reverse Provides: '
 testempty aptcache search nonexistentstring
 
 # packages without a description can't be found
-testequal 'apt-normal - commandline package manager
+testsuccessequal 'apt-normal - commandline package manager
 apt-both-below - commandline package manager
 apt-both-middle - commandline package manager
 apt-both-top - commandline package manager
diff --git a/test/integration/test-bug-717891-abolute-uris-for-proxies b/test/integration/test-bug-717891-abolute-uris-for-proxies
index 54a616686..ef948c2d5 100755
--- a/test/integration/test-bug-717891-abolute-uris-for-proxies
+++ b/test/integration/test-bug-717891-abolute-uris-for-proxies
@@ -19,7 +19,7 @@ echo 'Acquire::http::Proxy "http://localhost:8080";' > rootdir/etc/apt/apt.conf.
 msgtest 'Check that requests to proxies are' 'absolute uris'
 testsuccess --nomsg aptget update
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   unrelated
diff --git a/test/integration/test-bug-718329-support-data.tar-uncompressed b/test/integration/test-bug-718329-support-data.tar-uncompressed
new file mode 100755
index 000000000..d2845f768
--- /dev/null
+++ b/test/integration/test-bug-718329-support-data.tar-uncompressed
@@ -0,0 +1,31 @@
+#!/bin/sh
+set -e
+
+test_process_package_with_compression() {
+    COMPRESSOR="$1"
+    DATA_TAR="$2"
+
+    msgmsg "Testing apt-ftparchive with compression type: $COMPRESSOR"
+
+    buildsimplenativepackage 'pkg' 'all' '1.0' '' '' 'some descr' '' '' '' "$COMPRESSOR"
+    testsuccessequal "debian-binary
+control.tar.gz
+$DATA_TAR" ar t incoming/pkg_1.0_all.deb
+
+    testequal "Package: pkg" echo "$(aptftparchive packages incoming/|grep ^Package)"
+
+    testsuccessequal "usr/bin/pkg-all						    pkg
+usr/share/doc/pkg/FEATURES				    pkg
+usr/share/doc/pkg/changelog				    pkg
+usr/share/doc/pkg/copyright				    pkg" aptftparchive contents incoming/
+
+    rm -rf incoming/*
+}
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+test_process_package_with_compression "gzip" "data.tar.gz"
+test_process_package_with_compression "none" "data.tar"
+test_process_package_with_compression "xz" "data.tar.xz"
diff --git a/test/integration/test-bug-719263-print-uris-removes-authentication b/test/integration/test-bug-719263-print-uris-removes-authentication
index 5e674db0b..207bf4611 100755
--- a/test/integration/test-bug-719263-print-uris-removes-authentication
+++ b/test/integration/test-bug-719263-print-uris-removes-authentication
@@ -15,7 +15,7 @@ setupaptarchive
 testnoact() {
 	cp -a rootdir/var/lib/dpkg/status rootdir/var/lib/dpkg/status-backup-noact
 	touch rootdir/var/lib/apt/extended_states
-	testequal 'Reading package lists...
+	testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be upgraded:
diff --git a/test/integration/test-bug-720597-build-dep-purge b/test/integration/test-bug-720597-build-dep-purge
index 1e24ed5f1..6fa261fbd 100755
--- a/test/integration/test-bug-720597-build-dep-purge
+++ b/test/integration/test-bug-720597-build-dep-purge
@@ -13,7 +13,7 @@ buildsimplenativepackage 'pkgc' 'amd64' '1' 'stable' 'Build-Depends: pkgb'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   pkga
@@ -24,7 +24,7 @@ Remv pkga [1]
 Inst pkgb (1 stable [amd64])
 Conf pkgb (1 stable [amd64])' aptget build-dep pkgc -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   pkga*
diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
index 9a5685703..2cad929cc 100755
--- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet
+++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
@@ -16,15 +16,15 @@ setupaptarchive
 
 APTARCHIVE=$(readlink -f ./aptarchive)
 
-testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget upgrade -qq --print-uris
-testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris
-testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris
-testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris
-testequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
+testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget upgrade -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris
-testequal "'http://packages.debian.org/changelogs/pool/main/apt/apt_2/changelog'" aptget changelog apt -qq --print-uris
+testsuccessequal "'http://packages.debian.org/changelogs/pool/main/apt/apt_2/changelog'" aptget changelog apt -qq --print-uris
 
-testequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
 'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
 'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt apt2 -qq --print-uris
diff --git a/test/integration/test-bug-723586-any-stripped-in-single-arch b/test/integration/test-bug-723586-any-stripped-in-single-arch
index 392b88e9f..0cf3362cf 100755
--- a/test/integration/test-bug-723586-any-stripped-in-single-arch
+++ b/test/integration/test-bug-723586-any-stripped-in-single-arch
@@ -41,14 +41,14 @@ The following packages have unmet dependencies:
  python-mips : Depends: python3:mips but it is not installable
 E: Unable to correct problems, you have held broken packages.'
 
-testequal "$INSTALLLOG" aptget install python3-gnupg -s
+testsuccessequal "$INSTALLLOG" aptget install python3-gnupg -s
 aptcache showpkg python3 > showpkg.log
-testequal "$FAILLOG" aptget install python-mips -s
+testfailureequal "$FAILLOG" aptget install python-mips -s
 
 # same test, but this time in a multi-arch environment
 configarchitecture 'amd64' 'armhf'
 rm rootdir/var/cache/apt/*.bin
 
-testequal "$INSTALLLOG" aptget install python3-gnupg -s
-testequal "$(sed 's#3.3.2-16 - python3#3.3.2-16 - python3:any:armhf python3#' showpkg.log)" aptcache showpkg python3
-testequal "$FAILLOG" aptget install python-mips -s
+testsuccessequal "$INSTALLLOG" aptget install python3-gnupg -s
+testsuccessequal "$(sed 's#3.3.2-16 - python3#3.3.2-16 - python3:any:armhf python3#' showpkg.log)" aptcache showpkg python3
+testfailureequal "$FAILLOG" aptget install python-mips -s
diff --git a/test/integration/test-bug-728500-tempdir b/test/integration/test-bug-728500-tempdir
index 37e5a013e..3ae94c58f 100755
--- a/test/integration/test-bug-728500-tempdir
+++ b/test/integration/test-bug-728500-tempdir
@@ -26,5 +26,5 @@ else
 fi
 unset TMPDIR
 
-testequal 'coolstuff' aptcache pkgnames
+testsuccessequal 'coolstuff' aptcache pkgnames
 testsuccess ls rootdir/var/lib/apt/lists/*InRelease
diff --git a/test/integration/test-bug-732746-preferences b/test/integration/test-bug-732746-preferences
index b31f98aa0..ce73f1c17 100755
--- a/test/integration/test-bug-732746-preferences
+++ b/test/integration/test-bug-732746-preferences
@@ -25,7 +25,7 @@ Pin-Priority: 700
 #Pin: 800
 EOF
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree..." aptget check
 
 msgtest "Ensure policy is applied"
diff --git a/test/integration/test-bug-735967-lib32-to-i386-unavailable b/test/integration/test-bug-735967-lib32-to-i386-unavailable
index 826931fe4..eb6e1a331 100755
--- a/test/integration/test-bug-735967-lib32-to-i386-unavailable
+++ b/test/integration/test-bug-735967-lib32-to-i386-unavailable
@@ -31,7 +31,7 @@ setupaptarchive --no-update
 configarchitecture 'amd64'
 testsuccess aptget update
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
@@ -43,7 +43,7 @@ Remv lib32nss-mdns [0.9-1]
 Inst libnss-mdns [0.9-1] (0.10-6 unstable [amd64])
 Conf libnss-mdns (0.10-6 unstable [amd64])' aptget dist-upgrade -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -59,7 +59,7 @@ E: Unable to correct problems, you have held broken packages.' aptget install fo
 configarchitecture 'amd64' 'i386'
 testsuccess aptget update
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following NEW packages will be installed:
@@ -76,7 +76,7 @@ Conf libnss-mdns (0.10-6 unstable [amd64])
 Conf libnss-mdns-i386:i386 (0.10-6 unstable [i386])
 Conf lib32nss-mdns (0.10-6 unstable [amd64])' aptget dist-upgrade -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libfoo libfoo-bin:i386
diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index 0f458e099..f6336ffe3 100755
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -23,7 +23,7 @@ msgtest 'Test that the webserver does not answer' 'http requests'
 downloadfile 'http://localhost:8080/pool/apt_1.0/changelog' changelog >/dev/null 2>&1 && msgfail || msgpass
 
 echo 'Apt::Changelogs::Server "http://localhost:8080/redirectme";' > rootdir/etc/apt/apt.conf.d/changelog.conf
-testequal "'http://localhost:8080/redirectme/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
+testsuccessequal "'http://localhost:8080/redirectme/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
 
 cd downloaded
 testsuccess aptget changelog apt -d
@@ -52,7 +52,7 @@ cd - >/dev/null
 echo "Dir::Bin::Methods \"${COPYMETHODS}\";" >> aptconfig.conf
 
 cd downloaded
-testequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.
+testfailureequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.
 N: Is the package apt-transport-https installed?" aptget download apt -q=0
 testfailure test -e apt_1.0_all.deb
 cd - >/dev/null
diff --git a/test/integration/test-bug-745036-new-foreign-invalidates-cache b/test/integration/test-bug-745036-new-foreign-invalidates-cache
index 2b7ee06ad..bfa0f817c 100755
--- a/test/integration/test-bug-745036-new-foreign-invalidates-cache
+++ b/test/integration/test-bug-745036-new-foreign-invalidates-cache
@@ -14,7 +14,7 @@ setupaptarchive
 testsuccess aptget check -s
 
 configarchitecture 'amd64' 'i386'
-testequal 'E: The package cache was built for different architectures: amd64 vs amd64,i386' aptget check -s -o pkgCacheFile::Generate=false
+testfailureequal 'E: The package cache was built for different architectures: amd64 vs amd64,i386' aptget check -s -o pkgCacheFile::Generate=false
 
 testsuccess aptget check -s
 
diff --git a/test/integration/test-bug-745046-candidate-propagation-fails b/test/integration/test-bug-745046-candidate-propagation-fails
index e4aa67a72..f54e1872e 100755
--- a/test/integration/test-bug-745046-candidate-propagation-fails
+++ b/test/integration/test-bug-745046-candidate-propagation-fails
@@ -14,7 +14,7 @@ insertpackage 'experimental' 'gedit' 'amd64' '2' 'Depends: common (>= 2)'
 
 setupaptarchive
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2' (experimental [amd64]) for 'gedit'
 Some packages could not be installed. This may mean that you have
@@ -29,7 +29,7 @@ E: Unable to correct problems, you have held broken packages." aptget install ge
 
 insertinstalledpackage 'common' 'amd64' '2'
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Selected version '2' (experimental [amd64]) for 'gedit'
 The following packages will be upgraded:
diff --git a/test/integration/test-bug-753297-upgradable b/test/integration/test-bug-753297-upgradable
index 01395a095..53bf3361b 100755
--- a/test/integration/test-bug-753297-upgradable
+++ b/test/integration/test-bug-753297-upgradable
@@ -25,9 +25,9 @@ insertpackage 'testing' 'bar' 'all' '2'
 
 setupaptarchive
 
-testequal "Listing...
+testsuccessequal "Listing...
 bar/testing 2 all [upgradable from: 1]" apt list --upgradable
 
-testequal "Listing...
+testsuccessequal "Listing...
 bar/testing 2 all [upgradable from: 1]
 foo/testing,now 1 all [installed]" apt list
diff --git a/test/integration/test-bug-758153-versioned-provides-support b/test/integration/test-bug-758153-versioned-provides-support
index 21f9123c9..30bc921c3 100755
--- a/test/integration/test-bug-758153-versioned-provides-support
+++ b/test/integration/test-bug-758153-versioned-provides-support
@@ -24,9 +24,15 @@ insertpackage 'experimental' 'foreign-webserver' 'i386' '4' 'Multi-Arch: foreign
 Provides: httpd (= 4)'
 insertpackage 'experimental' 'cool-webapp' 'all' '4' 'Depends: httpd (>= 4)'
 
+# arch-qualified provides, see #777071
+insertpackage 'unstable' 'foo' 'all' '1' 'Provides: bar:i386'
+insertpackage 'unstable' 'baz' 'i386,amd64' '1' 'Depends: bar'
+insertpackage 'experimental' 'baz' 'i386,amd64' '2' 'Depends: bar:i386'
+insertpackage 'experimental' 'baz-broken' 'i386' '2' 'Depends: bar:amd64'
+
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be upgraded:
@@ -37,7 +43,7 @@ Inst webapp [1] (2 unstable [all])
 Conf webserver (2 unstable [amd64])
 Conf webapp (2 unstable [all])' aptget dist-upgrade -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be upgraded:
   webapp webserver
@@ -47,7 +53,7 @@ Inst webapp [1] (2 unstable [all])
 Conf webserver (2 unstable [amd64])
 Conf webapp (2 unstable [all])' aptget install webapp webserver -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be upgraded:
   webapp webserver
@@ -57,7 +63,7 @@ Inst webapp [1] (3 experimental [all])
 Conf webserver (2 unstable [amd64])
 Conf webapp (3 experimental [all])' aptget install webapp=3 webserver -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be upgraded:
   webapp webserver
@@ -67,7 +73,7 @@ Inst webapp [1] (2 unstable [all])
 Conf webserver (3 experimental [amd64])
 Conf webapp (2 unstable [all])' aptget install webapp webserver=3 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foreign-webserver:i386
@@ -79,7 +85,7 @@ Inst webapp [1] (2 unstable [all])
 Conf foreign-webserver:i386 (2 unstable [i386])
 Conf webapp (2 unstable [all])' aptget install webapp foreign-webserver:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foreign-webserver:i386
@@ -91,7 +97,7 @@ Inst webapp [1] (3 experimental [all])
 Conf foreign-webserver:i386 (2 unstable [i386])
 Conf webapp (3 experimental [all])' aptget install webapp=3 foreign-webserver:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foreign-webserver:i386
@@ -103,7 +109,7 @@ Inst webapp [1] (2 unstable [all])
 Conf foreign-webserver:i386 (4 experimental [i386])
 Conf webapp (2 unstable [all])' aptget install webapp foreign-webserver:i386=4 -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -115,7 +121,7 @@ The following packages have unmet dependencies:
  cool-webapp : Depends: httpd (>= 4)
 E: Unable to correct problems, you have held broken packages.' aptget install cool-webapp -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -127,7 +133,7 @@ The following packages have unmet dependencies:
  cool-webapp : Depends: httpd (>= 4)
 E: Unable to correct problems, you have held broken packages.' aptget install cool-webapp foreign-webserver:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   cool-webapp foreign-webserver:i386
@@ -136,3 +142,65 @@ Inst foreign-webserver:i386 (4 experimental [i386])
 Inst cool-webapp (4 experimental [all])
 Conf foreign-webserver:i386 (4 experimental [i386])
 Conf cool-webapp (4 experimental [all])' aptget install cool-webapp foreign-webserver:i386=4 -s
+
+testsuccessequal 'Reading package lists...
+Building dependency tree...
+The following extra packages will be installed:
+  foo
+The following NEW packages will be installed:
+  baz:i386 foo
+0 upgraded, 2 newly installed, 0 to remove and 2 not upgraded.
+Inst foo (1 unstable [all])
+Inst baz:i386 (1 unstable [i386])
+Conf foo (1 unstable [all])
+Conf baz:i386 (1 unstable [i386])' aptget install baz:i386 -s
+
+testfailureequal 'Reading package lists...
+Building dependency tree...
+Some packages could not be installed. This may mean that you have
+requested an impossible situation or if you are using the unstable
+distribution that some required packages have not yet been created
+or been moved out of Incoming.
+The following information may help to resolve the situation:
+
+The following packages have unmet dependencies:
+ baz : Depends: bar but it is not installable
+E: Unable to correct problems, you have held broken packages.' aptget install baz:amd64 -s
+
+testsuccessequal "Reading package lists...
+Building dependency tree...
+Selected version '2' (experimental [amd64]) for 'baz'
+The following extra packages will be installed:
+  foo
+The following NEW packages will be installed:
+  baz foo
+0 upgraded, 2 newly installed, 0 to remove and 2 not upgraded.
+Inst foo (1 unstable [all])
+Inst baz (2 experimental [amd64])
+Conf foo (1 unstable [all])
+Conf baz (2 experimental [amd64])" aptget install baz/experimental -s -q=0
+
+testsuccessequal "Reading package lists...
+Building dependency tree...
+Selected version '2' (experimental [i386]) for 'baz:i386'
+The following extra packages will be installed:
+  foo
+The following NEW packages will be installed:
+  baz:i386 foo
+0 upgraded, 2 newly installed, 0 to remove and 2 not upgraded.
+Inst foo (1 unstable [all])
+Inst baz:i386 (2 experimental [i386])
+Conf foo (1 unstable [all])
+Conf baz:i386 (2 experimental [i386])" aptget install baz:i386/experimental -s -q=0
+
+testfailureequal 'Reading package lists...
+Building dependency tree...
+Some packages could not be installed. This may mean that you have
+requested an impossible situation or if you are using the unstable
+distribution that some required packages have not yet been created
+or been moved out of Incoming.
+The following information may help to resolve the situation:
+
+The following packages have unmet dependencies:
+ baz-broken:i386 : Depends: bar but it is not installable
+E: Unable to correct problems, you have held broken packages.' aptget install baz-broken -s
diff --git a/test/integration/test-bug-770291-reinstall b/test/integration/test-bug-770291-reinstall
index ea1f57ede..a5b2aff07 100755
--- a/test/integration/test-bug-770291-reinstall
+++ b/test/integration/test-bug-770291-reinstall
@@ -39,7 +39,7 @@ Essential: yes'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 0 upgraded, 0 newly installed, 2 reinstalled, 0 to remove and 0 not upgraded.
 Inst systemd [215-5+b1] (215-5+b1 unstable [i386])
@@ -47,7 +47,7 @@ Conf systemd (215-5+b1 unstable [i386])
 Inst systemd-sysv [215-5+b1] (215-5+b1 unstable [i386])
 Conf systemd-sysv (215-5+b1 unstable [i386])' aptget install --reinstall systemd systemd-sysv -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 0 upgraded, 0 newly installed, 2 reinstalled, 0 to remove and 0 not upgraded.
 Inst dependsA [1] (1 unstable [i386])
@@ -68,7 +68,7 @@ testequal "E: Could not configure 'predependsdependsB:i386'. " tail -n1 rootdir/
 msgmsg 'While we are at it, lets try these loops without reinstall as well'
 cp dpkg.status rootdir/var/lib/dpkg/status
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   systemd systemd-sysv
@@ -78,7 +78,7 @@ Conf systemd (215-5+b1 unstable [i386])
 Inst systemd-sysv (215-5+b1 unstable [i386])
 Conf systemd-sysv (215-5+b1 unstable [i386])' aptget install systemd systemd-sysv -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   dependsA dependsB
diff --git a/test/integration/test-bug-multiarch-upgrade b/test/integration/test-bug-multiarch-upgrade
index c29e1f903..56071f184 100755
--- a/test/integration/test-bug-multiarch-upgrade
+++ b/test/integration/test-bug-multiarch-upgrade
@@ -16,7 +16,7 @@ insertpackage 'unstable' 'libcups2' 'i386' '2' 'Multi-Arch: same'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libcups2
diff --git a/test/integration/test-cachecontainer-architecture-specification b/test/integration/test-cachecontainer-architecture-specification
index 47abfb5b0..e5625e811 100755
--- a/test/integration/test-cachecontainer-architecture-specification
+++ b/test/integration/test-cachecontainer-architecture-specification
@@ -12,31 +12,31 @@ insertpackage 'unstable' 'foo' 'all' '1'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst libsame (1 unstable [amd64])
 Conf libsame (1 unstable [amd64])' aptget -s install libsame
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame:armel
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst libsame:armel (1 unstable [armel])
 Conf libsame:armel (1 unstable [armel])' aptget -s install libsame:armel
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 E: Unable to locate package libsame' aptget -s install libsame:armhf
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst libsame (1 unstable [amd64])
 Conf libsame (1 unstable [amd64])' aptget -s install libsame:amd64
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame libsame:armel
@@ -45,7 +45,7 @@ Inst libsame (1 unstable [amd64])
 Inst libsame:armel (1 unstable [armel])
 Conf libsame (1 unstable [amd64])
 Conf libsame:armel (1 unstable [armel])' aptget -s install libsame:armel libsame:amd64
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame libsame:armel
@@ -54,14 +54,14 @@ Inst libsame (1 unstable [amd64])
 Inst libsame:armel (1 unstable [armel])
 Conf libsame (1 unstable [amd64])
 Conf libsame:armel (1 unstable [armel])' aptget -s install libsame:*
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst libsame (1 unstable [amd64])
 Conf libsame (1 unstable [amd64])' aptget -s install libsame:any
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame libsame:armel
@@ -70,14 +70,14 @@ Inst libsame (1 unstable [amd64])
 Inst libsame:armel (1 unstable [armel])
 Conf libsame (1 unstable [amd64])
 Conf libsame:armel (1 unstable [armel])' aptget -s install libsame:a*
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst libsame (1 unstable [amd64])
 Conf libsame (1 unstable [amd64])' aptget -s install libsame:linux-any
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libsame libsame:armel
@@ -86,35 +86,35 @@ Inst libsame (1 unstable [amd64])
 Inst libsame:armel (1 unstable [armel])
 Conf libsame (1 unstable [amd64])
 Conf libsame:armel (1 unstable [armel])' aptget -s install libsame:linux-*
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 E: Unable to locate package libsame' aptget -s install libsame:windows-any
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 E: Unable to locate package foo' aptget -s install foo:armel
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst foo (1 unstable [all])
 Conf foo (1 unstable [all])' aptget -s install foo
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst foo (1 unstable [all])
 Conf foo (1 unstable [all])' aptget -s install foo:all
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst foo (1 unstable [all])
 Conf foo (1 unstable [all])' aptget -s install foo:amd64
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo
diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
index 92e7c0e84..5b966754c 100755
--- a/test/integration/test-compressed-indexes
+++ b/test/integration/test-compressed-indexes
@@ -63,7 +63,7 @@ testrun() {
 	msgtest '\tdeb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
 	rm testpkg_1.0_i386.deb
 	cd - >/dev/null
-	testequal 'Reading package lists...
+	testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   testpkg
@@ -71,14 +71,14 @@ The following NEW packages will be installed:
 Inst testpkg (1.0 unstable [i386])
 Conf testpkg (1.0 unstable [i386])' aptget install testpkg -s
 	rm -f rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
-	testequal "$GOODSHOW" aptcache show testpkg
-	testequal "$GOODSHOW" aptcache show testpkg
+	testsuccessequal "$GOODSHOW" aptcache show testpkg
+	testsuccessequal "$GOODSHOW" aptcache show testpkg
 	rm -f rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
-	testequal "$GOODPOLICY" aptcache policy testpkg
-	testequal "$GOODPOLICY" aptcache policy testpkg
+	testsuccessequal "$GOODPOLICY" aptcache policy testpkg
+	testsuccessequal "$GOODPOLICY" aptcache policy testpkg
 	rm -f rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
-	testequal "$GOODSHOWSRC" aptcache showsrc testpkg
-	testequal "$GOODSHOWSRC" aptcache showsrc testpkg
+	testsuccessequal "$GOODSHOWSRC" aptcache showsrc testpkg
+	testsuccessequal "$GOODSHOWSRC" aptcache showsrc testpkg
 	aptget clean
 	msgtest 'Check if the source is aptgetable'
 	cd downloaded
@@ -87,7 +87,7 @@ Conf testpkg (1.0 unstable [i386])' aptget install testpkg -s
 	testsuccess test -d testpkg-1.0
 	rm -rf testpkg-1.0*
 	cd - >/dev/null
-	testequal "$(aptcache show testpkg -o Acquire::Languages=none)
+	testsuccessequal "$(aptcache show testpkg -o Acquire::Languages=none)
 " aptcache dumpavail
 }
 
@@ -134,14 +134,14 @@ testsuccess aptget update
 GOODSHOW="$(aptcache show testpkg)
 "
 test $(echo "$GOODSHOW" | grep -e '^Package: testpkg' -e '^Version: 1.0' -e '^Architecture: i386' | wc -l) -eq 3 || msgdie 'show is broken'
-testequal "$GOODSHOW" aptcache show testpkg
+testsuccessequal "$GOODSHOW" aptcache show testpkg
 GOODSHOWSRC="$(aptcache showsrc testpkg)
 "
 test $(echo "$GOODSHOWSRC" | grep -e '^Package: testpkg' -e '^Format: 3.0 (native)' -e '^Files:' -e '^Checksums-Sha256:' | wc -l) -eq 4 || msgdie 'showsrc is broken'
-testequal "$GOODSHOWSRC" aptcache showsrc testpkg
+testsuccessequal "$GOODSHOWSRC" aptcache showsrc testpkg
 GOODPOLICY="$(aptcache policy testpkg)"
 test $(echo "$GOODPOLICY" | grep -e '^testpkg:' -e '^  Candidate:' -e '^  Installed: (none)' -e '500 file:/' | wc -l) -eq 4 || msgdie 'policy is broken'
-testequal "$GOODPOLICY" aptcache policy testpkg
+testsuccessequal "$GOODPOLICY" aptcache policy testpkg
 
 for COMPRESSOR in 'gzip' 'bzip2' 'lzma' 'xz'; do testovermethod 'file' $COMPRESSOR; done
 
@@ -150,7 +150,7 @@ rm -rf rootdir/var/lib/apt/lists
 testsuccess aptget update
 GOODPOLICY="$(aptcache policy testpkg)"
 test $(echo "$GOODPOLICY" | grep -e '^testpkg:' -e '^  Candidate:' -e '^  Installed: (none)' -e '500 http://' | wc -l) -eq 4 || msgdie 'policy is broken'
-testequal "$GOODPOLICY" aptcache policy testpkg
+testsuccessequal "$GOODPOLICY" aptcache policy testpkg
 
 for COMPRESSOR in 'gzip' 'bzip2' 'lzma' 'xz'; do testovermethod 'http' $COMPRESSOR; done
 
@@ -159,6 +159,6 @@ rm -rf rootdir/var/lib/apt/lists
 testsuccess aptcdrom add &1 | grep -E -q '(Writing more data than expected|Hash Sum mismatch)' > /dev/null && msgpass || msgfail
 
 # ensure there is no package
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 E: Unable to locate package bad-mitm' aptget install bad-mitm -s
 
 # and verify that its not picked up
-testequal 'N: Unable to locate package bad-mitm' aptcache policy bad-mitm -q=0
+testsuccessequal 'N: Unable to locate package bad-mitm' aptcache policy bad-mitm -q=0
 
 # and that the right one is used
-testequal "good-pkg:
+testsuccessequal "good-pkg:
   Installed: (none)
   Candidate: 1.0
   Version table:
diff --git a/test/integration/test-essential-force-loopbreak b/test/integration/test-essential-force-loopbreak
index 1493430d8..50c682d43 100755
--- a/test/integration/test-essential-force-loopbreak
+++ b/test/integration/test-essential-force-loopbreak
@@ -25,7 +25,7 @@ cp -a rootdir/var/lib/dpkg/status dpkg.status.backup
 testforcebreak() {
 	cp -a dpkg.status.backup rootdir/var/lib/dpkg/status
 	rm -f rootdir/var/lib/apt/extended_states
-	testequal "Reading package lists...
+	testfailureequal "Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   sysvinit
diff --git a/test/integration/test-external-dependency-solver-protocol b/test/integration/test-external-dependency-solver-protocol
index fd68578c5..3b9b38c39 100755
--- a/test/integration/test-external-dependency-solver-protocol
+++ b/test/integration/test-external-dependency-solver-protocol
@@ -25,7 +25,7 @@ insertpackage 'experimental' 'coolstuff' 'i386,amd64' '3' 'Depends: cool, stuff'
 setupaptarchive
 
 rm -f /tmp/dump.edsp
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Execute external solver...
 The solver encountered an error of type: ERR_JUST_DUMPING
@@ -38,7 +38,7 @@ testsuccess test -s /tmp/dump.edsp
 rm -f /tmp/dump.edsp
 
 #FIXME: this should be unstable, but we don't support pinning yet
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Execute external solver...
 The following NEW packages will be installed:
@@ -47,7 +47,7 @@ The following NEW packages will be installed:
 Inst coolstuff (3 experimental [amd64])
 Conf coolstuff (3 experimental [amd64])' aptget install --solver apt coolstuff -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Execute external solver...
 The following packages will be REMOVED:
@@ -86,7 +86,7 @@ rm -f /tmp/dump.edsp
 testfailure aptget install --solver apt awesomecoolstuff:i386 -s
 
 testsuccess aptinternalsolver scenario
-testequal 'Package: stuff
+testsuccessequal 'Package: stuff
 Source: stuff
 Architecture: all
 Version: 1
diff --git a/test/integration/test-handling-broken-orgroups b/test/integration/test-handling-broken-orgroups
index 20b314074..149f05fa9 100755
--- a/test/integration/test-handling-broken-orgroups
+++ b/test/integration/test-handling-broken-orgroups
@@ -23,7 +23,7 @@ Provides: stuff-abi-2'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   cool
@@ -35,7 +35,7 @@ Inst coolstuff (1.0-1 unstable [all])
 Conf cool (1.0-1 unstable [all])
 Conf coolstuff (1.0-1 unstable [all])' aptget install coolstuff -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   stuff
@@ -47,7 +47,7 @@ Inst coolstuff2 (1.0-1 unstable [all])
 Conf stuff (1.0-1 unstable [all])
 Conf coolstuff2 (1.0-1 unstable [all])' aptget install coolstuff2 -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -60,7 +60,7 @@ The following packages have unmet dependencies:
                              stuff2 but it is not installable
 E: Unable to correct problems, you have held broken packages.' aptget install coolstuff-broken -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Recommended packages:
   cool2 stuff2
@@ -70,7 +70,7 @@ The following NEW packages will be installed:
 Inst coolstuff-brokenrec (1.0-1 unstable [all])
 Conf coolstuff-brokenrec (1.0-1 unstable [all])' aptget install coolstuff-brokenrec -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   stuff
@@ -82,7 +82,7 @@ Inst coolstuff-conflict (1.0-1 unstable [all])
 Conf stuff (1.0-1 unstable [all])
 Conf coolstuff-conflict (1.0-1 unstable [all])' aptget install coolstuff-conflict -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   extrastuff
@@ -94,7 +94,7 @@ Inst coolstuff-provided (1.0-1 unstable [all])
 Conf extrastuff (1.0-1 unstable [all])
 Conf coolstuff-provided (1.0-1 unstable [all])' aptget install coolstuff-provided -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
diff --git a/test/integration/test-ignore-provides-if-versioned-breaks b/test/integration/test-ignore-provides-if-versioned-breaks
index 745f7d206..20424b942 100755
--- a/test/integration/test-ignore-provides-if-versioned-breaks
+++ b/test/integration/test-ignore-provides-if-versioned-breaks
@@ -33,7 +33,7 @@ insertpackage 'unstable' 'foo-same-breaker-none' 'i386' '1.0' 'Breaks: foo-same'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -45,7 +45,7 @@ The following packages have unmet dependencies:
  foo-breaker-none : Breaks: foo
 E: Unable to correct problems, you have held broken packages.' aptget install foo-provider foo-breaker-none -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-breaker-2 foo-provider
@@ -55,7 +55,7 @@ Inst foo-provider (1.0 unstable [i386])
 Conf foo-breaker-2 (1.0 unstable [i386])
 Conf foo-provider (1.0 unstable [i386])' aptget install foo-provider foo-breaker-2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -71,7 +71,7 @@ Conf foo (4.0 unstable [i386])
 Conf foo-breaker-3 (1.0 unstable [i386])
 Conf foo-provider (1.0 unstable [i386])' aptget install foo-provider foo-breaker-3 -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -83,7 +83,7 @@ The following packages have unmet dependencies:
  foo-foreign-breaker-none : Breaks: foo-foreign
 E: Unable to correct problems, you have held broken packages.' aptget install foo-foreign-provider foo-foreign-breaker-none -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-foreign-breaker-2 foo-foreign-provider
@@ -93,7 +93,7 @@ Inst foo-foreign-provider (1.0 unstable [i386])
 Conf foo-foreign-breaker-2 (1.0 unstable [i386])
 Conf foo-foreign-provider (1.0 unstable [i386])' aptget install foo-foreign-provider foo-foreign-breaker-2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo-foreign:amd64
@@ -109,7 +109,7 @@ Conf foo-foreign:amd64 (4.0 unstable [amd64])
 Conf foo-foreign-breaker-3 (1.0 unstable [i386])
 Conf foo-foreign-provider (1.0 unstable [i386])' aptget install foo-foreign-provider foo-foreign-breaker-3 -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -121,7 +121,7 @@ The following packages have unmet dependencies:
  foo-same-breaker-none : Breaks: foo-same
 E: Unable to correct problems, you have held broken packages.' aptget install foo-same-provider foo-same-breaker-none -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-same-breaker-2 foo-same-provider
@@ -131,7 +131,7 @@ Inst foo-same-provider (1.0 unstable [i386])
 Conf foo-same-breaker-2 (1.0 unstable [i386])
 Conf foo-same-provider (1.0 unstable [i386])' aptget install foo-same-provider foo-same-breaker-2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo-same:amd64 foo-same
diff --git a/test/integration/test-ignore-provides-if-versioned-conflicts b/test/integration/test-ignore-provides-if-versioned-conflicts
index a07252768..a781d8e44 100755
--- a/test/integration/test-ignore-provides-if-versioned-conflicts
+++ b/test/integration/test-ignore-provides-if-versioned-conflicts
@@ -33,7 +33,7 @@ insertpackage 'unstable' 'foo-same-breaker-none' 'i386' '1.0' 'Conflicts: foo-sa
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -45,7 +45,7 @@ The following packages have unmet dependencies:
  foo-breaker-none : Conflicts: foo
 E: Unable to correct problems, you have held broken packages.' aptget install foo-provider foo-breaker-none -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-breaker-2 foo-provider
@@ -55,7 +55,7 @@ Inst foo-provider (1.0 unstable [i386])
 Conf foo-breaker-2 (1.0 unstable [i386])
 Conf foo-provider (1.0 unstable [i386])' aptget install foo-provider foo-breaker-2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -71,7 +71,7 @@ Conf foo (4.0 unstable [i386])
 Conf foo-breaker-3 (1.0 unstable [i386])
 Conf foo-provider (1.0 unstable [i386])' aptget install foo-provider foo-breaker-3 -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -83,7 +83,7 @@ The following packages have unmet dependencies:
  foo-foreign-breaker-none : Conflicts: foo-foreign
 E: Unable to correct problems, you have held broken packages.' aptget install foo-foreign-provider foo-foreign-breaker-none -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-foreign-breaker-2 foo-foreign-provider
@@ -93,7 +93,7 @@ Inst foo-foreign-provider (1.0 unstable [i386])
 Conf foo-foreign-breaker-2 (1.0 unstable [i386])
 Conf foo-foreign-provider (1.0 unstable [i386])' aptget install foo-foreign-provider foo-foreign-breaker-2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo-foreign:amd64
@@ -109,7 +109,7 @@ Conf foo-foreign:amd64 (4.0 unstable [amd64])
 Conf foo-foreign-breaker-3 (1.0 unstable [i386])
 Conf foo-foreign-provider (1.0 unstable [i386])' aptget install foo-foreign-provider foo-foreign-breaker-3 -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -121,7 +121,7 @@ The following packages have unmet dependencies:
  foo-same-breaker-none : Conflicts: foo-same
 E: Unable to correct problems, you have held broken packages.' aptget install foo-same-provider foo-same-breaker-none -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-same-breaker-2 foo-same-provider
@@ -131,7 +131,7 @@ Inst foo-same-provider (1.0 unstable [i386])
 Conf foo-same-breaker-2 (1.0 unstable [i386])
 Conf foo-same-provider (1.0 unstable [i386])' aptget install foo-same-provider foo-same-breaker-2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo-same:amd64 foo-same
diff --git a/test/integration/test-implicit-conflicts-real-not-virtual b/test/integration/test-implicit-conflicts-real-not-virtual
index c9fca4edf..7c1365bdd 100755
--- a/test/integration/test-implicit-conflicts-real-not-virtual
+++ b/test/integration/test-implicit-conflicts-real-not-virtual
@@ -17,7 +17,7 @@ insertinstalledpackage 'wireless-crda' 'amd64' '1.14'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   crda
@@ -25,7 +25,7 @@ The following NEW packages will be installed:
 Inst crda (1.1.1-1ubuntu4m unstable-m [amd64])
 Conf crda (1.1.1-1ubuntu4m unstable-m [amd64])' aptget install crda -s -t unstable-m
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   crda
@@ -33,7 +33,7 @@ The following NEW packages will be installed:
 Inst crda (1.1.1-1ubuntu4p unstable-p [amd64])
 Conf crda (1.1.1-1ubuntu4p unstable-p [amd64])' aptget install crda -s -t unstable-p
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   crda
@@ -47,7 +47,7 @@ Conflicts: wireless-crda (<< 1.15)
 Replaces: wireless-crda ( << 1.15)
 Multi-arch: foreign'
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   wireless-crda
diff --git a/test/integration/test-kernel-helper-autoremove b/test/integration/test-kernel-helper-autoremove
index 1524ed4c7..c2fc37ee7 100755
--- a/test/integration/test-kernel-helper-autoremove
+++ b/test/integration/test-kernel-helper-autoremove
@@ -55,7 +55,7 @@ testprotected() {
 	grep -q "^\\^linux-image-$(uname -r | sed -e 's#\.#\\\\.#g')\\\$\$" protected.list && msgpass || catfail
 }
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
@@ -73,7 +73,7 @@ testprotected
 msgtest 'Check kernel autoremoval protection list does not include' 'old kernel'
 grep -q '^\^linux-image-1\\\.0\\\.0-2-generic\$$' protected.list && catfail || msgpass
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
@@ -85,7 +85,7 @@ Remv linux-image-1.0.0-2-generic [1.0.0-2]' aptget autoremove -s
 testprotected 1.0.0-2-generic
 msgtest 'Check kernel autoremoval protection list includes' 'installed kernel'
 grep -q '^\^linux-image-1\\\.0\\\.0-2-generic\$$' protected.list && msgpass || catfail
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
diff --git a/test/integration/test-multiarch-foreign b/test/integration/test-multiarch-foreign
index 332466d96..490abb873 100755
--- a/test/integration/test-multiarch-foreign
+++ b/test/integration/test-multiarch-foreign
@@ -15,7 +15,7 @@ Multi-Arch: foreign'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -27,7 +27,7 @@ Inst cool-foo:i386 (1.0 unstable [i386])
 Conf foo (1.0 unstable [amd64])
 Conf cool-foo:i386 (1.0 unstable [i386])' aptget install cool-foo:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -39,7 +39,7 @@ Inst cool-foo (1.0 unstable [amd64])
 Conf foo (1.0 unstable [amd64])
 Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   cool-foo foo
@@ -49,7 +49,7 @@ Inst cool-foo (1.0 unstable [amd64])
 Conf foo (1.0 unstable [amd64])
 Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 foo:amd64 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   cool-foo foo:i386
@@ -59,7 +59,7 @@ Inst cool-foo (1.0 unstable [amd64])
 Conf foo:i386 (1.0 unstable [i386])
 Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 foo:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   cool-foo foo:armel
@@ -73,7 +73,7 @@ Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 foo:armel -s
 
 
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   bar
@@ -85,7 +85,7 @@ Inst cool-bar:i386 (1.0 unstable [i386])
 Conf bar (1.0 unstable [amd64])
 Conf cool-bar:i386 (1.0 unstable [i386])' aptget install cool-bar:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   bar
@@ -97,7 +97,7 @@ Inst cool-bar (1.0 unstable [amd64])
 Conf bar (1.0 unstable [amd64])
 Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   bar cool-bar
@@ -107,7 +107,7 @@ Inst cool-bar (1.0 unstable [amd64])
 Conf bar (1.0 unstable [amd64])
 Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 bar:amd64 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   bar:i386 cool-bar
@@ -117,7 +117,7 @@ Inst cool-bar (1.0 unstable [amd64])
 Conf bar:i386 (1.0 unstable [i386])
 Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 bar:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   bar:armel cool-bar
@@ -127,7 +127,7 @@ Inst cool-bar (1.0 unstable [amd64])
 Conf bar:armel (1.0 unstable [armel])
 Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 bar:armel -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Note, selecting 'bar' instead of 'bar-provider'
 The following NEW packages will be installed:
@@ -138,7 +138,7 @@ Inst cool-bar (1.0 unstable [amd64])
 Conf bar (1.0 unstable [amd64])
 Conf cool-bar (1.0 unstable [amd64])" aptget install cool-bar bar-provider -s -q=0
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Note, selecting 'bar:i386' instead of 'bar-provider:i386'
 The following NEW packages will be installed:
diff --git a/test/integration/test-ordering-ignore-not-matching-breaks b/test/integration/test-ordering-ignore-not-matching-breaks
index c9fca4edf..7c1365bdd 100755
--- a/test/integration/test-ordering-ignore-not-matching-breaks
+++ b/test/integration/test-ordering-ignore-not-matching-breaks
@@ -17,7 +17,7 @@ insertinstalledpackage 'wireless-crda' 'amd64' '1.14'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   crda
@@ -25,7 +25,7 @@ The following NEW packages will be installed:
 Inst crda (1.1.1-1ubuntu4m unstable-m [amd64])
 Conf crda (1.1.1-1ubuntu4m unstable-m [amd64])' aptget install crda -s -t unstable-m
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   crda
@@ -33,7 +33,7 @@ The following NEW packages will be installed:
 Inst crda (1.1.1-1ubuntu4p unstable-p [amd64])
 Conf crda (1.1.1-1ubuntu4p unstable-p [amd64])' aptget install crda -s -t unstable-p
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   crda
@@ -47,7 +47,7 @@ Conflicts: wireless-crda (<< 1.15)
 Replaces: wireless-crda ( << 1.15)
 Multi-arch: foreign'
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   wireless-crda
diff --git a/test/integration/test-package-reinstallation b/test/integration/test-package-reinstallation
index b4f2061ec..f0412f98d 100755
--- a/test/integration/test-package-reinstallation
+++ b/test/integration/test-package-reinstallation
@@ -14,7 +14,7 @@ insertinstalledpackage 'apt' 'i386' '0.8.15' 'Depends: libc6'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 0 upgraded, 0 newly installed, 2 reinstalled, 0 to remove and 0 not upgraded.
 Inst libc-bin [2.13-8] (2.13-8 unstable [i386])
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index d773dcd66..4de07f1ad 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -60,7 +60,7 @@ testrun() {
 	testsuccess aptget update "$@"
 	cp -a rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists-bak
 	testnopackage newstuff
-	testequal "$(cat ${PKGFILE})
+	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt oldstuff
 
 	msgmsg "Testcase: apply with one patch: $*"
@@ -90,13 +90,13 @@ SHA256-Patches:
 	find aptarchive -name 'Packages*' -type f -delete
 	wasmergeused "$@"
 	testnopackage oldstuff
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
 
 	msgmsg "Testcase: index is already up-to-date: $*"
 	find rootdir/var/lib/apt/lists -name '*diff_Index' -type f -delete
 	testsuccess aptget update "$@"
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
 
 	msgmsg "Testcase: apply with two patches: $*"
@@ -146,7 +146,7 @@ SHA256-Patches:
 	cp -a rootdir/var/lib/apt/lists-bak rootdir/var/lib/apt/lists
 	wasmergeused "$@"
 	testnopackage oldstuff
-	testequal "$(cat Packages-future)
+	testsuccessequal "$(cat Packages-future)
 " aptcache show apt newstuff futurestuff
 
 	msgmsg "Testcase: patch applying fails, but successful fallback: $*"
@@ -178,7 +178,7 @@ SHA256-Patches:
 	signreleasefiles
 	testsuccess aptget update "$@"
 	testnopackage oldstuff
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
 
 	msgmsg "Testcase: pdiff patch bigger than index itself: $*"
@@ -212,7 +212,7 @@ SHA256-Patches:
 	cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output
 	testsuccess grep 'bytes (Limit is' rootdir/tmp/aptgetupdate.output
 	testnopackage oldstuff
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
 }
 echo 'Debug::pkgAcquire::Diffs "true";
diff --git a/test/integration/test-pin-non-existent-package b/test/integration/test-pin-non-existent-package
index c567e5285..5c839283f 100755
--- a/test/integration/test-pin-non-existent-package
+++ b/test/integration/test-pin-non-existent-package
@@ -23,8 +23,8 @@ testcandidate() {
 }
 
 testcandidate rapt '0.8.15'
-testequal 'N: Unable to locate package doesntexist' aptcache policy doesntexist -q=0
-testequal 'Reading package lists...
+testsuccessequal 'N: Unable to locate package doesntexist' aptcache policy doesntexist -q=0
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.' aptget dist-upgrade --trivial-only
@@ -34,8 +34,8 @@ Pin: release a=unstable
 Pin-Priority: -1' > rootdir/etc/apt/preferences
 
 testcandidate rapt '(none)'
-testequal 'N: Unable to locate package doesntexist' aptcache policy doesntexist -q=0
-testequal 'Reading package lists...
+testsuccessequal 'N: Unable to locate package doesntexist' aptcache policy doesntexist -q=0
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.' aptget dist-upgrade --trivial-only
@@ -53,9 +53,9 @@ Pin: release a=unstable
 Pin-Priority: 1000' >> rootdir/etc/apt/preferences
 
 testcandidate rapt '(none)'
-testequal 'N: Unable to locate package doesntexist' aptcache policy doesntexist -q=0
+testsuccessequal 'N: Unable to locate package doesntexist' aptcache policy doesntexist -q=0
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.' aptget dist-upgrade --trivial-only
diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning
index c08a2f103..15bf300ac 100755
--- a/test/integration/test-policy-pinning
+++ b/test/integration/test-policy-pinning
@@ -17,7 +17,7 @@ testequalpolicy() {
 	local SP="$1"
 	local AP="$2"
 	shift 2
-	testequal "Package files:
+	testsuccessequal "Package files:
  $(echo "$SP" | awk '{ printf("%3s\n",$0) }') ${STATUS}
      release a=now
  $(echo "$AP" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}/  Packages
@@ -126,7 +126,7 @@ testequalpolicycoolstuff() {
 		shift
 	fi
 	shift 6
-	testequal "coolstuff:
+	testsuccessequal "coolstuff:
   Installed: $INSTALLED
   Candidate: $CANDIDATE
   ${PINVERSION}Version table:${BPO2ARCHIVE}
diff --git a/test/integration/test-prefer-higher-priority-providers b/test/integration/test-prefer-higher-priority-providers
index 64b901dd0..85a302fb1 100755
--- a/test/integration/test-prefer-higher-priority-providers
+++ b/test/integration/test-prefer-higher-priority-providers
@@ -13,7 +13,7 @@ insertpackage 'unstable' 'awesome' 'all' '1' 'Depends: stuff'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -25,7 +25,7 @@ Inst awesome (1 unstable [all])
 Conf foo (1 unstable [all])
 Conf awesome (1 unstable [all])' aptget install awesome -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   awesome foo
@@ -35,7 +35,7 @@ Inst awesome (1 unstable [all])
 Conf foo (1 unstable [all])
 Conf awesome (1 unstable [all])' aptget install awesome foo -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'bar' is not installed, so not removed
 Package 'baz' is not installed, so not removed
@@ -49,7 +49,7 @@ Inst awesome (1 unstable [all])
 Conf foo (1 unstable [all])
 Conf awesome (1 unstable [all])" aptget install awesome bar- baz- -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'foo' is not installed, so not removed
 The following extra packages will be installed:
@@ -62,7 +62,7 @@ Inst awesome (1 unstable [all])
 Conf bar (1 unstable [all])
 Conf awesome (1 unstable [all])" aptget install awesome foo- -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'foo' is not installed, so not removed
 Package 'baz' is not installed, so not removed
@@ -76,7 +76,7 @@ Inst awesome (1 unstable [all])
 Conf bar (1 unstable [all])
 Conf awesome (1 unstable [all])" aptget install awesome foo- baz- -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'foo' is not installed, so not removed
 Package 'bar' is not installed, so not removed
@@ -90,7 +90,7 @@ Inst awesome (1 unstable [all])
 Conf baz (1 unstable [all])
 Conf awesome (1 unstable [all])" aptget install awesome foo- bar- -s
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Package 'foo' is not installed, so not removed
 Package 'bar' is not installed, so not removed
diff --git a/test/integration/test-prefer-native-architecture-over-higher-priority b/test/integration/test-prefer-native-architecture-over-higher-priority
index 2e5696376..7e4f8f34b 100755
--- a/test/integration/test-prefer-native-architecture-over-higher-priority
+++ b/test/integration/test-prefer-native-architecture-over-higher-priority
@@ -12,7 +12,7 @@ insertpackage 'unstable' 'autoconf' 'all' '1' 'Depends: m4'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   m4
diff --git a/test/integration/test-prevent-markinstall-multiarch-same-versionscrew b/test/integration/test-prevent-markinstall-multiarch-same-versionscrew
index db97687ce..5f67c0191 100755
--- a/test/integration/test-prevent-markinstall-multiarch-same-versionscrew
+++ b/test/integration/test-prevent-markinstall-multiarch-same-versionscrew
@@ -38,7 +38,7 @@ insertpackage 'unstable' 'libsame3' 'i386,amd64' '3' 'Multi-Arch: same'
 insertpackage 'unstable' 'depender3' 'all' '3' 'Depends: libsame3 (= 3)'
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
@@ -69,7 +69,7 @@ insertinstalledpackage 'libsame2' 'i386' '1' 'Multi-Arch: same'
 insertinstalledpackage 'libsame3' 'i386' '1' 'Multi-Arch: same'
 
 # the error message isn't great, but better than nothing, right?
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -81,7 +81,7 @@ The following packages have unmet dependencies:
  depender2 : Depends: libsame2 (= 2) but it is not going to be installed
 E: Unable to correct problems, you have held broken packages.' aptget install depender2 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libsame3:i386 libsame3
diff --git a/test/integration/test-provides-gone-with-upgrade b/test/integration/test-provides-gone-with-upgrade
index 3b4bc2d04..61d34fa57 100755
--- a/test/integration/test-provides-gone-with-upgrade
+++ b/test/integration/test-provides-gone-with-upgrade
@@ -13,7 +13,7 @@ Replaces: apt (<< 0.8.15)'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following NEW packages will be installed:
diff --git a/test/integration/test-release-candidate-switching b/test/integration/test-release-candidate-switching
index 0970cb935..a1a6a6142 100755
--- a/test/integration/test-release-candidate-switching
+++ b/test/integration/test-release-candidate-switching
@@ -54,7 +54,7 @@ insertpackage 'experimental' 'uninstallablepkg' 'all' '1.0' 'Depends: libmtp8 (>
 
 setupaptarchive
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
    amarok-common (2.3.1-1+sid)
@@ -73,7 +73,7 @@ The following NEW packages will be installed:
 After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok --trivial-only -V -q=0
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
    amarok-common (2.3.2-2+exp)
@@ -92,7 +92,7 @@ The following NEW packages will be installed:
 After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok -t experimental --trivial-only -V -q=0
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok'
@@ -114,7 +114,7 @@ The following NEW packages will be installed:
 After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok/experimental --trivial-only -V -q=0
 
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-null'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-null'
@@ -137,7 +137,7 @@ After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-null/experimental --trivial-only -V -q=0
 
 # do not select the same version multiple times
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok'
@@ -165,7 +165,7 @@ E: Trivial Only specified but this is not a trivial operation." aptget install a
 # in theory, the second line is wrong, but printing the right version is too much of a hassle
 # (we have to check if later in the Changed list is another change and if so use this version
 #  instead of the current candidate) - and it wouldn't be (really) useful anyway…
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental2 [i386]) for 'amarok-less'
 Selected version '5:4.6.0+exp' (experimental [i386]) for 'phonon-backend-xine' because of 'amarok-less'
@@ -192,7 +192,7 @@ After this operation, 301 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-less/experimental2 amarok-higher/experimental --trivial-only -V -q=0
 
 # phonon-backend-null can't be used directly, but as it provides it is still fine…
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-null2'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-null2'
@@ -215,7 +215,7 @@ After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-null2/experimental --trivial-only -V -q=0
 
 # if an or-group satisfier is already found, do not set others
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-xine'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-xine'
@@ -239,7 +239,7 @@ After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-xine/experimental --trivial-only -V -q=0
 
 # … but proceed testing if the first doesn't work out
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-xine2'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-xine2'
@@ -263,7 +263,7 @@ After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-xine2/experimental --trivial-only -V -q=0
 
 # sometimes, the second level need to be corrected, too
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-xine3'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-xine3'
@@ -290,7 +290,7 @@ After this operation, 301 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-xine3/experimental --trivial-only -V -q=0
 
 # … but proceed testing if the first doesn't work out even in second deep
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-xine4'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-xine4'
@@ -314,7 +314,7 @@ After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-xine4/experimental --trivial-only -V -q=0
 
 # providers can be broken, too
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-broken'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-broken'
@@ -338,7 +338,7 @@ After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-broken/experimental --trivial-only -V -q=0
 
 # switch the candidate for recommends too if they should be installed
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-recommends'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-recommends'
@@ -361,7 +361,7 @@ After this operation, 258 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-recommends/experimental --trivial-only -V -q=0 -o APT::Install-Recommends=1
 
 # … or not if not
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-recommends'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-recommends'
@@ -382,7 +382,7 @@ E: Trivial Only specified but this is not a trivial operation." aptget install a
 
 # but broken recommends are not the end of the world
 # FIXME: the version output for recommend packages is a bit strange… but what would be better?
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '2.3.2-2+exp' (experimental [i386]) for 'amarok-recommends2'
 Selected version '2.3.2-2+exp' (experimental [all]) for 'amarok-common' because of 'amarok-recommends2'
@@ -404,7 +404,7 @@ After this operation, 215 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation." aptget install amarok-recommends2/experimental --trivial-only -V -q=0 -o APT::Install-Recommends=1
 
 # if one depends doesn't work, we don't need to look deeper…
-testequal "Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
 Selected version '1.0' (experimental [all]) for 'uninstallablepkg'
 Some packages could not be installed. This may mean that you have
diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification
index 3765a4b1f..363b7fe5b 100755
--- a/test/integration/test-releasefile-verification
+++ b/test/integration/test-releasefile-verification
@@ -33,7 +33,7 @@ prepare() {
 }
 
 installaptold() {
-	testequal 'Reading package lists...
+	testsuccessequal 'Reading package lists...
 Building dependency tree...
 Suggested packages:
   aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt
@@ -46,7 +46,7 @@ Download complete and in download only mode' aptget install apt -dy
 }
 
 installaptnew() {
-	testequal 'Reading package lists...
+	testsuccessequal 'Reading package lists...
 Building dependency tree...
 Suggested packages:
   aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt
@@ -59,7 +59,7 @@ Download complete and in download only mode' aptget install apt -dy
 }
 
 failaptold() {
-	testequal 'Reading package lists...
+	testfailureequal 'Reading package lists...
 Building dependency tree...
 Suggested packages:
   aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt
@@ -73,7 +73,7 @@ E: There are problems and -y was used without --force-yes' aptget install apt -d
 }
 
 failaptnew() {
-	testequal 'Reading package lists...
+	testfailureequal 'Reading package lists...
 Building dependency tree...
 Suggested packages:
   aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt
@@ -119,7 +119,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Cold archive signed by' 'Joe Sixpack'
 	updatesuccess
-	testequal "$(cat ${PKGFILE})
+	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
 
@@ -128,7 +128,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Good warm archive signed by' 'Joe Sixpack'
 	updatesuccess
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
 
@@ -139,7 +139,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Cold archive signed by' 'Rex Expired'
 	updatefailure '^W: .* KEYEXPIRED'
-	testequal "$(cat ${PKGFILE})
+	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	failaptold
 	rm rootdir/etc/apt/trusted.gpg.d/rexexpired.gpg
@@ -150,7 +150,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Cold archive signed by' 'Marvin Paranoid'
 	updatefailure '^W: .* NO_PUBKEY'
-	testequal "$(cat ${PKGFILE})
+	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	failaptold
 
@@ -164,7 +164,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Bad warm archive signed by' 'Joe Sixpack'
 	updatesuccess
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
 
@@ -175,7 +175,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Cold archive signed by' 'Joe Sixpack'
 	updatesuccess
-	testequal "$(cat ${PKGFILE})
+	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
 
@@ -184,7 +184,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Good warm archive signed by' 'Marvin Paranoid'
 	updatefailure '^W: .* NO_PUBKEY'
-	testequal "$(cat ${PKGFILE})
+	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
 
@@ -194,7 +194,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Good warm archive signed by' 'Rex Expired'
 	updatefailure '^W: .* KEYEXPIRED'
-	testequal "$(cat ${PKGFILE})
+	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
 	rm rootdir/etc/apt/trusted.gpg.d/rexexpired.gpg
@@ -204,7 +204,7 @@ runtest() {
 	find aptarchive/ -name "$DELETEFILE" -delete
 	msgtest 'Good warm archive signed by' 'Joe Sixpack'
 	updatesuccess
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
 }
@@ -223,7 +223,7 @@ runtest2() {
 	find aptarchive/ -name Release.gpg -delete
 	msgtest 'Warm archive signed by' 'nobody'
 	updatesuccess
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	failaptnew
 
@@ -231,7 +231,7 @@ runtest2() {
 	rm -rf rootdir/var/lib/apt/lists
 	msgtest 'Cold archive signed by' 'nobody'
 	updatesuccess
-	testequal "$(cat ${PKGFILE}-new)
+	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	failaptnew
 }
diff --git a/test/integration/test-resolve-by-keep-new-recommends b/test/integration/test-resolve-by-keep-new-recommends
index 6b1772877..a8ab9057c 100755
--- a/test/integration/test-resolve-by-keep-new-recommends
+++ b/test/integration/test-resolve-by-keep-new-recommends
@@ -17,5 +17,5 @@ Calculating upgrade...
 The following packages have been kept back:
   foo
 0 upgraded, 0 newly installed, 0 to remove and 1 not upgraded."
-testequal "$UPGRADE_KEEP" aptget upgrade -s
+testsuccessequal "$UPGRADE_KEEP" aptget upgrade -s
 
diff --git a/test/integration/test-sourceslist-trusted-options b/test/integration/test-sourceslist-trusted-options
index 55d4e0233..5fe4933ce 100755
--- a/test/integration/test-sourceslist-trusted-options
+++ b/test/integration/test-sourceslist-trusted-options
@@ -15,7 +15,7 @@ setupaptarchive --no-update
 APTARCHIVE=$(readlink -f ./aptarchive)
 
 everythingsucceeds() {
-	testequal 'Listing...
+	testsuccessequal 'Listing...
 foo/testing 2 amd64
 foo/stable 1 amd64
 ' apt list foo -a
@@ -32,7 +32,7 @@ foo/stable 1 amd64
 }
 
 everythingfails() {
-	testequal 'Listing...
+	testsuccessequal 'Listing...
 foo/testing 2 amd64
 foo/stable 1 amd64
 ' apt list foo -a
diff --git a/test/integration/test-specific-architecture-dependencies b/test/integration/test-specific-architecture-dependencies
index 078a84654..cb6bc3cc2 100755
--- a/test/integration/test-specific-architecture-dependencies
+++ b/test/integration/test-specific-architecture-dependencies
@@ -30,7 +30,7 @@ insertpackage 'unstable' 'foo-no-conflictor' 'i386' '1' 'Conflicts: foo:i386'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libc6:i386
@@ -42,7 +42,7 @@ Conf libc6:i386 (1 unstable [i386])
 Inst pre-depender (1 unstable [all])
 Conf pre-depender (1 unstable [all])' aptget install pre-depender -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libc6:i386
@@ -54,7 +54,7 @@ Inst depender (1 unstable [all])
 Conf libc6:i386 (1 unstable [i386])
 Conf depender (1 unstable [all])' aptget install depender -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   libold libold:i386
@@ -66,7 +66,7 @@ Remv libold:i386 [1]
 Inst breaker (1 unstable [all])
 Conf breaker (1 unstable [all])' aptget install breaker -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   libold:i386
@@ -77,7 +77,7 @@ Remv libold:i386 [1]
 Inst breaker-x32 (1 unstable [amd64])
 Conf breaker-x32 (1 unstable [amd64])' aptget install breaker-x32 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   libold
@@ -88,7 +88,7 @@ Remv libold [1]
 Inst breaker-x64:i386 (1 unstable [i386])
 Conf breaker-x64:i386 (1 unstable [i386])' aptget install breaker-x64 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   oldconflictor
@@ -96,7 +96,7 @@ The following NEW packages will be installed:
 Inst oldconflictor (1 unstable [all])
 Conf oldconflictor (1 unstable [all])' aptget install oldconflictor -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   oldconflictor-x32
@@ -104,7 +104,7 @@ The following NEW packages will be installed:
 Inst oldconflictor-x32 (1 unstable [amd64])
 Conf oldconflictor-x32 (1 unstable [amd64])' aptget install oldconflictor-x32 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   oldconflictor-x64:i386
@@ -112,7 +112,7 @@ The following NEW packages will be installed:
 Inst oldconflictor-x64:i386 (1 unstable [i386])
 Conf oldconflictor-x64:i386 (1 unstable [i386])' aptget install oldconflictor-x64 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-depender
@@ -120,7 +120,7 @@ The following NEW packages will be installed:
 Inst foo-depender (1 unstable [amd64])
 Conf foo-depender (1 unstable [amd64])' aptget install foo-depender -s
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Some packages could not be installed. This may mean that you have
 requested an impossible situation or if you are using the unstable
@@ -132,7 +132,7 @@ The following packages have unmet dependencies:
  foo-depender:i386 : Depends: foo:i386 but it is not installable
 E: Unable to correct problems, you have held broken packages.' aptget install foo-depender:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-foreign-depender:i386
@@ -140,7 +140,7 @@ The following NEW packages will be installed:
 Inst foo-foreign-depender:i386 (1 unstable [i386])
 Conf foo-foreign-depender:i386 (1 unstable [i386])' aptget install foo-foreign-depender:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   provider
@@ -151,7 +151,7 @@ Remv provider [1]
 Inst foo-conflictor (1 unstable [amd64])
 Conf foo-conflictor (1 unstable [amd64])' aptget install foo-conflictor -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   provider
@@ -162,7 +162,7 @@ Remv provider [1]
 Inst foo-conflictor:i386 (1 unstable [i386])
 Conf foo-conflictor:i386 (1 unstable [i386])' aptget install foo-conflictor:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   provider
@@ -173,7 +173,7 @@ Remv provider [1]
 Inst foo-foreign-conflictor:i386 (1 unstable [i386])
 Conf foo-foreign-conflictor:i386 (1 unstable [i386])' aptget install foo-foreign-conflictor:i386 -s
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   foo-no-conflictor:i386
diff --git a/test/integration/test-suggest-installed-multiarch-silbing b/test/integration/test-suggest-installed-multiarch-silbing
index 89640a30c..f2b1db5eb 100755
--- a/test/integration/test-suggest-installed-multiarch-silbing
+++ b/test/integration/test-suggest-installed-multiarch-silbing
@@ -26,26 +26,26 @@ insertpackage 'unstable' 'mozplugger' 'i386,amd64' '1' 'Depends: iceweasel | fir
 
 setupaptarchive
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'foo' is not installed, so not removed. Did you mean 'foo:i386'?
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget remove foo -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   foo2:i386
 0 upgraded, 0 newly installed, 1 to remove and 0 not upgraded.
 Remv foo2:i386 [1]" aptget remove foo2 -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
   foo3
 0 upgraded, 0 newly installed, 1 to remove and 0 not upgraded.
 Remv foo3 [1]" aptget remove foo3 -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'foo3:i386' is not installed, so not removed. Did you mean 'foo3'?
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget remove foo3:i386 -s
@@ -58,22 +58,22 @@ Building dependency tree...
 Package 'samefoo:armel' is not installed, so not removed. Did you mean 'samefoo:i386'?
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget remove samefoo:armel -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'samefoo2' is not installed, so not removed. Did you mean 'samefoo2:i386'?
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget remove samefoo2 -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'samefoo2:armel' is not installed, so not removed. Did you mean 'samefoo2:i386'?
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget remove samefoo2:armel -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'iceweasel' is not installed, so not removed
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget remove iceweasel -s
 
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Package 'fireweasel' is not installed, so not removed. Did you mean 'fireweasel:i386'?
 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." aptget remove fireweasel:amd64 -s
diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
index 574183b0a..ec74a750b 100755
--- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
+++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
@@ -145,7 +145,7 @@ cd downloaded
 
 testok() {
 	rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz
-	testequal "Reading package lists...
+	testsuccessequal "Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080/  $1 1.0 (dsc) [3 B]
@@ -159,7 +159,7 @@ Download complete and in download only mode" aptget source -d "$@"
 testkeep() {
 	echo -n 'dsc' > ${1}_1.0.dsc
 	echo -n 'tar' > ${1}_1.0.tar.gz
-	testequal "Reading package lists...
+	testsuccessequal "Reading package lists...
 Building dependency tree...
 Skipping already downloaded file '${1}_1.0.dsc'
 Skipping already downloaded file '${1}_1.0.tar.gz'
@@ -172,7 +172,7 @@ Download complete and in download only mode" aptget source -d "$@"
 
 testmismatch() {
 	rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz
-	testequal "Reading package lists...
+	testfailureequal "Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080/  $1 1.0 (dsc) [3 B]
@@ -186,7 +186,7 @@ E: Failed to fetch some archives." aptget source -d "$@"
 	testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz
 
 	rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz
-	testequal "Reading package lists...
+	testsuccessequal "Reading package lists...
 Building dependency tree...
 Skipping download of file 'pkg-sha256-bad_1.0.dsc' as requested hashsum is not available for authentication
 Skipping download of file 'pkg-sha256-bad_1.0.tar.gz' as requested hashsum is not available for authentication
@@ -196,7 +196,7 @@ Download complete and in download only mode" aptget source -d "$@" -o Acquire::F
 	testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz
 
 	rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz
-	testequal "Reading package lists...
+	testsuccessequal "Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080/  $1 1.0 (dsc) [3 B]
@@ -221,7 +221,7 @@ testok pkg-sha256-bad -o Acquire::ForceHash=MD5Sum
 # not having MD5 sum doesn't mean the file doesn't exist at all …
 testok pkg-no-md5
 testok pkg-no-md5 -o Acquire::ForceHash=SHA256
-testequal "Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
 Skipping download of file 'pkg-no-md5_1.0.dsc' as requested hashsum is not available for authentication
 Skipping download of file 'pkg-no-md5_1.0.tar.gz' as requested hashsum is not available for authentication
@@ -233,7 +233,7 @@ testfailure --nomsg test -e pkg-no-md5_1.0.dsc -a -e pkg-no-md5_1.0.tar.gz
 # deal with cases in which we haven't for all files the same checksum type
 # mostly pathologic as this shouldn't happen, but just to be sure
 testok pkg-mixed-ok
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080/  pkg-mixed-sha1-bad 1.0 (tar) [3 B]
@@ -243,7 +243,7 @@ E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc  Hash Sum mi
 E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha1-bad
 msgtest 'Only tar file is downloaded as the dsc has hashsum mismatch' 'pkg-mixed-sha1-bad'
 testsuccess --nomsg test ! -e pkg-mixed-sha1-bad_1.0.dsc -a -e pkg-mixed-sha1-bad_1.0.tar.gz
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080/  pkg-mixed-sha2-bad 1.0 (tar) [3 B]
@@ -256,9 +256,9 @@ testsuccess --nomsg test -e pkg-mixed-sha2-bad_1.0.dsc -a ! -e pkg-mixed-sha2-ba
 
 # it gets even more pathologic: multiple entries for one file, some even disagreeing!
 testok pkg-md5-agree
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 E: Error parsing checksum in Files of source package pkg-md5-disagree' aptget source -d pkg-md5-disagree
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 E: Error parsing checksum in Checksums-SHA256 of source package pkg-sha256-disagree' aptget source -d pkg-sha256-disagree
diff --git a/test/integration/test-ubuntu-bug-1130419-prefer-installed-ma-same-siblings b/test/integration/test-ubuntu-bug-1130419-prefer-installed-ma-same-siblings
index af6b7b504..192ed5efc 100755
--- a/test/integration/test-ubuntu-bug-1130419-prefer-installed-ma-same-siblings
+++ b/test/integration/test-ubuntu-bug-1130419-prefer-installed-ma-same-siblings
@@ -20,7 +20,7 @@ insertpackage 'unstable' 'steam' 'i386' '2' 'Depends: libmesa'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libmesa:i386
@@ -31,7 +31,7 @@ Inst libmesa:i386 (1 stable [i386])
 Inst steam:i386 (1 stable [i386])
 Conf libmesa:i386 (1 stable [i386])
 Conf steam:i386 (1 stable [i386])' aptget install steam -st stable
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libmesa:i386
@@ -45,7 +45,7 @@ Conf steam:i386 (2 unstable [i386])' aptget install steam -st unstable
 
 cp rootdir/var/lib/dpkg/status default-status.dpkg
 insertinstalledpackage 'libmesa' 'amd64' '1' 'Multi-Arch: same'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libmesa:i386
@@ -56,7 +56,7 @@ Inst libmesa:i386 (1 stable [i386])
 Inst steam:i386 (1 stable [i386])
 Conf libmesa:i386 (1 stable [i386])
 Conf steam:i386 (1 stable [i386])' aptget install steam -st stable
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libmesa libmesa:i386
@@ -76,7 +76,7 @@ cp default-status.dpkg rootdir/var/lib/dpkg/status
 insertinstalledpackage 'libmesa-lts' 'amd64' '1' 'Provides: libmesa
 Conflicts: libmesa
 Multi-Arch: same'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libmesa-lts:i386
@@ -87,7 +87,7 @@ Inst libmesa-lts:i386 (1 stable [i386])
 Inst steam:i386 (1 stable [i386])
 Conf libmesa-lts:i386 (1 stable [i386])
 Conf steam:i386 (1 stable [i386])' aptget install steam -st stable
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   libmesa-lts libmesa-lts:i386
diff --git a/test/integration/test-ubuntu-bug-1304403-obsolete-priority-standard b/test/integration/test-ubuntu-bug-1304403-obsolete-priority-standard
index 45f70a898..b4f705d8b 100755
--- a/test/integration/test-ubuntu-bug-1304403-obsolete-priority-standard
+++ b/test/integration/test-ubuntu-bug-1304403-obsolete-priority-standard
@@ -25,7 +25,7 @@ insertinstalledpackage 'not-downloadable' 'all' '1.0' '' 'standard'
 setupaptarchive
 
 # discourage keeping obsolete high-priority packages …
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be REMOVED:
@@ -42,7 +42,7 @@ for i in $(seq 1 10); do
 insertinstalledpackage "depender$i" 'all' '1.0' 'Depends: not-downloadable'
 done
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages have been kept back:
diff --git a/test/integration/test-ubuntu-bug-614993 b/test/integration/test-ubuntu-bug-614993
index 49955f231..7067713e8 100755
--- a/test/integration/test-ubuntu-bug-614993
+++ b/test/integration/test-ubuntu-bug-614993
@@ -55,5 +55,5 @@ The following packages will be upgraded:
 Need to get 0 B/5505 kB of archives.
 After this operation, 2294 kB disk space will be freed.
 E: Trivial Only specified but this is not a trivial operation."
-testequal "$UPGRADE" aptget install xserver-xorg --trivial-only
+testfailureequal "$UPGRADE" aptget install xserver-xorg --trivial-only
 
diff --git a/test/integration/test-ubuntu-bug-761175-remove-purge b/test/integration/test-ubuntu-bug-761175-remove-purge
index 0b5a91246..c2e2aadf1 100755
--- a/test/integration/test-ubuntu-bug-761175-remove-purge
+++ b/test/integration/test-ubuntu-bug-761175-remove-purge
@@ -31,7 +31,7 @@ runtests() {
 	testdpkgnotinstalled compiz-core-$1
 	testdpkgstatus 'rc' '1' "compiz-core-$1"
 
-	testequal "Reading package lists...
+	testsuccessequal "Reading package lists...
 Building dependency tree...
 Reading state information...
 The following packages will be REMOVED:
@@ -39,7 +39,7 @@ The following packages will be REMOVED:
 0 upgraded, 0 newly installed, 1 to remove and 0 not upgraded.
 Purg compiz-core-$1" aptget purge compiz-core-$1 -s
 	testsuccess aptget purge compiz-core-$1 -y
-	testequal "dpkg-query: no packages found matching compiz-core-$1" dpkg -l compiz-core-$1
+	testfailureequal "dpkg-query: no packages found matching compiz-core-$1" dpkg -l compiz-core-$1
 }
 
 msgmsg 'Test in multi arch environment'
diff --git a/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only b/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only
index 09315868b..754487a90 100755
--- a/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only
+++ b/test/integration/test-ubuntu-bug-784473-InRelease-one-message-only
@@ -31,7 +31,7 @@ msgtest 'The unsigned garbage before signed block is' 'ignored'
 aptget update -qq 2>&1 | grep -q 'W:.*Does not start with a cleartext signature' && msgpass || msgfail
 
 ROOTDIR="$(readlink -f .)"
-testequal "Package files:
+testsuccessequal "Package files:
  100 ${ROOTDIR}/rootdir/var/lib/dpkg/status
      release a=now
 Pinned packages:" aptcache policy
diff --git a/test/integration/test-ubuntu-bug-802901-multiarch-early-remove b/test/integration/test-ubuntu-bug-802901-multiarch-early-remove
index bdb4e5e4f..f56ff020c 100755
--- a/test/integration/test-ubuntu-bug-802901-multiarch-early-remove
+++ b/test/integration/test-ubuntu-bug-802901-multiarch-early-remove
@@ -15,7 +15,7 @@ Conflicts: libgl1'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   libgl1-mesa-glx:i386
diff --git a/test/integration/test-ubuntu-bug-806274-install-suggests b/test/integration/test-ubuntu-bug-806274-install-suggests
index fb72f0999..3f02316f4 100755
--- a/test/integration/test-ubuntu-bug-806274-install-suggests
+++ b/test/integration/test-ubuntu-bug-806274-install-suggests
@@ -15,7 +15,7 @@ insertpackage 'unstable' 'baz' 'i386' '1.0'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   bar foo
@@ -32,7 +32,7 @@ Inst bar (1.0 unstable [i386])
 Conf bar (1.0 unstable [i386])' aptget install apt -s --install-recommends --no-install-suggests
 
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   bar baz foo
@@ -48,7 +48,7 @@ Inst baz (1.0 unstable [i386])
 Conf bar (1.0 unstable [i386])
 Conf baz (1.0 unstable [i386])' aptget install apt -s --install-recommends --install-suggests
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   foo
@@ -64,7 +64,7 @@ Conf foo (1.0 unstable [i386])
 Inst apt (0.8.15 unstable [i386])
 Conf apt (0.8.15 unstable [i386])' aptget install apt -s --no-install-recommends --no-install-suggests
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   baz foo
diff --git a/test/integration/test-ubuntu-bug-859188-multiarch-reinstall b/test/integration/test-ubuntu-bug-859188-multiarch-reinstall
index be86f2e91..9bb99305e 100755
--- a/test/integration/test-ubuntu-bug-859188-multiarch-reinstall
+++ b/test/integration/test-ubuntu-bug-859188-multiarch-reinstall
@@ -21,7 +21,7 @@ Inst libsame:i386 [1.0] (1.0 unstable [i386])
 Conf libsame (1.0 unstable [amd64])
 Conf libsame:i386 (1.0 unstable [i386])'
 
-testequal "$REINSTALL" aptget install --reinstall libsame -s
-testequal "$REINSTALL" aptget install --reinstall libsame:amd64 -s
-testequal "$REINSTALL" aptget install --reinstall libsame:i386 -s
-testequal "$REINSTALL" aptget install --reinstall libsame:amd64 libsame:i386 -s
+testsuccessequal "$REINSTALL" aptget install --reinstall libsame -s
+testsuccessequal "$REINSTALL" aptget install --reinstall libsame:amd64 -s
+testsuccessequal "$REINSTALL" aptget install --reinstall libsame:i386 -s
+testsuccessequal "$REINSTALL" aptget install --reinstall libsame:amd64 libsame:i386 -s
diff --git a/test/integration/test-ubuntu-bug-985852-pre-depends-or-group-ordering b/test/integration/test-ubuntu-bug-985852-pre-depends-or-group-ordering
index d2b6b9bad..e9cbf958e 100755
--- a/test/integration/test-ubuntu-bug-985852-pre-depends-or-group-ordering
+++ b/test/integration/test-ubuntu-bug-985852-pre-depends-or-group-ordering
@@ -12,7 +12,7 @@ insertpackage 'unstable' 'custom' 'amd64' '2.0' 'Pre-Depends: grub-pc | grub'
 
 setupaptarchive
 
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be upgraded:
diff --git a/test/integration/test-unpack-different-version-unpacked b/test/integration/test-unpack-different-version-unpacked
index 952f6e6b2..ae121cf4e 100755
--- a/test/integration/test-unpack-different-version-unpacked
+++ b/test/integration/test-unpack-different-version-unpacked
@@ -19,7 +19,7 @@ cleanstatus() {
 
 #FIXME: the reported version is wrong, it should be 1, not 2
 insertinstalledpackage 'libqtcore4' 'i386,amd64' '1' 'Multi-Arch: same' '' 'install ok unpacked'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 0 upgraded, 0 newly installed, 0 to remove and 2 not upgraded.
 2 not fully installed or removed.
@@ -29,7 +29,7 @@ Conf libqtcore4:i386 (2 unstable [i386])' aptget install -s -f
 cleanstatus
 insertinstalledpackage 'libqtcore4' 'amd64' '2' 'Multi-Arch: same' '' 'install ok unpacked'
 insertinstalledpackage 'libqtcore4' 'i386' '1' 'Multi-Arch: same' '' 'install ok unpacked'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Correcting dependencies... Done
 The following extra packages will be installed:
@@ -45,7 +45,7 @@ Conf libqtcore4 (2 unstable [amd64])' aptget install -s -f
 cleanstatus
 insertinstalledpackage 'libqtcore4' 'i386' '2' 'Multi-Arch: same' '' 'install ok unpacked'
 insertinstalledpackage 'libqtcore4' 'amd64' '1' 'Multi-Arch: same' '' 'install ok unpacked'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Correcting dependencies... Done
 The following extra packages will be installed:
@@ -61,7 +61,7 @@ Conf libqtcore4:i386 (2 unstable [i386])' aptget install -s -f
 cleanstatus
 insertinstalledpackage 'libqtcore4' 'amd64' '2' 'Multi-Arch: same' '' 'install ok unpacked'
 insertinstalledpackage 'libqtcore4' 'i386' '1' 'Multi-Arch: same'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Correcting dependencies... Done
 The following extra packages will be installed:
@@ -77,7 +77,7 @@ Conf libqtcore4 (2 unstable [amd64])' aptget install -s -f
 cleanstatus
 insertinstalledpackage 'libqtcore4' 'i386' '2' 'Multi-Arch: same' '' 'install ok unpacked'
 insertinstalledpackage 'libqtcore4' 'amd64' '1' 'Multi-Arch: same'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Correcting dependencies... Done
 The following extra packages will be installed:
@@ -93,7 +93,7 @@ Conf libqtcore4:i386 (2 unstable [i386])' aptget install -s -f
 cleanstatus
 insertinstalledpackage 'libqtcore4' 'amd64' '2' 'Multi-Arch: same'
 insertinstalledpackage 'libqtcore4' 'i386' '1' 'Multi-Arch: same' '' 'install ok unpacked'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Correcting dependencies... Done
 The following extra packages will be installed:
@@ -108,7 +108,7 @@ Conf libqtcore4:i386 (2 unstable [i386])' aptget install -s -f
 cleanstatus
 insertinstalledpackage 'libqtcore4' 'i386' '2' 'Multi-Arch: same'
 insertinstalledpackage 'libqtcore4' 'amd64' '1' 'Multi-Arch: same' '' 'install ok unpacked'
-testequal 'Reading package lists...
+testsuccessequal 'Reading package lists...
 Building dependency tree...
 Correcting dependencies... Done
 The following extra packages will be installed:
diff --git a/test/integration/test-xorg-break-providers b/test/integration/test-xorg-break-providers
index 0be57d979..ff1f3b077 100755
--- a/test/integration/test-xorg-break-providers
+++ b/test/integration/test-xorg-break-providers
@@ -13,7 +13,7 @@ setupaptarchive
 # The test ensures that only -intel will be upgraded
 # (together with -core of course) and -vesa not touched.
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
   xserver-xorg-video-intel
@@ -24,7 +24,7 @@ Need to get 0 B/2992 kB of archives.
 After this operation, 24.6 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation.' aptget install xserver-xorg-core --trivial-only
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be upgraded:
@@ -34,7 +34,7 @@ Need to get 0 B/2992 kB of archives.
 After this operation, 24.6 kB of additional disk space will be used.
 E: Trivial Only specified but this is not a trivial operation.' aptget upgrade --trivial-only
 
-testequal 'Reading package lists...
+testfailureequal 'Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages will be upgraded:
-- 
cgit v1.2.3-70-g09d2


From 249aec3b7397662a678ea0014f94392085477b09 Mon Sep 17 00:00:00 2001
From: Jérémy Bobbio 
Date: Tue, 10 Mar 2015 10:09:44 +0100
Subject: stop displaying time of build in online help
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As part of the “reproducible builds” effort [1], we have noticed that
apt could not be built reproducibly.

One issue is that it uses the __DATE__ and __TIME__ macros of the C
preprocessor to display the time of build in the online help. We believe
this information not to be really useful to users as they can always
look at the package data and metadata to figure it out.

The attached patch simply removes this information. All
non-documentation packages can then be built reproducibly with our
current experimental framework.

[David: changed the string slightly to be untranslateable as well]

Closes: 774342
---
 cmdline/apt-cache.cc                        | 5 ++---
 cmdline/apt-cdrom.cc                        | 4 ++--
 cmdline/apt-config.cc                       | 5 ++---
 cmdline/apt-dump-solver.cc                  | 3 +--
 cmdline/apt-extracttemplates.cc             | 3 +--
 cmdline/apt-get.cc                          | 9 ++++-----
 cmdline/apt-helper.cc                       | 3 +--
 cmdline/apt-internal-solver.cc              | 3 +--
 cmdline/apt-mark.cc                         | 3 +--
 cmdline/apt-sortpkgs.cc                     | 3 +--
 cmdline/apt.cc                              | 3 +--
 ftparchive/apt-ftparchive.cc                | 3 +--
 test/integration/test-00-commands-have-help | 2 +-
 13 files changed, 19 insertions(+), 30 deletions(-)

(limited to 'test')

diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index a5024c581..e2cf7e8b7 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -1824,9 +1824,8 @@ static bool GenCaches(CommandLine &)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
-   
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
      return true;
 
diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc
index 327039e00..d95c169cd 100644
--- a/cmdline/apt-cdrom.cc
+++ b/cmdline/apt-cdrom.cc
@@ -205,8 +205,8 @@ static bool DoIdent(CommandLine &)
 // ShowHelp - Show the help screen					/*{{{*/
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
       return true;
    
diff --git a/cmdline/apt-config.cc b/cmdline/apt-config.cc
index e0b8a624e..4479b84a7 100644
--- a/cmdline/apt-config.cc
+++ b/cmdline/apt-config.cc
@@ -80,11 +80,10 @@ static bool DoDump(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
-   
+
    cout <<
     _("Usage: apt-config [options] command\n"
       "\n"
diff --git a/cmdline/apt-dump-solver.cc b/cmdline/apt-dump-solver.cc
index f765234c5..4729eac55 100644
--- a/cmdline/apt-dump-solver.cc
+++ b/cmdline/apt-dump-solver.cc
@@ -21,9 +21,8 @@
 // ---------------------------------------------------------------------
 /* */
 static bool ShowHelp() {
-
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 	std::cout <<
-		PACKAGE " " PACKAGE_VERSION " for " COMMON_ARCH " compiled on " __DATE__ " " __TIME__ << std::endl <<
 		"Usage: apt-dump-resolver\n"
 		"\n"
 		"apt-dump-resolver is a dummy solver who just dumps its input to the\n"
diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc
index 5211ee027..3e4f89286 100644
--- a/cmdline/apt-extracttemplates.cc
+++ b/cmdline/apt-extracttemplates.cc
@@ -219,8 +219,7 @@ bool DebFile::ParseInfo()
 /* */
 static bool ShowHelp(CommandLine &)
 {
-	ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
 	if (_config->FindB("version") == true)
 		return true;
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index 33bbdd861..a0d78257c 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1607,13 +1607,12 @@ static bool DoChangelog(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
-	    
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
    {
       cout << _("Supported modules:") << endl;
-      
+
       for (unsigned I = 0; I != pkgVersioningSystem::GlobalListLen; I++)
       {
 	 pkgVersioningSystem *VS = pkgVersioningSystem::GlobalList[I];
@@ -1622,7 +1621,7 @@ static bool ShowHelp(CommandLine &)
 	 else
 	    cout << ' ';
 	 cout << "Ver: " << VS->Label << endl;
-	 
+
 	 /* Print out all the packaging systems that will work with 
 	    this VS */
 	 for (unsigned J = 0; J != pkgSystem::GlobalListLen; J++)
diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index a05ae90a2..65db857ad 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -83,8 +83,7 @@ static bool DoDownloadFile(CommandLine &CmdL)
 
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(std::cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    if (_config->FindB("version") == true)
      return true;
diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc
index 4fabeb02f..939061b93 100644
--- a/cmdline/apt-internal-solver.cc
+++ b/cmdline/apt-internal-solver.cc
@@ -44,8 +44,7 @@
 // ---------------------------------------------------------------------
 /* */
 static bool ShowHelp(CommandLine &) {
-	ioprintf(std::cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-		 COMMON_ARCH,__DATE__,__TIME__);
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
 	std::cout <<
 		_("Usage: apt-internal-solver\n"
diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc
index 47777009b..de1c80309 100644
--- a/cmdline/apt-mark.cc
+++ b/cmdline/apt-mark.cc
@@ -443,8 +443,7 @@ static bool ShowHold(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    cout <<
     _("Usage: apt-mark [options] {auto|manual} pkg1 [pkg2 ...]\n"
diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc
index 9b66ad4db..971900e4f 100644
--- a/cmdline/apt-sortpkgs.cc
+++ b/cmdline/apt-sortpkgs.cc
@@ -146,8 +146,7 @@ static bool DoIt(string InFile)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
    
diff --git a/cmdline/apt.cc b/cmdline/apt.cc
index 056cd213f..2f7eddb61 100644
--- a/cmdline/apt.cc
+++ b/cmdline/apt.cc
@@ -39,8 +39,7 @@
 
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(c1out,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(c1out, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    // FIXME: generate from CommandLine
    c1out << 
diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index adf1b6d73..69b936dff 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -616,8 +616,7 @@ static void LoadBinDir(vector &PkgList,Configuration &Setup)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
 
diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help
index bbd1475eb..f2317dbdf 100755
--- a/test/integration/test-00-commands-have-help
+++ b/test/integration/test-00-commands-have-help
@@ -13,7 +13,7 @@ configarchitecture 'amd64'
 export LD_BIND_NOW=1
 
 checkversionmessage() {
-	testsuccess grep '^apt .* compiled on ' ${1}-help.output
+	testsuccess grep '^apt .* (' ${1}-help.output
 }
 
 checkhelpmessage() {
-- 
cgit v1.2.3-70-g09d2


From d01fa806061db7f5d227a0784405987c78998c84 Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Fri, 20 Mar 2015 17:38:51 +0100
Subject: test/integration/test-apt-download-progress: fix test failure on fast
 hardware

---
 test/integration/test-apt-download-progress | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'test')

diff --git a/test/integration/test-apt-download-progress b/test/integration/test-apt-download-progress
index 0a9020bec..7092e163e 100755
--- a/test/integration/test-apt-download-progress
+++ b/test/integration/test-apt-download-progress
@@ -30,13 +30,13 @@ testsuccess dd if=/dev/zero of=./aptarchive/$TESTFILE bs=800k count=1
 msgtest 'download progress works via' 'http'
 printf '\n'
 exec 3> apt-progress.log
-testsuccess apthelper download-file "http://localhost:8080/$TESTFILE" http-$TESTFILE -o APT::Status-Fd=3 -o Acquire::http::Dl-Limit=800
+testsuccess apthelper download-file "http://localhost:8080/$TESTFILE" http-$TESTFILE -o APT::Status-Fd=3 -o Acquire::http::Dl-Limit=600
 assertprogress apt-progress.log
 
 msgtest 'download progress works via' 'https'
 printf '\n'
 exec 3> apt-progress.log
-testsuccess apthelper download-file "https://localhost:4433/$TESTFILE" https-$TESTFILE -o APT::Status-Fd=3 -o Acquire::https::Dl-Limit=800
+testsuccess apthelper download-file "https://localhost:4433/$TESTFILE" https-$TESTFILE -o APT::Status-Fd=3 -o Acquire::https::Dl-Limit=600
 assertprogress apt-progress.log
 
 # cleanup
-- 
cgit v1.2.3-70-g09d2


From d5cf8851753dde4f45bfd3b48fcdf34247a8752a Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 7 Apr 2015 22:34:34 +0200
Subject: keyids in "apt-key del" should be case-insensitive

gnupg is case-insensitive about keyids, so back then apt-key called it
directly any keyid was accepted, but now that we work more with the
keyid ourself we regressed to require uppercase keyids by accident.

This is also inconsistent with other apt-key commands which still use
gnupg directly. A single case-insensitive grep and we are fine again.

Closes: 781696
---
 cmdline/apt-key.in            | 2 +-
 test/integration/test-apt-key | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'test')

diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in
index b4e071000..1da311d35 100644
--- a/cmdline/apt-key.in
+++ b/cmdline/apt-key.in
@@ -180,7 +180,7 @@ update() {
 remove_key_from_keyring() {
     local GPG="$GPG_CMD --keyring $1"
     # check if the key is in this keyring: the key id is in the 5 column at the end
-    if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then
+    if ! $GPG --with-colons --list-keys 2>&1 | grep -iq "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then
 	return
     fi
     if [ ! -w "$1" ]; then
diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key
index 47230cb55..b4f823ef1 100755
--- a/test/integration/test-apt-key
+++ b/test/integration/test-apt-key
@@ -111,3 +111,9 @@ cleanplate
 cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 testsuccess --nomsg aptkey --fakeroot del 5A90D141DBAC8DAE
 testempty aptkey list
+
+msgtest 'Test key removal with' 'lowercase key ID' #keylength somewher between 8byte and short
+cleanplate
+cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
+testsuccess --nomsg aptkey --fakeroot del d141dbac8dae
+testempty aptkey list
-- 
cgit v1.2.3-70-g09d2


From b8eba208daebe3e3f235983e44da9c398d6f7a57 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 10 Mar 2015 14:11:54 +0100
Subject: reimplement the last uses of sprintf

Working with strings c-style is complicated and error-prune,
so by converting to c++ style we gain some simplicity and
avoid buffer overflows by later extensions.

Git-Dch: Ignore
---
 apt-pkg/contrib/cdromutl.cc |  31 ++++----
 apt-pkg/contrib/strutl.cc   | 126 ++++++++++++++------------------
 apt-private/acqprogress.cc  | 173 ++++++++++++++++++++++----------------------
 apt-private/acqprogress.h   |   4 +-
 ftparchive/writer.cc        |   8 +-
 methods/ftp.cc              |  16 ++--
 test/libapt/strutil_test.cc |  22 ++++++
 test/libapt/uri_test.cc     |  12 +++
 8 files changed, 206 insertions(+), 186 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/contrib/cdromutl.cc b/apt-pkg/contrib/cdromutl.cc
index 936e377fb..6eb917457 100644
--- a/apt-pkg/contrib/cdromutl.cc
+++ b/apt-pkg/contrib/cdromutl.cc
@@ -207,7 +207,6 @@ bool IdentCdrom(string CD,string &Res,unsigned int Version)
    /* Run over the directory, we assume that the reader order will never
       change as the media is read-only. In theory if the kernel did
       some sort of wacked caching this might not be true.. */
-   char S[300];
    for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
    {
       // Skip some files..
@@ -215,30 +214,32 @@ bool IdentCdrom(string CD,string &Res,unsigned int Version)
 	  strcmp(Dir->d_name,"..") == 0)
 	 continue;
 
+      std::string S;
       if (Version <= 1)
       {
-	 sprintf(S,"%lu",(unsigned long)Dir->d_ino);
+	 strprintf(S, "%lu", (unsigned long)Dir->d_ino);
       }
       else
       {
 	 struct stat Buf;
 	 if (stat(Dir->d_name,&Buf) != 0)
 	    continue;
-	 sprintf(S,"%lu",(unsigned long)Buf.st_mtime);
+	 strprintf(S, "%lu", (unsigned long)Buf.st_mtime);
       }
-      
-      Hash.Add(S);
+
+      Hash.Add(S.c_str());
       Hash.Add(Dir->d_name);
    };
-   
+
    if (chdir(StartDir.c_str()) != 0) {
       _error->Errno("chdir",_("Unable to change to %s"),StartDir.c_str());
       closedir(D);
       return false;
    }
    closedir(D);
-   
+
    // Some stats from the fsys
+   std::string S;
    if (_config->FindB("Debug::identcdrom",false) == false)
    {
       struct statvfs Buf;
@@ -248,19 +249,19 @@ bool IdentCdrom(string CD,string &Res,unsigned int Version)
       // We use a kilobyte block size to advoid overflow
       if (writable_media)
       {
-         sprintf(S,"%lu",(long)(Buf.f_blocks*(Buf.f_bsize/1024)));
+         strprintf(S, "%lu", (unsigned long)(Buf.f_blocks*(Buf.f_bsize/1024)));
       } else {
-         sprintf(S,"%lu %lu",(long)(Buf.f_blocks*(Buf.f_bsize/1024)),
-                 (long)(Buf.f_bfree*(Buf.f_bsize/1024)));
+         strprintf(S, "%lu %lu", (unsigned long)(Buf.f_blocks*(Buf.f_bsize/1024)),
+                 (unsigned long)(Buf.f_bfree*(Buf.f_bsize/1024)));
       }
-      Hash.Add(S);
-      sprintf(S,"-%u",Version);
+      Hash.Add(S.c_str());
+      strprintf(S, "-%u", Version);
    }
    else
-      sprintf(S,"-%u.debug",Version);
-   
+      strprintf(S, "-%u.debug", Version);
+
    Res = Hash.Result().Value() + S;
-   return true;   
+   return true;
 }
 									/*}}}*/
 // FindMountPointForDevice - Find mountpoint for the given device	/*{{{*/
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index 0ac587a9e..0db4c57b2 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -324,21 +324,19 @@ bool ParseCWord(const char *&String,string &Res)
 /* */
 string QuoteString(const string &Str, const char *Bad)
 {
-   string Res;
+   std::stringstream Res;
    for (string::const_iterator I = Str.begin(); I != Str.end(); ++I)
    {
-      if (strchr(Bad,*I) != 0 || isprint(*I) == 0 || 
+      if (strchr(Bad,*I) != 0 || isprint(*I) == 0 ||
 	  *I == 0x25 || // percent '%' char
 	  *I <= 0x20 || *I >= 0x7F) // control chars
       {
-	 char Buf[10];
-	 sprintf(Buf,"%%%02x",(int)*I);
-	 Res += Buf;
+	 ioprintf(Res,"%%%02x",(int)*I);
       }
       else
-	 Res += *I;
+	 Res << *I;
    }
-   return Res;
+   return Res.str();
 }
 									/*}}}*/
 // DeQuoteString - Convert a string from quoted from                    /*{{{*/
@@ -379,13 +377,12 @@ string DeQuoteString(string::const_iterator const &begin,
    YottaBytes (E24) */
 string SizeToStr(double Size)
 {
-   char S[300];
    double ASize;
    if (Size >= 0)
       ASize = Size;
    else
       ASize = -1*Size;
-   
+
    /* bytes, KiloBytes, MegaBytes, GigaBytes, TeraBytes, PetaBytes, 
       ExaBytes, ZettaBytes, YottaBytes */
    char Ext[] = {'\0','k','M','G','T','P','E','Z','Y'};
@@ -394,20 +391,21 @@ string SizeToStr(double Size)
    {
       if (ASize < 100 && I != 0)
       {
-         sprintf(S,"%'.1f %c",ASize,Ext[I]);
-	 break;
+	 std::string S;
+	 strprintf(S, "%'.1f %c", ASize, Ext[I]);
+	 return S;
       }
-      
+
       if (ASize < 10000)
       {
-         sprintf(S,"%'.0f %c",ASize,Ext[I]);
-	 break;
+	 std::string S;
+	 strprintf(S, "%'.0f %c", ASize, Ext[I]);
+	 return S;
       }
       ASize /= 1000.0;
       I++;
    }
-   
-   return S;
+   return "";
 }
 									/*}}}*/
 // TimeToStr - Convert the time into a string				/*{{{*/
@@ -415,36 +413,27 @@ string SizeToStr(double Size)
 /* Converts a number of seconds to a hms format */
 string TimeToStr(unsigned long Sec)
 {
-   char S[300];
-   
-   while (1)
+   std::string S;
+   if (Sec > 60*60*24)
    {
-      if (Sec > 60*60*24)
-      {
-	 //d means days, h means hours, min means minutes, s means seconds
-	 sprintf(S,_("%lid %lih %limin %lis"),Sec/60/60/24,(Sec/60/60) % 24,(Sec/60) % 60,Sec % 60);
-	 break;
-      }
-      
-      if (Sec > 60*60)
-      {
-	 //h means hours, min means minutes, s means seconds
-	 sprintf(S,_("%lih %limin %lis"),Sec/60/60,(Sec/60) % 60,Sec % 60);
-	 break;
-      }
-      
-      if (Sec > 60)
-      {
-	 //min means minutes, s means seconds
-	 sprintf(S,_("%limin %lis"),Sec/60,Sec % 60);
-	 break;
-      }
-
-      //s means seconds
-      sprintf(S,_("%lis"),Sec);
-      break;
+      //TRANSLATOR: d means days, h means hours, min means minutes, s means seconds
+      strprintf(S,_("%lid %lih %limin %lis"),Sec/60/60/24,(Sec/60/60) % 24,(Sec/60) % 60,Sec % 60);
+   }
+   else if (Sec > 60*60)
+   {
+      //TRANSLATOR: h means hours, min means minutes, s means seconds
+      strprintf(S,_("%lih %limin %lis"),Sec/60/60,(Sec/60) % 60,Sec % 60);
+   }
+   else if (Sec > 60)
+   {
+      //TRANSLATOR: min means minutes, s means seconds
+      strprintf(S,_("%limin %lis"),Sec/60,Sec % 60);
+   }
+   else
+   {
+      //TRANSLATOR: s means seconds
+      strprintf(S,_("%lis"),Sec);
    }
-   
    return S;
 }
 									/*}}}*/
@@ -1423,7 +1412,7 @@ size_t strv_length(const char **str_array)
       ;
    return i;
 }
-
+									/*}}}*/
 // DeEscapeString - unescape (\0XX and \xXX) from a string		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -1605,51 +1594,46 @@ void URI::CopyFrom(const string &U)
 /* */
 URI::operator string()
 {
-   string Res;
-   
+   std::stringstream Res;
+
    if (Access.empty() == false)
-      Res = Access + ':';
-   
+      Res << Access << ':';
+
    if (Host.empty() == false)
-   {	 
+   {
       if (Access.empty() == false)
-	 Res += "//";
-          
+	 Res << "//";
+
       if (User.empty() == false)
       {
 	 // FIXME: Technically userinfo is permitted even less
 	 // characters than these, but this is not conveniently
 	 // expressed with a blacklist.
-	 Res += QuoteString(User, ":/?#[]@");
+	 Res << QuoteString(User, ":/?#[]@");
 	 if (Password.empty() == false)
-	    Res += ":" + QuoteString(Password, ":/?#[]@");
-	 Res += "@";
+	    Res << ":" << QuoteString(Password, ":/?#[]@");
+	 Res << "@";
       }
-      
+
       // Add RFC 2732 escaping characters
-      if (Access.empty() == false &&
-	  (Host.find('/') != string::npos || Host.find(':') != string::npos))
-	 Res += '[' + Host + ']';
+      if (Access.empty() == false && Host.find_first_of("/:") != string::npos)
+	 Res << '[' << Host << ']';
       else
-	 Res += Host;
-      
+	 Res << Host;
+
       if (Port != 0)
-      {
-	 char S[30];
-	 sprintf(S,":%u",Port);
-	 Res += S;
-      }	 
+	 Res << ':' << Port;
    }
-   
+
    if (Path.empty() == false)
    {
       if (Path[0] != '/')
-	 Res += "/" + Path;
+	 Res << "/" << Path;
       else
-	 Res += Path;
+	 Res << Path;
    }
-   
-   return Res;
+
+   return Res.str();
 }
 									/*}}}*/
 // URI::SiteOnly - Return the schema and site for the URI		/*{{{*/
diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index 14a53eacb..e834d7d6a 100644
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include 
@@ -34,9 +35,8 @@ using namespace std;
 // ---------------------------------------------------------------------
 /* */
 AcqTextStatus::AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet) :
-    pkgAcquireStatus(), ScreenWidth(ScreenWidth), ID(0), Quiet(Quiet)
+    pkgAcquireStatus(), ScreenWidth(ScreenWidth), LastLineLength(0), ID(0), Quiet(Quiet)
 {
-   BlankLine[0] = 0;
    // testcases use it to disable pulses without disabling other user messages
    if (Quiet == 0 && _config->FindB("quiet::NoUpdate", false) == true)
       this->Quiet = 1;
@@ -48,7 +48,7 @@ AcqTextStatus::AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet)
 void AcqTextStatus::Start()
 {
    pkgAcquireStatus::Start();
-   BlankLine[0] = 0;
+   LastLineLength = 0;
    ID = 1;
 }
 									/*}}}*/
@@ -60,8 +60,7 @@ void AcqTextStatus::IMSHit(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r';
+   clearLastLine();
 
    cout << _("Hit ") << Itm.Description;
    cout << endl;
@@ -82,8 +81,7 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r';
+   clearLastLine();
 
    cout << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
    if (Itm.Owner->FileSize != 0)
@@ -111,8 +109,7 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
    if (Itm.Owner->Status == pkgAcquire::Item::StatIdle)
       return;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r';
+   clearLastLine();
 
    if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
    {
@@ -140,8 +137,7 @@ void AcqTextStatus::Stop()
    if (Quiet > 1)
       return;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r' << flush;
+   clearLastLine();
 
    if (_config->FindB("quiet::NoStatistic", false) == true)
       return;
@@ -167,77 +163,66 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 
    enum {Long = 0,Medium,Short} Mode = Medium;
 
-   char Buffer[sizeof(BlankLine)];
-   char *End = Buffer + sizeof(Buffer);
-   char *S = Buffer;
-   if (ScreenWidth >= sizeof(Buffer))
-      ScreenWidth = sizeof(Buffer)-1;
-
-   // Put in the percent done
-   sprintf(S,"%.0f%%", Percent);
-
-   bool Shown = false;
-   for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
-	I = Owner->WorkerStep(I))
+   std::string Line;
    {
-      S += strlen(S);
-
-      // There is no item running
-      if (I->CurrentItem == 0)
+      std::stringstream S;
+      for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
+	    I = Owner->WorkerStep(I))
       {
-	 if (I->Status.empty() == false)
+	 // There is no item running
+	 if (I->CurrentItem == 0)
 	 {
-	    snprintf(S,End-S," [%s]",I->Status.c_str());
-	    Shown = true;
-	 }
-
-	 continue;
-      }
-
-      Shown = true;
+	    if (I->Status.empty() == false)
+	       S << " [" << I->Status << "]";
 
-      // Add in the short description
-      if (I->CurrentItem->Owner->ID != 0)
-	 snprintf(S,End-S," [%lu %s",I->CurrentItem->Owner->ID,
-		  I->CurrentItem->ShortDesc.c_str());
-      else
-	 snprintf(S,End-S," [%s",I->CurrentItem->ShortDesc.c_str());
-      S += strlen(S);
+	    continue;
+	 }
 
-      // Show the short mode string
-      if (I->CurrentItem->Owner->ActiveSubprocess.empty() == false)
-      {
-	 snprintf(S,End-S, " %s", I->CurrentItem->Owner->ActiveSubprocess.c_str());
-	 S += strlen(S);
-      }
+	 // Add in the short description
+	 S << " [";
+	 if (I->CurrentItem->Owner->ID != 0)
+	    S << I->CurrentItem->Owner->ID << " ";
+	 S << I->CurrentItem->ShortDesc;
 
-      // Add the current progress
-      if (Mode == Long)
-	 snprintf(S,End-S," %llu",I->CurrentSize);
-      else
-      {
-	 if (Mode == Medium || I->TotalSize == 0)
-	    snprintf(S,End-S," %sB",SizeToStr(I->CurrentSize).c_str());
-      }
-      S += strlen(S);
+	 // Show the short mode string
+	 if (I->CurrentItem->Owner->ActiveSubprocess.empty() == false)
+	    S << " " << I->CurrentItem->Owner->ActiveSubprocess;
 
-      // Add the total size and percent
-      if (I->TotalSize > 0 && I->CurrentItem->Owner->Complete == false)
-      {
-	 if (Mode == Short)
-	    snprintf(S,End-S," %.0f%%",
-		     (I->CurrentSize*100.0)/I->TotalSize);
+	 // Add the current progress
+	 if (Mode == Long)
+	    S << " " << I->CurrentSize;
 	 else
-	    snprintf(S,End-S,"/%sB %.0f%%",SizeToStr(I->TotalSize).c_str(),
+	 {
+	    if (Mode == Medium || I->TotalSize == 0)
+	       S << " " << SizeToStr(I->CurrentSize) << "B";
+	 }
+
+	 // Add the total size and percent
+	 if (I->TotalSize > 0 && I->CurrentItem->Owner->Complete == false)
+	 {
+	    if (Mode == Short)
+	       ioprintf(S, " %.0f%%", (I->CurrentSize*100.0)/I->TotalSize);
+	    else
+	       ioprintf(S, "/%sB %.0f%%", SizeToStr(I->TotalSize).c_str(),
 		     (I->CurrentSize*100.0)/I->TotalSize);
+	 }
+	 S << "]";
       }
-      S += strlen(S);
-      snprintf(S,End-S,"]");
-   }
 
-   // Show something..
-   if (Shown == false)
-      snprintf(S,End-S,_(" [Working]"));
+      // Show at least something
+      Line = S.str();
+      S.clear();
+      if (Line.empty() == true)
+	 Line = _(" [Working]");
+   }
+   // Put in the percent done
+   {
+      std::stringstream S;
+      ioprintf(S, "%.0f%%", Percent);
+      S << Line;
+      Line = S.str();
+      S.clear();
+   }
 
    /* Put in the ETA and cps meter, block off signals to prevent strangeness
       during resizing */
@@ -248,34 +233,33 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 
    if (CurrentCPS != 0)
    {
-      char Tmp[300];
       unsigned long long ETA = (TotalBytes - CurrentBytes)/CurrentCPS;
-      sprintf(Tmp," %sB/s %s",SizeToStr(CurrentCPS).c_str(),TimeToStr(ETA).c_str());
-      unsigned int Len = strlen(Buffer);
-      unsigned int LenT = strlen(Tmp);
-      if (Len + LenT < ScreenWidth)
+      std::string Tmp = " " + SizeToStr(CurrentCPS) + "B/s " + TimeToStr(ETA);
+      size_t alignment = Line.length() + Tmp.length();
+      if (alignment < ScreenWidth)
       {
-	 memset(Buffer + Len,' ',ScreenWidth - Len);
-	 strcpy(Buffer + ScreenWidth - LenT,Tmp);
+	 alignment = ScreenWidth - alignment;
+	 for (size_t i = 0; i < alignment; ++i)
+	    Line.append(" ");
+	 Line.append(Tmp);
       }
    }
-   Buffer[ScreenWidth] = 0;
-   BlankLine[ScreenWidth] = 0;
+   if (Line.length() > ScreenWidth)
+      Line.erase(ScreenWidth);
    sigprocmask(SIG_SETMASK,&OldSigs,0);
 
    // Draw the current status
    if (_config->FindB("Apt::Color", false) == true)
       cout << _config->Find("APT::Color::Yellow");
-   if (strlen(Buffer) == strlen(BlankLine))
-      cout << '\r' << Buffer << flush;
+   if (LastLineLength > Line.length())
+      clearLastLine();
    else
-      cout << '\r' << BlankLine << '\r' << Buffer << flush;
+      cout << '\r';
+   cout << Line << flush;
    if (_config->FindB("Apt::Color", false) == true)
       cout << _config->Find("APT::Color::Neutral") << flush;
 
-   memset(BlankLine,' ',strlen(Buffer));
-   BlankLine[strlen(Buffer)] = 0;
-
+   LastLineLength = Line.length();
    Update = false;
 
    return true;
@@ -296,8 +280,7 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
 
       return false;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r';
+   clearLastLine();
    ioprintf(cout,_("Media change: please insert the disc labeled\n"
 		   " '%s'\n"
 		   "in the drive '%s' and press enter\n"),
@@ -317,3 +300,17 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
    return bStatus;
 }
 									/*}}}*/
+void AcqTextStatus::clearLastLine() {					/*{{{*/
+   if (Quiet > 0)
+      return;
+
+   // do not try to clear more than the (now smaller) screen
+   if (LastLineLength > ScreenWidth)
+      LastLineLength = ScreenWidth;
+
+   std::cout << '\r';
+   for (size_t i = 0; i < LastLineLength; ++i)
+      std::cout << ' ';
+   std::cout << '\r' << std::flush;
+}
+									/*}}}*/
diff --git a/apt-private/acqprogress.h b/apt-private/acqprogress.h
index 71a10d78a..ab8170126 100644
--- a/apt-private/acqprogress.h
+++ b/apt-private/acqprogress.h
@@ -17,10 +17,12 @@
 class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
 {
    unsigned int &ScreenWidth;
-   char BlankLine[1024];
+   size_t LastLineLength;
    unsigned long ID;
    unsigned long Quiet;
 
+   void clearLastLine();
+
    public:
 
    virtual bool MediaChange(std::string Media,std::string Drive);
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index a63d8846b..db68c21f0 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -440,9 +440,6 @@ bool PackagesWriter::DoPackage(string FileName)
       OverItem->Priority = Tags.FindS("Priority");
    }
 
-   char Size[40];
-   sprintf(Size,"%llu", (unsigned long long) FileSize);
-   
    // Strip the DirStrip prefix from the FileName and add the PathPrefix
    string NewFileName;
    if (DirStrip.empty() == false &&
@@ -466,7 +463,10 @@ bool PackagesWriter::DoPackage(string FileName)
    // This lists all the changes to the fields we are going to make.
    std::vector Changes;
 
-   Changes.push_back(SetTFRewriteData("Size", Size));
+   std::string Size;
+   strprintf(Size, "%llu", (unsigned long long) FileSize);
+   Changes.push_back(SetTFRewriteData("Size", Size.c_str()));
+
    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
    {
       if (hs->HashType() == "MD5Sum")
diff --git a/methods/ftp.cc b/methods/ftp.cc
index 0504e5872..7764acf6a 100644
--- a/methods/ftp.cc
+++ b/methods/ftp.cc
@@ -259,19 +259,21 @@ bool FTPConn::Login()
       {
 	 if (Opts->Value.empty() == true)
 	    continue;
-	 
+
 	 // Substitute the variables into the command
-	 char SitePort[20];
-	 if (ServerName.Port != 0)
-	    sprintf(SitePort,"%u",ServerName.Port);
-	 else
-	    strcpy(SitePort,"21");
 	 string Tmp = Opts->Value;
 	 Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 	 Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 	 Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 	 Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
-	 Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
+	 if (ServerName.Port != 0)
+	 {
+	    std::string SitePort;
+	    strprintf(SitePort, "%u", ServerName.Port);
+	    Tmp = SubstVar(Tmp,"$(SITE_PORT)", SitePort);
+	 }
+	 else
+	    Tmp = SubstVar(Tmp,"$(SITE_PORT)", "21");
 	 Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 
 	 // Send the command
diff --git a/test/libapt/strutil_test.cc b/test/libapt/strutil_test.cc
index 9bc3c76fd..23dc08727 100644
--- a/test/libapt/strutil_test.cc
+++ b/test/libapt/strutil_test.cc
@@ -97,6 +97,28 @@ TEST(StrUtilTest,StartsWith)
    EXPECT_FALSE(Startswith("abcd", "x"));
    EXPECT_FALSE(Startswith("abcd", "abcndefg"));
 }
+TEST(StrUtilTest,TimeToStr)
+{
+   EXPECT_EQ("0s", TimeToStr(0));
+   EXPECT_EQ("42s", TimeToStr(42));
+   EXPECT_EQ("9min 21s", TimeToStr((9*60) + 21));
+   EXPECT_EQ("20min 42s", TimeToStr((20*60) + 42));
+   EXPECT_EQ("10h 42min 21s", TimeToStr((10*3600) + (42*60) + 21));
+   EXPECT_EQ("10h 42min 21s", TimeToStr((10*3600) + (42*60) + 21));
+   EXPECT_EQ("1988d 3h 29min 7s", TimeToStr((1988*86400) + (3*3600) + (29*60) + 7));
+
+   EXPECT_EQ("59s", TimeToStr(59));
+   EXPECT_EQ("60s", TimeToStr(60));
+   EXPECT_EQ("1min 1s", TimeToStr(61));
+   EXPECT_EQ("59min 59s", TimeToStr(3599));
+   EXPECT_EQ("60min 0s", TimeToStr(3600));
+   EXPECT_EQ("1h 0min 1s", TimeToStr(3601));
+   EXPECT_EQ("1h 1min 0s", TimeToStr(3660));
+   EXPECT_EQ("23h 59min 59s", TimeToStr(86399));
+   EXPECT_EQ("24h 0min 0s", TimeToStr(86400));
+   EXPECT_EQ("1d 0h 0min 1s", TimeToStr(86401));
+   EXPECT_EQ("1d 0h 1min 0s", TimeToStr(86460));
+}
 TEST(StrUtilTest,SubstVar)
 {
    EXPECT_EQ("", SubstVar("", "fails", "passes"));
diff --git a/test/libapt/uri_test.cc b/test/libapt/uri_test.cc
index 1662f51f0..5d5ae9679 100644
--- a/test/libapt/uri_test.cc
+++ b/test/libapt/uri_test.cc
@@ -12,6 +12,7 @@ TEST(URITest, BasicHTTP)
    EXPECT_EQ(90, U.Port);
    EXPECT_EQ("www.debian.org", U.Host);
    EXPECT_EQ("/temp/test", U.Path);
+   EXPECT_EQ("http://www.debian.org:90/temp/test", (std::string)U);
    // Login data
    U = URI("http://jgg:foo@ualberta.ca/blah");
    EXPECT_EQ("http", U.Access);
@@ -20,6 +21,7 @@ TEST(URITest, BasicHTTP)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("ualberta.ca", U.Host);
    EXPECT_EQ("/blah", U.Path);
+   EXPECT_EQ("http://jgg:foo@ualberta.ca/blah", (std::string)U);
 }
 TEST(URITest, SingeSlashFile)
 {
@@ -30,6 +32,7 @@ TEST(URITest, SingeSlashFile)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("", U.Host);
    EXPECT_EQ("/usr/bin/foo", U.Path);
+   EXPECT_EQ("file:/usr/bin/foo", (std::string)U);
 }
 TEST(URITest, BasicCDROM)
 {
@@ -40,6 +43,7 @@ TEST(URITest, BasicCDROM)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("Moo Cow Rom", U.Host);
    EXPECT_EQ("/debian", U.Path);
+   EXPECT_EQ("cdrom://Moo Cow Rom/debian", (std::string)U);
 }
 TEST(URITest, RelativeGzip)
 {
@@ -50,6 +54,7 @@ TEST(URITest, RelativeGzip)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ(".", U.Host);
    EXPECT_EQ("/bar/cow", U.Path);
+   EXPECT_EQ("gzip://./bar/cow", (std::string)U);
 }
 TEST(URITest, NoSlashFTP)
 {
@@ -60,6 +65,7 @@ TEST(URITest, NoSlashFTP)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("ftp.fr.debian.org", U.Host);
    EXPECT_EQ("/debian/pool/main/x/xtel/xtel_3.2.1-15_i386.deb", U.Path);
+   EXPECT_EQ("ftp://ftp.fr.debian.org/debian/pool/main/x/xtel/xtel_3.2.1-15_i386.deb", (std::string)U);
 }
 TEST(URITest, RFC2732)
 {
@@ -70,6 +76,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("1080::8:800:200C:417A", U.Host);
    EXPECT_EQ("/foo", U.Path);
+   EXPECT_EQ("http://[1080::8:800:200C:417A]/foo", (std::string)U);
    // with port
    U = URI("http://[::FFFF:129.144.52.38]:80/index.html");
    EXPECT_EQ("http", U.Access);
@@ -78,6 +85,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(80, U.Port);
    EXPECT_EQ("::FFFF:129.144.52.38", U.Host);
    EXPECT_EQ("/index.html", U.Path);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38]:80/index.html", (std::string)U);
    // extra colon
    U = URI("http://[::FFFF:129.144.52.38:]:80/index.html");
    EXPECT_EQ("http", U.Access);
@@ -86,6 +94,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(80, U.Port);
    EXPECT_EQ("::FFFF:129.144.52.38:", U.Host);
    EXPECT_EQ("/index.html", U.Path);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]:80/index.html", (std::string)U);
    // extra colon port
    U = URI("http://[::FFFF:129.144.52.38:]/index.html");
    EXPECT_EQ("http", U.Access);
@@ -94,6 +103,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("::FFFF:129.144.52.38:", U.Host);
    EXPECT_EQ("/index.html", U.Path);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]/index.html", (std::string)U);
    // My Evil Corruption of RFC 2732 to handle CDROM names!
    // Fun for the whole family! */
    U = URI("cdrom:[The Debian 1.2 disk, 1/2 R1:6]/debian/");
@@ -103,6 +113,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("The Debian 1.2 disk, 1/2 R1:6", U.Host);
    EXPECT_EQ("/debian/", U.Path);
+   EXPECT_EQ("cdrom://[The Debian 1.2 disk, 1/2 R1:6]/debian/", (std::string)U);
    // no brackets
    U = URI("cdrom:Foo Bar Cow/debian/");
    EXPECT_EQ("cdrom", U.Access);
@@ -111,6 +122,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("Foo Bar Cow", U.Host);
    EXPECT_EQ("/debian/", U.Path);
+   EXPECT_EQ("cdrom://Foo Bar Cow/debian/", (std::string)U);
    // percent encoded
    U = URI("ftp://foo:b%40r@example.org");
    EXPECT_EQ("foo", U.User);
-- 
cgit v1.2.3-70-g09d2


From dfad5beea77d75983f6ff8a1b8296b74dd48203e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sun, 15 Mar 2015 22:34:54 +0100
Subject: add a simple unit test for acquire progress

This isn't testing much of the 'complex' parts,
but its better than nothing for now.

Git-Dch: Ignore
---
 apt-private/acqprogress.cc      |  53 ++++++-------
 apt-private/acqprogress.h       |   4 +-
 apt-private/private-install.cc  |   2 +-
 apt-private/private-update.cc   |   4 +-
 cmdline/apt-get.cc              |   6 +-
 cmdline/apt-helper.cc           |   2 +-
 test/libapt/acqprogress_test.cc | 170 ++++++++++++++++++++++++++++++++++++++++
 7 files changed, 205 insertions(+), 36 deletions(-)
 create mode 100644 test/libapt/acqprogress_test.cc

(limited to 'test')

diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index e834d7d6a..0c606e48e 100644
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -1,10 +1,9 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acqprogress.cc,v 1.24 2003/04/27 01:56:48 doogie Exp $
 /* ######################################################################
 
-   Acquire Progress - Command line progress meter 
-   
+   Acquire Progress - Command line progress meter
+
    ##################################################################### */
 									/*}}}*/
 // Include files							/*{{{*/
@@ -29,13 +28,11 @@
 #include 
 									/*}}}*/
 
-using namespace std;
-
 // AcqTextStatus::AcqTextStatus - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-AcqTextStatus::AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet) :
-    pkgAcquireStatus(), ScreenWidth(ScreenWidth), LastLineLength(0), ID(0), Quiet(Quiet)
+AcqTextStatus::AcqTextStatus(std::ostream &out, unsigned int &ScreenWidth,unsigned int const Quiet) :
+    pkgAcquireStatus(), out(out), ScreenWidth(ScreenWidth), LastLineLength(0), ID(0), Quiet(Quiet)
 {
    // testcases use it to disable pulses without disabling other user messages
    if (Quiet == 0 && _config->FindB("quiet::NoUpdate", false) == true)
@@ -62,8 +59,8 @@ void AcqTextStatus::IMSHit(pkgAcquire::ItemDesc &Itm)
 
    clearLastLine();
 
-   cout << _("Hit ") << Itm.Description;
-   cout << endl;
+   out << _("Hit ") << Itm.Description;
+   out << std::endl;
    Update = true;
 }
 									/*}}}*/
@@ -83,10 +80,10 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
 
    clearLastLine();
 
-   cout << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
+   out << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
    if (Itm.Owner->FileSize != 0)
-      cout << " [" << SizeToStr(Itm.Owner->FileSize) << "B]";
-   cout << endl;
+      out << " [" << SizeToStr(Itm.Owner->FileSize) << "B]";
+   out << std::endl;
 }
 									/*}}}*/
 // AcqTextStatus::Done - Completed a download				/*{{{*/
@@ -113,15 +110,15 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
 
    if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
    {
-      cout << _("Ign ") << Itm.Description << endl;
+      out << _("Ign ") << Itm.Description << std::endl;
       if (Itm.Owner->ErrorText.empty() == false &&
 	    _config->FindB("Acquire::Progress::Ignore::ShowErrorText", false) == true)
-	 cout << "  " << Itm.Owner->ErrorText << endl;
+	 out << "  " << Itm.Owner->ErrorText << std::endl;
    }
    else
    {
-      cout << _("Err ") << Itm.Description << endl;
-      cout << "  " << Itm.Owner->ErrorText << endl;
+      out << _("Err ") << Itm.Description << std::endl;
+      out << "  " << Itm.Owner->ErrorText << std::endl;
    }
 
    Update = true;
@@ -143,7 +140,7 @@ void AcqTextStatus::Stop()
       return;
 
    if (FetchedBytes != 0 && _error->PendingError() == false)
-      ioprintf(cout,_("Fetched %sB in %s (%sB/s)\n"),
+      ioprintf(out,_("Fetched %sB in %s (%sB/s)\n"),
 	       SizeToStr(FetchedBytes).c_str(),
 	       TimeToStr(ElapsedTime).c_str(),
 	       SizeToStr(CurrentCPS).c_str());
@@ -152,7 +149,7 @@ void AcqTextStatus::Stop()
 // AcqTextStatus::Pulse - Regular event pulse				/*{{{*/
 // ---------------------------------------------------------------------
 /* This draws the current progress. Each line has an overall percent
-   meter and a per active item status meter along with an overall 
+   meter and a per active item status meter along with an overall
    bandwidth and ETA indicator. */
 bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 {
@@ -250,14 +247,14 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 
    // Draw the current status
    if (_config->FindB("Apt::Color", false) == true)
-      cout << _config->Find("APT::Color::Yellow");
+      out << _config->Find("APT::Color::Yellow");
    if (LastLineLength > Line.length())
       clearLastLine();
    else
-      cout << '\r';
-   cout << Line << flush;
+      out << '\r';
+   out << Line << std::flush;
    if (_config->FindB("Apt::Color", false) == true)
-      cout << _config->Find("APT::Color::Neutral") << flush;
+      out << _config->Find("APT::Color::Neutral") << std::flush;
 
    LastLineLength = Line.length();
    Update = false;
@@ -268,7 +265,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 // AcqTextStatus::MediaChange - Media need to be swapped		/*{{{*/
 // ---------------------------------------------------------------------
 /* Prompt for a media swap */
-bool AcqTextStatus::MediaChange(string Media,string Drive)
+bool AcqTextStatus::MediaChange(std::string Media, std::string Drive)
 {
    // If we do not output on a terminal and one of the options to avoid user
    // interaction is given, we assume that no user is present who could react
@@ -281,7 +278,7 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
       return false;
 
    clearLastLine();
-   ioprintf(cout,_("Media change: please insert the disc labeled\n"
+   ioprintf(out,_("Media change: please insert the disc labeled\n"
 		   " '%s'\n"
 		   "in the drive '%s' and press enter\n"),
 	    Media.c_str(),Drive.c_str());
@@ -301,16 +298,16 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
 }
 									/*}}}*/
 void AcqTextStatus::clearLastLine() {					/*{{{*/
-   if (Quiet > 0)
+   if (Quiet > 0 || LastLineLength == 0)
       return;
 
    // do not try to clear more than the (now smaller) screen
    if (LastLineLength > ScreenWidth)
       LastLineLength = ScreenWidth;
 
-   std::cout << '\r';
+   out << '\r';
    for (size_t i = 0; i < LastLineLength; ++i)
-      std::cout << ' ';
-   std::cout << '\r' << std::flush;
+      out << ' ';
+   out << '\r' << std::flush;
 }
 									/*}}}*/
diff --git a/apt-private/acqprogress.h b/apt-private/acqprogress.h
index ab8170126..7cf990c65 100644
--- a/apt-private/acqprogress.h
+++ b/apt-private/acqprogress.h
@@ -13,9 +13,11 @@
 #include 
 
 #include 
+#include 
 
 class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
 {
+   std::ostream &out;
    unsigned int &ScreenWidth;
    size_t LastLineLength;
    unsigned long ID;
@@ -35,7 +37,7 @@ class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
 
    bool Pulse(pkgAcquire *Owner);
 
-   AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet);
+   AcqTextStatus(std::ostream &out, unsigned int &ScreenWidth,unsigned int const Quiet);
 };
 
 #endif
diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc
index 79a22edf4..acc6d42c2 100644
--- a/apt-private/private-install.cc
+++ b/apt-private/private-install.cc
@@ -116,7 +116,7 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
       return false;
 
    // Create the download object
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
    if (_config->FindB("APT::Get::Print-URIs", false) == true)
    {
diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc
index df77ac33a..73a82e988 100644
--- a/apt-private/private-update.cc
+++ b/apt-private/private-update.cc
@@ -38,8 +38,8 @@ bool DoUpdate(CommandLine &CmdL)
    pkgSourceList *List = Cache.GetSourceList();
 
    // Create the progress
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
-      
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
+
    // Just print out the uris an exit if the --print-uris flag was used
    if (_config->FindB("APT::Get::Print-URIs") == true)
    {
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index a0d78257c..1a4f4c5a8 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -624,7 +624,7 @@ static bool DoDownload(CommandLine &CmdL)
    if (verset.empty() == true)
       return false;
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet", 0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    pkgRecords Recs(Cache);
@@ -729,7 +729,7 @@ static bool DoSource(CommandLine &CmdL)
       return false;
 
    // Create the download object
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    SPtrArray Dsc = new DscFile[CmdL.FileSize()];
@@ -1550,7 +1550,7 @@ static bool DoChangelog(CommandLine &CmdL)
       return Success;
    }
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    Fetcher.SetLog(&Stat);
 
    bool const downOnly = _config->FindB("APT::Get::Download-Only", false);
diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index 65db857ad..a97fc903f 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,7 +48,7 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    size_t fileind = 0;
diff --git a/test/libapt/acqprogress_test.cc b/test/libapt/acqprogress_test.cc
new file mode 100644
index 000000000..288e05aca
--- /dev/null
+++ b/test/libapt/acqprogress_test.cc
@@ -0,0 +1,170 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+class TestItem: public pkgAcquire::Item
+{
+public:
+   TestItem(pkgAcquire * const Acq) : pkgAcquire::Item(Acq, "", NULL) {}
+
+   virtual std::string DescURI() { return ""; }
+
+};
+
+TEST(AcqProgress, IMSHit)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire::ItemDesc hit;
+   hit.URI = "http://example.org/file";
+   hit.Description = "Example File from example.org";
+   hit.ShortDesc = "Example File";
+   hit.Owner = NULL;
+
+   EXPECT_EQ("", out.str());
+   Stat.IMSHit(hit);
+   EXPECT_EQ("Hit Example File from example.org\n", out.str());
+   Stat.IMSHit(hit);
+   EXPECT_EQ("Hit Example File from example.org\n"
+	     "Hit Example File from example.org\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Hit Example File from example.org\n"
+	     "Hit Example File from example.org\n", out.str());
+}
+TEST(AcqProgress, FetchNoFileSize)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org\n", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org\n"
+	     "Get:2 Example File from example.org\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Get:1 Example File from example.org\n"
+	     "Get:2 Example File from example.org\n", out.str());
+}
+TEST(AcqProgress, FetchFileSize)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n", out.str());
+   fetchO.FileSize = 42;
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n"
+	     "Get:2 Example File from example.org [42 B]\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n"
+	     "Get:2 Example File from example.org [42 B]\n", out.str());
+}
+TEST(AcqProgress, Fail)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetchO.Status = pkgAcquire::Item::StatIdle;
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fail(fetch);
+   EXPECT_EQ("", out.str());
+   fetchO.Status = pkgAcquire::Item::StatDone;
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n", out.str());
+   fetchO.Status = pkgAcquire::Item::StatError;
+   fetchO.ErrorText = "An error test!";
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n", out.str());
+   _config->Set("Acquire::Progress::Ignore::ShowErrorText", true);
+   fetchO.Status = pkgAcquire::Item::StatDone;
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n"
+	     "Ign Example File from example.org\n"
+	     "  An error test!\n", out.str());
+   _config->Set("Acquire::Progress::Ignore::ShowErrorText", true);
+   Stat.Stop();
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n"
+	     "Ign Example File from example.org\n"
+	     "  An error test!\n", out.str());
+}
+TEST(AcqProgress, Pulse)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   _config->Set("APT::Sandbox::User", ""); // ensure we aren't sandboxing
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetchO.Status = pkgAcquire::Item::StatFetching;
+   fetch.Owner = &fetchO;
+
+   // make screen smaller and bigger again while running
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]", out.str());
+   width = 8;
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]"
+	     "\r        "
+	     "\r0% [Work", out.str());
+   width = 80;
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]"
+	     "\r        "
+	     "\r0% [Work"
+	     "\r0% [Working]", out.str());
+}
-- 
cgit v1.2.3-70-g09d2


From f51401eb51939bfc932519601c43822a0aa5e343 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 10 Mar 2015 21:05:02 +0100
Subject: test for or-group choice behaviour in upgrade

In #780028 we were discussing how the or-group order should be more
important than keep-back decisions of 'upgrade'. We have this behaviour,
but to ensure it stays this way lets add a test for it.

Git-Dch: Ignore
---
 test/integration/test-apt-get-upgrade | 90 ++++++++++++++++++++++++++++++-----
 1 file changed, 79 insertions(+), 11 deletions(-)

(limited to 'test')

diff --git a/test/integration/test-apt-get-upgrade b/test/integration/test-apt-get-upgrade
index fcee95023..09cbcdb7e 100755
--- a/test/integration/test-apt-get-upgrade
+++ b/test/integration/test-apt-get-upgrade
@@ -21,37 +21,47 @@ insertpackage 'stable,installed' 'upgrade-with-conflict' 'all' '1.0'
 insertpackage 'unstable' 'upgrade-with-conflict' 'all' '2.0' 'Conflicts: conflicting-dep' 'standard'
 insertpackage 'stable,installed' 'conflicting-dep' 'all' '1.0'
 
+# upgrade with conflict and a new pkg with higher priority than conflict
+insertpackage 'stable,installed' 'init' 'all' '1'
+insertpackage 'unstable' 'init' 'all' '2' 'Pre-Depends: systemd | sysvinit'
+insertpackage 'unstable' 'systemd' 'all' '2' 'Conflicts: conflicting-dep'
+insertpackage 'unstable' 'sysvinit' 'all' '2'
+
 setupaptarchive
 
 # Test if normal upgrade works as expected
-testsuccessequal 'Reading package lists...
+UPGRADE='Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following packages have been kept back:
-  upgrade-with-conflict upgrade-with-new-dep
+  init upgrade-with-conflict upgrade-with-new-dep
 The following packages will be upgraded:
   upgrade-simple
-1 upgraded, 0 newly installed, 0 to remove and 2 not upgraded.
+1 upgraded, 0 newly installed, 0 to remove and 3 not upgraded.
 Inst upgrade-simple [1.0] (2.0 unstable [all])
-Conf upgrade-simple (2.0 unstable [all])' aptget -s upgrade
+Conf upgrade-simple (2.0 unstable [all])'
+testsuccessequal "$UPGRADE" aptget upgrade -s
+testsuccessequal "$UPGRADE" apt upgrade -s --without-new-pkgs
 
 # Test if apt-get upgrade --with-new-pkgs works
-testsuccessequal 'Reading package lists...
+UPGRADENEW='Reading package lists...
 Building dependency tree...
 Calculating upgrade...
 The following NEW packages will be installed:
   new-dep
 The following packages have been kept back:
-  upgrade-with-conflict
+  init upgrade-with-conflict
 The following packages will be upgraded:
   upgrade-simple upgrade-with-new-dep
-2 upgraded, 1 newly installed, 0 to remove and 1 not upgraded.
+2 upgraded, 1 newly installed, 0 to remove and 2 not upgraded.
 Inst new-dep (1.0 stable [all])
 Inst upgrade-simple [1.0] (2.0 unstable [all])
 Inst upgrade-with-new-dep [1.0] (2.0 unstable [all])
 Conf new-dep (1.0 stable [all])
 Conf upgrade-simple (2.0 unstable [all])
-Conf upgrade-with-new-dep (2.0 unstable [all])' aptget -s upgrade --with-new-pkgs
+Conf upgrade-with-new-dep (2.0 unstable [all])'
+testsuccessequal "$UPGRADENEW" aptget upgrade -s --with-new-pkgs
+testsuccessequal "$UPGRADENEW" apt upgrade -s
 
 # Test if apt-get dist-upgrade works
 testsuccessequal 'Reading package lists...
@@ -60,17 +70,75 @@ Calculating upgrade...
 The following packages will be REMOVED:
   conflicting-dep
 The following NEW packages will be installed:
-  new-dep
+  new-dep systemd
 The following packages will be upgraded:
-  upgrade-simple upgrade-with-conflict upgrade-with-new-dep
-3 upgraded, 1 newly installed, 1 to remove and 0 not upgraded.
+  init upgrade-simple upgrade-with-conflict upgrade-with-new-dep
+4 upgraded, 2 newly installed, 1 to remove and 0 not upgraded.
 Remv conflicting-dep [1.0]
+Inst systemd (2 unstable [all])
+Conf systemd (2 unstable [all])
+Inst init [1] (2 unstable [all])
 Inst upgrade-with-conflict [1.0] (2.0 unstable [all])
 Inst new-dep (1.0 stable [all])
 Inst upgrade-simple [1.0] (2.0 unstable [all])
 Inst upgrade-with-new-dep [1.0] (2.0 unstable [all])
+Conf init (2 unstable [all])
 Conf upgrade-with-conflict (2.0 unstable [all])
 Conf new-dep (1.0 stable [all])
 Conf upgrade-simple (2.0 unstable [all])
 Conf upgrade-with-new-dep (2.0 unstable [all])' aptget -s dist-upgrade
 
+msgmsg 'make systemd a non-choice in the or-group and try again'
+echo 'Package: systemd
+Pin: release unstable
+Pin-Priority: -1' > rootdir/etc/apt/preferences.d/nosystemd.pref
+
+testsuccessequal "$UPGRADE" aptget upgrade -s
+testsuccessequal "$UPGRADE" apt upgrade -s --without-new-pkgs
+
+UPGRADENEW='Reading package lists...
+Building dependency tree...
+Calculating upgrade...
+The following NEW packages will be installed:
+  new-dep sysvinit
+The following packages have been kept back:
+  upgrade-with-conflict
+The following packages will be upgraded:
+  init upgrade-simple upgrade-with-new-dep
+3 upgraded, 2 newly installed, 0 to remove and 1 not upgraded.
+Inst sysvinit (2 unstable [all])
+Conf sysvinit (2 unstable [all])
+Inst init [1] (2 unstable [all])
+Inst new-dep (1.0 stable [all])
+Inst upgrade-simple [1.0] (2.0 unstable [all])
+Inst upgrade-with-new-dep [1.0] (2.0 unstable [all])
+Conf init (2 unstable [all])
+Conf new-dep (1.0 stable [all])
+Conf upgrade-simple (2.0 unstable [all])
+Conf upgrade-with-new-dep (2.0 unstable [all])'
+testsuccessequal "$UPGRADENEW" aptget upgrade -s --with-new-pkgs
+testsuccessequal "$UPGRADENEW" apt upgrade -s
+
+testsuccessequal 'Reading package lists...
+Building dependency tree...
+Calculating upgrade...
+The following packages will be REMOVED:
+  conflicting-dep
+The following NEW packages will be installed:
+  new-dep sysvinit
+The following packages will be upgraded:
+  init upgrade-simple upgrade-with-conflict upgrade-with-new-dep
+4 upgraded, 2 newly installed, 1 to remove and 0 not upgraded.
+Remv conflicting-dep [1.0]
+Inst sysvinit (2 unstable [all])
+Conf sysvinit (2 unstable [all])
+Inst init [1] (2 unstable [all])
+Inst upgrade-with-conflict [1.0] (2.0 unstable [all])
+Inst new-dep (1.0 stable [all])
+Inst upgrade-simple [1.0] (2.0 unstable [all])
+Inst upgrade-with-new-dep [1.0] (2.0 unstable [all])
+Conf init (2 unstable [all])
+Conf upgrade-with-conflict (2.0 unstable [all])
+Conf new-dep (1.0 stable [all])
+Conf upgrade-simple (2.0 unstable [all])
+Conf upgrade-with-new-dep (2.0 unstable [all])' aptget -s dist-upgrade
-- 
cgit v1.2.3-70-g09d2


From 596ec43ce34421080a58b28299c1ed9cb0dbaa25 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sun, 12 Apr 2015 19:16:01 +0200
Subject: parse specific-arch dependencies correctly on single-arch systems
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

On single-arch the parsing was creating groupnames like 'apt:amd64' even
through it should be 'apt' and a package in it belonging to architecture
amd64. The result for foreign architectures was as expected: The
dependency isn't satisfiable, but for native architecture it means the
wrong package (ala apt:amd64:amd64) is linked so this is also not
satisfiable, which is very much not expected.

No longer excluding single-arch from this codepath allows the generation
of the correct links, which still link to non-exisiting packages for
foreign dependencies, but natives link to the expected native package
just as if no architecture was given.

For negative arch-specific dependencies ala Conflicts this matter was
worse as apt will believe there isn't a Conflict to resolve, tricking it
into calculating a solution dpkg will refuse.

Architecture specific positive dependencies are rare in jessie – the
only one in amd64 main is foreign –, negative dependencies do not even
exist. Neither class has a native specimen, so no package in jessie is
effected by this bug, but it might be interesting for stretch upgrades.
This also means the regression potential is very low.

Closes: 777760
---
 apt-pkg/deb/deblistparser.cc                       |   2 +-
 test/integration/test-multiarch-foreign            | 159 ++++++++++++++-------
 .../test-specific-architecture-dependencies        | 148 ++++++++++++++++++-
 3 files changed, 254 insertions(+), 55 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index 4eef66c2b..213235c2b 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -770,7 +770,7 @@ bool debListParser::ParseDepends(pkgCache::VerIterator &Ver,
 	 if (NewDepends(Ver,Package,"none",Version,Op,Type) == false)
 	    return false;
       }
-      else if (MultiArchEnabled == true && found != string::npos &&
+      else if (found != string::npos &&
 	       strcmp(Package.c_str() + found, ":any") != 0)
       {
 	 string Arch = Package.substr(found+1, string::npos);
diff --git a/test/integration/test-multiarch-foreign b/test/integration/test-multiarch-foreign
index 332466d96..240f1a4d1 100755
--- a/test/integration/test-multiarch-foreign
+++ b/test/integration/test-multiarch-foreign
@@ -7,9 +7,13 @@ setupenvironment
 configarchitecture 'amd64' 'i386' 'armel'
 
 insertpackage 'unstable' 'cool-foo' 'amd64,i386' '1.0' 'Depends: foo'
+insertpackage 'unstable' 'cool-foo-x64' 'amd64' '1.0' 'Depends: foo:amd64'
+insertpackage 'unstable' 'cool-foo-x32' 'amd64' '1.0' 'Depends: foo:i386'
 insertpackage 'unstable' 'foo' 'amd64,i386,armel' '1.0' 'Multi-Arch: foreign'
 
 insertpackage 'unstable' 'cool-bar' 'amd64,i386' '1.0' 'Depends: bar-provider'
+insertpackage 'unstable' 'cool-bar-x64' 'amd64' '1.0' 'Depends: bar-provider:amd64'
+insertpackage 'unstable' 'cool-bar-x32' 'amd64' '1.0' 'Depends: bar-provider:i386'
 insertpackage 'unstable' 'bar' 'amd64,i386,armel' '1.0' 'Provides: bar-provider
 Multi-Arch: foreign'
 
@@ -27,28 +31,6 @@ Inst cool-foo:i386 (1.0 unstable [i386])
 Conf foo (1.0 unstable [amd64])
 Conf cool-foo:i386 (1.0 unstable [i386])' aptget install cool-foo:i386 -s
 
-testequal 'Reading package lists...
-Building dependency tree...
-The following extra packages will be installed:
-  foo
-The following NEW packages will be installed:
-  cool-foo foo
-0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
-Inst foo (1.0 unstable [amd64])
-Inst cool-foo (1.0 unstable [amd64])
-Conf foo (1.0 unstable [amd64])
-Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 -s
-
-testequal 'Reading package lists...
-Building dependency tree...
-The following NEW packages will be installed:
-  cool-foo foo
-0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
-Inst foo (1.0 unstable [amd64])
-Inst cool-foo (1.0 unstable [amd64])
-Conf foo (1.0 unstable [amd64])
-Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 foo:amd64 -s
-
 testequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
@@ -69,10 +51,6 @@ Inst cool-foo (1.0 unstable [amd64])
 Conf foo:armel (1.0 unstable [armel])
 Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 foo:armel -s
 
-
-
-
-
 testequal 'Reading package lists...
 Building dependency tree...
 The following extra packages will be installed:
@@ -87,6 +65,60 @@ Conf cool-bar:i386 (1.0 unstable [i386])' aptget install cool-bar:i386 -s
 
 testequal 'Reading package lists...
 Building dependency tree...
+The following NEW packages will be installed:
+  bar:i386 cool-bar
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst bar:i386 (1.0 unstable [i386])
+Inst cool-bar (1.0 unstable [amd64])
+Conf bar:i386 (1.0 unstable [i386])
+Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 bar:i386 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  bar:armel cool-bar
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst bar:armel (1.0 unstable [armel])
+Inst cool-bar (1.0 unstable [amd64])
+Conf bar:armel (1.0 unstable [armel])
+Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 bar:armel -s
+
+testequal "Reading package lists...
+Building dependency tree...
+Note, selecting 'bar:i386' instead of 'bar-provider:i386'
+The following NEW packages will be installed:
+  bar:i386 cool-bar
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst bar:i386 (1.0 unstable [i386])
+Inst cool-bar (1.0 unstable [amd64])
+Conf bar:i386 (1.0 unstable [i386])
+Conf cool-bar (1.0 unstable [amd64])" aptget install cool-bar bar-provider:i386 -s -q=0
+
+satisfiable_in_singlearch() {
+	testequal 'Reading package lists...
+Building dependency tree...
+The following extra packages will be installed:
+  foo
+The following NEW packages will be installed:
+  cool-foo foo
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst foo (1.0 unstable [amd64])
+Inst cool-foo (1.0 unstable [amd64])
+Conf foo (1.0 unstable [amd64])
+Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 -s
+
+	testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  cool-foo foo
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst foo (1.0 unstable [amd64])
+Inst cool-foo (1.0 unstable [amd64])
+Conf foo (1.0 unstable [amd64])
+Conf cool-foo (1.0 unstable [amd64])' aptget install cool-foo:amd64 foo:amd64 -s
+
+	testequal 'Reading package lists...
+Building dependency tree...
 The following extra packages will be installed:
   bar
 The following NEW packages will be installed:
@@ -97,7 +129,7 @@ Inst cool-bar (1.0 unstable [amd64])
 Conf bar (1.0 unstable [amd64])
 Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 -s
 
-testequal 'Reading package lists...
+	testequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
   bar cool-bar
@@ -107,44 +139,71 @@ Inst cool-bar (1.0 unstable [amd64])
 Conf bar (1.0 unstable [amd64])
 Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 bar:amd64 -s
 
-testequal 'Reading package lists...
+	testequal "Reading package lists...
 Building dependency tree...
+Note, selecting 'bar' instead of 'bar-provider'
 The following NEW packages will be installed:
-  bar:i386 cool-bar
+  bar cool-bar
 0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
-Inst bar:i386 (1.0 unstable [i386])
+Inst bar (1.0 unstable [amd64])
 Inst cool-bar (1.0 unstable [amd64])
-Conf bar:i386 (1.0 unstable [i386])
-Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 bar:i386 -s
+Conf bar (1.0 unstable [amd64])
+Conf cool-bar (1.0 unstable [amd64])" aptget install cool-bar bar-provider -s -q=0
 
-testequal 'Reading package lists...
+	testequal 'Reading package lists...
 Building dependency tree...
+The following extra packages will be installed:
+  foo
 The following NEW packages will be installed:
-  bar:armel cool-bar
+  cool-foo-x64 foo
 0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
-Inst bar:armel (1.0 unstable [armel])
-Inst cool-bar (1.0 unstable [amd64])
-Conf bar:armel (1.0 unstable [armel])
-Conf cool-bar (1.0 unstable [amd64])' aptget install cool-bar:amd64 bar:armel -s
+Inst foo (1.0 unstable [amd64])
+Inst cool-foo-x64 (1.0 unstable [amd64])
+Conf foo (1.0 unstable [amd64])
+Conf cool-foo-x64 (1.0 unstable [amd64])' aptget install cool-foo-x64 -s
+}
 
-testequal "Reading package lists...
+#FIXME: do not work in single-arch as i386 isn't known at cache generation time
+	testequal 'Reading package lists...
 Building dependency tree...
-Note, selecting 'bar' instead of 'bar-provider'
+The following extra packages will be installed:
+  foo
 The following NEW packages will be installed:
-  bar cool-bar
+  cool-foo-x32 foo
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst foo (1.0 unstable [amd64])
+Inst cool-foo-x32 (1.0 unstable [amd64])
+Conf foo (1.0 unstable [amd64])
+Conf cool-foo-x32 (1.0 unstable [amd64])' aptget install cool-foo-x32 -s
+
+	testequal 'Reading package lists...
+Building dependency tree...
+The following extra packages will be installed:
+  bar
+The following NEW packages will be installed:
+  bar cool-bar-x32
 0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
 Inst bar (1.0 unstable [amd64])
-Inst cool-bar (1.0 unstable [amd64])
+Inst cool-bar-x32 (1.0 unstable [amd64])
 Conf bar (1.0 unstable [amd64])
-Conf cool-bar (1.0 unstable [amd64])" aptget install cool-bar bar-provider -s -q=0
+Conf cool-bar-x32 (1.0 unstable [amd64])' aptget install cool-bar-x32 -s -q=0
 
-testequal "Reading package lists...
+	testequal 'Reading package lists...
 Building dependency tree...
-Note, selecting 'bar:i386' instead of 'bar-provider:i386'
+The following extra packages will be installed:
+  bar
 The following NEW packages will be installed:
-  bar:i386 cool-bar
+  bar cool-bar-x64
 0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
-Inst bar:i386 (1.0 unstable [i386])
-Inst cool-bar (1.0 unstable [amd64])
-Conf bar:i386 (1.0 unstable [i386])
-Conf cool-bar (1.0 unstable [amd64])" aptget install cool-bar bar-provider:i386 -s -q=0
+Inst bar (1.0 unstable [amd64])
+Inst cool-bar-x64 (1.0 unstable [amd64])
+Conf bar (1.0 unstable [amd64])
+Conf cool-bar-x64 (1.0 unstable [amd64])' aptget install cool-bar-x64 -s -q=0
+
+
+satisfiable_in_singlearch
+
+msgmsg 'switch to single architecture'
+configarchitecture 'amd64'
+
+satisfiable_in_singlearch
diff --git a/test/integration/test-specific-architecture-dependencies b/test/integration/test-specific-architecture-dependencies
index 078a84654..ccfced150 100755
--- a/test/integration/test-specific-architecture-dependencies
+++ b/test/integration/test-specific-architecture-dependencies
@@ -12,16 +12,19 @@ insertinstalledpackage 'provider' 'amd64' '1' 'Provides: foo'
 
 insertpackage 'unstable' 'pre-depender' 'all' '1' 'Pre-Depends: libc6:i386'
 insertpackage 'unstable' 'depender' 'all' '1' 'Depends: libc6:i386'
+insertpackage 'unstable' 'depender-x32' 'i386,amd64' '1' 'Depends: libc6:i386'
+insertpackage 'unstable' 'depender-x64' 'i386,amd64' '1' 'Depends: libc6:amd64'
 
 insertpackage 'unstable' 'breaker' 'all' '1' 'Breaks: libold (<< 2)'
-insertpackage 'unstable' 'breaker-x32' 'amd64' '1' 'Breaks: libold:i386 (<< 2)'
-insertpackage 'unstable' 'breaker-x64' 'i386' '1' 'Breaks: libold:amd64 (<< 2)'
+insertpackage 'unstable' 'breaker-x32' 'i386,amd64' '1' 'Breaks: libold:i386 (<< 2)'
+insertpackage 'unstable' 'breaker-x64' 'i386,amd64' '1' 'Breaks: libold:amd64 (<< 2)'
 # conflicts with no effect
 insertpackage 'unstable' 'oldconflictor' 'all' '1' 'Conflicts: libold (<< 0)'
 insertpackage 'unstable' 'oldconflictor-x32' 'amd64' '1' 'Conflicts: libold:i386 (<< 0)'
 insertpackage 'unstable' 'oldconflictor-x64' 'i386' '1' 'Conflicts: libold:amd64 (<< 0)'
 
 insertpackage 'unstable' 'foo-depender' 'i386,amd64' '1' 'Depends: foo'
+insertpackage 'unstable' 'foo-native-depender' 'amd64' '1' 'Depends: foo:amd64'
 insertpackage 'unstable' 'foo-foreign-depender' 'i386' '1' 'Depends: foo:amd64'
 
 insertpackage 'unstable' 'foo-conflictor' 'i386,amd64' '1' 'Conflicts: foo'
@@ -54,6 +57,54 @@ Inst depender (1 unstable [all])
 Conf libc6:i386 (1 unstable [i386])
 Conf depender (1 unstable [all])' aptget install depender -s
 
+testequal 'Reading package lists...
+Building dependency tree...
+The following extra packages will be installed:
+  libc6:i386
+The following NEW packages will be installed:
+  depender-x32:i386 libc6:i386
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst libc6:i386 (1 unstable [i386])
+Inst depender-x32:i386 (1 unstable [i386])
+Conf libc6:i386 (1 unstable [i386])
+Conf depender-x32:i386 (1 unstable [i386])' aptget install depender-x32:i386 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following extra packages will be installed:
+  libc6:i386
+The following NEW packages will be installed:
+  depender-x32 libc6:i386
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst libc6:i386 (1 unstable [i386])
+Inst depender-x32 (1 unstable [amd64])
+Conf libc6:i386 (1 unstable [i386])
+Conf depender-x32 (1 unstable [amd64])' aptget install depender-x32:amd64 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following extra packages will be installed:
+  libc6
+The following NEW packages will be installed:
+  depender-x64 libc6
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst libc6 (1 unstable [amd64])
+Inst depender-x64 (1 unstable [amd64])
+Conf libc6 (1 unstable [amd64])
+Conf depender-x64 (1 unstable [amd64])' aptget install depender-x64:amd64 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following extra packages will be installed:
+  libc6
+The following NEW packages will be installed:
+  depender-x64:i386 libc6
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst libc6 (1 unstable [amd64])
+Inst depender-x64:i386 (1 unstable [i386])
+Conf libc6 (1 unstable [amd64])
+Conf depender-x64:i386 (1 unstable [i386])' aptget install depender-x64:i386 -s
+
 testequal 'Reading package lists...
 Building dependency tree...
 The following packages will be REMOVED:
@@ -75,7 +126,29 @@ The following NEW packages will be installed:
 0 upgraded, 1 newly installed, 1 to remove and 0 not upgraded.
 Remv libold:i386 [1]
 Inst breaker-x32 (1 unstable [amd64])
-Conf breaker-x32 (1 unstable [amd64])' aptget install breaker-x32 -s
+Conf breaker-x32 (1 unstable [amd64])' aptget install breaker-x32:amd64 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following packages will be REMOVED:
+  libold:i386
+The following NEW packages will be installed:
+  breaker-x32:i386
+0 upgraded, 1 newly installed, 1 to remove and 0 not upgraded.
+Remv libold:i386 [1]
+Inst breaker-x32:i386 (1 unstable [i386])
+Conf breaker-x32:i386 (1 unstable [i386])' aptget install breaker-x32:i386 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following packages will be REMOVED:
+  libold
+The following NEW packages will be installed:
+  breaker-x64
+0 upgraded, 1 newly installed, 1 to remove and 0 not upgraded.
+Remv libold [1]
+Inst breaker-x64 (1 unstable [amd64])
+Conf breaker-x64 (1 unstable [amd64])' aptget install breaker-x64:amd64 -s
 
 testequal 'Reading package lists...
 Building dependency tree...
@@ -86,7 +159,7 @@ The following NEW packages will be installed:
 0 upgraded, 1 newly installed, 1 to remove and 0 not upgraded.
 Remv libold [1]
 Inst breaker-x64:i386 (1 unstable [i386])
-Conf breaker-x64:i386 (1 unstable [i386])' aptget install breaker-x64 -s
+Conf breaker-x64:i386 (1 unstable [i386])' aptget install breaker-x64:i386 -s
 
 testequal 'Reading package lists...
 Building dependency tree...
@@ -132,6 +205,14 @@ The following packages have unmet dependencies:
  foo-depender:i386 : Depends: foo:i386 but it is not installable
 E: Unable to correct problems, you have held broken packages.' aptget install foo-depender:i386 -s
 
+testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  foo-native-depender
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+Inst foo-native-depender (1 unstable [amd64])
+Conf foo-native-depender (1 unstable [amd64])' aptget install foo-native-depender -s
+
 testequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
@@ -180,3 +261,62 @@ The following NEW packages will be installed:
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 Inst foo-no-conflictor:i386 (1 unstable [i386])
 Conf foo-no-conflictor:i386 (1 unstable [i386])' aptget install foo-no-conflictor:i386 -s
+
+msgmsg 'switch to single architecture'
+configarchitecture 'amd64'
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following extra packages will be installed:
+  libc6
+The following NEW packages will be installed:
+  depender-x64 libc6
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst libc6 (1 unstable [amd64])
+Inst depender-x64 (1 unstable [amd64])
+Conf libc6 (1 unstable [amd64])
+Conf depender-x64 (1 unstable [amd64])' aptget install depender-x64 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+E: Unable to locate package depender-x64' aptget install depender-x64:i386 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  foo-native-depender
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+Inst foo-native-depender (1 unstable [amd64])
+Conf foo-native-depender (1 unstable [amd64])' aptget install foo-native-depender -s
+
+# libold:i386 is installed, but we don't see it as i386 isn't configured
+testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  breaker-x32
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+Inst breaker-x32 (1 unstable [amd64])
+Conf breaker-x32 (1 unstable [amd64])' aptget install breaker-x32:amd64 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following packages will be REMOVED:
+  libold
+The following NEW packages will be installed:
+  breaker-x64
+0 upgraded, 1 newly installed, 1 to remove and 0 not upgraded.
+Remv libold [1]
+Inst breaker-x64 (1 unstable [amd64])
+Conf breaker-x64 (1 unstable [amd64])' aptget install breaker-x64:amd64 -s
+
+testequal 'Reading package lists...
+Building dependency tree...
+Some packages could not be installed. This may mean that you have
+requested an impossible situation or if you are using the unstable
+distribution that some required packages have not yet been created
+or been moved out of Incoming.
+The following information may help to resolve the situation:
+
+The following packages have unmet dependencies:
+ depender-x32 : Depends: libc6:i386 but it is not installable
+E: Unable to correct problems, you have held broken packages.' aptget install depender-x32 -s
-- 
cgit v1.2.3-70-g09d2


From 42fab5fbc85e243b958beff95912c46fb81abcc0 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 17 Mar 2015 16:51:29 +0100
Subject: demote missing gtest to a buildtime warning
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We just need it for unit tests and our debian/rules file actually skips
calling them if nocheck is given… but this fails anyhow as we declared a
hard-dependency on it. Demoting the error to a warning in configuration
and adding a test in the 'make test' path with a friendly message allows
nocheck to be useful again.

(Running unit tests is fully encouraged of course, but bootstrappers and
 co do not need to be burdened with this stuff)
---
 configure.ac         | 2 +-
 test/libapt/makefile | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

(limited to 'test')

diff --git a/configure.ac b/configure.ac
index 06879f839..3fb393335 100644
--- a/configure.ac
+++ b/configure.ac
@@ -91,7 +91,7 @@ AC_CHECK_LIB(curl, curl_easy_init,
 
 AC_LANG_PUSH([C++])
 AC_CHECK_HEADER(gtest/gtest.h,,
-     AC_MSG_ERROR([failed: I need gtest to build tests]),
+     AC_MSG_WARN([failed: I need gtest (packaged as libgtest-dev) for unit testing]),
 )
 AC_LANG_POP([C++])
 
diff --git a/test/libapt/makefile b/test/libapt/makefile
index 7f23ace46..0f8df291f 100644
--- a/test/libapt/makefile
+++ b/test/libapt/makefile
@@ -8,6 +8,7 @@ APT_DOMAIN=none
 include ../../buildlib/defaults.mak
 
 .PHONY: test
+ifeq (file-okay,$(shell $(CC) -M gtest_runner.cc >/dev/null 2>&1 && echo 'file-okay'))
 test: $(BIN)/gtest$(BASENAME)
 	MALLOC_PERTURB_=21 MALLOC_CHECK_=2 LD_LIBRARY_PATH=$(LIB) $(BIN)/gtest$(BASENAME)
 
@@ -71,3 +72,11 @@ $(LIB)/gtest.a: $(OBJ)/gtest-all.o
 	echo Building static library $@
 	-rm -f $@
 	$(AR) $(ARFLAGS) $@ $^
+
+else
+test:
+	@echo "APT uses Googles C++ testing framework for its unit tests"
+	@echo "On Debian systems this is available in the 'libgtest-dev' package."
+	@echo "Please install it before attempting to run the unit tests."
+	exit 100
+endif
-- 
cgit v1.2.3-70-g09d2


From 27925d82dd0cbae74d48040363fe6f6c2bae5215 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 27 Mar 2015 15:53:43 +0100
Subject: improve https method queue progress reporting

The worker expects that the methods tell him when they start or finish
downloading a file. Various information pieces are passed along in this
report including the (expected) filesize. https was using a "global"
struct for reporting which made it 'reuse' incorrect values in some
cases like a non-existent InRelease fallbacking to Release{,.gpg}
resulting in a size-mismatch warning. Reducing the scope and redesigning
the setting of the values we can fix this and related issues.

Closes: 777565, 781509
Thanks: Robert Edmonds and Anders Kaseorg for initial patchs
---
 methods/https.cc                                  | 82 ++++++++++-----------
 methods/https.h                                   |  2 +-
 methods/server.h                                  |  3 +-
 test/integration/framework                        | 20 +++++-
 test/integration/test-apt-https-no-redirect       | 15 ++--
 test/integration/test-apt-update-expected-size    | 86 +++++++++++++++--------
 test/integration/test-bug-602412-dequote-redirect |  3 +-
 7 files changed, 121 insertions(+), 90 deletions(-)

(limited to 'test')

diff --git a/methods/https.cc b/methods/https.cc
index c69e84d3a..70f6a1046 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -37,11 +37,17 @@
 									/*}}}*/
 using namespace std;
 
+struct APT_HIDDEN CURLUserPointer {
+   HttpsMethod * const https;
+   HttpsMethod::FetchResult * const Res;
+   CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res) : https(https), Res(Res) {}
+};
+
 size_t
 HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    size_t len = size * nmemb;
-   HttpsMethod *me = (HttpsMethod *)userp;
+   CURLUserPointer *me = (CURLUserPointer *)userp;
    std::string line((char*) buffer, len);
    for (--len; len > 0; --len)
       if (isspace(line[len]) == 0)
@@ -53,23 +59,33 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
 
    if (line.empty() == true)
    {
-      if (me->Server->Result != 416 && me->Server->StartPos != 0)
+      if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0)
 	 ;
-      else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize())
+      else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize())
       {
-         me->Server->Result = 200;
-	 me->Server->StartPos = me->Server->Size;
+         me->https->Server->Result = 200;
+	 me->https->Server->StartPos = me->https->Server->Size;
 	 // the actual size is not important for https as curl will deal with it
 	 // by itself and e.g. doesn't bother us with transport-encoding…
-	 me->Server->JunkSize = std::numeric_limits::max();
+	 me->https->Server->JunkSize = std::numeric_limits::max();
       }
       else
-	 me->Server->StartPos = 0;
+	 me->https->Server->StartPos = 0;
+
+      me->https->File->Truncate(me->https->Server->StartPos);
+      me->https->File->Seek(me->https->Server->StartPos);
+
+      me->Res->LastModified = me->https->Server->Date;
+      me->Res->Size = me->https->Server->Size;
+      me->Res->ResumePoint = me->https->Server->StartPos;
 
-      me->File->Truncate(me->Server->StartPos);
-      me->File->Seek(me->Server->StartPos);
+      // we expect valid data, so tell our caller we get the file now
+      if (me->https->Server->Result >= 200 && me->https->Server->Result < 300 &&
+	    me->https->Server->JunkSize == 0 &&
+	    me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
+	 me->https->URIStart(*me->Res);
    }
-   else if (me->Server->HeaderLine(line) == false)
+   else if (me->https->Server->HeaderLine(line) == false)
       return 0;
 
    return size*nmemb;
@@ -85,12 +101,6 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
    if (me->Server->JunkSize != 0)
       return buffer_size;
 
-   if (me->Server->ReceivedData == false)
-   {
-      me->URIStart(me->Res);
-      me->Server->ReceivedData = true;
-   }
-
    if(me->File->Write(buffer, buffer_size) != true)
       return 0;
 
@@ -109,27 +119,15 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
    return buffer_size;
 }
 
-int
-HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/,
-			       double /*ultotal*/, double /*ulnow*/)
-{
-   HttpsMethod *me = (HttpsMethod *)clientp;
-   if(dltotal > 0 && me->Res.Size == 0) {
-      me->Res.Size = (unsigned long long)dltotal;
-   }
-   return 0;
-}
-
 // HttpsServerState::HttpsServerState - Constructor			/*{{{*/
 HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner)
 {
    TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
-   ReceivedData = false;
    Reset();
 }
 									/*}}}*/
 
-void HttpsMethod::SetupProxy()  					/*{{{*/
+void HttpsMethod::SetupProxy()						/*{{{*/
 {
    URI ServerName = Queue->Uri;
 
@@ -207,16 +205,16 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
 
    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 
+   FetchResult Res;
+   CURLUserPointer userp(this, &Res);
    // callbacks
    curl_easy_setopt(curl, CURLOPT_URL, static_cast(Uri).c_str());
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
-   curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this);
+   curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &userp);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
-   curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
-   curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
    // options
-   curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+   curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
    curl_easy_setopt(curl, CURLOPT_FILETIME, true);
    // only allow curl to handle https, not the other stuff it supports
    curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
@@ -414,24 +412,23 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       return false;
    }
 
-   struct stat resultStat;
-   if (unlikely(stat(File->Name().c_str(), &resultStat) != 0))
-   {
-      _error->Errno("stat", "Unable to access file %s", File->Name().c_str());
-      return false;
-   }
-   Res.Size = resultStat.st_size;
-
    // invalid range-request
    if (Server->Result == 416)
    {
       unlink(File->Name().c_str());
-      Res.Size = 0;
       delete File;
       Redirect(Itm->Uri);
       return true;
    }
 
+   struct stat resultStat;
+   if (unlikely(stat(File->Name().c_str(), &resultStat) != 0))
+   {
+      _error->Errno("stat", "Unable to access file %s", File->Name().c_str());
+      return false;
+   }
+   Res.Size = resultStat.st_size;
+
    // Timestamp
    curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified);
    if (Res.LastModified != -1)
@@ -455,7 +452,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    URIDone(Res);
 
    // cleanup
-   Res.Size = 0;
    delete File;
 
    return true;
diff --git a/methods/https.h b/methods/https.h
index 433a84680..4cc48fc34 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -65,7 +65,6 @@ class HttpsMethod : public ServerMethod
 				 double ultotal, double ulnow);
    void SetupProxy();
    CURL *curl;
-   FetchResult Res;
    ServerState *Server;
 
    // Used by ServerMethods unused by https
@@ -77,6 +76,7 @@ class HttpsMethod : public ServerMethod
 
    virtual bool Configuration(std::string Message);
    virtual ServerState * CreateServerState(URI uri);
+   using pkgAcqMethod::FetchResult;
 
    HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL)
    {
diff --git a/methods/server.h b/methods/server.h
index 3b232dcac..b974ec89a 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -37,7 +37,6 @@ struct ServerState
    unsigned long long Size; // size of the usable content (aka: the file)
    unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
-   bool ReceivedData;
    time_t Date;
    bool HaveContent;
    enum {Chunked,Stream,Closes} Encoding;
@@ -76,7 +75,7 @@ struct ServerState
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
    virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
-		 StartPos = 0; ReceivedData = false; Encoding = Closes; time(&Date); HaveContent = false;
+		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;};
    virtual bool WriteResponse(std::string const &Data) = 0;
 
diff --git a/test/integration/framework b/test/integration/framework
index ec23e41e6..50c027a2c 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1125,8 +1125,10 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 
 downloadfile() {
 	local PROTO="${1%%:*}"
-	apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
-		download-file "$1" "$2" 2>&1 || true
+	if ! apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
+		download-file "$1" "$2" 2>&1 ; then
+		return 1
+	fi
 	# only if the file exists the download was successful
 	if [ -r "$2" ]; then
 		return 0
@@ -1407,6 +1409,20 @@ testfailureequal() {
 	testfileequal "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output" "$CMP"
 }
 
+testfailuremsg() {
+	local CMP="$1"
+	shift
+	testfailure "$@"
+	msgtest 'Check that the output of the previous failed command has expected' 'failures and warnings'
+	grep '^\(W\|E\):' "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output" > "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailureequal.output" 2>&1 || true
+	if echo "$CMP" | checkdiff - "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailureequal.output"; then
+		msgpass
+	else
+		echo '### Complete output ###'
+		cat "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output"
+		msgfail
+	fi
+}
 
 testfilestats() {
 	msgtest "Test that file $1 has $2 $3" "$4"
diff --git a/test/integration/test-apt-https-no-redirect b/test/integration/test-apt-https-no-redirect
index bc744d6f2..c91c78916 100755
--- a/test/integration/test-apt-https-no-redirect
+++ b/test/integration/test-apt-https-no-redirect
@@ -14,22 +14,15 @@ echo 'alright' > aptarchive/working
 changetohttpswebserver  -o 'aptwebserver::redirect::replace::/redirectme/=http://localhost:8080/'
 
 msgtest 'download of a file works via' 'http'
-downloadfile 'http://localhost:8080/working' httpfile >/dev/null 2>&1 && msgpass || msgfail
+testsuccess --nomsg downloadfile 'http://localhost:8080/working' httpfile
 testfileequal httpfile 'alright'
 
 msgtest 'download of a file works via' 'https'
-downloadfile 'https://localhost:4433/working' httpsfile >/dev/null 2>&1 && msgpass || msgfail
+testsuccess --nomsg downloadfile 'https://localhost:4433/working' httpsfile
 testfileequal httpsfile 'alright'
 
 msgtest 'download of a file does not work if' 'https redirected to http'
-downloadfile 'https://localhost:4433/redirectme/working' redirectfile >curloutput 2>&1 && msgfail || msgpass
+testfailure --nomsg downloadfile 'https://localhost:4433/redirectme/working' redirectfile
 
 msgtest 'libcurl has forbidden access in last request to' 'http resource'
-if grep -q -E -- 'Protocol "?http"? not supported or disabled in libcurl' curloutput; then
-	msgpass
-else
-	cat curloutput
-	msgfail
-fi
-
-
+testsuccess --nomsg grep -q -E -- 'Protocol "?http"? not supported or disabled in libcurl' rootdir/tmp/testfailure.output
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 9711c293a..22de13ea5 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -10,35 +10,61 @@ configarchitecture "i386"
 insertpackage 'unstable' 'apt' 'all' '1.0'
 
 setupaptarchive --no-update
+cp -a aptarchive/dists aptarchive/dists.good
+
+test_inreleasetoobig() {
+	# make InRelease really big to trigger fallback
+	dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null
+	touch -d '+1hour' aptarchive/dists/unstable/InRelease
+	testsuccess aptget update -o Apt::Get::List-Cleanup=0  -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0
+	msgtest 'Check that the max write warning is triggered'
+	cp rootdir/tmp/testsuccess.output update.output
+	testsuccess --nomsg grep -q 'Writing more data than expected' update.output
+	rm -f update.output
+	# ensure the failed InRelease file got renamed
+	testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED
+}
+
+test_packagestoobig() {
+	# append junk at the end of the Packages.gz/Packages
+	SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
+	find aptarchive/dists -name 'Packages*' | while read pkg; do
+		echo "1234567890" >> "$pkg"
+		touch -d '+1hour' "$pkg"
+	done
+	NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
+	testfailuremsg "W: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages  Writing more data than expected ($NEW_SIZE > $SIZE)
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0
+}
+
+methodtest() {
+	msgmsg 'Test with' "$1" 'and clean start'
+	rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
+	# normal update works fine
+	testsuccess aptget update
+	mv rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
+
+	# starting fresh works
+	test_inreleasetoobig "$1"
+	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists.good aptarchive/dists
+	test_packagestoobig "$1"
+	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists.good aptarchive/dists
+
+	msgmsg 'Test with' "$1" 'and existing old data'
+	cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists
+	test_inreleasetoobig "$1"
+	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
+	cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists.good aptarchive/dists
+	test_packagestoobig "$1"
+	rm -rf aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
+}
+
 changetowebserver
+methodtest 'http://localhost:8080'
 
-# normal update works fine
-testsuccess aptget update
-
-# make InRelease really big to trigger fallback
-mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good
-dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null
-touch -d '+1hour' aptarchive/dists/unstable/InRelease
-testsuccess aptget update -o Apt::Get::List-Cleanup=0  -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0
-msgtest 'Check that the max write warning is triggered'
-if grep -q "Writing more data than expected" rootdir/tmp/testsuccess.output; then
-    msgpass
-else
-    cat rootdir/tmp/testsuccess.output
-    msgfail
-fi
-# ensure the failed InRelease file got renamed
-testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED
-mv aptarchive/dists/unstable/InRelease.good aptarchive/dists/unstable/InRelease
-
-
-# append junk at the end of the Packages.gz/Packages
-SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
-find aptarchive -name 'Packages*' | while read pkg; do
-	echo "1234567890" >> "$pkg"
-done
-NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
-rm -f rootdir/var/lib/apt/lists/localhost*
-testfailureequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages  Writing more data than expected ($NEW_SIZE > $SIZE)
-
-E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+changetohttpswebserver
+methodtest 'https://localhost:4433'
diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect
index d3573a79a..384c8b113 100755
--- a/test/integration/test-bug-602412-dequote-redirect
+++ b/test/integration/test-bug-602412-dequote-redirect
@@ -8,7 +8,7 @@ configarchitecture 'amd64'
 
 buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
 
-setupaptarchive
+setupaptarchive --no-update
 changetowebserver -o aptwebserver::redirect::replace::/pool/=/newpool/ \
 		  -o aptwebserver::redirect::replace::/dists/=/newdists/
 
@@ -16,6 +16,7 @@ mv aptarchive/pool aptarchive/newpool
 mv aptarchive/dists aptarchive/newdists
 
 testrun() {
+	msgmsg 'Test redirection works in method boundaries' "$1"
 	msgtest 'Test redirection works in' 'apt-get update'
 	testsuccess --nomsg aptget update
 
-- 
cgit v1.2.3-70-g09d2


From a09f6eb8fc67cd2d836019f448f18580396185e5 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 27 Mar 2015 19:59:44 +0100
Subject: send Alt-* info for uncompressed based on any compressions

file sends information about the uncompressed file if it can find it as
well as for the compressed file. This was done only for gzip so far, but
we support more compression types. That this information isn't used a
lot is a different story.

Git-Dch: Ignore
---
 methods/file.cc                          | 43 ++++++++++++++++++--------------
 test/integration/framework               |  2 +-
 test/integration/test-compressed-indexes |  2 +-
 3 files changed, 26 insertions(+), 21 deletions(-)

(limited to 'test')

diff --git a/methods/file.cc b/methods/file.cc
index 12db62203..5d9d7b951 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -16,6 +16,7 @@
 #include 
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -33,7 +34,7 @@ class FileMethod : public pkgAcqMethod
    
    public:
    
-   FileMethod() : pkgAcqMethod("1.0",SingleInstance | LocalOnly) {};
+   FileMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig | LocalOnly) {};
 };
 
 // FileMethod::Fetch - Fetch a file					/*{{{*/
@@ -58,27 +59,31 @@ bool FileMethod::Fetch(FetchItem *Itm)
       if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
 	 Res.IMSHit = true;
    }
-   
-   // See if we can compute a file without a .gz exentsion
-   std::string::size_type Pos = File.rfind(".gz");
-   if (Pos + 3 == File.length())
+
+   // See if the uncompressed file exists and reuse it
+   std::vector extensions = APT::Configuration::getCompressorExtensions();
+   for (std::vector::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext)
    {
-      File = std::string(File,0,Pos);
-      if (stat(File.c_str(),&Buf) == 0)
+      if (APT::String::Endswith(File, *ext) == true)
       {
-	 FetchResult AltRes;
-	 AltRes.Size = Buf.st_size;
-	 AltRes.Filename = File;
-	 AltRes.LastModified = Buf.st_mtime;
-	 AltRes.IMSHit = false;
-	 if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
-	    AltRes.IMSHit = true;
-	 
-	 URIDone(Res,&AltRes);
-	 return true;
-      }      
+	 std::string const unfile = File.substr(0, File.length() - ext->length() - 1);
+	 if (stat(unfile.c_str(),&Buf) == 0)
+	 {
+	    FetchResult AltRes;
+	    AltRes.Size = Buf.st_size;
+	    AltRes.Filename = unfile;
+	    AltRes.LastModified = Buf.st_mtime;
+	    AltRes.IMSHit = false;
+	    if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+	       AltRes.IMSHit = true;
+
+	    URIDone(Res,&AltRes);
+	    return true;
+	 }
+	 // no break here as we could have situations similar to '.gz' vs '.tar.gz' here
+      }
    }
-   
+
    if (Res.Filename.empty() == true)
       return _error->Error(_("File not found"));
 
diff --git a/test/integration/framework b/test/integration/framework
index 50c027a2c..994956b74 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -3,7 +3,7 @@
 EXIT_CODE=0
 
 # we all like colorful messages
-if [ "$MSGCOLOR" != 'NO' ]; then
+if [ "$MSGCOLOR" != 'NO' ] && [ "$MSGCOLOR" != 'ALWAYS' ]; then
 	if [ ! -t 1 ]; then # but check that we output to a terminal
 		export MSGCOLOR='NO'
 	fi
diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
index 5b966754c..c6b292baa 100755
--- a/test/integration/test-compressed-indexes
+++ b/test/integration/test-compressed-indexes
@@ -61,7 +61,7 @@ testrun() {
 	cd downloaded
 	testsuccess --nomsg aptget download testpkg
 	msgtest '\tdeb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
-	rm testpkg_1.0_i386.deb
+	rm -f testpkg_1.0_i386.deb
 	cd - >/dev/null
 	testsuccessequal 'Reading package lists...
 Building dependency tree...
-- 
cgit v1.2.3-70-g09d2


From 117038bac90261351518870b3f48136f134d4bfc Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 30 Mar 2015 19:52:32 +0200
Subject: handle servers closing encoded connections correctly

Servers who advertise that they close the connection get the 'Closes'
encoding flag, but this conflicts with servers who response with a
transfer-encoding (e.g. encoding) as it is saved in the same flag.

We have a better flag for the keep-alive (or not) of the connection
anyway, so we check this instead of the encoding.

This is in practice not much of a problem as real servers we talk to are
HTTP1.1 servers (with keep-alive) and there isn't much point in doing
chunked encoding if you are going to close anyway, but our simple
testserver stumbles over this if pressed and its a bit cleaner, too.

Git-Dch: Ignore
---
 methods/http.cc                         |  6 +++---
 test/interactive-helper/aptwebserver.cc | 28 +++++++++++++++++++++-------
 2 files changed, 24 insertions(+), 10 deletions(-)

(limited to 'test')

diff --git a/methods/http.cc b/methods/http.cc
index 021b284d0..947002cc6 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -442,7 +442,7 @@ bool HttpServerState::RunData(FileFd * const File)
    {
       /* Closes encoding is used when the server did not specify a size, the
          loss of the connection means we are done */
-      if (Encoding == Closes)
+      if (Persistent == false)
 	 In.Limit(-1);
       else if (JunkSize != 0)
 	 In.Limit(JunkSize);
@@ -524,7 +524,7 @@ bool HttpServerState::Die(FileFd &File)
 
    // See if this is because the server finished the data stream
    if (In.IsLimit() == false && State != HttpServerState::Header &&
-       Encoding != HttpServerState::Closes)
+       Persistent == true)
    {
       Close();
       if (LErrno == 0)
@@ -571,7 +571,7 @@ bool HttpServerState::Flush(FileFd * const File)
 	    return true;
       }
 
-      if (In.IsLimit() == true || Encoding == ServerState::Closes)
+      if (In.IsLimit() == true || Persistent == false)
 	 return true;
    }
    return false;
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 644629a33..86d5c06f0 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -598,17 +598,24 @@ static void * handleClient(void * voidclient)				/*{{{*/
 {
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
-   std::vector messages;
    bool closeConnection = false;
-   std::list headers;
-   while (closeConnection == false && ReadMessages(client, messages))
+   while (closeConnection == false)
    {
-      // if we announced a closing, do the close
-      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+      std::vector messages;
+      if (ReadMessages(client, messages) == false)
 	 break;
-      headers.clear();
+
+      std::list headers;
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
+	 // if we announced a closing in previous response, do the close now
+	 if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 {
+	    closeConnection = true;
+	    break;
+	 }
+	 headers.clear();
+
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
 	 std::string filename;
@@ -760,9 +767,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 else
 	    sendError(client, 404, *m, sendContent, "", headers);
       }
+
+      // if we announced a closing in the last response, do the close now
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 closeConnection = true;
+
+      if (_error->PendingError() == true)
+	 break;
       _error->DumpErrors(std::cerr);
-      messages.clear();
    }
+   _error->DumpErrors(std::cerr);
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
    return NULL;
-- 
cgit v1.2.3-70-g09d2


From 9224ce3d4d1ea0428a70e75134998e08aa45b1e6 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 30 Mar 2015 20:47:13 +0200
Subject: calculate only expected hashes in methods

Methods get told which hashes are expected by the acquire system, which
means we can use this list to restrict what we calculate in the methods
as any extra we are calculating is wasted effort as we can't compare it
with anything anyway.

Adding support for a new hash algorithm is therefore 'free' now and if a
algorithm is no longer provided in a repository for a file, we
automatically stop calculating it.

In practice this results in a speed-up in Debian as we don't have SHA512
here (so far), so we practically stop calculating it.
---
 apt-pkg/contrib/hashes.cc    | 63 +++++++++++++++++++++++++++++++++-----------
 apt-pkg/contrib/hashes.h     | 21 ++++++++++++---
 ftparchive/cachedb.cc        |  4 +--
 ftparchive/writer.cc         |  4 +--
 methods/cdrom.cc             |  2 +-
 methods/copy.cc              | 10 +++----
 methods/file.cc              |  2 +-
 methods/ftp.cc               |  2 +-
 methods/gzip.cc              |  2 +-
 methods/http.cc              | 14 +++++-----
 methods/http.h               |  2 +-
 methods/https.cc             |  2 +-
 methods/https.h              |  2 +-
 methods/rred.cc              |  2 +-
 methods/rsh.cc               |  2 +-
 methods/server.cc            |  2 +-
 methods/server.h             |  2 +-
 test/libapt/hashsums_test.cc | 24 +++++++++++++++++
 18 files changed, 116 insertions(+), 46 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index 6e7080bc9..953465091 100644
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -250,28 +250,34 @@ bool HashStringList::operator!=(HashStringList const &other) const
 class PrivateHashes {
 public:
    unsigned long long FileSize;
+   unsigned int CalcHashes;
 
-   PrivateHashes() : FileSize(0) {}
+   PrivateHashes(unsigned int const CalcHashes) : FileSize(0), CalcHashes(CalcHashes) {}
 };
 									/*}}}*/
 // Hashes::Add* - Add the contents of data or FD			/*{{{*/
-bool Hashes::Add(const unsigned char * const Data,unsigned long long const Size, unsigned int const Hashes)
+bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size)
 {
    bool Res = true;
 APT_IGNORE_DEPRECATED_PUSH
-   if ((Hashes & MD5SUM) == MD5SUM)
+   if ((d->CalcHashes & MD5SUM) == MD5SUM)
       Res &= MD5.Add(Data, Size);
-   if ((Hashes & SHA1SUM) == SHA1SUM)
+   if ((d->CalcHashes & SHA1SUM) == SHA1SUM)
       Res &= SHA1.Add(Data, Size);
-   if ((Hashes & SHA256SUM) == SHA256SUM)
+   if ((d->CalcHashes & SHA256SUM) == SHA256SUM)
       Res &= SHA256.Add(Data, Size);
-   if ((Hashes & SHA512SUM) == SHA512SUM)
+   if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
       Res &= SHA512.Add(Data, Size);
 APT_IGNORE_DEPRECATED_POP
    d->FileSize += Size;
    return Res;
 }
-bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes)
+bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes)
+{
+   d->CalcHashes = Hashes;
+   return Add(Data, Size);
+}
+bool Hashes::AddFD(int const Fd,unsigned long long Size)
 {
    unsigned char Buf[64*64];
    bool const ToEOF = (Size == UntilEOF);
@@ -285,12 +291,17 @@ bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hash
       if (ToEOF && Res == 0) // EOF
 	 break;
       Size -= Res;
-      if (Add(Buf, Res, Hashes) == false)
+      if (Add(Buf, Res) == false)
 	 return false;
    }
    return true;
 }
-bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes)
+bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes)
+{
+   d->CalcHashes = Hashes;
+   return AddFD(Fd, Size);
+}
+bool Hashes::AddFD(FileFd &Fd,unsigned long long Size)
 {
    unsigned char Buf[64*64];
    bool const ToEOF = (Size == 0);
@@ -309,20 +320,29 @@ bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes
       else if (a == 0) // EOF
 	 break;
       Size -= a;
-      if (Add(Buf, a, Hashes) == false)
+      if (Add(Buf, a) == false)
 	 return false;
    }
    return true;
+}
+bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes)
+{
+   d->CalcHashes = Hashes;
+   return AddFD(Fd, Size);
 }
 									/*}}}*/
 HashStringList Hashes::GetHashStringList()
 {
    HashStringList hashes;
 APT_IGNORE_DEPRECATED_PUSH
-   hashes.push_back(HashString("MD5Sum", MD5.Result().Value()));
-   hashes.push_back(HashString("SHA1", SHA1.Result().Value()));
-   hashes.push_back(HashString("SHA256", SHA256.Result().Value()));
-   hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
+   if ((d->CalcHashes & MD5SUM) == MD5SUM)
+      hashes.push_back(HashString("MD5Sum", MD5.Result().Value()));
+   if ((d->CalcHashes & SHA1SUM) == SHA1SUM)
+      hashes.push_back(HashString("SHA1", SHA1.Result().Value()));
+   if ((d->CalcHashes & SHA256SUM) == SHA256SUM)
+      hashes.push_back(HashString("SHA256", SHA256.Result().Value()));
+   if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
+      hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
 APT_IGNORE_DEPRECATED_POP
    std::string SizeStr;
    strprintf(SizeStr, "%llu", d->FileSize);
@@ -330,6 +350,19 @@ APT_IGNORE_DEPRECATED_POP
    return hashes;
 }
 APT_IGNORE_DEPRECATED_PUSH
-Hashes::Hashes() { d = new PrivateHashes(); }
+Hashes::Hashes() { d = new PrivateHashes(~0); }
+Hashes::Hashes(unsigned int const Hashes) { d = new PrivateHashes(Hashes); }
+Hashes::Hashes(HashStringList const &Hashes) {
+   unsigned int calcHashes = Hashes.usable() ? 0 : ~0;
+   if (Hashes.find("MD5Sum") != NULL)
+      calcHashes |= MD5SUM;
+   if (Hashes.find("SHA1") != NULL)
+      calcHashes |= SHA1SUM;
+   if (Hashes.find("SHA256") != NULL)
+      calcHashes |= SHA256SUM;
+   if (Hashes.find("SHA512") != NULL)
+      calcHashes |= SHA512SUM;
+   d = new PrivateHashes(calcHashes);
+}
 Hashes::~Hashes() { delete d; }
 APT_IGNORE_DEPRECATED_POP
diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h
index 154862457..ac13c8ace 100644
--- a/apt-pkg/contrib/hashes.h
+++ b/apt-pkg/contrib/hashes.h
@@ -178,7 +178,8 @@ class Hashes
 
    static const int UntilEOF = 0;
 
-   bool Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes = ~0);
+   bool Add(const unsigned char * const Data, unsigned long long const Size);
+   APT_DEPRECATED bool Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes);
    inline bool Add(const char * const Data)
    {return Add((unsigned char const * const)Data,strlen(Data));};
    inline bool Add(const unsigned char * const Beg,const unsigned char * const End)
@@ -186,13 +187,24 @@ class Hashes
 
    enum SupportedHashes { MD5SUM = (1 << 0), SHA1SUM = (1 << 1), SHA256SUM = (1 << 2),
       SHA512SUM = (1 << 3) };
-   bool AddFD(int const Fd,unsigned long long Size = 0, unsigned int const Hashes = ~0);
-   bool AddFD(FileFd &Fd,unsigned long long Size = 0, unsigned int const Hashes = ~0);
+   bool AddFD(int const Fd,unsigned long long Size = 0);
+   APT_DEPRECATED bool AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes);
+   bool AddFD(FileFd &Fd,unsigned long long Size = 0);
+   APT_DEPRECATED bool AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes);
 
    HashStringList GetHashStringList();
 
 APT_IGNORE_DEPRECATED_PUSH
+   /** create a Hashes object to calculate all supported hashes
+    *
+    * If ALL is too much, you can limit which Hashes are calculated
+    * with the following other constructors which mention explicitly
+    * which hashes to generate. */
    Hashes();
+   /** @param Hashes bitflag composed of #SupportedHashes */
+   Hashes(unsigned int const Hashes);
+   /** @param Hashes is a list of hashes */
+   Hashes(HashStringList const &Hashes);
    virtual ~Hashes();
 APT_IGNORE_DEPRECATED_POP
 
@@ -208,15 +220,16 @@ APT_IGNORE_DEPRECATED_POP
    }
 
    public:
+APT_IGNORE_DEPRECATED_PUSH
    APT_DEPRECATED bool AddFD(int const Fd, unsigned long long Size, bool const addMD5,
 	 bool const addSHA1, bool const addSHA256, bool const addSHA512) {
       return AddFD(Fd, Size, boolsToFlag(addMD5, addSHA1, addSHA256, addSHA512));
    };
-
    APT_DEPRECATED bool AddFD(FileFd &Fd, unsigned long long Size, bool const addMD5,
 	 bool const addSHA1, bool const addSHA256, bool const addSHA512) {
       return AddFD(Fd, Size, boolsToFlag(addMD5, addSHA1, addSHA256, addSHA512));
    };
+APT_IGNORE_DEPRECATED_POP
 };
 
 #endif
diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc
index 1dc268594..cc3527ea4 100644
--- a/ftparchive/cachedb.cc
+++ b/ftparchive/cachedb.cc
@@ -441,8 +441,8 @@ bool CacheDB::GetHashes(bool const GenOnly, unsigned int const DoHashes)
       if (OpenFile() == false)
 	 return false;
 
-      Hashes hashes;
-      if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize, FlHashes) == false)
+      Hashes hashes(FlHashes);
+      if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize) == false)
 	 return false;
 
       HashStringList hl = hashes.GetHashStringList();
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index db68c21f0..593278590 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -1075,8 +1075,8 @@ bool ReleaseWriter::DoPackage(string FileName)
 
    CheckSums[NewFileName].size = fd.Size();
 
-   Hashes hs;
-   hs.AddFD(fd, 0, DoHashes);
+   Hashes hs(DoHashes);
+   hs.AddFD(fd);
    CheckSums[NewFileName].Hashes = hs.GetHashStringList();
    fd.Close();
 
diff --git a/methods/cdrom.cc b/methods/cdrom.cc
index 74e2ecc6b..10cb29f66 100644
--- a/methods/cdrom.cc
+++ b/methods/cdrom.cc
@@ -266,7 +266,7 @@ bool CDROMMethod::Fetch(FetchItem *Itm)
    Res.LastModified = Buf.st_mtime;
    Res.Size = Buf.st_size;
 
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    FileFd Fd(Res.Filename, FileFd::ReadOnly);
    Hash.AddFD(Fd);
    Res.TakeHashes(Hash);
diff --git a/methods/copy.cc b/methods/copy.cc
index f3cd84215..a8e289df5 100644
--- a/methods/copy.cc
+++ b/methods/copy.cc
@@ -28,16 +28,16 @@
 class CopyMethod : public pkgAcqMethod
 {
    virtual bool Fetch(FetchItem *Itm);
-   void CalculateHashes(FetchResult &Res);
+   void CalculateHashes(FetchItem const * const Itm, FetchResult &Res);
    
    public:
    
    CopyMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {};
 };
 
-void CopyMethod::CalculateHashes(FetchResult &Res)
+void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
 {
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    FileFd::CompressMode CompressMode = FileFd::None;
    if (_config->FindB("Acquire::GzipIndexes", false) == true)
       CompressMode = FileFd::Extension;
@@ -71,7 +71,7 @@ bool CopyMethod::Fetch(FetchItem *Itm)
    // just calc the hashes if the source and destination are identical
    if (File == Itm->DestFile)
    {
-      CalculateHashes(Res);
+      CalculateHashes(Itm, Res);
       URIDone(Res);
       return true;
    }
@@ -104,7 +104,7 @@ bool CopyMethod::Fetch(FetchItem *Itm)
    if (utimes(Res.Filename.c_str(), times) != 0)
       return _error->Errno("utimes",_("Failed to set modification time"));
 
-   CalculateHashes(Res);
+   CalculateHashes(Itm, Res);
 
    URIDone(Res);
    return true;
diff --git a/methods/file.cc b/methods/file.cc
index 5d9d7b951..043ab04b8 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -87,7 +87,7 @@ bool FileMethod::Fetch(FetchItem *Itm)
    if (Res.Filename.empty() == true)
       return _error->Error(_("File not found"));
 
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    FileFd Fd(Res.Filename, FileFd::ReadOnly);
    Hash.AddFD(Fd);
    Res.TakeHashes(Hash);
diff --git a/methods/ftp.cc b/methods/ftp.cc
index 7764acf6a..92d8573f1 100644
--- a/methods/ftp.cc
+++ b/methods/ftp.cc
@@ -1064,7 +1064,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
    }
    
    // Open the file
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    {
       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
       if (_error->PendingError() == true)
diff --git a/methods/gzip.cc b/methods/gzip.cc
index 387c05f2e..65519633c 100644
--- a/methods/gzip.cc
+++ b/methods/gzip.cc
@@ -91,7 +91,7 @@ bool GzipMethod::Fetch(FetchItem *Itm)
       return false;
 
    // Read data from source, generate checksums and write
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    bool Failed = false;
    while (1) 
    {
diff --git a/methods/http.cc b/methods/http.cc
index 947002cc6..e4773b0e2 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -64,8 +64,8 @@ const unsigned int CircleBuf::BW_HZ=10;
 // CircleBuf::CircleBuf - Circular input buffer				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-CircleBuf::CircleBuf(unsigned long long Size) 
-   : Size(Size), Hash(0), TotalWriten(0)
+CircleBuf::CircleBuf(unsigned long long Size)
+   : Size(Size), Hash(NULL), TotalWriten(0)
 {
    Buf = new unsigned char[Size];
    Reset();
@@ -84,10 +84,10 @@ void CircleBuf::Reset()
    TotalWriten = 0;
    MaxGet = (unsigned long long)-1;
    OutQueue = string();
-   if (Hash != 0)
+   if (Hash != NULL)
    {
       delete Hash;
-      Hash = new Hashes;
+      Hash = NULL;
    }
 }
 									/*}}}*/
@@ -222,7 +222,7 @@ bool CircleBuf::Write(int Fd)
 
       TotalWriten += Res;
       
-      if (Hash != 0)
+      if (Hash != NULL)
 	 Hash->Add(Buf + (OutP%Size),Res);
       
       OutP += Res;
@@ -484,10 +484,10 @@ APT_PURE bool HttpServerState::IsOpen()					/*{{{*/
    return (ServerFd != -1);
 }
 									/*}}}*/
-bool HttpServerState::InitHashes(FileFd &File)				/*{{{*/
+bool HttpServerState::InitHashes(FileFd &File, HashStringList const &ExpectedHashes)/*{{{*/
 {
    delete In.Hash;
-   In.Hash = new Hashes;
+   In.Hash = new Hashes(ExpectedHashes);
 
    // Set the expected size and read file for the hashes
    File.Truncate(StartPos);
diff --git a/methods/http.h b/methods/http.h
index 40a88a7be..6dc872659 100644
--- a/methods/http.h
+++ b/methods/http.h
@@ -111,7 +111,7 @@ struct HttpServerState: public ServerState
    virtual bool Open();
    virtual bool IsOpen();
    virtual bool Close();
-   virtual bool InitHashes(FileFd &File);
+   virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes);
    virtual Hashes * GetHashes();
    virtual bool Die(FileFd &File);
    virtual bool Flush(FileFd * const File);
diff --git a/methods/https.cc b/methods/https.cc
index 70f6a1046..81903b239 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -443,7 +443,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       Res.LastModified = resultStat.st_mtime;
 
    // take hashes
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    FileFd Fd(Res.Filename, FileFd::ReadOnly);
    Hash.AddFD(Fd);
    Res.TakeHashes(Hash);
diff --git a/methods/https.h b/methods/https.h
index 4cc48fc34..dc0ff3322 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -42,7 +42,7 @@ class HttpsServerState : public ServerState
    virtual bool Open() { return false; }
    virtual bool IsOpen() { return false; }
    virtual bool Close() { return false; }
-   virtual bool InitHashes(FileFd &/*File*/) { return false; }
+   virtual bool InitHashes(FileFd &/*File*/, HashStringList const &/*ExpectedHashes*/) { return false; }
    virtual Hashes * GetHashes() { return NULL; }
    virtual bool Die(FileFd &/*File*/) { return false; }
    virtual bool Flush(FileFd * const /*File*/) { return false; }
diff --git a/methods/rred.cc b/methods/rred.cc
index 774b58a40..554ac99b4 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -581,7 +581,7 @@ class RredMethod : public pkgAcqMethod {
 	 FILE *inp = fopen(Path.c_str(), "r");
 	 FILE *out = fopen(Itm->DestFile.c_str(), "w");
 
-	 Hashes hash;
+	 Hashes hash(Itm->ExpectedHashes);
 
 	 patch.apply_against_file(out, inp, &hash);
 
diff --git a/methods/rsh.cc b/methods/rsh.cc
index 0e949160b..52349c61c 100644
--- a/methods/rsh.cc
+++ b/methods/rsh.cc
@@ -477,7 +477,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
    }
 
    // Open the file
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    {
       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
       if (_error->PendingError() == true)
diff --git a/methods/server.cc b/methods/server.cc
index 91ec824d1..e403f1071 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -357,7 +357,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
    FailFd = File->Fd();
    FailTime = Server->Date;
 
-   if (Server->InitHashes(*File) == false)
+   if (Server->InitHashes(*File, Queue->ExpectedHashes) == false)
    {
       _error->Errno("read",_("Problem hashing file"));
       return ERROR_NOT_FROM_SERVER;
diff --git a/methods/server.h b/methods/server.h
index b974ec89a..45622dd34 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -85,7 +85,7 @@ struct ServerState
    virtual bool Open() = 0;
    virtual bool IsOpen() = 0;
    virtual bool Close() = 0;
-   virtual bool InitHashes(FileFd &File) = 0;
+   virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes) = 0;
    virtual Hashes * GetHashes() = 0;
    virtual bool Die(FileFd &File) = 0;
    virtual bool Flush(FileFd * const File) = 0;
diff --git a/test/libapt/hashsums_test.cc b/test/libapt/hashsums_test.cc
index a19a0befd..edcd8a11a 100644
--- a/test/libapt/hashsums_test.cc
+++ b/test/libapt/hashsums_test.cc
@@ -193,6 +193,30 @@ TEST(HashSumsTest, FileBased)
       EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
    }
    fd.Seek(0);
+   {
+      Hashes hashes(Hashes::MD5SUM | Hashes::SHA512SUM);
+      hashes.AddFD(fd);
+      HashStringList list = hashes.GetHashStringList();
+      EXPECT_FALSE(list.empty());
+      EXPECT_EQ(3, list.size());
+      EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue());
+      EXPECT_EQ(NULL, list.find("SHA1"));
+      EXPECT_EQ(NULL, list.find("SHA256"));
+      EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue());
+      EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
+      fd.Seek(0);
+      Hashes hashes2(list);
+      hashes2.AddFD(fd);
+      list = hashes2.GetHashStringList();
+      EXPECT_FALSE(list.empty());
+      EXPECT_EQ(3, list.size());
+      EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue());
+      EXPECT_EQ(NULL, list.find("SHA1"));
+      EXPECT_EQ(NULL, list.find("SHA256"));
+      EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue());
+      EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
+   }
+   fd.Seek(0);
    {
       MD5Summation MD5;
       MD5.AddFD(fd.Fd());
-- 
cgit v1.2.3-70-g09d2


From 34faa8f7ae2526f46cd1f84bb6962ad06d841e5e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 11 Apr 2015 10:23:52 +0200
Subject: calculate hashes while downloading in https
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We do this in HTTP already to give the CPU some exercise while the disk
is heavily spinning (or flashing?) to store the data avoiding the need
to reread the entire file again later on to calculate the hashes – which
happens outside of the eyes of progress reporting, so you might ended up
with a bunch of https workers 'stuck' at 100% while they were busy
calculating hashes.

This is a bummer for everyone using apt as a connection speedtest as the
https method works slower now (not really, it just isn't reporting done
too early anymore).
---
 methods/http.cc                             |  8 +++---
 methods/http.h                              |  2 +-
 methods/https.cc                            | 38 ++++++++++++++++++++---------
 methods/https.h                             |  6 +++--
 methods/server.cc                           |  8 +++++-
 methods/server.h                            |  3 ++-
 test/integration/test-apt-download-progress |  6 +++--
 7 files changed, 47 insertions(+), 24 deletions(-)

(limited to 'test')

diff --git a/methods/http.cc b/methods/http.cc
index e4773b0e2..af3d5ccb6 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -484,16 +484,14 @@ APT_PURE bool HttpServerState::IsOpen()					/*{{{*/
    return (ServerFd != -1);
 }
 									/*}}}*/
-bool HttpServerState::InitHashes(FileFd &File, HashStringList const &ExpectedHashes)/*{{{*/
+bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes)	/*{{{*/
 {
    delete In.Hash;
    In.Hash = new Hashes(ExpectedHashes);
-
-   // Set the expected size and read file for the hashes
-   File.Truncate(StartPos);
-   return In.Hash->AddFD(File, StartPos);
+   return true;
 }
 									/*}}}*/
+
 APT_PURE Hashes * HttpServerState::GetHashes()				/*{{{*/
 {
    return In.Hash;
diff --git a/methods/http.h b/methods/http.h
index 6dc872659..e73871931 100644
--- a/methods/http.h
+++ b/methods/http.h
@@ -111,7 +111,7 @@ struct HttpServerState: public ServerState
    virtual bool Open();
    virtual bool IsOpen();
    virtual bool Close();
-   virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes);
+   virtual bool InitHashes(HashStringList const &ExpectedHashes);
    virtual Hashes * GetHashes();
    virtual bool Die(FileFd &File);
    virtual bool Flush(FileFd * const File);
diff --git a/methods/https.cc b/methods/https.cc
index 81903b239..c6b75d9ad 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -72,18 +72,18 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       else
 	 me->https->Server->StartPos = 0;
 
-      me->https->File->Truncate(me->https->Server->StartPos);
-      me->https->File->Seek(me->https->Server->StartPos);
-
       me->Res->LastModified = me->https->Server->Date;
       me->Res->Size = me->https->Server->Size;
       me->Res->ResumePoint = me->https->Server->StartPos;
 
       // we expect valid data, so tell our caller we get the file now
-      if (me->https->Server->Result >= 200 && me->https->Server->Result < 300 &&
-	    me->https->Server->JunkSize == 0 &&
-	    me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
-	 me->https->URIStart(*me->Res);
+      if (me->https->Server->Result >= 200 && me->https->Server->Result < 300)
+      {
+	 if (me->https->Server->JunkSize == 0 && me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
+	    me->https->URIStart(*me->Res);
+	 if (me->https->Server->AddPartialFileToHashes(*(me->https->File)) == false)
+	    return 0;
+      }
    }
    else if (me->https->Server->HeaderLine(line) == false)
       return 0;
@@ -116,16 +116,31 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
       }
    }
 
+   if (me->Server->GetHashes()->Add((unsigned char const * const)buffer, buffer_size) == false)
+      return 0;
+
    return buffer_size;
 }
 
 // HttpsServerState::HttpsServerState - Constructor			/*{{{*/
-HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner)
+HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner), Hash(NULL)
 {
    TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
    Reset();
 }
 									/*}}}*/
+bool HttpsServerState::InitHashes(HashStringList const &ExpectedHashes)	/*{{{*/
+{
+   delete Hash;
+   Hash = new Hashes(ExpectedHashes);
+   return true;
+}
+									/*}}}*/
+APT_PURE Hashes * HttpsServerState::GetHashes()				/*{{{*/
+{
+   return Hash;
+}
+									/*}}}*/
 
 void HttpsMethod::SetupProxy()						/*{{{*/
 {
@@ -365,6 +380,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    // go for it - if the file exists, append on it
    File = new FileFd(Itm->DestFile, FileFd::WriteAny);
    Server = CreateServerState(Itm->Uri);
+   if (Server->InitHashes(Itm->ExpectedHashes) == false)
+      return false;
 
    // keep apt updated
    Res.Filename = Itm->DestFile;
@@ -443,10 +460,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       Res.LastModified = resultStat.st_mtime;
 
    // take hashes
-   Hashes Hash(Itm->ExpectedHashes);
-   FileFd Fd(Res.Filename, FileFd::ReadOnly);
-   Hash.AddFD(Fd);
-   Res.TakeHashes(Hash);
+   Res.TakeHashes(*(Server->GetHashes()));
 
    // keep apt updated
    URIDone(Res);
diff --git a/methods/https.h b/methods/https.h
index dc0ff3322..6e32e8d3d 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -29,6 +29,8 @@ class FileFd;
 
 class HttpsServerState : public ServerState
 {
+   Hashes * Hash;
+
    protected:
    virtual bool ReadHeaderLines(std::string &/*Data*/) { return false; }
    virtual bool LoadNextResponse(bool const /*ToFile*/, FileFd * const /*File*/) { return false; }
@@ -42,8 +44,8 @@ class HttpsServerState : public ServerState
    virtual bool Open() { return false; }
    virtual bool IsOpen() { return false; }
    virtual bool Close() { return false; }
-   virtual bool InitHashes(FileFd &/*File*/, HashStringList const &/*ExpectedHashes*/) { return false; }
-   virtual Hashes * GetHashes() { return NULL; }
+   virtual bool InitHashes(HashStringList const &ExpectedHashes);
+   virtual Hashes * GetHashes();
    virtual bool Die(FileFd &/*File*/) { return false; }
    virtual bool Flush(FileFd * const /*File*/) { return false; }
    virtual bool Go(bool /*ToFile*/, FileFd * const /*File*/) { return false; }
diff --git a/methods/server.cc b/methods/server.cc
index e403f1071..2116926b0 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -235,6 +235,12 @@ ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOu
    Reset();
 }
 									/*}}}*/
+bool ServerState::AddPartialFileToHashes(FileFd &File)			/*{{{*/
+{
+   File.Truncate(StartPos);
+   return GetHashes()->AddFD(File, StartPos);
+}
+									/*}}}*/
 
 bool ServerMethod::Configuration(string Message)			/*{{{*/
 {
@@ -357,7 +363,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
    FailFd = File->Fd();
    FailTime = Server->Date;
 
-   if (Server->InitHashes(*File, Queue->ExpectedHashes) == false)
+   if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
    {
       _error->Errno("read",_("Problem hashing file"));
       return ERROR_NOT_FROM_SERVER;
diff --git a/methods/server.h b/methods/server.h
index 45622dd34..1b1f754a3 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -72,6 +72,7 @@ struct ServerState
    };
    /** \brief Get the headers before the data */
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
+   bool AddPartialFileToHashes(FileFd &File);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
    virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
@@ -85,7 +86,7 @@ struct ServerState
    virtual bool Open() = 0;
    virtual bool IsOpen() = 0;
    virtual bool Close() = 0;
-   virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes) = 0;
+   virtual bool InitHashes(HashStringList const &ExpectedHashes) = 0;
    virtual Hashes * GetHashes() = 0;
    virtual bool Die(FileFd &File) = 0;
    virtual bool Flush(FileFd * const File) = 0;
diff --git a/test/integration/test-apt-download-progress b/test/integration/test-apt-download-progress
index b2c9effe6..07c5e09c5 100755
--- a/test/integration/test-apt-download-progress
+++ b/test/integration/test-apt-download-progress
@@ -26,14 +26,16 @@ assertprogress() {
 TESTFILE=testfile.big
 testsuccess dd if=/dev/zero of=./aptarchive/$TESTFILE bs=800k count=1 
 
+OPT='-o APT::Status-Fd=3 -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 -o Debug::Acquire::https=1'
+
 msgtest 'download progress works via' 'http'
 exec 3> apt-progress.log
-testsuccess --nomsg apthelper download-file "http://localhost:8080/$TESTFILE" http-$TESTFILE -o APT::Status-Fd=3 -o Acquire::http::Dl-Limit=800
+testsuccess --nomsg apthelper download-file "http://localhost:8080/$TESTFILE" http-$TESTFILE $OPT -o Acquire::http::Dl-Limit=800
 assertprogress apt-progress.log
 
 msgtest 'download progress works via' 'https'
 exec 3> apt-progress.log
-testsuccess --nomsg apthelper download-file "https://localhost:4433/$TESTFILE" https-$TESTFILE -o APT::Status-Fd=3 -o Acquire::https::Dl-Limit=800
+testsuccess --nomsg apthelper download-file "https://localhost:4433/$TESTFILE" https-$TESTFILE $OPT -o Acquire::https::Dl-Limit=800
 assertprogress apt-progress.log
 
 # cleanup
-- 
cgit v1.2.3-70-g09d2


From d84da4995df24329e96d57a22136683a9e370f4e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 11 Apr 2015 20:13:19 +0200
Subject: ensure lists/ files have correct permissions after apt-cdrom add

Its a bit unpredictable which permissons and owners we will encounter on
a CD-ROM (or a USB stick, as apt-cdrom is responsible for those too),
so we have to ensure in this codepath as well that everything is nicely
setup without waiting for a 'apt-get update' to fix up the (potential)
mess.
---
 apt-pkg/acquire-worker.cc       | 14 --------------
 apt-pkg/cdrom.cc                | 13 ++++---------
 apt-pkg/contrib/fileutl.cc      | 19 +++++++++++++++++++
 apt-pkg/contrib/fileutl.h       | 13 ++++++++++++-
 apt-pkg/indexcopy.cc            |  5 ++++-
 test/integration/framework      |  1 +
 test/integration/test-apt-cdrom |  5 +++--
 7 files changed, 43 insertions(+), 27 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index f4d1ad412..bee01e620 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -43,20 +43,6 @@
 
 using namespace std;
 
-static void ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) /*{{{*/
-{
-   if (getuid() == 0 && strlen(user) != 0 && strlen(group) != 0) // if we aren't root, we can't chown, so don't try it
-   {
-      // ensure the file is owned by root and has good permissions
-      struct passwd const * const pw = getpwnam(user);
-      struct group const * const gr = getgrnam(group);
-      if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0)
-	 _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
-   }
-   if (chmod(file, mode) != 0)
-      _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
-}
-									/*}}}*/
 // Worker::Worker - Constructor for Queue startup			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc
index 5eccbe5de..8cec4b78e 100644
--- a/apt-pkg/cdrom.cc
+++ b/apt-pkg/cdrom.cc
@@ -927,8 +927,7 @@ pkgUdevCdromDevices::pkgUdevCdromDevices()				/*{{{*/
 }
 									/*}}}*/
 
-bool
-pkgUdevCdromDevices::Dlopen()                     		        /*{{{*/
+bool pkgUdevCdromDevices::Dlopen()					/*{{{*/
 {
    // alread open
    if(libudev_handle != NULL)
@@ -957,18 +956,14 @@ pkgUdevCdromDevices::Dlopen()                     		        /*{{{*/
    return true;
 }
 									/*}}}*/
-                                                                        /*{{{*/
-// convenience interface, this will just call ScanForRemovable
-vector
-pkgUdevCdromDevices::Scan()
+// convenience interface, this will just call ScanForRemovable		/*{{{*/
+vector pkgUdevCdromDevices::Scan()
 {
    bool CdromOnly = _config->FindB("APT::cdrom::CdromOnly", true);
    return ScanForRemovable(CdromOnly);
 }
 									/*}}}*/
-                                                                        /*{{{*/
-vector
-pkgUdevCdromDevices::ScanForRemovable(bool CdromOnly)
+vector pkgUdevCdromDevices::ScanForRemovable(bool CdromOnly)/*{{{*/
 {
    vector cdrom_devices;
    struct udev_enumerate *enumerate;
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index 47033eadf..afc243b7f 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -874,6 +874,25 @@ bool StartsWithGPGClearTextSignature(string const &FileName)
    return true;
 }
 									/*}}}*/
+// ChangeOwnerAndPermissionOfFile - set file attributes to requested values /*{{{*/
+bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode)
+{
+   if (strcmp(file, "/dev/null") == 0)
+      return true;
+   bool Res = true;
+   if (getuid() == 0 && strlen(user) != 0 && strlen(group) != 0) // if we aren't root, we can't chown, so don't try it
+   {
+      // ensure the file is owned by root and has good permissions
+      struct passwd const * const pw = getpwnam(user);
+      struct group const * const gr = getgrnam(group);
+      if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0)
+	 Res &= _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
+   }
+   if (chmod(file, mode) != 0)
+      Res &= _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
+   return Res;
+}
+									/*}}}*/
 
 class FileFdPrivate {							/*{{{*/
 	public:
diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h
index a64d6cb98..97cb05c56 100644
--- a/apt-pkg/contrib/fileutl.h
+++ b/apt-pkg/contrib/fileutl.h
@@ -195,10 +195,21 @@ pid_t ExecFork(std::set keep_fds);
 void MergeKeepFdsFromConfiguration(std::set &keep_fds);
 bool ExecWait(pid_t Pid,const char *Name,bool Reap = false);
 
-
 // check if the given file starts with a PGP cleartext signature
 bool StartsWithGPGClearTextSignature(std::string const &FileName);
 
+/** change file attributes to requested known good values
+ *
+ * The method skips the user:group setting if not root.
+ *
+ * @param requester is printed as functionname in error cases
+ * @param file is the file to be modified
+ * @param user is the (new) owner of the file, e.g. _apt
+ * @param group is the (new) group owning the file, e.g. root
+ * @param mode is the access mode of the file, e.g. 0644
+ */
+bool ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode);
+
 /**
  * \brief Drop privileges
  *
diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc
index 5fa57fd8b..144c508be 100644
--- a/apt-pkg/indexcopy.cc
+++ b/apt-pkg/indexcopy.cc
@@ -216,6 +216,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	 FinalF += URItoFileName(S);
 	 if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 	    return _error->Errno("rename","Failed to rename");
+	 ChangeOwnerAndPermissionOfFile("CopyPackages", FinalF.c_str(), "root", "root", 0644);
       }
 	 
       /* Mangle the source to be in the proper notation with
@@ -546,8 +547,9 @@ bool SigVerify::CopyMetaIndex(string CDROM, string CDName,		/*{{{*/
       FileFd Rel;
       Target.Open(TargetF,FileFd::WriteAtomic);
       Rel.Open(prefix + file,FileFd::ReadOnly);
-      if (CopyFile(Rel,Target) == false)
+      if (CopyFile(Rel,Target) == false || Target.Close() == false)
 	 return _error->Error("Copying of '%s' for '%s' from '%s' failed", file.c_str(), CDName.c_str(), prefix.c_str());
+      ChangeOwnerAndPermissionOfFile("CopyPackages", TargetF.c_str(), "root", "root", 0644);
 
       return true;
 }
@@ -760,6 +762,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
 	 FinalF += URItoFileName(S);
 	 if (rename(TargetF.c_str(),FinalF.c_str()) != 0)
 	    return _error->Errno("rename","Failed to rename");
+	 ChangeOwnerAndPermissionOfFile("CopyTranslations", FinalF.c_str(), "root", "root", 0644);
       }
       
       
diff --git a/test/integration/framework b/test/integration/framework
index 994956b74..642c5f0d0 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1525,6 +1525,7 @@ aptautotest_aptget_update() {
 	done
 }
 aptautotest_apt_update() { aptautotest_aptget_update "$@"; }
+aptautotest_aptcdrom_add() { aptautotest_aptget_update "$@"; }
 
 testaptautotestnodpkgwarning() {
 	local TESTCALL="$1"
diff --git a/test/integration/test-apt-cdrom b/test/integration/test-apt-cdrom
index 9906795ca..34b35f745 100755
--- a/test/integration/test-apt-cdrom
+++ b/test/integration/test-apt-cdrom
@@ -33,6 +33,7 @@ aptcdromlog() {
 	test ! -e rootdir/media/cdrom || echo "CD-ROM is mounted, but shouldn't be!"
 	test -e rootdir/media/cdrom-unmounted || echo "Unmounted CD-ROM doesn't exist, but it should!"
 }
+aptautotest_aptcdromlog_add() { aptautotest_aptget_update "$@"; }
 
 CDROM_PRE="Using CD-ROM mount point $(readlink -f ./rootdir/media)/cdrom/
 Unmounting CD-ROM...
@@ -133,13 +134,13 @@ aptcache show testing -o Acquire::Languages=en | grep -q '^Description-en: ' &&
 
 # ensure cdrom method isn't trying to mount the cdrom
 mv rootdir/media/cdrom-unmounted rootdir/media/cdrom-ejected
-# ensure an update doesn't mess with cdrom sources
+msgmsg "ensure an update doesn't mess with cdrom sources"
 testsuccess aptget update
 testfileequal rootdir/tmp/testsuccess.output 'Reading package lists...'
 mv rootdir/media/cdrom-ejected rootdir/media/cdrom-unmounted
 testcdromusage
 
-# and again to check that it withstands the temptation even if it could mount
+msgmsg 'and again to check that it withstands the temptation even if it could mount'
 testsuccess aptget update
 testfileequal rootdir/tmp/testsuccess.output 'Reading package lists...'
 testcdromusage
-- 
cgit v1.2.3-70-g09d2


From ba6b79bd0090077724fa1272ea4d3a31706fcd5a Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sun, 12 Apr 2015 17:08:46 +0200
Subject: a hit on Release files means the indexes will be hits too

If we get a IMSHit for the Transaction-Manager (= the InRelease file or
as its still supported fallback Release + Release.gpg combo) we can
assume that every file we would queue based on this manager, but already
have locally is current and hence would get an IMSHit, too. We therefore
save us and the server the trouble and skip the queuing in this case.
Beside speeding up repetative executions of 'apt-get update' this way we
also avoid hitting hashsum errors if the indexes are in fact already
updated, but the Release file isn't yet as it is the case on well
behaving mirrors as Release files is updated last.

The implementation is a bit harder than the theory makes it sound as we
still have to keep reverifying the Release files (e.g. to detect now expired
once to avoid an attacker being able to silently stale us) and have to
handle cases in which the Release file hits, but some indexes aren't
present (e.g. user added a new foreign architecture).
---
 apt-pkg/acquire-item.cc                           |  90 ++++++++++---
 apt-pkg/acquire-item.h                            |  28 ++--
 apt-pkg/acquire.h                                 |   1 +
 test/integration/framework                        |  19 +--
 test/integration/test-apt-update-expected-size    |   4 +
 test/integration/test-apt-update-ims              | 150 +++++++++++++++-------
 test/integration/test-apt-update-not-modified     |  45 +++++++
 test/integration/test-apt-update-transactions     |   7 +
 test/integration/test-bug-602412-dequote-redirect |   3 -
 test/integration/test-pdiff-usage                 |  15 +--
 test/interactive-helper/aptwebserver.cc           |   9 +-
 11 files changed, 273 insertions(+), 98 deletions(-)
 create mode 100755 test/integration/test-apt-update-not-modified

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index f30529066..750e7daca 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -213,7 +213,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringLis
    step */
 bool pkgAcquire::Item::Rename(string From,string To)
 {
-   if (rename(From.c_str(),To.c_str()) == 0)
+   if (From == To || rename(From.c_str(),To.c_str()) == 0)
       return true;
 
    std::string S;
@@ -224,9 +224,41 @@ bool pkgAcquire::Item::Rename(string From,string To)
    return false;
 }
 									/*}}}*/
-void pkgAcquire::Item::QueueURI(ItemDesc &Item)				/*{{{*/
+// Acquire::Item::QueueURI and specialisations from child classes	/*{{{*/
+/* The idea here is that an item isn't queued if it exists on disk and the
+   transition manager was a hit as this means that the files it contains
+   the checksums for can't be updated either (or they are and we are asking
+   for a hashsum mismatch to happen which helps nobody) */
+bool pkgAcquire::Item::QueueURI(ItemDesc &Item)
 {
+   std::string const FinalFile = GetFinalFilename();
+   if (TransactionManager != NULL && TransactionManager->IMSHit == true &&
+	 FileExists(FinalFile) == true)
+   {
+      PartialFile = DestFile = FinalFile;
+      Status = StatDone;
+      return false;
+   }
+
    Owner->Enqueue(Item);
+   return true;
+}
+/* The transition manager InRelease itself (or its older sisters-in-law
+   Release & Release.gpg) is always queued as this allows us to rerun gpgv
+   on it to verify that we aren't stalled with old files */
+bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item)
+{
+   Owner->Enqueue(Item);
+   return true;
+}
+/* the Diff/Index needs to queue also the up-to-date complete index file
+   to ensure that the list cleaner isn't eating it */
+bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item)
+{
+   if (pkgAcquire::Item::QueueURI(Item) == true)
+      return true;
+   QueueOnIMSHit();
+   return false;
 }
 									/*}}}*/
 void pkgAcquire::Item::Dequeue()					/*{{{*/
@@ -408,6 +440,14 @@ string pkgAcqDiffIndex::Custom600Headers()
    return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 }
 									/*}}}*/
+void pkgAcqDiffIndex::QueueOnIMSHit() const				/*{{{*/
+{
+   // list cleanup needs to know that this file as well as the already
+   // present index is ours, so we create an empty diff to save it for us
+   new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
+	 ExpectedHashes, MetaIndexParser);
+}
+									/*}}}*/
 bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
 {
    // failing here is fine: our caller will take care of trying to
@@ -470,11 +510,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
       // we have the same sha1 as the server so we are done here
       if(Debug)
 	 std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
-
-      // list cleanup needs to know that this file as well as the already
-      // present index is ours, so we create an empty diff to save it for us
-      new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
-                           ExpectedHashes, MetaIndexParser);
+      QueueOnIMSHit();
       return true;
    }
 
@@ -1096,6 +1132,17 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri
       if(Debug)
 	 std::clog << "allDone: " << DestFile << "\n" << std::endl;
    }
+}
+									/*}}}*/
+// AcqBaseIndex - Constructor						/*{{{*/
+pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire *Owner,
+      pkgAcqMetaBase *TransactionManager,
+      struct IndexTarget const * const Target,
+      HashStringList const &ExpectedHashes,
+      indexRecords *MetaIndexParser)
+: Item(Owner, ExpectedHashes, TransactionManager), Target(Target),
+   MetaIndexParser(MetaIndexParser)
+{
 }
 									/*}}}*/
 // AcqBaseIndex::VerifyHashByMetaKey - verify hash for the given metakey /*{{{*/
@@ -1489,6 +1536,19 @@ void pkgAcqIndex::StageDecompressDone(string Message,
    TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
 
    return;
+}
+									/*}}}*/
+// AcqMetaBase - Constructor						/*{{{*/
+pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire *Owner,
+      const std::vector* IndexTargets,
+      indexRecords* MetaIndexParser,
+      std::string const &RealURI,
+      HashStringList const &ExpectedHashes,
+      pkgAcqMetaBase *TransactionManager)
+: Item(Owner, ExpectedHashes, TransactionManager),
+   MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets),
+   AuthPass(false), RealURI(RealURI), IMSHit(false)
+{
 }
 									/*}}}*/
 // AcqMetaBase::Add - Add a item to the current Transaction		/*{{{*/
@@ -1831,13 +1891,7 @@ bool pkgAcqMetaBase::CheckAuthDone(string Message)			/*{{{*/
       std::cerr << "Signature verification succeeded: "
                 << DestFile << std::endl;
 
-   // Download further indexes with verification 
-   //
-   // it would be really nice if we could simply do
-   //    if (IMSHit == false) QueueIndexes(true)
-   // and skip the download if the Release file has not changed
-   // - but right now the list cleaner will needs to be tricked
-   //   to not delete all our packages/source indexes in this case
+   // Download further indexes with verification
    QueueIndexes(true);
 
    return true;
@@ -1908,7 +1962,13 @@ bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message)
    // make sure to verify against the right file on I-M-S hit
    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
    if(IMSHit)
+   {
+      // for simplicity, the transaction manager is always InRelease
+      // even if it doesn't exist.
+      if (TransactionManager != NULL)
+	 TransactionManager->IMSHit = true;
       DestFile = GetFinalFilename();
+   }
 
    // set Item to complete as the remaining work is all local (verify etc)
    Complete = true;
@@ -1951,7 +2011,7 @@ void pkgAcqMetaBase::QueueIndexes(bool verify)				/*{{{*/
                    << "Expected Hash:" << std::endl;
          for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs)
             std::cerr <<  "\t- " << hs->toStr() << std::endl;
-         std::cerr << "For: " << Record->MetaKeyFilename << std::endl;
+         std::cerr << "For: " << ((Record == NULL) ? "" : Record->MetaKeyFilename) << std::endl;
 
       }
       if (verify == true && ExpectedIndexHashes.empty() == true)
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index c2759b792..148728b3c 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -73,11 +73,16 @@ class pkgAcquire::Item : public WeakPointable
    pkgAcquire *Owner;
 
    /** \brief Insert this item into its owner's queue.
+    *
+    *  The method is designed to check if the request would end
+    *  in an IMSHit and if it determines that it would, it isn't
+    *  queueing the Item and instead sets it to completion instantly.
     *
     *  \param Item Metadata about this item (its URI and
     *  description).
+    *  \return true if the item was inserted, false if IMSHit was detected
     */
-   void QueueURI(ItemDesc &Item);
+   virtual bool QueueURI(ItemDesc &Item);
 
    /** \brief Remove this item from its owner's queue. */
    void Dequeue();
@@ -392,9 +397,6 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     */
    bool AuthPass;
 
-   // required to deal gracefully with problems caused by incorrect ims hits
-   bool IMSHit;
-
    /** \brief The URI of the signature file.  Unlike Desc.URI, this is
     *  never modified; it is used to determine the file that is being
     *  downloaded.
@@ -457,7 +459,11 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
    virtual std::string GetFinalFilename() const;
 
  public:
+   // This refers more to the Transaction-Manager than the actual file
+   bool IMSHit;
+
    virtual std::string DescURI() {return RealURI; };
+   virtual bool QueueURI(pkgAcquire::ItemDesc &Item);
 
    // transaction code
    void Add(Item *I);
@@ -479,10 +485,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
                   indexRecords* MetaIndexParser,
 		  std::string const &RealURI,
                   HashStringList const &ExpectedHashes=HashStringList(),
-                  pkgAcqMetaBase *TransactionManager=NULL)
-      : Item(Owner, ExpectedHashes, TransactionManager),
-        MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets),
-        AuthPass(false), IMSHit(false), RealURI(RealURI) {};
+                  pkgAcqMetaBase *TransactionManager=NULL);
 };
 									/*}}}*/
 /** \brief An acquire item that downloads the detached signature	{{{
@@ -652,9 +655,7 @@ class pkgAcqBaseIndex : public pkgAcquire::Item
                    pkgAcqMetaBase *TransactionManager,
                    struct IndexTarget const * const Target,
                    HashStringList const &ExpectedHashes,
-                   indexRecords *MetaIndexParser)
-      : Item(Owner, ExpectedHashes, TransactionManager), Target(Target), 
-        MetaIndexParser(MetaIndexParser) {};
+                   indexRecords *MetaIndexParser);
 };
 									/*}}}*/
 /** \brief An item that is responsible for fetching an index file of	{{{
@@ -691,6 +692,7 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
    /** \brief Get the full pathname of the final file for the current URI */
    virtual std::string GetFinalFilename() const;
 
+   virtual bool QueueURI(pkgAcquire::ItemDesc &Item);
  public:
    // Specialized action members
    virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
@@ -714,7 +716,6 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
     *  false otherwise.
     */
    bool ParseDiffIndex(std::string IndexDiffFile);
-   
 
    /** \brief Create a new pkgAcqDiffIndex.
     *
@@ -733,6 +734,8 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
                    struct IndexTarget const * const Target,
                    HashStringList const &ExpectedHashes,
                    indexRecords *MetaIndexParser);
+ private:
+   APT_HIDDEN void QueueOnIMSHit() const;
 };
 									/*}}}*/
 /** \brief An item that is responsible for fetching client-merge patches {{{
@@ -1008,7 +1011,6 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
    virtual std::string GetFinalFilename() const;
 
    public:
-   
    // Specialized action members
    virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
    virtual void Done(std::string Message,unsigned long long Size, 
diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index f33362922..fc90624e1 100644
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -111,6 +111,7 @@ class pkgAcquire
    struct MethodConfig;
    struct ItemDesc;
    friend class Item;
+   friend class pkgAcqMetaBase;
    friend class Queue;
 
    typedef std::vector::iterator ItemIterator;
diff --git a/test/integration/framework b/test/integration/framework
index 642c5f0d0..4229ae162 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1005,6 +1005,7 @@ signreleasefiles() {
 }
 
 webserverconfig() {
+	local WEBSERVER="${3:-http://localhost:8080}"
 	local NOCHECK=false
 	if [ "$1" = '--no-check' ]; then
 		NOCHECK=true
@@ -1016,10 +1017,10 @@ webserverconfig() {
 	local URI
 	if [ -n "$2" ]; then
 		msgtest "Set webserver config option '${1}' to" "$2"
-		URI="http://localhost:8080/_config/set/${1}/${2}"
+		URI="${WEBSERVER}/_config/set/${1}/${2}"
 	else
 		msgtest 'Clear webserver config option' "${1}"
-		URI="http://localhost:8080/_config/clear/${1}"
+		URI="${WEBSERVER}/_config/clear/${1}"
 	fi
 	if downloadfile "$URI" "$STATUS" > "$DOWNLOG"; then
 		msgpass
@@ -1467,12 +1468,14 @@ pause() {
 }
 
 listcurrentlistsdirectory() {
-	find rootdir/var/lib/apt/lists -maxdepth 1 -type d | while read line; do
-		stat --format '%U:%G:%a:%n' "$line"
-	done
-	find rootdir/var/lib/apt/lists -maxdepth 1 \! -type d | while read line; do
-		stat --format '%U:%G:%a:%s:%y:%n' "$line"
-	done
+	{
+		find rootdir/var/lib/apt/lists -maxdepth 1 -type d | while read line; do
+			stat --format '%U:%G:%a:%n' "$line"
+		done
+		find rootdir/var/lib/apt/lists -maxdepth 1 \! -type d | while read line; do
+			stat --format '%U:%G:%a:%s:%y:%n' "$line"
+		done
+	} | sort
 }
 
 ### convinience hacks ###
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 22de13ea5..9f58308aa 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -38,6 +38,10 @@ E: Some index files failed to download. They have been ignored, or old ones used
 }
 
 methodtest() {
+	# less complicated test setup this way
+	webserverconfig 'aptwebserver::support::modified-since' 'false' "$1"
+	webserverconfig 'aptwebserver::support::last-modified' 'false' "$1"  # curl is clever and sees hits here also
+
 	msgmsg 'Test with' "$1" 'and clean start'
 	rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
 	# normal update works fine
diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 0fa882d78..f091bffaa 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -6,85 +6,145 @@ TESTDIR=$(readlink -f $(dirname $0))
 setupenvironment
 configarchitecture 'amd64'
 
-buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
+insertpackage 'unstable' 'unrelated' 'all' '0.5~squeeze1'
+insertsource 'unstable' 'unrelated' 'all' '0.5~squeeze1'
 
 setupaptarchive --no-update
 changetowebserver
 
 runtest() {
-    configallowinsecurerepositories "${1:-false}"
+    if [ -n "$1" ]; then
+	configallowinsecurerepositories 'true'
+    else
+	configallowinsecurerepositories 'false'
+    fi
 
-    rm -f rootdir/var/lib/apt/lists/localhost*
+    rm -rf rootdir/var/lib/apt/lists/
 
-    if [ "$1" = 'true' ]; then
-	testwarning aptget update
-    else
-	testsuccess aptget update
+    local TEST="test${1:-success}"
+    $TEST aptget update
+    if [ "$1" = 'failure' ]; then
+	# accept the outdated Release file so we can check Hit behaviour
+	"test${2:-success}" aptget update -o Acquire::Min-ValidTime=99999999999
     fi
+    listcurrentlistsdirectory > listsdir.lst
+    testsuccess grep '_Packages\(\.gz\)\?$' listsdir.lst
+    testsuccess grep '_Sources\(\.gz\)\?$' listsdir.lst
+    testsuccess grep '_Translation-en\(\.gz\)\?$' listsdir.lst
 
     # ensure no leftovers in partial
-    testfailure ls "rootdir/var/lib/apt/lists/partial/*"
+    testfailure ls 'rootdir/var/lib/apt/lists/partial/*'
 
     # check that I-M-S header is kept in redirections
-    testequal "$EXPECT" aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
-    
-    # ensure that we still do a hash check on ims hit
-    msgtest 'Test I-M-S' 'reverify'
-    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A2 'ReceivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
+    echo "$EXPECT" | sed -e 's#(invalid since [^)]\+)#(invalid since)#' > expected.output
+    $TEST aptget update  -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::http=0
+    sed -i -e 's#(invalid since [^)]\+)#(invalid since)#' rootdir/tmp/${TEST}.output
+    testequal "$(cat expected.output)" cat rootdir/tmp/${TEST}.output
+    testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+    # ensure that we still do a hash check for other files on ims hit of Release
+    if grep -q '^Hit .* \(InRelease\|Release.gpg\)$' expected.output ; then
+	    $TEST aptget update -o Debug::Acquire::gpgv=1
+	    cp rootdir/tmp/${TEST}.output goodsign.output
+	    testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+	    testsuccess grep '^Got GOODSIG, key ID:GOODSIG' goodsign.output
+    fi
 
     # ensure no leftovers in partial
-    testfailure ls "rootdir/var/lib/apt/lists/partial/*"
+    testfailure ls 'rootdir/var/lib/apt/lists/partial/*'
 }
 
-msgmsg "InRelease"
-EXPECT="Hit http://localhost:8080 unstable InRelease
-Hit http://localhost:8080 unstable/main Sources
-Hit http://localhost:8080 unstable/main amd64 Packages
-Hit http://localhost:8080 unstable/main Translation-en
-Reading package lists..."
-# with InRelease
+msgmsg 'InRelease'
+EXPECT='Hit http://localhost:8080 unstable InRelease
+Reading package lists...'
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
-
-# with gzip
-echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
-msgmsg "Release/Release.gpg"
-# with Release/Release.gpg
-EXPECT="Ign http://localhost:8080 unstable InRelease
+msgmsg 'Release/Release.gpg'
+EXPECT='Ign http://localhost:8080 unstable InRelease
   404  Not Found
 Hit http://localhost:8080 unstable Release
 Hit http://localhost:8080 unstable Release.gpg
-Hit http://localhost:8080 unstable/main Sources
-Hit http://localhost:8080 unstable/main amd64 Packages
-Hit http://localhost:8080 unstable/main Translation-en
-Reading package lists..."
-
+Reading package lists...'
 find aptarchive -name 'InRelease' -delete
-
-echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
-
-echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
-# no Release.gpg or InRelease
-msgmsg "Release only"
+msgmsg 'Release only'
 EXPECT="Ign http://localhost:8080 unstable InRelease
   404  Not Found
 Hit http://localhost:8080 unstable Release
 Ign http://localhost:8080 unstable Release.gpg
   404  Not Found
-Hit http://localhost:8080 unstable/main Sources
-Hit http://localhost:8080 unstable/main amd64 Packages
-Hit http://localhost:8080 unstable/main Translation-en
 Reading package lists...
 W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated."
+find aptarchive -name 'Release.gpg' -delete
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'warning'
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'warning'
+
+
+# make the release file old
+find aptarchive -name '*Release' -exec sed -i \
+	-e "s#^Date: .*\$#Date: $(date -d '-2 weeks' '+%a, %d %b %Y %H:%M:%S %Z')#" \
+	-e '/^Valid-Until: / d' -e "/^Date: / a\
+Valid-Until: $(date -d '-1 weeks' '+%a, %d %b %Y %H:%M:%S %Z')" '{}' \;
+signreleasefiles
+
+msgmsg 'expired InRelease'
+EXPECT='Hit http://localhost:8080 unstable InRelease
+E: Release file for http://localhost:8080/dists/unstable/InRelease is expired (invalid since). Updates for this repository will not be applied.'
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure'
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure'
+
+msgmsg 'expired Release/Release.gpg'
+EXPECT='Ign http://localhost:8080 unstable InRelease
+  404  Not Found
+Hit http://localhost:8080 unstable Release
+Hit http://localhost:8080 unstable Release.gpg
+E: Release file for http://localhost:8080/dists/unstable/Release.gpg is expired (invalid since). Updates for this repository will not be applied.'
+find aptarchive -name 'InRelease' -delete
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure'
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure'
 
+msgmsg 'expired Release only'
+EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
+Hit http://localhost:8080 unstable Release
+Ign http://localhost:8080 unstable Release.gpg
+  404  Not Found
+W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated.
+E: Release file for http://localhost:8080/dists/unstable/InRelease is expired (invalid since). Updates for this repository will not be applied."
 find aptarchive -name 'Release.gpg' -delete
+echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure' 'warning'
+echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'failure' 'warning'
 
-echo "Acquire::GzipIndexes "0";" > rootdir/etc/apt/apt.conf.d/02compressindex
-runtest "true"
 
-echo "Acquire::GzipIndexes "1";" > rootdir/etc/apt/apt.conf.d/02compressindex
-runtest "true"
+msgmsg 'no Release at all'
+EXPECT="Ign http://localhost:8080 unstable InRelease
+  404  Not Found
+Ign http://localhost:8080 unstable Release
+  404  Not Found
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists...
+W: The repository 'http://localhost:8080 unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository."
+find aptarchive -name '*Release*' -delete
+echo 'Acquire::GzipIndexes "0";
+Acquire::PDiffs "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'warning'
+echo 'Acquire::GzipIndexes "1";
+Acquire::PDiffs "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
+runtest 'warning'
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
new file mode 100755
index 000000000..2dc56e76c
--- /dev/null
+++ b/test/integration/test-apt-update-not-modified
@@ -0,0 +1,45 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'amd64' 'i386'
+
+insertpackage 'unstable' 'apt' 'all' '1.0'
+
+setupaptarchive --no-update
+
+methodtest() {
+	msgmsg 'Test with' "$1"
+	rm -rf rootdir/var/lib/apt/lists
+	# get our cache populated
+	testsuccess aptget update
+	listcurrentlistsdirectory > listsdir.lst
+
+	# hit again with a good cache
+	testsuccessequal "Hit $1 unstable InRelease
+Reading package lists..." aptget update
+	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+	# drop an architecture, which means the file should be gone now
+	configarchitecture 'i386'
+	sed '/_binary-amd64_Packages/ d' listsdir.lst > listsdir-without-amd64.lst
+	testsuccessequal "Hit $1 unstable InRelease
+Reading package lists..." aptget update
+	testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)"
+
+	# readd arch so its downloaded again
+	configarchitecture 'amd64' 'i386'
+	testsuccessequal "Hit $1 unstable InRelease
+Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Reading package lists..." aptget update
+	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+}
+
+changetowebserver
+methodtest 'http://localhost:8080'
+
+changetohttpswebserver
+methodtest 'https://localhost:4433'
diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions
index b325733ac..f028ac0c7 100755
--- a/test/integration/test-apt-update-transactions
+++ b/test/integration/test-apt-update-transactions
@@ -59,6 +59,13 @@ testsetup() {
 }
 
 testsetup 'file'
+
 changetowebserver
+webserverconfig 'aptwebserver::support::modified-since' 'false' "$1"
+webserverconfig 'aptwebserver::support::last-modified' 'false' "$1"  # curl is clever and sees hits here also
+
 testsetup 'http'
 
+changetohttpswebserver
+
+testsetup 'https'
diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect
index 384c8b113..ca2378c19 100755
--- a/test/integration/test-bug-602412-dequote-redirect
+++ b/test/integration/test-bug-602412-dequote-redirect
@@ -22,9 +22,6 @@ testrun() {
 
 	# check that I-M-S header is kept in redirections
 	testsuccessequal "Hit $1 unstable InRelease
-Hit $1 unstable/main Sources
-Hit $1 unstable/main amd64 Packages
-Hit $1 unstable/main Translation-en
 Reading package lists..." aptget update
 
 	msgtest 'Test redirection works in' 'package download'
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 4de07f1ad..5e759e50e 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -14,16 +14,7 @@ changetowebserver
 PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')"
 
 wasmergeused() {
-	msgtest 'Test for successful execution of' "$*"
-	local OUTPUT=$(mktemp)
-	addtrap "rm $OUTPUT;"
-	if aptget update "$@" >${OUTPUT} 2>&1; then
-		msgpass
-	else
-		echo
-		cat $OUTPUT
-		msgfail
-	fi
+	testsuccess aptget update "$@"
 
 	msgtest 'No intermediate patch files' 'still exist'
 	local EDS="$(find rootdir/var/lib/apt/lists -name '*.ed' -o -name '*.ed.*')"
@@ -36,7 +27,7 @@ wasmergeused() {
 	fi
 
 	msgtest 'Check if the right pdiff merger was used'
-	if grep -q '^pkgAcqIndexMergeDiffs::Done(): rred' $OUTPUT; then
+	if grep -q '^pkgAcqIndexMergeDiffs::Done(): rred' rootdir/tmp/testsuccess.output; then
 		if echo "$*" | grep -q -- '-o Acquire::PDiffs::Merge=1'; then
 			msgpass
 		else
@@ -96,6 +87,8 @@ SHA256-Patches:
 	msgmsg "Testcase: index is already up-to-date: $*"
 	find rootdir/var/lib/apt/lists -name '*diff_Index' -type f -delete
 	testsuccess aptget update "$@"
+	testequal 'Hit http://localhost:8080  InRelease
+Reading package lists...' aptget update "$@" -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Diffs=0
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
 
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 86d5c06f0..6a411e24e 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -96,9 +96,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
       contentlength << "Content-Length: " << data.FileSize();
       headers.push_back(contentlength.str());
    }
-   std::string lastmodified("Last-Modified: ");
-   lastmodified.append(TimeRFC1123(data.ModificationTime()));
-   headers.push_back(lastmodified);
+   if (_config->FindB("aptwebserver::support::last-modified", true) == true)
+   {
+      std::string lastmodified("Last-Modified: ");
+      lastmodified.append(TimeRFC1123(data.ModificationTime()));
+      headers.push_back(lastmodified);
+   }
 }
 									/*}}}*/
 static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/
-- 
cgit v1.2.3-70-g09d2


From 6c9937da76b9155d166092b9dda22d06200510c1 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Wed, 22 Apr 2015 12:06:40 +0200
Subject: remove "first package seen is native package" assumption

The fix for #777760 causes packages of foreign (and the native)
architectures, to be created correctly, but invalidates (like the
previously existing, but policy-forbidden architecture-less packages
we had to support for some upgrade scenarios) the assumption that the
first (and only) package in the cache for a single architecture system
must be the package for the native architecture (as, where should the
other architectures come from, right? Wrong.).

Depending on the order of parsing sources more or less packages can be
effected by this. The effects are strange (for apt it mostly effects
simulation/debug output, but also apt-mark on these specific packages),
which complicates debugging, but relatively harmless if understood as
most actions do not need direct named access to packages.

The problem is fixed by removing the single-arch special casing in the
paths who had them (Cache.FindPkg), so they use the same code as
multi-arch systems, which use them as a wrapper for Grp.FindPkg.

Note that single-arch system code was using Grp.FindPkg before as well
if a Grp structure was handily available, so we don't introduce new
untested code here: We remove more brittle special cases which are less
tested instead (this was planed to be done for Stretch anyhow).

Note further that the method with the assumption itself isn't fixed. As
it is a private method I opted for declaring it deprecated instead and
remove all its call positions. As it is private no-one can call this
method legally (thanks to how c++ works by default its still an exported
symbol through) and fixing it basically means reimplementing code we
already have in Grp.FindPkg.

Removing rather than fixing seems hence like a good solution.

Closes: 782777
Thanks: Axel Beckert for testing
---
 apt-pkg/pkgcache.cc                                | 14 +----
 apt-pkg/pkgcache.h                                 |  2 +-
 .../test-bug-782777-single-arch-weirdness          | 72 ++++++++++++++++++++++
 3 files changed, 74 insertions(+), 14 deletions(-)
 create mode 100755 test/integration/test-bug-782777-single-arch-weirdness

(limited to 'test')

diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index d7c9656b9..a7b75dae8 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -230,12 +230,7 @@ pkgCache::PkgIterator pkgCache::SingleArchFindPkg(const string &Name)
 pkgCache::PkgIterator pkgCache::FindPkg(const string &Name) {
 	size_t const found = Name.find(':');
 	if (found == string::npos)
-	{
-		if (MultiArchCache() == false)
-			return SingleArchFindPkg(Name);
-		else
-			return FindPkg(Name, "native");
-	}
+	   return FindPkg(Name, "native");
 	string const Arch = Name.substr(found+1);
 	/* Beware: This is specialcased to handle pkg:any in dependencies as
 	   these are linked to virtual pkg:any named packages with all archs.
@@ -249,13 +244,6 @@ pkgCache::PkgIterator pkgCache::FindPkg(const string &Name) {
 // ---------------------------------------------------------------------
 /* Returns 0 on error, pointer to the package otherwise */
 pkgCache::PkgIterator pkgCache::FindPkg(const string &Name, string const &Arch) {
-	if (MultiArchCache() == false && Arch != "none") {
-		if (Arch == "native" || Arch == "all" || Arch == "any" ||
-		    Arch == NativeArch())
-			return SingleArchFindPkg(Name);
-		else
-			return PkgIterator(*this,0);
-	}
 	/* We make a detour via the GrpIterator here as
 	   on a multi-arch environment a group is easier to
 	   find than a package (less entries in the buckets) */
diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h
index 5e8a9630a..a7e520bc9 100644
--- a/apt-pkg/pkgcache.h
+++ b/apt-pkg/pkgcache.h
@@ -218,7 +218,7 @@ class pkgCache								/*{{{*/
 
 private:
    bool MultiArchEnabled;
-   PkgIterator SingleArchFindPkg(const std::string &Name);
+   APT_DEPRECATED PkgIterator SingleArchFindPkg(const std::string &Name);
 };
 									/*}}}*/
 // Header structure							/*{{{*/
diff --git a/test/integration/test-bug-782777-single-arch-weirdness b/test/integration/test-bug-782777-single-arch-weirdness
new file mode 100755
index 000000000..004903385
--- /dev/null
+++ b/test/integration/test-bug-782777-single-arch-weirdness
@@ -0,0 +1,72 @@
+#!/bin/sh
+# Ensure that the order in which packages are in the binary cache
+# does not effect if they can be found or not
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'i386'
+
+insertpackage 'unstable' 'abar' 'i386' '1'
+insertpackage 'unstable' 'foobar' 'i386' '1' 'Depends: abar:amd64, zfoo:amd64'
+insertpackage 'unstable' 'zfoo' 'i386' '1'
+
+setupaptarchive
+
+testrun() {
+	rm -f rootdir/var/lib/apt/extended_states
+
+	testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  abar zfoo
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst abar (1 unstable [i386])
+Inst zfoo (1 unstable [i386])
+Conf abar (1 unstable [i386])
+Conf zfoo (1 unstable [i386])' aptget install abar zfoo -s
+
+	testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  abar zfoo
+0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded.
+Inst abar (1 unstable [i386])
+Inst zfoo (1 unstable [i386])
+Conf abar (1 unstable [i386])
+Conf zfoo (1 unstable [i386])' aptget install abar:i386 zfoo:i386 -s
+
+	testequal "Reading package lists...
+Building dependency tree...
+Package abar:amd64 is not available, but is referred to by another package.
+This may mean that the package is missing, has been obsoleted, or
+is only available from another source
+
+Package zfoo:amd64 is not available, but is referred to by another package.
+This may mean that the package is missing, has been obsoleted, or
+is only available from another source
+
+E: Package 'abar:amd64' has no installation candidate
+E: Package 'zfoo:amd64' has no installation candidate" aptget install abar:amd64 zfoo:amd64 -s
+
+	cp -f rootdir/var/lib/dpkg/status status.backup
+	insertinstalledpackage 'abar' 'i386' '1'
+	insertinstalledpackage 'zfoo' 'i386' '1'
+
+	testequal 'abar
+zfoo' aptmark showmanual abar zfoo
+	testequal 'abar set to automatically installed.
+zfoo set to automatically installed.' aptmark auto abar zfoo
+	testempty aptmark showmanual abar zfoo
+	testequal 'abar
+zfoo' aptmark showauto abar zfoo
+
+	mv -f status.backup rootdir/var/lib/dpkg/status
+}
+
+msgmsg 'Single-Arch testrun'
+testrun
+msgmsg 'Multi-Arch testrun'
+configarchitecture 'i386' 'amd64'
+testrun
-- 
cgit v1.2.3-70-g09d2


From 146f7715a9f36d246b461255b3c683b479296915 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 27 Apr 2015 10:59:27 +0200
Subject: improve partial/ cleanup in abort and failure cases

Especially pdiff-enhanced downloads have the tendency to fail for
various reasons from which we can recover and even a successful download
used to leave the old unpatched index in partial/.

By adding a new method responsible for making the transaction of an
individual file happen we can at specialisations especially for abort
cases to deal with the cleanup.

This also helps in keeping the compressed indexes around if another
index failed instead of keeping the decompressed files, which we
wouldn't pick up in the next call.
---
 apt-pkg/acquire-item.cc                        | 193 +++++++++++++++++--------
 apt-pkg/acquire-item.h                         |  17 ++-
 test/integration/framework                     |   6 +-
 test/integration/test-apt-update-expected-size |   2 +-
 test/integration/test-partial-file-support     |  23 ++-
 test/integration/test-pdiff-usage              |   2 +-
 6 files changed, 161 insertions(+), 82 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 00862fe23..01ce0e650 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -173,6 +173,39 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       ReportMirrorFailure(ErrorText);
 }
 									/*}}}*/
+bool pkgAcquire::Item::TransactionState(TransactionStates const state)	/*{{{*/
+{
+   bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
+   switch(state)
+   {
+      case TransactionAbort:
+	 if(Debug == true)
+	    std::clog << "  Cancel: " << DestFile << std::endl;
+	 if (Status == pkgAcquire::Item::StatIdle)
+	 {
+	    Status = pkgAcquire::Item::StatDone;
+	    Dequeue();
+	 }
+	 break;
+      case TransactionCommit:
+	 if(PartialFile != "")
+	 {
+	    if(Debug == true)
+	       std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl;
+
+	    Rename(PartialFile, DestFile);
+	 } else {
+	    if(Debug == true)
+	       std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
+	    unlink(DestFile.c_str());
+	 }
+	 // mark that this transaction is finished
+	 TransactionManager = 0;
+	 break;
+   }
+   return true;
+}
+									/*}}}*/
 // Acquire::Item::Start - Item has begun to download			/*{{{*/
 // ---------------------------------------------------------------------
 /* Stash status and the file size. Note that setting Complete means 
@@ -300,6 +333,9 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const
 	 // the method is expected to report a good error for this
 	 Status = StatError;
 	 break;
+      case PDiffError:
+	 // no handling here, done by callers
+	 break;
    }
    return false;
 }
@@ -374,7 +410,7 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
 				 HashStringList const &ExpectedHashes,
                                  indexRecords *MetaIndexParser)
    : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, 
-                     MetaIndexParser), PackagesFileReadyInPartial(false)
+                     MetaIndexParser)
 {
    
    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
@@ -671,20 +707,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
       return false;
    }
 
-   // FIXME: make this use the method
-   PackagesFileReadyInPartial = true;
-   std::string const Partial = GetPartialFileNameFromURI(RealURI);
-   
-   FileFd From(CurrentPackagesFile, FileFd::ReadOnly);
-   FileFd To(Partial, FileFd::WriteEmpty);
-   if(CopyFile(From, To) == false)
-      return _error->Errno("CopyFile", "failed to copy");
-   
-   if(Debug)
-      std::cerr << "Done copying " << CurrentPackagesFile
-                << " -> " << Partial
-                << std::endl;
-
    // we have something, queue the diffs
    string::size_type const last_space = Description.rfind(" ");
    if(last_space != string::npos)
@@ -738,6 +760,24 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{
    new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
 }
 									/*}}}*/
+bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)	/*{{{*/
+{
+   if (pkgAcquire::Item::TransactionState(state) == false)
+      return false;
+
+   switch (state)
+   {
+      case TransactionCommit:
+	 break;
+      case TransactionAbort:
+	 std::string const Partial = GetPartialFileNameFromURI(RealURI);
+	 unlink(Partial.c_str());
+	 break;
+   }
+
+   return true;
+}
+									/*}}}*/
 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,	/*{{{*/
 			   pkgAcquire::MethodConfig *Cnf)
 {
@@ -765,15 +805,21 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList
    if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
       DestFile = FinalFile;
 
-   if(!ParseDiffIndex(DestFile))
-      return Failed("Message: Couldn't parse pdiff index", Cnf);
+   if(ParseDiffIndex(DestFile) == false)
+   {
+      Failed("Message: Couldn't parse pdiff index", Cnf);
+      // queue for final move - this should happen even if we fail
+      // while parsing (e.g. on sizelimit) and download the complete file.
+      TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
+      return;
+   }
 
-   // queue for final move
    TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
 
    Complete = true;
    Status = StatDone;
    Dequeue();
+
    return;
 }
 									/*}}}*/
@@ -808,6 +854,17 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
    }
    else
    {
+      // patching needs to be bootstrapped with the 'old' version
+      std::string const PartialFile = GetPartialFileNameFromURI(RealURI);
+      if (RealFileExists(PartialFile) == false)
+      {
+	 if (symlink(GetFinalFilename().c_str(), PartialFile.c_str()) != 0)
+	 {
+	    Failed("Link creation of " + PartialFile + " to " + GetFinalFilename() + " failed", NULL);
+	    return;
+	 }
+      }
+
       // get the next diff
       State = StateFetchDiff;
       QueueNextDiff();
@@ -822,6 +879,8 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{
    if(Debug)
       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 		<< "Falling back to normal index file acquire" << std::endl;
+   DestFile = GetPartialFileNameFromURI(Target->URI);
+   RenameOnError(PDiffError);
    new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
    Finish();
 }
@@ -950,6 +1009,8 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringLi
       if (fd.Size() != available_patches[0].patch_size ||
 	    available_patches[0].patch_hashes != LocalHashes)
       {
+	 // patchfiles are dated, so bad indicates a bad download, so kill it
+	 unlink(DestFile.c_str());
 	 Failed("Patch has Size/Hashsum mismatch", NULL);
 	 return;
       }
@@ -1046,6 +1107,8 @@ void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf
    State = StateErrorDiff;
    if (Debug)
       std::clog << "Falling back to normal index file acquire" << std::endl;
+   DestFile = GetPartialFileNameFromURI(Target->URI);
+   RenameOnError(PDiffError);
    new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
 }
 									/*}}}*/
@@ -1069,6 +1132,8 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri
 
       if (fd.Size() != patch.patch_size || patch.patch_hashes != LocalHashes)
       {
+	 // patchfiles are dated, so bad indicates a bad download, so kill it
+	 unlink(DestFile.c_str());
 	 Failed("Patch has Size/Hashsum mismatch", NULL);
 	 return;
       }
@@ -1090,6 +1155,13 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri
       // this is the last completed diff, so we are ready to apply now
       State = StateApplyDiff;
 
+      // patching needs to be bootstrapped with the 'old' version
+      if (symlink(GetFinalFilename().c_str(), FinalFile.c_str()) != 0)
+      {
+	 Failed("Link creation of " + FinalFile + " to " + GetFinalFilename() + " failed", NULL);
+	 return;
+      }
+
       if(Debug)
 	 std::clog << "Sending to rred method: " << FinalFile << std::endl;
 
@@ -1109,15 +1181,14 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri
 	 return;
       }
 
-
       // move the result into place
-      std::string const FinalFile = GetFinalFilename();
+      std::string const Final = GetFinalFilename();
       if(Debug)
 	 std::clog << "Queue patched file in place: " << std::endl
-		   << DestFile << " -> " << FinalFile << std::endl;
+		   << DestFile << " -> " << Final << std::endl;
 
       // queue for copy by the transaction manager
-      TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
+      TransactionManager->TransactionStageCopy(this, DestFile, Final);
 
       // ensure the ed's are gone regardless of list-cleanup
       for (std::vector::const_iterator I = allPatches->begin();
@@ -1127,6 +1198,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStri
 	 std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz";
 	 unlink(patch.c_str());
       }
+      unlink(FinalFile.c_str());
 
       // all set and done
       Complete = true;
@@ -1337,12 +1409,6 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       return;
    }
 
-   // on decompression failure, remove bad versions in partial/
-   if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
-   {
-      unlink(EraseFileName.c_str());
-   }
-
    Item::Failed(Message,Cnf);
 
    if(Target->IsOptional() && ExpectedHashes.empty() && Stage == STAGE_DOWNLOAD)
@@ -1351,6 +1417,30 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       TransactionManager->AbortTransaction();
 }
 									/*}}}*/
+bool pkgAcqIndex::TransactionState(TransactionStates const state)	/*{{{*/
+{
+   if (pkgAcquire::Item::TransactionState(state) == false)
+      return false;
+
+   switch (state)
+   {
+      case TransactionAbort:
+	 if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
+	 {
+	    // keep the compressed file, but drop the decompressed
+	    EraseFileName.clear();
+	    if (PartialFile.empty() == false && flExtension(PartialFile) == "decomp")
+	       unlink(PartialFile.c_str());
+	 }
+	 break;
+      case TransactionCommit:
+	 if (EraseFileName.empty() == false)
+	    unlink(EraseFileName.c_str());
+	 break;
+   }
+   return true;
+}
+									/*}}}*/
 // pkgAcqIndex::GetFinalFilename - Return the full final file path	/*{{{*/
 std::string pkgAcqIndex::GetFinalFilename() const
 {
@@ -1530,9 +1620,6 @@ void pkgAcqIndex::StageDecompressDone(string Message,
       return;
    }
 
-   // remove the compressed version of the file
-   unlink(EraseFileName.c_str());
-
    // Done, queue for rename on transaction finished
    TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
 
@@ -1568,14 +1655,7 @@ void pkgAcqMetaBase::AbortTransaction()
    for (std::vector::iterator I = Transaction.begin();
         I != Transaction.end(); ++I)
    {
-      if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-         std::clog << "  Cancel: " << (*I)->DestFile << std::endl;
-      // the transaction will abort, so stop anything that is idle
-      if ((*I)->Status == pkgAcquire::Item::StatIdle)
-      {
-         (*I)->Status = pkgAcquire::Item::StatDone;
-         (*I)->Dequeue();
-      }
+      (*I)->TransactionState(TransactionAbort);
    }
    Transaction.clear();
 }
@@ -1585,10 +1665,16 @@ bool pkgAcqMetaBase::TransactionHasError()
 {
    for (pkgAcquire::ItemIterator I = Transaction.begin();
         I != Transaction.end(); ++I)
-      if((*I)->Status != pkgAcquire::Item::StatDone &&
-         (*I)->Status != pkgAcquire::Item::StatIdle)
-         return true;
-
+   {
+      switch((*I)->Status) {
+	 case StatDone: break;
+	 case StatIdle: break;
+	 case StatAuthError: return true;
+	 case StatError: return true;
+	 case StatTransientNetworkError: return true;
+	 case StatFetching: break;
+      }
+   }
    return false;
 }
 									/*}}}*/
@@ -1603,24 +1689,7 @@ void pkgAcqMetaBase::CommitTransaction()
    for (std::vector::iterator I = Transaction.begin();
         I != Transaction.end(); ++I)
    {
-      if((*I)->PartialFile != "")
-      {
-	 if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-	    std::clog << "mv " << (*I)->PartialFile << " -> "<< (*I)->DestFile << " "
-	       << (*I)->DescURI() << std::endl;
-
-	 Rename((*I)->PartialFile, (*I)->DestFile);
-      } else {
-         if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-            std::clog << "rm "
-                      <<  (*I)->DestFile
-                      << " "
-                      << (*I)->DescURI()
-                      << std::endl;
-         unlink((*I)->DestFile.c_str());
-      }
-      // mark that this transaction is finished
-      (*I)->TransactionManager = 0;
+      (*I)->TransactionState(TransactionCommit);
    }
    Transaction.clear();
 }
@@ -1634,7 +1703,7 @@ void pkgAcqMetaBase::TransactionStageCopy(Item *I,
    I->DestFile = To;
 }
 									/*}}}*/
-// AcqMetaBase::TransactionStageRemoval - Sage a file for removal	/*{{{*/
+// AcqMetaBase::TransactionStageRemoval - Stage a file for removal	/*{{{*/
 void pkgAcqMetaBase::TransactionStageRemoval(Item *I,
                                              const std::string &FinalFile)
 {
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 148728b3c..33a28671c 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -344,7 +344,8 @@ class pkgAcquire::Item : public WeakPointable
       InvalidFormat,
       SignatureError,
       NotClearsigned,
-      MaximumSizeExceeded
+      MaximumSizeExceeded,
+      PDiffError,
    };
 
    /** \brief Rename failed file and set error
@@ -353,6 +354,12 @@ class pkgAcquire::Item : public WeakPointable
     */
    bool RenameOnError(RenameOnErrorState const state);
 
+   enum TransactionStates {
+      TransactionCommit,
+      TransactionAbort,
+   };
+   virtual bool TransactionState(TransactionStates const state);
+
    /** \brief The HashSums of the item is supposed to have than done */
    HashStringList ExpectedHashes;
 
@@ -685,14 +692,12 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
     */
    std::string Description;
 
-   /** \brief If the copy step of the packages file is done
-    */
-   bool PackagesFileReadyInPartial;
-
    /** \brief Get the full pathname of the final file for the current URI */
    virtual std::string GetFinalFilename() const;
 
    virtual bool QueueURI(pkgAcquire::ItemDesc &Item);
+
+   virtual bool TransactionState(TransactionStates const state);
  public:
    // Specialized action members
    virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
@@ -1010,6 +1015,8 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
    /** \brief Get the full pathname of the final file for the current URI */
    virtual std::string GetFinalFilename() const;
 
+   virtual bool TransactionState(TransactionStates const state);
+
    public:
    // Specialized action members
    virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
diff --git a/test/integration/framework b/test/integration/framework
index 4229ae162..7564a0faf 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1523,9 +1523,13 @@ aptautotest_aptget_update() {
 	testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755"
 	testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755"
 	# all copied files are properly chmodded
-	for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f ! -name 'lock'); do
+	for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -type f ! -name 'lock'); do
 		testfilestats "$file" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
 	done
+	if [ "$1" = 'testsuccess' ]; then
+		# failure cases can retain partial files and such
+		testempty find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" -mindepth 1 ! \( -name 'lock' -o -name '*.FAILED' \)
+	fi
 }
 aptautotest_apt_update() { aptautotest_aptget_update "$@"; }
 aptautotest_aptcdrom_add() { aptautotest_aptget_update "$@"; }
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 9f58308aa..7efccaa57 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -34,7 +34,7 @@ test_packagestoobig() {
 	done
 	NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
 	testfailuremsg "W: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages  Writing more data than expected ($NEW_SIZE > $SIZE)
-E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::Transaction=0
 }
 
 methodtest() {
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index b6b305d25..85046b3eb 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -126,18 +126,17 @@ testrun() {
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 
-msgmsg 'http: Test with Content-Length'
-webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
-testrun 'http://localhost:8080'
-msgmsg 'http: Test with Transfer-Encoding: chunked'
-webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
-testrun 'http://localhost:8080'
+serverconfigs() {
+	msgmsg "${1%%:*}: Test with Content-Length"
+	webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+	testrun "$1"
+	msgmsg "${1%%:*}: Test with Transfer-Encoding: chunked"
+	webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
+	testrun "$1"
+}
+
+serverconfigs 'http://localhost:8080'
 
 changetohttpswebserver
 
-msgmsg 'https: Test with Content-Length'
-webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
-testrun 'https://localhost:4433'
-msgmsg 'https: Test with Transfer-Encoding: chunked'
-webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
-testrun 'https://localhost:4433'
+serverconfigs 'https://localhost:4433'
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 5e759e50e..7d72a6944 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -47,7 +47,7 @@ testrun() {
 	compressfile 'aptarchive/Packages'
 	generatereleasefiles
 	signreleasefiles
-	rm -rf aptarchive/Packages.diff rootdir/var/lib/apt/lists
+	rm -rf aptarchive/Packages.diff rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists-bak
 	testsuccess aptget update "$@"
 	cp -a rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists-bak
 	testnopackage newstuff
-- 
cgit v1.2.3-70-g09d2


From f492283b4369ed0711f5f538b6d5fe61a8a75a0f Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 28 Apr 2015 15:06:12 +0200
Subject: do not require installed libapt-pkg-dev for gtest

Git-Dch: Ignore
---
 test/libapt/makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'test')

diff --git a/test/libapt/makefile b/test/libapt/makefile
index 0f8df291f..61a8aaf31 100644
--- a/test/libapt/makefile
+++ b/test/libapt/makefile
@@ -8,7 +8,7 @@ APT_DOMAIN=none
 include ../../buildlib/defaults.mak
 
 .PHONY: test
-ifeq (file-okay,$(shell $(CC) -M gtest_runner.cc >/dev/null 2>&1 && echo 'file-okay'))
+ifeq (file-okay,$(shell $(CC) -I $(BASE)/build/include -M gtest_runner.cc >/dev/null 2>&1 && echo 'file-okay'))
 test: $(BIN)/gtest$(BASENAME)
 	MALLOC_PERTURB_=21 MALLOC_CHECK_=2 LD_LIBRARY_PATH=$(LIB) $(BIN)/gtest$(BASENAME)
 
@@ -78,5 +78,6 @@ test:
 	@echo "APT uses Googles C++ testing framework for its unit tests"
 	@echo "On Debian systems this is available in the 'libgtest-dev' package."
 	@echo "Please install it before attempting to run the unit tests."
+	$(CC) -I $(BASE)/build/include -M gtest_runner.cc
 	exit 100
 endif
-- 
cgit v1.2.3-70-g09d2


From 3f732aa6ad0a81b6a6942a61fd5ed26a26590e8e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 28 Apr 2015 23:42:03 +0200
Subject: a pin of 1000 always means downgrade allowed

The documentation says this, but the code only agreed while evaluating
specific packages, but not generics. These needed a pin above 1000 to
have the same effect.

The code causing this makes references to a 'second pesduo status file',
but nowhere is explained what this might stand for and/or what it was,
so we do the only reasonable thing: Remove all references and do as
documented.
---
 apt-pkg/policy.cc                                  | 32 +++-----
 .../test-bug-543966-downgrade-below-1000-pin       | 84 +++++++++++---------
 test/integration/test-policy-pinning               | 92 ++++++++++------------
 3 files changed, 96 insertions(+), 112 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index 3cfc32829..9a1d2977e 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -4,22 +4,12 @@
 /* ######################################################################
 
    Package Version Policy implementation
-   
+
    This is just a really simple wrapper around pkgVersionMatch with
    some added goodies to manage the list of things..
-   
-   Priority Table:
-   
-   1000 -> inf = Downgradeable priorities
-   1000        = The 'no downgrade' pseduo-status file
-   100 -> 1000 = Standard priorities
-   990         = Config file override package files
-   989         = Start for preference auto-priorities
-   500         = Default package files
-   100         = The status file and ButAutomaticUpgrades sources
-   0 -> 100    = NotAutomatic sources like experimental
-   -inf -> 0   = Never selected   
-   
+
+   See man apt_preferences for what value means what.
+
    ##################################################################### */
 									/*}}}*/
 // Include Files							/*{{{*/
@@ -56,7 +46,7 @@ using namespace std;
    file matches the V0 policy engine. */
 pkgPolicy::pkgPolicy(pkgCache *Owner) : Pins(0), PFPriority(0), Cache(Owner)
 {
-   if (Owner == 0 || &(Owner->Head()) == 0)
+   if (Owner == 0)
       return;
    PFPriority = new signed short[Owner->Head().PackageFileCount];
    Pins = new Pin[Owner->Head().PackageCount];
@@ -125,7 +115,7 @@ bool pkgPolicy::InitDefaults()
 	    else
 	       PFPriority[F->ID] = Cur;
 	    
-	    if (PFPriority[F->ID] > 1000)
+	    if (PFPriority[F->ID] >= 1000)
 	       StatusOverride = true;
 	    
 	    Fixed[F->ID] = true;
@@ -166,9 +156,7 @@ pkgCache::VerIterator pkgPolicy::GetCandidateVer(pkgCache::PkgIterator const &Pk
       effectively excludes everything <= 0 which are the non-automatic
       priorities.. The status file is given a prio of 100 which will exclude
       not-automatic sources, except in a single shot not-installed mode.
-      The second pseduo-status file is at prio 1000, above which will permit
-      the user to force-downgrade things.
-      
+
       The user pin is subject to the same priority rules as default 
       selections. Thus there are two ways to create a pin - a pin that
       tracks the default when the default is taken away, and a permanent
@@ -218,9 +206,9 @@ pkgCache::VerIterator pkgPolicy::GetCandidateVer(pkgCache::PkgIterator const &Pk
 	    Pref = Ver;
 	    PrefSeen = true;
 	 }
-	 /* Elevate our current selection (or the status file itself)
-	    to the Pseudo-status priority. */
-	 Max = 1000;
+	 /* Elevate our current selection (or the status file itself) so that only
+	    a downgrade can override it from now on */
+	 Max = 999;
 
 	 // Fast path optimize.
 	 if (StatusOverride == false)
diff --git a/test/integration/test-bug-543966-downgrade-below-1000-pin b/test/integration/test-bug-543966-downgrade-below-1000-pin
index d37539b9f..485df999c 100755
--- a/test/integration/test-bug-543966-downgrade-below-1000-pin
+++ b/test/integration/test-bug-543966-downgrade-below-1000-pin
@@ -24,58 +24,66 @@ testsuccessequal "base-files:
      5.0.0 0
         500 file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=0
 
-echo 'Package: base-files
+writepin() {
+	echo "Package: $1
 Pin: release a=unstable
-Pin-Priority: 99' > rootdir/etc/apt/preferences
+Pin-Priority: $2" > rootdir/etc/apt/preferences
+}
 
-testsuccessequal "base-files:
+
+
+testpinning() {
+	local PKGPIN=''
+	local PKGPINPRIO='0'
+	local REPPINPRIO=''
+	if [ "$1" != '*' ]; then
+		PKGPINPRIO=''
+		REPPINPRIO=' 500'
+		PKGPIN='Package pin: 5.0.0
+  '
+	fi
+	writepin "$1" '99'
+	testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0-1
-  Package pin: 5.0.0
-  Version table:
- *** 5.0.0-1 99
+  ${PKGPIN}Version table:
+ *** 5.0.0-1 ${PKGPINPRIO:-99}
         100 $STATUS
-     5.0.0 99
-        500 file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=99
+     5.0.0 ${PKGPINPRIO:-99}
+       ${REPPINPRIO:-  99} file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=99
 
-echo 'Package: base-files
-Pin: release a=unstable
-Pin-Priority: 100' > rootdir/etc/apt/preferences
-
-testsuccessequal "base-files:
+	writepin "$1" '100'
+	testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0-1
-  Package pin: 5.0.0
-  Version table:
- *** 5.0.0-1 100
+  ${PKGPIN}Version table:
+ *** 5.0.0-1 ${PKGPINPRIO:-100}
         100 $STATUS
-     5.0.0 100
-        500 file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=100
+     5.0.0 ${PKGPINPRIO:-100}
+       ${REPPINPRIO:- 100} file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=100
 
-echo 'Package: base-files
-Pin: release a=unstable
-Pin-Priority: 999' > rootdir/etc/apt/preferences
-
-testsuccessequal "base-files:
+	writepin "$1" '999'
+	testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0-1
-  Package pin: 5.0.0
-  Version table:
- *** 5.0.0-1 999
+  ${PKGPIN}Version table:
+ *** 5.0.0-1 ${PKGPINPRIO:-999}
         100 $STATUS
-     5.0.0 999
-        500 file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=999
-
-echo 'Package: base-files
-Pin: release a=unstable
-Pin-Priority: 1000' > rootdir/etc/apt/preferences
+     5.0.0 ${PKGPINPRIO:-999}
+       ${REPPINPRIO:- 999} file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=999
 
-testsuccessequal "base-files:
+	writepin "$1" '1000'
+	testsuccessequal "base-files:
   Installed: 5.0.0-1
   Candidate: 5.0.0
-  Package pin: 5.0.0
-  Version table:
- *** 5.0.0-1 1000
+  ${PKGPIN}Version table:
+ *** 5.0.0-1 ${PKGPINPRIO:-1000}
         100 $STATUS
-     5.0.0 1000
-        500 file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=1000
+     5.0.0 ${PKGPINPRIO:-1000}
+       ${REPPINPRIO:-1000} file:${APTARCHIVE} unstable/main i386 Packages" aptcache policy base-files -o apt::pin=1000
+}
+
+msgmsg 'Tests with generic-form pin'
+testpinning '*'
+msgmsg 'Tests with specific-form pin'
+testpinning 'base-files'
diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning
index 15bf300ac..2675b51bc 100755
--- a/test/integration/test-policy-pinning
+++ b/test/integration/test-policy-pinning
@@ -25,70 +25,58 @@ testequalpolicy() {
 Pinned packages:" aptcache policy $*
 }
 
-aptgetupdate() {
-	# just to be sure that no old files are used
-	rm -rf rootdir/var/lib/apt
-	if aptget update  --allow-insecure-repositories -qq 2>&1 | grep '^E: '; then
-		msgwarn 'apt-get update failed with an error'
-	fi
-}
+testglobalpolicy() {
+	aptgetupdate
 
-### not signed archive
+	testequalpolicy 100 500
+	testequalpolicy 990 500 -t now
 
-aptgetupdate
+	sed -i aptarchive/Release -e 1i"NotAutomatic: yes"
+	aptgetupdate
 
-testequalpolicy 100 500
-testequalpolicy 990 500 -t now
+	testequalpolicy 100 1 -o Test=NotAutomatic
+	testequalpolicy 990 1 -o Test=NotAutomatic -t now
 
-sed -i aptarchive/Release -e 1i"NotAutomatic: yes"
-aptgetupdate
+	sed -i aptarchive/Release -e 1i"ButAutomaticUpgrades: yes"
+	aptgetupdate
 
-testequalpolicy 100 1 -o Test=NotAutomatic
-testequalpolicy 990 1 -o Test=NotAutomatic -t now
+	testequalpolicy 100 100 -o Test=ButAutomaticUpgrades
+	testequalpolicy 990 100 -o Test=ButAutomaticUpgrades -t now
 
-sed -i aptarchive/Release -e 1i"ButAutomaticUpgrades: yes"
-aptgetupdate
+	sed -i aptarchive/Release -e 's#NotAutomatic: yes#NotAutomatic: no#' -e '/ButAutomaticUpgrades: / d'
+	aptgetupdate
 
-testequalpolicy 100 100 -o Test=ButAutomaticUpgrades
-testequalpolicy 990 100 -o Test=ButAutomaticUpgrades -t now
+	testequalpolicy 100 500 -o Test=Automatic
+	testequalpolicy 990 500 -o Test=Automatic -t now
 
-sed -i aptarchive/Release -e 's#NotAutomatic: yes#NotAutomatic: no#' -e '/ButAutomaticUpgrades: / d'
-aptgetupdate
-
-testequalpolicy 100 500 -o Test=Automatic
-testequalpolicy 990 500 -o Test=Automatic -t now
-
-sed -i aptarchive/Release -e '/NotAutomatic: / d' -e '/ButAutomaticUpgrades: / d'
-
-### signed but no key in trusted
-
-signreleasefiles 'Marvin Paranoid'
-aptgetupdate
-testequalpolicy 100 500
-testequalpolicy 990 500 -t now
-
-sed -i aptarchive/Release -e 1i"NotAutomatic: yes"
-signreleasefiles 'Marvin Paranoid'
-aptgetupdate
-
-testequalpolicy 100 1 -o Test=NotAutomatic
-testequalpolicy 990 1 -o Test=NotAutomatic -t now
-
-sed -i aptarchive/Release -e 1i"ButAutomaticUpgrades: yes"
-signreleasefiles 'Marvin Paranoid'
-aptgetupdate
+	sed -i aptarchive/Release -e '/NotAutomatic: / d' -e '/ButAutomaticUpgrades: / d'
+}
 
-testequalpolicy 100 100 -o Test=ButAutomaticUpgrades
-testequalpolicy 990 100 -o Test=ButAutomaticUpgrades -t now
+msgmsg 'Test with not signed archive'
+aptgetupdate() {
+	rm -rf rootdir/var/lib/apt
+	testwarning aptget update  --allow-insecure-repositories
+}
+testglobalpolicy
 
-sed -i aptarchive/Release -e 's#NotAutomatic: yes#NotAutomatic: no#' -e '/ButAutomaticUpgrades: / d'
-signreleasefiles 'Marvin Paranoid'
-aptgetupdate
+msgmsg 'Test with signed but no key in trusted'
+aptgetupdate() {
+	rm -rf rootdir/var/lib/apt
+	signreleasefiles 'Marvin Paranoid'
+	testwarning aptget update  --allow-insecure-repositories
+}
+testglobalpolicy
 
-testequalpolicy 100 500 -o Test=Automatic
-testequalpolicy 990 500 -o Test=Automatic -t now
+# much the same tests will be executed below in more detail again for this one
+msgmsg 'Test with signed and valid key'
+aptgetupdate() {
+	rm -rf rootdir/var/lib/apt
+	signreleasefiles 'Joe Sixpack'
+	testsuccess aptget update
+}
+testglobalpolicy
 
-### signed and valid key
+msgmsg 'Test with specific packages'
 
 buildsimplenativepackage "coolstuff" "all" "1.0" "stable"
 buildsimplenativepackage "coolstuff" "all" "2.0~bpo1" "backports"
-- 
cgit v1.2.3-70-g09d2


From ab640001ee772fa27ee789f6717805f9d46d5bce Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 28 Apr 2015 23:59:00 +0200
Subject: remove unused and strange default-value for pins
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If the pin for a generic pin is 0, it get a value by strange looking
rules, if the pin is specific the rules are at least not strange, but
the value 989 is a magic number without any direct meaning… but both
never happens in practice as the parsing skips such entries with a
warning, so there always is a priority != 0 and the code therefore never
used.
---
 apt-pkg/policy.cc                                  | 34 +++++++---------------
 .../test-bug-543966-downgrade-below-1000-pin       |  4 +++
 2 files changed, 14 insertions(+), 24 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index 9a1d2977e..00693ce54 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -97,37 +97,29 @@ bool pkgPolicy::InitDefaults()
    // Apply the defaults..
    SPtrArray Fixed = new bool[Cache->HeaderP->PackageFileCount];
    memset(Fixed,0,sizeof(*Fixed)*Cache->HeaderP->PackageFileCount);
-   signed Cur = 989;
    StatusOverride = false;
-   for (vector::const_iterator I = Defaults.begin(); I != Defaults.end();
-	++I, --Cur)
+   for (vector::const_iterator I = Defaults.begin(); I != Defaults.end(); ++I)
    {
       pkgVersionMatch Match(I->Data,I->Type);
       for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
       {
-	 if (Match.FileMatch(F) == true && Fixed[F->ID] == false)
+	 if (Fixed[F->ID] == false && Match.FileMatch(F) == true)
 	 {
-	    if (I->Priority != 0 && I->Priority > 0)
-	       Cur = I->Priority;
-	    
-	    if (I->Priority < 0)
-	       PFPriority[F->ID] =  I->Priority;
-	    else
-	       PFPriority[F->ID] = Cur;
-	    
+	    PFPriority[F->ID] = I->Priority;
+
 	    if (PFPriority[F->ID] >= 1000)
 	       StatusOverride = true;
-	    
+
 	    Fixed[F->ID] = true;
-	 }      
-      }      
+	 }
+      }
    }
 
    if (_config->FindB("Debug::pkgPolicy",false) == true)
       for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
-	 std::clog << "Prio of " << F.FileName() << ' ' << PFPriority[F->ID] << std::endl; 
-   
-   return true;   
+	 std::clog << "Prio of " << F.FileName() << ' ' << PFPriority[F->ID] << std::endl;
+
+   return true;
 }
 									/*}}}*/
 // Policy::GetCandidateVer - Get the candidate install version		/*{{{*/
@@ -324,13 +316,7 @@ pkgCache::VerIterator pkgPolicy::GetMatch(pkgCache::PkgIterator const &Pkg)
 APT_PURE signed short pkgPolicy::GetPriority(pkgCache::PkgIterator const &Pkg)
 {
    if (Pins[Pkg->ID].Type != pkgVersionMatch::None)
-   {
-      // In this case 0 means default priority
-      if (Pins[Pkg->ID].Priority == 0)
-	 return 989;
       return Pins[Pkg->ID].Priority;
-   }
-   
    return 0;
 }
 APT_PURE signed short pkgPolicy::GetPriority(pkgCache::PkgFileIterator const &File)
diff --git a/test/integration/test-bug-543966-downgrade-below-1000-pin b/test/integration/test-bug-543966-downgrade-below-1000-pin
index 485df999c..e59231608 100755
--- a/test/integration/test-bug-543966-downgrade-below-1000-pin
+++ b/test/integration/test-bug-543966-downgrade-below-1000-pin
@@ -87,3 +87,7 @@ msgmsg 'Tests with generic-form pin'
 testpinning '*'
 msgmsg 'Tests with specific-form pin'
 testpinning 'base-files'
+msgmsg 'Tests with specific-form pin with glob'
+testpinning 'base-fil*'
+msgmsg 'Tests with specific-form pin with regex'
+testpinning '/^base-f[iI]les$/'
-- 
cgit v1.2.3-70-g09d2


From 41fadd29f136627fb5c9e733130b2a57c49fafaf Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Wed, 29 Apr 2015 10:55:08 +0200
Subject: remove available file to have same dpkg -l behavior

dpkg -l < 1.16.2 loads the available file and hence sees a package which
later versions do not see, leading to failures on travis-ci.
The different versions also have slightly different messages.

Git-Dch: Ignore
---
 test/integration/test-ubuntu-bug-761175-remove-purge | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'test')

diff --git a/test/integration/test-ubuntu-bug-761175-remove-purge b/test/integration/test-ubuntu-bug-761175-remove-purge
index c2e2aadf1..d5dfa2acf 100755
--- a/test/integration/test-ubuntu-bug-761175-remove-purge
+++ b/test/integration/test-ubuntu-bug-761175-remove-purge
@@ -39,7 +39,8 @@ The following packages will be REMOVED:
 0 upgraded, 0 newly installed, 1 to remove and 0 not upgraded.
 Purg compiz-core-$1" aptget purge compiz-core-$1 -s
 	testsuccess aptget purge compiz-core-$1 -y
-	testfailureequal "dpkg-query: no packages found matching compiz-core-$1" dpkg -l compiz-core-$1
+	echo -n '' > rootdir/var/lib/dpkg/available # dpkg -l < 1.16.2 reads the available file by default, where the package can be found
+	testequalor2 "dpkg-query: no packages found matching compiz-core-$1" "No packages found matching compiz-core-$1." dpkg -l compiz-core-$1
 }
 
 msgmsg 'Test in multi arch environment'
-- 
cgit v1.2.3-70-g09d2


From e8fb1cdfdd13e45f2b3abbd57a28b57ae6137f14 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 9 May 2015 18:55:41 +0200
Subject: sync TFRewrite*Order arrays with dpkg and dak

dpkg and dak know various field names and order them in their output,
while we have yet another order and have to play catch up with them as
we are sitting between chairs here and neither order is ideal for us,
too.

A little testcase is from now on supposed to help ensureing that we do
not derivate to far away from which fields dpkg knows and orders.
---
 apt-pkg/tagfile-order.c                            | 109 +++++++++++++++++++++
 apt-pkg/tagfile.cc                                 |  63 +-----------
 debian/tests/run-tests                             |   1 +
 test/integration/framework                         |   1 +
 test/integration/test-apt-ftparchive-cachedb       |   6 +-
 .../test-apt-ftparchive-cachedb-lp1274466          |   6 +-
 test/integration/test-apt-ftparchive-src-cachedb   |  18 ++--
 test/integration/test-apt-tagfile-fields-order     |  82 ++++++++++++++++
 8 files changed, 211 insertions(+), 75 deletions(-)
 create mode 100644 apt-pkg/tagfile-order.c
 create mode 100755 test/integration/test-apt-tagfile-fields-order

(limited to 'test')

diff --git a/apt-pkg/tagfile-order.c b/apt-pkg/tagfile-order.c
new file mode 100644
index 000000000..10c61ab94
--- /dev/null
+++ b/apt-pkg/tagfile-order.c
@@ -0,0 +1,109 @@
+/* In this file is the order defined in which e.g. apt-ftparchive will write stanzas in.
+   Other commands might or might not use this. 'apt-cache show' e.g. does NOT!
+
+   The order we chose here is inspired by both dpkg and dak.
+   The testcase test/integration/test-apt-tagfile-fields-order intends to ensure that
+   this file isn't lacking (too far) behind dpkg over time. */
+
+static const char *iTFRewritePackageOrder[] = {
+   "Package",
+   "Package-Type",
+   "Architecture",
+   "Subarchitecture", // Used only by d-i
+   "Version",
+   "Revision",         // Obsolete (warning in dpkg)
+   "Package-Revision", // Obsolete (warning in dpkg)
+   "Package_Revision", // Obsolete (warning in dpkg)
+   "Kernel-Version", // Used only by d-i
+   "Built-Using",
+   "Built-For-Profiles",
+   "Multi-Arch",
+   "Status",
+   "Priority",
+   "Class", // dpkg nickname for Priority
+   "Essential",
+   "Installer-Menu-Item", // Used only by d-i
+   "Section",
+   "Source",
+   "Origin",
+   "Maintainer",
+   "Original-Maintainer", // unknown in dpkg order
+   "Bugs",
+   "Config-Version", // Internal of dpkg
+   "Conffiles",
+   "Triggers-Awaited",
+   "Triggers-Pending",
+   "Installed-Size",
+   "Provides",
+   "Pre-Depends",
+   "Depends",
+   "Recommends",
+   "Recommended", // dpkg nickname for Recommends
+   "Suggests",
+   "Optional", // dpkg nickname for Suggests
+   "Conflicts",
+   "Breaks",
+   "Replaces",
+   "Enhances",
+   "Filename",
+   "MSDOS-Filename", // Obsolete (used by dselect)
+   "Size",
+   "MD5sum",
+   "SHA1",
+   "SHA256",
+   "SHA512",
+   "Homepage",
+   "Description",
+   "Tag",
+   "Task",
+   0
+};
+static const char *iTFRewriteSourceOrder[] = {
+   "Package",
+   "Source", // dsc file, renamed to Package in Sources
+   "Format",
+   "Binary",
+   "Architecture",
+   "Version",
+   "Priority",
+   "Class", // dpkg nickname for Priority
+   "Section",
+   "Origin",
+   "Maintainer",
+   "Original-Maintainer", // unknown in dpkg order
+   "Uploaders",
+   "Dm-Upload-Allowed", // Obsolete (ignored by dak)
+   "Standards-Version",
+   "Build-Depends",
+   "Build-Depends-Arch",
+   "Build-Depends-Indep",
+   "Build-Conflicts",
+   "Build-Conflicts-Arch",
+   "Build-Conflicts-Indep",
+   "Testsuite",
+   "Homepage",
+   "Vcs-Browser",
+   "Vcs-Browse", // dak only (nickname?)
+   "Vcs-Arch",
+   "Vcs-Bzr",
+   "Vcs-Cvs",
+   "Vcs-Darcs",
+   "Vcs-Git",
+   "Vcs-Hg",
+   "Vcs-Mtn",
+   "Vcs-Svn",
+   "Directory",
+   "Package-List",
+   "Files",
+   "Checksums-Md5",
+   "Checksums-Sha1",
+   "Checksums-Sha256",
+   "Checksums-Sha512",
+   0
+};
+
+/* Two levels of initialization are used because gcc will set the symbol
+   size of an array to the length of the array, causing dynamic relinking
+   errors. Doing this makes the symbol size constant */
+const char **TFRewritePackageOrder = iTFRewritePackageOrder;
+const char **TFRewriteSourceOrder = iTFRewriteSourceOrder;
diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc
index 9c40c8c92..2f7900d93 100644
--- a/apt-pkg/tagfile.cc
+++ b/apt-pkg/tagfile.cc
@@ -623,71 +623,14 @@ APT_PURE unsigned int pkgTagSection::Count() const {			/*{{{*/
    return d->Tags.size() - 1;
 }
 									/*}}}*/
+
+#include "tagfile-order.c"
+
 // TFRewrite - Rewrite a control record					/*{{{*/
 // ---------------------------------------------------------------------
 /* This writes the control record to stdout rewriting it as necessary. The
    override map item specificies the rewriting rules to follow. This also
    takes the time to sort the feild list. */
-
-/* The order of this list is taken from dpkg source lib/parse.c the fieldinfos
-   array. */
-static const char *iTFRewritePackageOrder[] = {
-                          "Package",
-                          "Essential",
-                          "Status",
-                          "Priority",
-                          "Section",
-                          "Installed-Size",
-                          "Maintainer",
-                          "Original-Maintainer",
-                          "Architecture",
-                          "Source",
-                          "Version",
-                           "Revision",         // Obsolete
-                           "Config-Version",   // Obsolete
-                          "Replaces",
-                          "Provides",
-                          "Depends",
-                          "Pre-Depends",
-                          "Recommends",
-                          "Suggests",
-                          "Conflicts",
-                          "Breaks",
-                          "Conffiles",
-                          "Filename",
-                          "Size",
-                          "MD5sum",
-                          "SHA1",
-                          "SHA256",
-                          "SHA512",
-                           "MSDOS-Filename",   // Obsolete
-                          "Description",
-                          0};
-static const char *iTFRewriteSourceOrder[] = {"Package",
-                                      "Source",
-                                      "Binary",
-                                      "Version",
-                                      "Priority",
-                                      "Section",
-                                      "Maintainer",
-				      "Original-Maintainer",
-                                      "Build-Depends",
-                                      "Build-Depends-Indep",
-                                      "Build-Conflicts",
-                                      "Build-Conflicts-Indep",
-                                      "Architecture",
-                                      "Standards-Version",
-                                      "Format",
-                                      "Directory",
-                                      "Files",
-                                      0};   
-
-/* Two levels of initialization are used because gcc will set the symbol
-   size of an array to the length of the array, causing dynamic relinking 
-   errors. Doing this makes the symbol size constant */
-const char **TFRewritePackageOrder = iTFRewritePackageOrder;
-const char **TFRewriteSourceOrder = iTFRewriteSourceOrder;
-   
 bool TFRewrite(FILE *Output,pkgTagSection const &Tags,const char *Order[],
 	       TFRewriteData *Rewrite)
 {
diff --git a/debian/tests/run-tests b/debian/tests/run-tests
index 7cc37f618..e459d7093 100644
--- a/debian/tests/run-tests
+++ b/debian/tests/run-tests
@@ -11,6 +11,7 @@ make -C test/interactive-helper/
 # run tests against the installed apt, use "env -i" to ensure
 # the host env does not pollute our environment
 env -i \
+APT_INTEGRATION_TESTS_SOURCE_DIR=$(pwd) \
 APT_INTEGRATION_TESTS_WEBSERVER_BIN_DIR=$(pwd)/build/bin \
 APT_INTEGRATION_TESTS_METHODS_DIR=/usr/lib/apt/methods \
 APT_INTEGRATION_TESTS_LIBEXEC_DIR=/usr/lib/apt/ \
diff --git a/test/integration/framework b/test/integration/framework
index 7564a0faf..03c188189 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -189,6 +189,7 @@ setupenvironment() {
 
 	TESTDIRECTORY=$(readlink -f $(dirname $0))
         # allow overriding the default BUILDDIR location
+	SOURCEDIRECTORY=${APT_INTEGRATION_TESTS_SOURCE_DIR:-"${TESTDIRECTORY}/../../"}
 	BUILDDIRECTORY=${APT_INTEGRATION_TESTS_BUILD_DIR:-"${TESTDIRECTORY}/../../build/bin"}
 	LIBRARYPATH=${APT_INTEGRATION_TESTS_LIBRARY_PATH:-"${BUILDDIRECTORY}"}
         METHODSDIR=${APT_INTEGRATION_TESTS_METHODS_DIR:-"${BUILDDIRECTORY}/methods"}
diff --git a/test/integration/test-apt-ftparchive-cachedb b/test/integration/test-apt-ftparchive-cachedb
index 3454ee36a..962095320 100755
--- a/test/integration/test-apt-ftparchive-cachedb
+++ b/test/integration/test-apt-ftparchive-cachedb
@@ -3,12 +3,12 @@ set -e
 
 ensure_correct_packages_file() {
     testequal "Package: foo
+Architecture: i386
+Version: 1
 Priority: optional
 Section: others
-$(dpkg-deb -I ./aptarchive/pool/main/foo_1_i386.deb | grep 'Installed-Size:' | sed 's#^ ##')
 Maintainer: Joe Sixpack 
-Architecture: i386
-Version: 1
+$(dpkg-deb -I ./aptarchive/pool/main/foo_1_i386.deb | grep 'Installed-Size:' | sed 's#^ ##')
 Filename: pool/main/foo_1_i386.deb" head -n8 ./aptarchive/dists/test/main/binary-i386/Packages
 }
 
diff --git a/test/integration/test-apt-ftparchive-cachedb-lp1274466 b/test/integration/test-apt-ftparchive-cachedb-lp1274466
index 1f86e367f..8b768441a 100755
--- a/test/integration/test-apt-ftparchive-cachedb-lp1274466
+++ b/test/integration/test-apt-ftparchive-cachedb-lp1274466
@@ -27,12 +27,12 @@ testfailure grep 7da58ff901a40ecf42a730dc33198b182e9ba9ec98799fc2c2b6fabeeee40cc
 
 # regression test for corruption with previous generation of cachedb
 testsuccessequal "Package: foo
+Architecture: i386
+Version: 1
 Priority: optional
 Section: others
-Installed-Size: 29
 Maintainer: Joe Sixpack 
-Architecture: i386
-Version: 1
+Installed-Size: 29
 Filename: ./foo_1_i386.deb
 Size: 1270
 MD5sum: 85d0e908c1a897700e2c5dea72d7e3c0
diff --git a/test/integration/test-apt-ftparchive-src-cachedb b/test/integration/test-apt-ftparchive-src-cachedb
index 2a361ecc0..66a3b7845 100755
--- a/test/integration/test-apt-ftparchive-src-cachedb
+++ b/test/integration/test-apt-ftparchive-src-cachedb
@@ -3,16 +3,16 @@ set -e
 
 assert_correct_sources_file() {
     testsuccessequal "Package: bar
+Format: 3.0 (native)
+Binary: bar
 Architecture: all
 Version: 1.0
-Binary: bar
-Format: 3.0 (native)
 Directory: pool/main
+Package-List:
+ bar deb admin extra
 Files:
  7b57dd065e51de5905288a5104d4bef5 406 bar_1.0.dsc
  d41d8cd98f00b204e9800998ecf8427e 0 bar_1.0.tar.gz
-Package-List:
- bar deb admin extra
 Checksums-Sha1:
  17a40b76715f393ab7fd6485c9392a02f1adf903 406 bar_1.0.dsc
  da39a3ee5e6b4b0d3255bfef95601890afd80709 0 bar_1.0.tar.gz
@@ -24,16 +24,16 @@ Checksums-Sha512:
  cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e 0 bar_1.0.tar.gz
 
 Package: foo
+Format: 3.0 (native)
+Binary: foo
 Architecture: all
 Version: 1.0
-Binary: foo
-Format: 3.0 (native)
 Directory: pool/main
+Package-List: 
+ foo deb admin extra
 Files:
  d144826e6f02831c1933e910c92cd7e0 171 foo_1.0.dsc
  d41d8cd98f00b204e9800998ecf8427e 0 foo_1.0.tar.gz
-Package-List: 
- foo deb admin extra
 Checksums-Sha1:
  979306aa3ccff3d61bba062bb6977e2493c6f907 171 foo_1.0.dsc
  da39a3ee5e6b4b0d3255bfef95601890afd80709 0 foo_1.0.tar.gz
@@ -43,7 +43,7 @@ Checksums-Sha256:
 Checksums-Sha512:
  3da0240fd764657c2f3661b4d750578a9a99b0580591b133756379d48117ebda87a5ed2467f513200d6e7eaf51422cbe91c15720eef7fb4bba2cc8ff81ebc547 171 foo_1.0.dsc
  cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e 0 foo_1.0.tar.gz
-" aptsortpkgs ./aptarchive/dists/test/main/source/Sources
+" aptsortpkgs ./aptarchive/dists/test/main/source/Sources -o APT::SortPkgs::Source=true
 }
 
 create_source_files() {
diff --git a/test/integration/test-apt-tagfile-fields-order b/test/integration/test-apt-tagfile-fields-order
new file mode 100755
index 000000000..27d5c14ff
--- /dev/null
+++ b/test/integration/test-apt-tagfile-fields-order
@@ -0,0 +1,82 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+
+dpkg_field_ordered_list() {
+	local FIELDS="$(perl -e "
+use Dpkg::Control;
+use Dpkg::Control::Fields;
+foreach \$f (field_ordered_list(${1})) {
+	print \"\$f\\n\";
+}" | sort -u)"
+	if [ -z "$FIELDS" ]; then
+		msgfail 'Could not get fields via libdpkg-perl'
+	fi
+	echo "$FIELDS"
+}
+
+comparelsts() {
+	local DIFFOUTPUT="$(diff -u apt.lst dpkg.lst || true)"
+	if echo "$DIFFOUTPUT" | grep -q '^+[^+]'; then
+		echo
+		echo "$DIFFOUTPUT" | grep '^[+-][^+-]'
+		msgfail
+	else
+		msgpass
+	fi
+}
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'Packages'
+dpkg_field_ordered_list 'CTRL_INDEX_PKG' > dpkg.lst
+sed -ne 's#^   "\(.*\)",.*$#\1#p' ${SOURCEDIRECTORY}/apt-pkg/tagfile-order.c | sed -n '/^Package$/,/^Package$/ p' | head -n -1 | sort -u > apt.lst
+comparelsts
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'status'
+dpkg_field_ordered_list 'CTRL_FILE_STATUS' > dpkg.lst
+comparelsts
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'DEBIAN/control'
+dpkg_field_ordered_list 'CTRL_PKG_DEB' > dpkg.lst
+comparelsts
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'Sources'
+dpkg_field_ordered_list 'CTRL_INDEX_SRC' > dpkg.lst
+echo 'Package' > apt.tmp
+sed -ne 's#^   "\(.*\)",.*$#\1#p' ${SOURCEDIRECTORY}/apt-pkg/tagfile-order.c | sed '/^Package$/,/^Package$/ d' >> apt.tmp
+sort -u apt.tmp > apt.lst
+comparelsts
+
+msgtest 'Check that apt knows all fields dpkg orders in' 'dsc'
+dpkg_field_ordered_list 'CTRL_PKG_SRC' > dpkg.lst
+comparelsts
+
+# HACK, but there is no good way to acquire sources in tests and/or to remember to run this regular manually
+if [ "$USER" = 'david' ]; then
+	msgtest 'Check if we have somewhere the sources of' 'dpkg'
+	DPKGSOURCE="$(locate dpkg/lib/dpkg/parse.c  | head -n 1 || true)"
+	if [ -z "$DPKGSOURCE" ]; then
+		msgskip 'Not found'
+	else
+		msgpass
+		msgtest 'Check that apt knows about all fields' 'dpkg parses'
+		sed -n 's#^.*FIELD("\(.*\)").*$#\1#p' "${DPKGSOURCE}" | sort -u > dpkg.lst
+		sed -ne 's#^   "\(.*\)",.*$#\1#p' ${SOURCEDIRECTORY}/apt-pkg/tagfile-order.c | sed -n '/^Package$/,/^Package$/ p' | head -n -1 | sort -u > apt.lst
+		comparelsts
+	fi
+
+	msgtest 'Check if we have somewhere the sources of' 'dak'
+	DAKSOURCE="$(locate dak/setup/core-init.d/080_metadatakeys | head -n 1 || true)"
+	if [ -z "$DAKSOURCE" ]; then
+		msgskip 'Not found'
+	else
+		msgpass
+		msgtest 'Check that apt knows about all fields' 'dak knows'
+		# dak mixes both, so we can only check with the mixed one as well
+		sed -ne "s#^.* VALUES ('\(.*\)', \(.*\)).*\$#\1 \2#p" "${DAKSOURCE}" | cut -d ' ' -f 1 | sort -u > dpkg.lst
+		sed -ne 's#^   "\(.*\)",.*$#\1#p' ${SOURCEDIRECTORY}/apt-pkg/tagfile-order.c | sort -u > apt.lst
+		comparelsts
+	fi
+fi
-- 
cgit v1.2.3-70-g09d2


From 8d058ea53b18348f81229049a27d14282bd8d8c1 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sun, 10 May 2015 22:53:15 +0200
Subject: implement a more c++-style TFRewrite alternative

TFRewrite is okay, but it has obscure limitations (256 Tags), even more
obscure bugs (order for renames is defined by the old name) and the
interface is very c-style encouraging bad usage like we do it in
apt-ftparchive passing massive amounts of c_str() from std::string in.

The old-style is marked as deprecated accordingly. The next commit will
fix all places in the apt code to not use the old-style anymore.
---
 apt-pkg/tagfile.cc          | 148 +++++++++++++++++++++++++++++++++++++++++++-
 apt-pkg/tagfile.h           |  47 ++++++++++----
 test/libapt/tagfile_test.cc |   6 ++
 3 files changed, 187 insertions(+), 14 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc
index 2f7900d93..5ff495fbd 100644
--- a/apt-pkg/tagfile.cc
+++ b/apt-pkg/tagfile.cc
@@ -509,8 +509,6 @@ bool pkgTagSection::Find(const char *Tag,const char *&Start,
 }
 									/*}}}*/
 // TagSection::FindS - Find a string					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 string pkgTagSection::FindS(const char *Tag) const
 {
    const char *Start;
@@ -520,6 +518,24 @@ string pkgTagSection::FindS(const char *Tag) const
    return string(Start,End);      
 }
 									/*}}}*/
+// TagSection::FindRawS - Find a string					/*{{{*/
+string pkgTagSection::FindRawS(const char *Tag) const
+{
+   unsigned int Pos;
+   if (Find(Tag, Pos) == false)
+      return "";
+
+   char const *Start = (char const *) memchr(Section + d->Tags[Pos].EndTag, ':', d->Tags[Pos].StartValue - d->Tags[Pos].EndTag);
+   ++Start;
+   char const *End = Section + d->Tags[Pos + 1].StartTag;
+   if (unlikely(Start > End))
+      return "";
+
+   for (; isspace(End[-1]) != 0 && End > Start; --End);
+
+   return std::string(Start, End - Start);
+}
+									/*}}}*/
 // TagSection::FindI - Find an integer					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -623,6 +639,132 @@ APT_PURE unsigned int pkgTagSection::Count() const {			/*{{{*/
    return d->Tags.size() - 1;
 }
 									/*}}}*/
+// TagSection::Write - Ordered (re)writing of fields			/*{{{*/
+pkgTagSection::Tag pkgTagSection::Tag::Remove(std::string const &Name)
+{
+   return Tag(REMOVE, Name, "");
+}
+pkgTagSection::Tag pkgTagSection::Tag::Rename(std::string const &OldName, std::string const &NewName)
+{
+   return Tag(RENAME, OldName, NewName);
+}
+pkgTagSection::Tag pkgTagSection::Tag::Rewrite(std::string const &Name, std::string const &Data)
+{
+   if (Data.empty() == true)
+      return Tag(REMOVE, Name, "");
+   else
+      return Tag(REWRITE, Name, Data);
+}
+static bool WriteTag(FileFd &File, std::string Tag, std::string const &Value)
+{
+   if (Value.empty() || isspace(Value[0]) != 0)
+      Tag.append(":");
+   else
+      Tag.append(": ");
+   Tag.append(Value);
+   Tag.append("\n");
+   return File.Write(Tag.c_str(), Tag.length());
+}
+static bool RewriteTags(FileFd &File, pkgTagSection const * const This, char const * const Tag,
+      std::vector::const_iterator &R,
+      std::vector::const_iterator const &REnd)
+{
+   size_t const TagLen = strlen(Tag);
+   for (; R != REnd; ++R)
+   {
+      std::string data;
+      if (R->Name.length() == TagLen && strncasecmp(R->Name.c_str(), Tag, R->Name.length()) == 0)
+      {
+	 if (R->Action != pkgTagSection::Tag::REWRITE)
+	    break;
+	 data = R->Data;
+      }
+      else if(R->Action == pkgTagSection::Tag::RENAME && R->Data.length() == TagLen &&
+	    strncasecmp(R->Data.c_str(), Tag, R->Data.length()) == 0)
+	 data = This->FindRawS(R->Name.c_str());
+      else
+	 continue;
+
+      return WriteTag(File, Tag, data);
+   }
+   return true;
+}
+bool pkgTagSection::Write(FileFd &File, char const * const * const Order, std::vector const &Rewrite) const
+{
+   // first pass: Write everything we have an order for
+   if (Order != NULL)
+   {
+      for (unsigned int I = 0; Order[I] != 0; ++I)
+      {
+	 std::vector::const_iterator R = Rewrite.begin();
+	 if (RewriteTags(File, this, Order[I], R, Rewrite.end()) == false)
+	    return false;
+	 if (R != Rewrite.end())
+	    continue;
+
+	 if (Exists(Order[I]) == false)
+	    continue;
+
+	 if (WriteTag(File, Order[I], FindRawS(Order[I])) == false)
+	    return false;
+      }
+   }
+   // second pass: See if we have tags which aren't ordered
+   if (d->Tags.empty() == false)
+   {
+      for (std::vector::const_iterator T = d->Tags.begin(); T != d->Tags.end() - 1; ++T)
+      {
+	 char const * const fieldname = Section + T->StartTag;
+	 size_t fieldnamelen = T->EndTag - T->StartTag;
+	 if (Order != NULL)
+	 {
+	    unsigned int I = 0;
+	    for (; Order[I] != 0; ++I)
+	    {
+	       if (fieldnamelen == strlen(Order[I]) && strncasecmp(fieldname, Order[I], fieldnamelen) == 0)
+		  break;
+	    }
+	    if (Order[I] != 0)
+	       continue;
+	 }
+
+	 std::string const name(fieldname, fieldnamelen);
+	 std::vector::const_iterator R = Rewrite.begin();
+	 if (RewriteTags(File, this, name.c_str(), R, Rewrite.end()) == false)
+	    return false;
+	 if (R != Rewrite.end())
+	    continue;
+
+	 if (WriteTag(File, name, FindRawS(name.c_str())) == false)
+	    return false;
+      }
+   }
+   // last pass: see if there are any rewrites remaining we haven't done yet
+   for (std::vector::const_iterator R = Rewrite.begin(); R != Rewrite.end(); ++R)
+   {
+      if (R->Action == Tag::REMOVE)
+	 continue;
+      std::string const name = ((R->Action == Tag::RENAME) ? R->Data : R->Name);
+      if (Exists(name.c_str()))
+	 continue;
+      if (Order != NULL)
+      {
+	 unsigned int I = 0;
+	 for (; Order[I] != 0; ++I)
+	 {
+	    if (strncasecmp(name.c_str(), Order[I], name.length()) == 0 && name.length() == strlen(Order[I]))
+	       break;
+	 }
+	 if (Order[I] != 0)
+	    continue;
+      }
+
+      if (WriteTag(File, name, ((R->Action == Tag::RENAME) ? FindRawS(R->Name.c_str()) : R->Data)) == false)
+	 return false;
+   }
+   return true;
+}
+									/*}}}*/
 
 #include "tagfile-order.c"
 
@@ -631,6 +773,7 @@ APT_PURE unsigned int pkgTagSection::Count() const {			/*{{{*/
 /* This writes the control record to stdout rewriting it as necessary. The
    override map item specificies the rewriting rules to follow. This also
    takes the time to sort the feild list. */
+APT_IGNORE_DEPRECATED_PUSH
 bool TFRewrite(FILE *Output,pkgTagSection const &Tags,const char *Order[],
 	       TFRewriteData *Rewrite)
 {
@@ -754,6 +897,7 @@ bool TFRewrite(FILE *Output,pkgTagSection const &Tags,const char *Order[],
       
    return true;
 }
+APT_IGNORE_DEPRECATED_POP
 									/*}}}*/
 
 pkgTagSection::~pkgTagSection() { delete d; }
diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h
index d09e7046c..118954541 100644
--- a/apt-pkg/tagfile.h
+++ b/apt-pkg/tagfile.h
@@ -54,13 +54,14 @@ class pkgTagSection
    const char *Stop;
 
    public:
-   
+
    inline bool operator ==(const pkgTagSection &rhs) {return Section == rhs.Section;};
    inline bool operator !=(const pkgTagSection &rhs) {return Section != rhs.Section;};
-   
+
    bool Find(const char *Tag,const char *&Start, const char *&End) const;
    bool Find(const char *Tag,unsigned int &Pos) const;
    std::string FindS(const char *Tag) const;
+   std::string FindRawS(const char *Tag) const;
    signed int FindI(const char *Tag,signed long Default = 0) const;
    bool FindB(const char *Tag, bool const &Default = false) const;
    unsigned long long FindULL(const char *Tag, unsigned long long const &Default = 0) const;
@@ -115,9 +116,32 @@ class pkgTagSection
       Start = Section;
       Stop = this->Stop;
    };
-   
+
    pkgTagSection();
    virtual ~pkgTagSection();
+
+   struct Tag
+   {
+      enum ActionType { REMOVE, RENAME, REWRITE } Action;
+      std::string Name;
+      std::string Data;
+
+      static Tag Remove(std::string const &Name);
+      static Tag Rename(std::string const &OldName, std::string const &NewName);
+      static Tag Rewrite(std::string const &Name, std::string const &Data);
+      private:
+      Tag(ActionType const Action, std::string const &Name, std::string const &Data) :
+	 Action(Action), Name(Name), Data(Data) {}
+   };
+
+   /** Write this section (with optional rewrites) to a file
+    *
+    * @param File to write the section to
+    * @param Order in which tags should appear in the file
+    * @param Rewrite is a set of tags to be renamed, rewitten and/or removed
+    * @return \b true if successful, otherwise \b false
+    */
+   bool Write(FileFd &File, char const * const * const Order = NULL, std::vector const &Rewrite = std::vector()) const;
 };
 
 class pkgTagFilePrivate;
@@ -141,20 +165,19 @@ class pkgTagFile
    virtual ~pkgTagFile();
 };
 
-/* This is the list of things to rewrite. The rewriter
-   goes through and changes or adds each of these headers
-   to suit. A zero forces the header to be erased, an empty string
-   causes the old value to be used. (rewrite rule ignored) */
-struct TFRewriteData
+extern const char **TFRewritePackageOrder;
+extern const char **TFRewriteSourceOrder;
+
+// Use pkgTagSection::Tag and pkgTagSection::Write() instead
+APT_IGNORE_DEPRECATED_PUSH
+struct APT_DEPRECATED TFRewriteData
 {
    const char *Tag;
    const char *Rewrite;
    const char *NewTag;
 };
-extern const char **TFRewritePackageOrder;
-extern const char **TFRewriteSourceOrder;
-
-bool TFRewrite(FILE *Output,pkgTagSection const &Tags,const char *Order[],
+APT_DEPRECATED bool TFRewrite(FILE *Output,pkgTagSection const &Tags,const char *Order[],
 	       TFRewriteData *Rewrite);
+APT_IGNORE_DEPRECATED_POP
 
 #endif
diff --git a/test/libapt/tagfile_test.cc b/test/libapt/tagfile_test.cc
index df618ea16..d7030f41a 100644
--- a/test/libapt/tagfile_test.cc
+++ b/test/libapt/tagfile_test.cc
@@ -34,6 +34,12 @@ TEST(TagFileTest,SingleField)
    EXPECT_FALSE(section.Exists("FieldB-12345678"));
    // There is only one section in this tag file
    EXPECT_FALSE(tfile.Step(section));
+
+   // Now we scan an empty section to test reset
+   ASSERT_TRUE(section.Scan("\n\n", 2, true));
+   EXPECT_EQ(0, section.Count());
+   EXPECT_FALSE(section.Exists("FieldA-12345678"));
+   EXPECT_FALSE(section.Exists("FieldB-12345678"));
 }
 
 TEST(TagFileTest,MultipleSections)
-- 
cgit v1.2.3-70-g09d2


From 88593886a42025d51d76051da5929b044e42efee Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 11 May 2015 15:08:08 +0200
Subject: rewrite all TFRewrite instances to use the new pkgTagSection::Write
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

While it is mostly busywork to rewrite all instances it actually fixes
bugs as the data storage used by the new method is std::string rather
than a char*, the later mostly created by c_str() from a std::string
which the caller has to ensure keeps in scope – something apt-ftparchive
actually didn't ensure and relied on copy-on-write behavior instead
which c++11 forbids and hence the new default gcc abi doesn't use it.
---
 apt-pkg/depcache.cc                       |  50 +++---
 apt-pkg/indexcopy.cc                      | 125 ++++++--------
 apt-pkg/indexcopy.h                       |   7 +-
 apt-private/private-show.cc               |  46 ++---
 cmdline/apt-cache.cc                      |  21 ++-
 cmdline/apt-sortpkgs.cc                   |  14 +-
 ftparchive/apt-ftparchive.cc              |  51 +++---
 ftparchive/contents.cc                    |  34 ++--
 ftparchive/contents.h                     |   7 +-
 ftparchive/multicompress.cc               |  19 +--
 ftparchive/multicompress.h                |   2 +-
 ftparchive/writer.cc                      | 204 +++++++++++-----------
 ftparchive/writer.h                       |  45 +++--
 test/integration/test-apt-cli-show        |   4 +-
 test/libapt/indexcopytosourcelist_test.cc |   2 +-
 test/libapt/tagsection_test.cc            | 270 ++++++++++++++++++++++++++++++
 16 files changed, 559 insertions(+), 342 deletions(-)
 create mode 100644 test/libapt/tagsection_test.cc

(limited to 'test')

diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc
index 73c5bb320..b73c336db 100644
--- a/apt-pkg/depcache.cc
+++ b/apt-pkg/depcache.cc
@@ -33,7 +33,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 
 #include 
@@ -252,17 +251,14 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
       return _error->Error(_("Failed to open StateFile %s"),
 			   state.c_str());
 
-   FILE *OutFile;
-   string const outfile = state + ".tmp";
-   if((OutFile = fopen(outfile.c_str(),"w")) == NULL)
-      return _error->Error(_("Failed to write temporary StateFile %s"),
-			   outfile.c_str());
+   FileFd OutFile(state, FileFd::ReadWrite | FileFd::Atomic);
+   if (OutFile.IsOpen() == false || OutFile.Failed() == true)
+      return _error->Error(_("Failed to write temporary StateFile %s"), state.c_str());
 
    // first merge with the existing sections
    pkgTagFile tagfile(&StateFile);
    pkgTagSection section;
    std::set pkgs_seen;
-   const char *nullreorderlist[] = {0};
    while(tagfile.Step(section)) {
 	 string const pkgname = section.FindS("Package");
 	 string pkgarch = section.FindS("Architecture");
@@ -271,7 +267,7 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	 // Silently ignore unknown packages and packages with no actual
 	 // version.
 	 pkgCache::PkgIterator pkg = Cache->FindPkg(pkgname, pkgarch);
-	 if(pkg.end() || pkg.VersionList().end()) 
+	 if(pkg.end() || pkg.VersionList().end())
 	    continue;
 	 StateCache const &P = PkgState[pkg->ID];
 	 bool newAuto = (P.Flags & Flag::Auto);
@@ -292,21 +288,17 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	 if(_config->FindB("Debug::pkgAutoRemove",false))
 	    std::clog << "Update existing AutoInstall info: " 
 		      << pkg.FullName() << std::endl;
-	 TFRewriteData rewrite[3];
-	 rewrite[0].Tag = "Architecture";
-	 rewrite[0].Rewrite = pkg.Arch();
-	 rewrite[0].NewTag = 0;
-	 rewrite[1].Tag = "Auto-Installed";
-	 rewrite[1].Rewrite = newAuto ? "1" : "0";
-	 rewrite[1].NewTag = 0;
-	 rewrite[2].Tag = 0;
-	 TFRewrite(OutFile, section, nullreorderlist, rewrite);
-	 fprintf(OutFile,"\n");
+
+	 std::vector rewrite;
+	 rewrite.push_back(pkgTagSection::Tag::Rewrite("Architecture", pkg.Arch()));
+	 rewrite.push_back(pkgTagSection::Tag::Rewrite("Auto-Installed", newAuto ? "1" : "0"));
+	 section.Write(OutFile, NULL, rewrite);
+	 if (OutFile.Write("\n", 1) == false)
+	    return false;
 	 pkgs_seen.insert(pkg.FullName());
    }
-   
+
    // then write the ones we have not seen yet
-   std::ostringstream ostr;
    for(pkgCache::PkgIterator pkg=Cache->PkgBegin(); !pkg.end(); ++pkg) {
       StateCache const &P = PkgState[pkg->ID];
       if(P.Flags & Flag::Auto) {
@@ -325,19 +317,17 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	    continue;
 	 if(debug_autoremove)
 	    std::clog << "Writing new AutoInstall: " << pkg.FullName() << std::endl;
-	 ostr.str(string(""));
-	 ostr << "Package: " << pkg.Name()
-	      << "\nArchitecture: " << pkgarch
-	      << "\nAuto-Installed: 1\n\n";
-	 fprintf(OutFile,"%s",ostr.str().c_str());
+	 std::string stanza = "Package: ";
+	 stanza.append(pkg.Name())
+	      .append("\nArchitecture: ").append(pkgarch)
+	      .append("\nAuto-Installed: 1\n\n");
+	 if (OutFile.Write(stanza.c_str(), stanza.length()) == false)
+	    return false;
       }
    }
-   fclose(OutFile);
-
-   // move the outfile over the real file and set permissions
-   rename(outfile.c_str(), state.c_str());
+   if (OutFile.Close() == false)
+      return false;
    chmod(state.c_str(), 0644);
-
    return true;
 }
 									/*}}}*/
diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc
index 144c508be..461aa4217 100644
--- a/apt-pkg/indexcopy.cc
+++ b/apt-pkg/indexcopy.cc
@@ -108,10 +108,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
       }
       if (_error->PendingError() == true)
 	 return false;
-      FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
-      if (TargetFl == 0)
-	 return _error->Errno("fdopen","Failed to reopen fd");
-      
+
       // Setup the progress meter
       if(Progress)
 	 Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
@@ -132,14 +129,11 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	 string File;
 	 unsigned long long Size;
 	 if (GetFile(File,Size) == false)
-	 {
-	    fclose(TargetFl);
 	    return false;
-	 }
-	 
+
 	 if (Chop != 0)
 	    File = OrigPath + ChopDirs(File,Chop);
-	 
+
 	 // See if the file exists
 	 if (NoStat == false || Hits < 10)
 	 {
@@ -157,10 +151,10 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	       if (Chop != 0)
 		  File = OrigPath + ChopDirs(File,Chop);
 	    }
-	    
+
 	    // Get the size
 	    struct stat Buf;
-	    if (stat((CDROM + Prefix + File).c_str(),&Buf) != 0 || 
+	    if (stat((CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 		Buf.st_size == 0)
 	    {
 	       bool Mangled = false;
@@ -173,7 +167,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 		  File.replace(Start,End-Start,"binary-all");
 		  Mangled = true;
 	       }
-	       
+
 	       if (Mangled == false ||
 		   stat((CDROM + Prefix + File).c_str(),&Buf) != 0)
 	       {
@@ -181,9 +175,9 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 		     clog << "Missed(2): " << OrigFile << endl;
 		  NotFound++;
 		  continue;
-	       }	       
-	    }	    
-	    			    	    
+	       }
+	    }
+
 	    // Size match
 	    if ((unsigned long long)Buf.st_size != Size)
 	    {
@@ -193,21 +187,17 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	       continue;
 	    }
 	 }
-	 
+
 	 Packages++;
 	 Hits++;
-	 
-	 if (RewriteEntry(TargetFl,File) == false)
-	 {
-	    fclose(TargetFl);
+
+	 if (RewriteEntry(Target, File) == false)
 	    return false;
-	 }
       }
-      fclose(TargetFl);
 
       if (Debug == true)
 	 cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
-	 
+
       if (_config->FindB("APT::CDROM::NoAct",false) == false)
       {
 	 // Move out of the partial directory
@@ -218,39 +208,38 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	    return _error->Errno("rename","Failed to rename");
 	 ChangeOwnerAndPermissionOfFile("CopyPackages", FinalF.c_str(), "root", "root", 0644);
       }
-	 
+
       /* Mangle the source to be in the proper notation with
-       	 prefix dist [component] */ 
+	 prefix dist [component] */
       *I = string(*I,Prefix.length());
       ConvertToSourceList(CDROM,*I);
       *I = Prefix + ' ' + *I;
-      
+
       CurrentSize += FileSize;
-   }   
+   }
    if(Progress)
       Progress->Done();
-   
+
    // Some stats
    if(log) {
       stringstream msg;
       if(NotFound == 0 && WrongSize == 0)
 	 ioprintf(msg, _("Wrote %i records.\n"), Packages);
       else if (NotFound != 0 && WrongSize == 0)
-	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 		  Packages, NotFound);
       else if (NotFound == 0 && WrongSize != 0)
-	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 		  Packages, WrongSize);
       if (NotFound != 0 && WrongSize != 0)
 	 ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
    }
-   
+
    if (Packages == 0)
       _error->Warning("No valid records were found.");
 
    if (NotFound + WrongSize > 10)
       _error->Warning("A lot of entries were discarded, something may be wrong.\n");
-   
 
    return true;
 }
@@ -267,10 +256,10 @@ string IndexCopy::ChopDirs(string Path,unsigned int Depth)
       Depth--;
    }
    while (I != string::npos && Depth != 0);
-   
+
    if (I == string::npos)
       return string();
-   
+
    return string(Path,I+1);
 }
 									/*}}}*/
@@ -433,17 +422,15 @@ bool PackageCopy::GetFile(string &File,unsigned long long &Size)
 }
 									/*}}}*/
 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool PackageCopy::RewriteEntry(FILE *Target,string File)
+bool PackageCopy::RewriteEntry(FileFd &Target,string const &File)
 {
-   TFRewriteData Changes[] = {{ "Filename", File.c_str(), NULL },
-                              { NULL, NULL, NULL }};
-   
-   if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
+   string const Dir(File,0,File.rfind('/'));
+   std::vector Changes;
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", File));
+
+   if (Section->Write(Target, TFRewritePackageOrder, Changes) == false)
       return false;
-   fputc('\n',Target);
-   return true;
+   return Target.Write("\n", 1);
 }
 									/*}}}*/
 // SourceCopy::GetFile - Get the file information from the section	/*{{{*/
@@ -478,23 +465,18 @@ bool SourceCopy::GetFile(string &File,unsigned long long &Size)
 }
 									/*}}}*/
 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool SourceCopy::RewriteEntry(FILE *Target,string File)
+bool SourceCopy::RewriteEntry(FileFd &Target, std::string const &File)
 {
-   string Dir(File,0,File.rfind('/'));
-   TFRewriteData Changes[] = {{ "Directory", Dir.c_str(), NULL },
-                              { NULL, NULL, NULL }};
-   
-   if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
+   string const Dir(File,0,File.rfind('/'));
+   std::vector Changes;
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Dir));
+
+   if (Section->Write(Target, TFRewriteSourceOrder, Changes) == false)
       return false;
-   fputc('\n',Target);
-   return true;
+   return Target.Write("\n", 1);
 }
 									/*}}}*/
-// SigVerify::Verify - Verify a files md5sum against its metaindex     	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// SigVerify::Verify - Verify a files md5sum against its metaindex	/*{{{*/
 bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex)
 {
    const indexRecords::checkSum *Record = MetaIndex->Lookup(file);
@@ -702,7 +684,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
       pkgTagFile Parser(&Pkg);
       if (_error->PendingError() == true)
 	 return false;
-      
+
       // Open the output file
       char S[400];
       snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(),
@@ -719,10 +701,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
       }
       if (_error->PendingError() == true)
 	 return false;
-      FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
-      if (TargetFl == 0)
-	 return _error->Errno("fdopen","Failed to reopen fd");
-      
+
       // Setup the progress meter
       if(Progress)
 	 Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
@@ -740,20 +719,16 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
 	 if(Progress)
 	    Progress->Progress(Parser.Offset());
 
-	 const char *Start;
-	 const char *Stop;
-	 Section.GetSection(Start,Stop);
-	 fwrite(Start,Stop-Start, 1, TargetFl);
-	 fputc('\n',TargetFl);
+	 if (Section.Write(Target) == false || Target.Write("\n", 1) == false)
+	    return false;
 
 	 Packages++;
 	 Hits++;
       }
-      fclose(TargetFl);
 
       if (Debug == true)
 	 cout << " Processed by using Prefix '" << Prefix << "' and chop " << endl;
-	 
+
       if (_config->FindB("APT::CDROM::NoAct",false) == false)
       {
 	 // Move out of the partial directory
@@ -764,34 +739,32 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
 	    return _error->Errno("rename","Failed to rename");
 	 ChangeOwnerAndPermissionOfFile("CopyTranslations", FinalF.c_str(), "root", "root", 0644);
       }
-      
-      
+
       CurrentSize += FileSize;
-   }   
+   }
    if(Progress)
       Progress->Done();
-   
+
    // Some stats
    if(log) {
       stringstream msg;
       if(NotFound == 0 && WrongSize == 0)
 	 ioprintf(msg, _("Wrote %i records.\n"), Packages);
       else if (NotFound != 0 && WrongSize == 0)
-	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 		  Packages, NotFound);
       else if (NotFound == 0 && WrongSize != 0)
-	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 		  Packages, WrongSize);
       if (NotFound != 0 && WrongSize != 0)
 	 ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
    }
-   
+
    if (Packages == 0)
       _error->Warning("No valid records were found.");
 
    if (NotFound + WrongSize > 10)
       _error->Warning("A lot of entries were discarded, something may be wrong.\n");
-   
 
    return true;
 }
diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h
index 701beb075..729b0c8cb 100644
--- a/apt-pkg/indexcopy.h
+++ b/apt-pkg/indexcopy.h
@@ -28,6 +28,7 @@ using std::vector;
 class pkgTagSection;
 class indexRecords;
 class pkgCdromStatus;
+class FileFd;
 
 class IndexCopy								/*{{{*/
 {
@@ -45,7 +46,7 @@ class IndexCopy								/*{{{*/
    void ConvertToSourceList(std::string CD,std::string &Path);
    bool GrabFirst(std::string Path,std::string &To,unsigned int Depth);
    virtual bool GetFile(std::string &Filename,unsigned long long &Size) = 0;
-   virtual bool RewriteEntry(FILE *Target,std::string File) = 0;
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File) = 0;
    virtual const char *GetFileName() = 0;
    virtual const char *Type() = 0;
    
@@ -61,7 +62,7 @@ class PackageCopy : public IndexCopy					/*{{{*/
    protected:
    
    virtual bool GetFile(std::string &Filename,unsigned long long &Size);
-   virtual bool RewriteEntry(FILE *Target,std::string File);
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File);
    virtual const char *GetFileName() {return "Packages";};
    virtual const char *Type() {return "Package";};
    
@@ -72,7 +73,7 @@ class SourceCopy : public IndexCopy					/*{{{*/
    protected:
    
    virtual bool GetFile(std::string &Filename,unsigned long long &Size);
-   virtual bool RewriteEntry(FILE *Target,std::string File);
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File);
    virtual const char *GetFileName() {return "Sources";};
    virtual const char *Type() {return "Source";};
    
diff --git a/apt-private/private-show.cc b/apt-private/private-show.cc
index 289f035a6..790bc0092 100644
--- a/apt-private/private-show.cc
+++ b/apt-private/private-show.cc
@@ -97,28 +97,30 @@ static bool DisplayRecord(pkgCacheFile &CacheFile, pkgCache::VerIterator V,
       manual_installed = 0;
 
    // FIXME: add verbose that does not do the removal of the tags?
-   TFRewriteData RW[] = {
-      // delete, apt-cache show has this info and most users do not care
-      {"MD5sum", NULL, NULL},
-      {"SHA1", NULL, NULL},
-      {"SHA256", NULL, NULL},
-      {"Filename", NULL, NULL},
-      {"Multi-Arch", NULL, NULL},
-      {"Architecture", NULL, NULL},
-      {"Conffiles", NULL, NULL},
-      // we use the translated description
-      {"Description", NULL, NULL},
-      {"Description-md5", NULL, NULL},
-      // improve
-      {"Installed-Size", installed_size.c_str(), NULL},
-      {"Size", package_size.c_str(), "Download-Size"},
-      // add
-      {"APT-Manual-Installed", manual_installed, NULL},
-      {"APT-Sources", source_index_file.c_str(), NULL},
-      {NULL, NULL, NULL}
-   };
-
-   if(TFRewrite(stdout, Tags, NULL, RW) == false)
+   std::vector RW;
+   // delete, apt-cache show has this info and most users do not care
+   RW.push_back(pkgTagSection::Tag::Remove("MD5sum"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA1"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA256"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA512"));
+   RW.push_back(pkgTagSection::Tag::Remove("Filename"));
+   RW.push_back(pkgTagSection::Tag::Remove("Multi-Arch"));
+   RW.push_back(pkgTagSection::Tag::Remove("Architecture"));
+   RW.push_back(pkgTagSection::Tag::Remove("Conffiles"));
+   // we use the translated description
+   RW.push_back(pkgTagSection::Tag::Remove("Description"));
+   RW.push_back(pkgTagSection::Tag::Remove("Description-md5"));
+   // improve
+   RW.push_back(pkgTagSection::Tag::Rewrite("Installed-Size", installed_size));
+   RW.push_back(pkgTagSection::Tag::Remove("Size"));
+   RW.push_back(pkgTagSection::Tag::Rewrite("Download-Size", package_size));
+   // add
+   RW.push_back(pkgTagSection::Tag::Rewrite("APT-Manual-Installed", manual_installed));
+   RW.push_back(pkgTagSection::Tag::Rewrite("APT-Sources", source_index_file));
+
+   FileFd stdoutfd;
+   if (stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false) == false ||
+	 Tags.Write(stdoutfd, TFRewritePackageOrder, RW) == false || stdoutfd.Close() == false)
       return _error->Error("Internal Error, Unable to parse a package record");
 
    // write the description
diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index e2cf7e8b7..690b03bcc 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -580,6 +580,12 @@ static bool DumpAvail(CommandLine &)
    
    LocalitySort(VFList,Count,sizeof(*VFList));
 
+   std::vector RW;
+   RW.push_back(pkgTagSection::Tag::Remove("Status"));
+   RW.push_back(pkgTagSection::Tag::Remove("Config-Version"));
+   FileFd stdoutfd;
+   stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
+
    // Iterate over all the package files and write them out.
    char *Buffer = new char[Cache->HeaderP->MaxVerFileSize+10];
    for (pkgCache::VerFile **J = VFList; *J != 0;)
@@ -620,35 +626,32 @@ static bool DumpAvail(CommandLine &)
 	 if (PkgF.Read(Buffer,VF.Size + Jitter) == false)
 	    break;
 	 Buffer[VF.Size + Jitter] = '\n';
-	 
+
 	 // See above..
 	 if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
 	 {
 	    pkgTagSection Tags;
-	    TFRewriteData RW[] = {{"Status", NULL, NULL},{"Config-Version", NULL, NULL},{NULL, NULL, NULL}};
-	    const char *Zero = 0;
 	    if (Tags.Scan(Buffer+Jitter,VF.Size+1) == false ||
-		TFRewrite(stdout,Tags,&Zero,RW) == false)
+		Tags.Write(stdoutfd, NULL, RW) == false ||
+		stdoutfd.Write("\n", 1) == false)
 	    {
 	       _error->Error("Internal Error, Unable to parse a package record");
 	       break;
 	    }
-	    fputc('\n',stdout);
 	 }
 	 else
 	 {
-	    if (fwrite(Buffer+Jitter,VF.Size+1,1,stdout) != 1)
+	    if (stdoutfd.Write(Buffer + Jitter, VF.Size + 1) == false)
 	       break;
 	 }
-	 
+
 	 Pos = VF.Offset + VF.Size;
       }
 
-      fflush(stdout);
       if (_error->PendingError() == true)
          break;
    }
-   
+
    delete [] Buffer;
    delete [] VFList;
    return !_error->PendingError();
diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc
index 971900e4f..12ef8dda0 100644
--- a/cmdline/apt-sortpkgs.cc
+++ b/cmdline/apt-sortpkgs.cc
@@ -108,8 +108,10 @@ static bool DoIt(string InFile)
    const char **Order = TFRewritePackageOrder;
    if (Source == true)
       Order = TFRewriteSourceOrder;
-   
+
    // Emit
+   FileFd stdoutfd;
+   stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
    unsigned char *Buffer = new unsigned char[Largest+1];
    for (vector::iterator I = List.begin(); I != List.end(); ++I)
    {
@@ -119,8 +121,8 @@ static bool DoIt(string InFile)
 	 delete [] Buffer;
 	 return false;
       }
-      
-      Buffer[I->Length] = '\n';      
+
+      Buffer[I->Length] = '\n';
       if (Section.Scan((char *)Buffer,I->Length+1) == false)
       {
 	 delete [] Buffer;
@@ -128,15 +130,13 @@ static bool DoIt(string InFile)
       }
 
       // Sort the section
-      if (TFRewrite(stdout,Section,Order,0) == false)
+      if (Section.Write(stdoutfd, Order) == false || stdoutfd.Write("\n", 1) == false)
       {
 	 delete [] Buffer;
 	 return _error->Error("Internal error, failed to sort fields");
       }
-      
-      fputc('\n',stdout);      
    }
-   
+
    delete [] Buffer;
    return true;
 }
diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index 69b936dff..62108f7ca 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -178,7 +178,9 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    PkgDone = true;
    
    // Create a package writer object.
-   PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
+   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
+		      PkgCompress,Permissions);
+   PackagesWriter Packages(&Comp.Input, flCombine(CacheDir,BinCacheDB),
 			   flCombine(OverrideDir,BinOverride),
 			   flCombine(OverrideDir,ExtraOverride),
 			   Arch);
@@ -197,10 +199,6 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
    Packages.DeLinkLimit = DeLinkLimit;
 
-   // Create a compressor object
-   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
-		      PkgCompress,Permissions);
-   Packages.Output = Comp.Input;
    if (_error->PendingError() == true)
       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
    
@@ -272,7 +270,9 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    SrcDone = true;
    
    // Create a package writer object.
-   SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
+   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
+		      SrcCompress,Permissions);
+   SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 			 flCombine(OverrideDir,BinOverride),
 			 flCombine(OverrideDir,SrcOverride),
 			 flCombine(OverrideDir,SrcExtraOverride));
@@ -287,11 +287,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 
    Sources.DeLinkLimit = DeLinkLimit;
    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
-   
-   // Create a compressor object
-   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
-		      SrcCompress,Permissions);
-   Sources.Output = Comp.Input;
+
    if (_error->PendingError() == true)
       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 
@@ -365,16 +361,15 @@ bool PackageMap::GenContents(Configuration &Setup,
    gettimeofday(&StartTime,0);   
    
    // Create a package writer object.
-   ContentsWriter Contents("", Arch);
+   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
+		      CntCompress,Permissions);
+   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
+   ContentsWriter Contents(&Comp.Input, "", Arch);
    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
       return false;
 
-   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
-		      CntCompress,Permissions);
-   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
-   Contents.Output = Comp.Input;
    if (_error->PendingError() == true)
       return false;
 
@@ -384,7 +379,7 @@ bool PackageMap::GenContents(Configuration &Setup,
       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
       if (_error->PendingError() == true)
 	 return false;
-      
+
       unsigned long long Size = Head.Size();
       unsigned char Buf[4096];
       while (Size != 0)
@@ -392,17 +387,17 @@ bool PackageMap::GenContents(Configuration &Setup,
 	 unsigned long long ToRead = Size;
 	 if (Size > sizeof(Buf))
 	    ToRead = sizeof(Buf);
-	 
+
 	 if (Head.Read(Buf,ToRead) == false)
 	    return false;
-	 
-	 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
+
+	 if (Comp.Input.Write(Buf, ToRead) == false)
 	    return _error->Errno("fwrite",_("Error writing header to contents file"));
-	 
+
 	 Size -= ToRead;
-      }            
-   }  
-      
+      }
+   }
+
    /* Go over all the package file records and parse all the package
       files associated with this contents file into one great big honking
       memory structure, then dump the sorted version */
@@ -676,7 +671,7 @@ static bool SimpleGenPackages(CommandLine &CmdL)
       Override = CmdL.FileList[2];
    
    // Create a package writer object.
-   PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
+   PackagesWriter Packages(NULL, _config->Find("APT::FTPArchive::DB"),
 			   Override, "", _config->Find("APT::FTPArchive::Architecture"));
    if (_error->PendingError() == true)
       return false;
@@ -704,7 +699,7 @@ static bool SimpleGenContents(CommandLine &CmdL)
       return ShowHelp(CmdL);
    
    // Create a package writer object.
-   ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
+   ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
    if (_error->PendingError() == true)
       return false;
    
@@ -737,7 +732,7 @@ static bool SimpleGenSources(CommandLine &CmdL)
 			     SOverride.c_str());
        
    // Create a package writer object.
-   SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
+   SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
    if (_error->PendingError() == true)
       return false;
    
@@ -764,7 +759,7 @@ static bool SimpleGenRelease(CommandLine &CmdL)
 
    string Dir = CmdL.FileList[1];
 
-   ReleaseWriter Release("");
+   ReleaseWriter Release(NULL, "");
    Release.DirStrip = Dir;
 
    if (_error->PendingError() == true)
diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc
index 8c4181eda..145f3910e 100644
--- a/ftparchive/contents.cc
+++ b/ftparchive/contents.cc
@@ -38,6 +38,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -238,19 +239,19 @@ void GenContents::Add(const char *Dir,const char *Package)
 // GenContents::WriteSpace - Write a given number of white space chars	/*{{{*/
 // ---------------------------------------------------------------------
 /* We mod 8 it and write tabs where possible. */
-void GenContents::WriteSpace(FILE *Out,unsigned int Current,unsigned int Target)
+void GenContents::WriteSpace(std::string &out, size_t Current, size_t Target)
 {
    if (Target <= Current)
       Target = Current + 1;
-   
+
    /* Now we write tabs so long as the next tab stop would not pass
       the target */
    for (; (Current/8 + 1)*8 < Target; Current = (Current/8 + 1)*8)
-      fputc('\t',Out);
+      out.append("\t");
 
    // Fill the last bit with spaces
    for (; Current < Target; Current++)
-      fputc(' ',Out);
+      out.append(" ");
 }
 									/*}}}*/
 // GenContents::Print - Display the tree				/*{{{*/
@@ -259,13 +260,13 @@ void GenContents::WriteSpace(FILE *Out,unsigned int Current,unsigned int Target)
    calls itself and runs over each section of the tree printing out
    the pathname and the hit packages. We use Buf to build the pathname
    summed over all the directory parents of this node. */
-void GenContents::Print(FILE *Out)
+void GenContents::Print(FileFd &Out)
 {
    char Buffer[1024];
    Buffer[0] = 0;
    DoPrint(Out,&Root,Buffer);
 }
-void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf)
+void GenContents::DoPrint(FileFd &Out,GenContents::Node *Top, char *Buf)
 {
    if (Top == 0)
       return;
@@ -278,26 +279,27 @@ void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf)
    if (Top->Path != 0)
    {
       strcat(Buf,Top->Path);
-      
+
       // Do not show the item if it is a directory with dups
       if (Top->Path[strlen(Top->Path)-1] != '/' /*|| Top->Dups == 0*/)
       {
-	 fputs(Buf,Out);
-	 WriteSpace(Out,strlen(Buf),60);
+	 std::string out = Buf;
+	 WriteSpace(out, out.length(), 60);
 	 for (Node *I = Top; I != 0; I = I->Dups)
 	 {
 	    if (I != Top)
-	       fputc(',',Out);
-	    fputs(I->Package,Out);
+	       out.append(",");
+	    out.append(I->Package);
 	 }
-         fputc('\n',Out);
-      }      
-   }   
-   
+         out.append("\n");
+	 Out.Write(out.c_str(), out.length());
+      }
+   }
+
    // Go along the directory link
    DoPrint(Out,Top->DirDown,Buf);
    *OldEnd = 0;
-   
+
    // Go right
    DoPrint(Out,Top->BTreeRight,Buf);  
 }
diff --git a/ftparchive/contents.h b/ftparchive/contents.h
index f58e3278e..953d0d54b 100644
--- a/ftparchive/contents.h
+++ b/ftparchive/contents.h
@@ -17,6 +17,7 @@
 #include 
 
 class debDebFile;
+class FileFd;
 
 class GenContents
 {
@@ -54,14 +55,14 @@ class GenContents
    unsigned long NodeLeft;
    
    Node *Grab(Node *Top,const char *Name,const char *Package);
-   void WriteSpace(FILE *Out,unsigned int Current,unsigned int Target);
-   void DoPrint(FILE *Out,Node *Top, char *Buf);
+   void WriteSpace(std::string &out, size_t Current, size_t Target);
+   void DoPrint(FileFd &Out,Node *Top, char *Buf);
    
    public:
    
    char *Mystrdup(const char *From);
    void Add(const char *Dir,const char *Package);   
-   void Print(FILE *Out);
+   void Print(FileFd &Out);
 
    GenContents() : BlockList(0), StrPool(0), StrLeft(0), 
                    NodePool(0), NodeLeft(0) {};
diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc
index f35d5304a..08a3cff5a 100644
--- a/ftparchive/multicompress.cc
+++ b/ftparchive/multicompress.cc
@@ -46,7 +46,6 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress,
 {
    Outputs = 0;
    Outputter = -1;
-   Input = 0;
    UpdateMTime = 0;
 
    /* Parse the compression string, a space separated lists of compresison
@@ -187,12 +186,11 @@ bool MultiCompress::Start()
    };
 
    close(Pipe[0]);
-   Input = fdopen(Pipe[1],"w");
-   if (Input == 0)
-      return _error->Errno("fdopen",_("Failed to create FILE*"));
-   
+   if (Input.OpenDescriptor(Pipe[1], FileFd::WriteOnly, true) == false)
+      return false;
+
    if (Outputter == -1)
-      return _error->Errno("fork",_("Failed to fork"));   
+      return _error->Errno("fork",_("Failed to fork"));
    return true;
 }
 									/*}}}*/
@@ -201,11 +199,10 @@ bool MultiCompress::Start()
 /* */
 bool MultiCompress::Die()
 {
-   if (Input == 0)
+   if (Input.IsOpen() == false)
       return true;
-   
-   fclose(Input);
-   Input = 0;
+
+   Input.Close();
    bool Res = ExecWait(Outputter,_("Compress child"),false);
    Outputter = -1;
    return Res;
@@ -217,7 +214,7 @@ bool MultiCompress::Die()
 bool MultiCompress::Finalize(unsigned long long &OutSize)
 {
    OutSize = 0;
-   if (Input == 0 || Die() == false)
+   if (Input.IsOpen() == false || Die() == false)
       return false;
    
    time_t Now;
diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h
index ddd1815a3..161716b86 100644
--- a/ftparchive/multicompress.h
+++ b/ftparchive/multicompress.h
@@ -48,7 +48,7 @@ class MultiCompress
    public:
    
    // The FD to write to for compression.
-   FILE *Input;
+   FileFd Input;
    unsigned long UpdateMTime;
    
    bool Finalize(unsigned long long &OutSize);
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index 855e0ef79..7cf7e6efc 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -51,20 +51,6 @@
 using namespace std;
 FTWScanner *FTWScanner::Owner;
 
-// SetTFRewriteData - Helper for setting rewrite lists			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-static inline TFRewriteData SetTFRewriteData(const char *tag,
-			     const char *rewrite,
-			     const char *newtag = 0)
-{
-   TFRewriteData tfrd;
-   tfrd.Tag = tag;
-   tfrd.Rewrite = rewrite;
-   tfrd.NewTag = newtag;
-   return tfrd;
-}
-									/*}}}*/
 // ConfigToDoHashes - which hashes to generate				/*{{{*/
 static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
 {
@@ -85,8 +71,15 @@ static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
 // FTWScanner::FTWScanner - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-FTWScanner::FTWScanner(string const &Arch): Arch(Arch), DoHashes(~0)
+FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch): Arch(Arch), DoHashes(~0)
 {
+   if (GivenOutput == NULL)
+   {
+      Output = new FileFd;
+      Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
+   }
+   else
+      Output = GivenOutput;
    ErrorPrinted = false;
    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
    ConfigToDoHashes(DoHashes, "APT::FTPArchive");
@@ -331,11 +324,10 @@ bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 // PackagesWriter::PackagesWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
+PackagesWriter::PackagesWriter(FileFd * const GivenOutput, string const &DB,string const &Overrides,string const &ExtOverrides,
 			       string const &Arch) :
-   FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
+   FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 {
-   Output = stdout;
    SetExts(".deb .udeb");
    DeLinkLimit = 0;
 
@@ -461,32 +453,32 @@ bool PackagesWriter::DoPackage(string FileName)
    }
 
    // This lists all the changes to the fields we are going to make.
-   std::vector Changes;
+   std::vector Changes;
 
    std::string Size;
    strprintf(Size, "%llu", (unsigned long long) FileSize);
-   Changes.push_back(SetTFRewriteData("Size", Size.c_str()));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Size", Size));
 
    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
    {
       if (hs->HashType() == "MD5Sum")
-	 Changes.push_back(SetTFRewriteData("MD5sum", hs->HashValue().c_str()));
+	 Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue()));
       else if (hs->HashType() == "Checksum-FileSize")
 	 continue;
       else
-	 Changes.push_back(SetTFRewriteData(hs->HashType().c_str(), hs->HashValue().c_str()));
+	 Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue()));
    }
-   Changes.push_back(SetTFRewriteData("Filename", NewFileName.c_str()));
-   Changes.push_back(SetTFRewriteData("Priority", OverItem->Priority.c_str()));
-   Changes.push_back(SetTFRewriteData("Status", 0));
-   Changes.push_back(SetTFRewriteData("Optional", 0));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority));
+   Changes.push_back(pkgTagSection::Tag::Remove("Status"));
+   Changes.push_back(pkgTagSection::Tag::Remove("Optional"));
 
    string DescriptionMd5;
    if (LongDescription == false) {
       MD5Summation descmd5;
       descmd5.Add(desc.c_str());
       DescriptionMd5 = descmd5.Result().Value();
-      Changes.push_back(SetTFRewriteData("Description-md5", DescriptionMd5.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5));
       if (TransWriter != NULL)
 	 TransWriter->DoPackage(Package, desc, DescriptionMd5);
    }
@@ -505,7 +497,7 @@ bool PackagesWriter::DoPackage(string FileName)
    }
 
    if (NewMaint.empty() == false)
-      Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint));
 
    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
@@ -517,19 +509,17 @@ bool PackagesWriter::DoPackage(string FileName)
    {
       if (Tags.FindS("Suggests").empty() == false)
 	 OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
-      Changes.push_back(SetTFRewriteData("Suggests", OptionalStr.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Suggests", OptionalStr));
    }
 
    for (map::const_iterator I = OverItem->FieldOverride.begin();
         I != OverItem->FieldOverride.end(); ++I)
-      Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
-
-   Changes.push_back(SetTFRewriteData( 0, 0));
+      Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 
    // Rewrite and store the fields.
-   if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes.data()) == false)
+   if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false ||
+	 Output->Write("\n", 1) == false)
       return false;
-   fprintf(Output,"\n");
 
    return Db.Finish();
 }
@@ -539,14 +529,13 @@ bool PackagesWriter::DoPackage(string FileName)
 // ---------------------------------------------------------------------
 /* Create a Translation-Master file for this Packages file */
 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
-					mode_t const &Permissions) : Output(NULL),
-							RefCounter(0)
+					mode_t const &Permissions) : RefCounter(0)
 {
    if (File.empty() == true)
       return;
 
    Comp = new MultiCompress(File, TransCompress, Permissions);
-   Output = Comp->Input;
+   Output = &Comp->Input;
 }
 									/*}}}*/
 // TranslationWriter::DoPackage - Process a single package		/*{{{*/
@@ -565,8 +554,10 @@ bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
    if (Included.find(Record) != Included.end())
       return true;
 
-   fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
+   std::string out;
+   strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 	   Pkg.c_str(), MD5.c_str(), Desc.c_str());
+   Output->Write(out.c_str(), out.length());
 
    Included.insert(Record);
    return true;
@@ -587,11 +578,10 @@ TranslationWriter::~TranslationWriter()
 // SourcesWriter::SourcesWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
+SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides,
 			     string const &ExtOverrides) :
-   Db(DB), Stats(Db.Stats)
+   FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats)
 {
-   Output = stdout;
    AddPattern("*.dsc");
    DeLinkLimit = 0;
    Buffer = 0;
@@ -620,15 +610,16 @@ SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string c
 }
 									/*}}}*/
 // SourcesWriter::DoPackage - Process a single package			/*{{{*/
-static std::ostream& addDscHash(std::ostream &out, unsigned int const DoHashes,
+static std::string getDscHash(unsigned int const DoHashes,
       Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, char const * const FieldName,
       HashString const * const Hash, unsigned long long Size, std::string FileName)
 {
    if ((DoHashes & DoIt) != DoIt || Tags.Exists(FieldName) == false || Hash == NULL)
-      return out;
+      return "";
+   std::ostringstream out;
    out << "\n " << Hash->HashValue() << " " << Size << " " << FileName
       << "\n " << Tags.FindS(FieldName);
-   return out;
+   return out.str();
 }
 bool SourcesWriter::DoPackage(string FileName)
 {
@@ -721,16 +712,10 @@ bool SourcesWriter::DoPackage(string FileName)
 
    // Add the dsc to the files hash list
    string const strippedName = flNotDir(FileName);
-   std::ostringstream ostreamFiles;
-   addDscHash(ostreamFiles, DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
-   string const Files = ostreamFiles.str();
-
-   std::ostringstream ostreamSha1;
-   addDscHash(ostreamSha1, DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
-   std::ostringstream ostreamSha256;
-   addDscHash(ostreamSha256, DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
-   std::ostringstream ostreamSha512;
-   addDscHash(ostreamSha512, DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
+   std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
+   std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
+   std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
+   std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
 
    // Strip the DirStrip prefix from the FileName and add the PathPrefix
    string NewFileName;
@@ -752,7 +737,7 @@ bool SourcesWriter::DoPackage(string FileName)
    char *RealPath = NULL;
    for (;isspace(*C); C++);
    while (*C != 0)
-   {   
+   {
       // Parse each of the elements
       if (ParseQuoteWord(C,ParseJnk) == false ||
 	  ParseQuoteWord(C,ParseJnk) == false ||
@@ -782,21 +767,21 @@ bool SourcesWriter::DoPackage(string FileName)
 	    if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
 	       continue;
 	    char const * fieldname;
-	    std::ostream * out;
+	    std::string * out;
 	    if (hs->HashType() == "SHA1")
 	    {
 	       fieldname = "Checksums-Sha1";
-	       out = &ostreamSha1;
+	       out = &ChecksumsSha1;
 	    }
 	    else if (hs->HashType() == "SHA256")
 	    {
 	       fieldname = "Checksums-Sha256";
-	       out = &ostreamSha256;
+	       out = &ChecksumsSha256;
 	    }
 	    else if (hs->HashType() == "SHA512")
 	    {
 	       fieldname = "Checksums-Sha512";
-	       out = &ostreamSha512;
+	       out = &ChecksumsSha512;
 	    }
 	    else
 	    {
@@ -805,10 +790,12 @@ bool SourcesWriter::DoPackage(string FileName)
 	    }
 	    if (Tags.Exists(fieldname) == true)
 	       continue;
-	    (*out) << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
+	    std::ostringstream streamout;
+	    streamout << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
+	    out->append(streamout.str());
 	 }
 
-	 // write back the GetFileInfo() stats data 
+	 // write back the GetFileInfo() stats data
 	 Db.Finish();
       }
 
@@ -829,54 +816,48 @@ bool SourcesWriter::DoPackage(string FileName)
    if (Directory.length() > 2)
       Directory.erase(Directory.end()-1);
 
-   string const ChecksumsSha1 = ostreamSha1.str();
-   string const ChecksumsSha256 = ostreamSha256.str();
-   string const ChecksumsSha512 = ostreamSha512.str();
-
    // This lists all the changes to the fields we are going to make.
    // (5 hardcoded + checksums + maintainer + end marker)
-   std::vector Changes;
+   std::vector Changes;
 
-   Changes.push_back(SetTFRewriteData("Source", 0));
-   Changes.push_back(SetTFRewriteData("Package",Package.c_str()));
+   Changes.push_back(pkgTagSection::Tag::Remove("Source"));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package));
    if (Files.empty() == false)
-      Changes.push_back(SetTFRewriteData("Files",Files.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files));
    if (ChecksumsSha1.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha1",ChecksumsSha1.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1));
    if (ChecksumsSha256.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha256",ChecksumsSha256.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256));
    if (ChecksumsSha512.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha512",ChecksumsSha512.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512));
    if (Directory != "./")
-      Changes.push_back(SetTFRewriteData("Directory",Directory.c_str()));
-   Changes.push_back(SetTFRewriteData("Priority",BestPrio.c_str()));
-   Changes.push_back(SetTFRewriteData("Status",0));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio));
+   Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 
    // Rewrite the maintainer field if necessary
    bool MaintFailed;
-   string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
+   string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"), MaintFailed);
    if (MaintFailed == true)
    {
       if (NoOverride == false)
       {
-	 NewLine(1);	 
+	 NewLine(1);
 	 ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 	       Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
-      }      
+      }
    }
    if (NewMaint.empty() == false)
-      Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
-   
-   for (map::const_iterator I = SOverItem->FieldOverride.begin(); 
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str()));
+
+   for (map::const_iterator I = SOverItem->FieldOverride.begin();
         I != SOverItem->FieldOverride.end(); ++I)
-      Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 
-   Changes.push_back(SetTFRewriteData(0, 0));
-      
    // Rewrite and store the fields.
-   if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes.data()) == false)
+   if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false ||
+	 Output->Write("\n", 1) == false)
       return false;
-   fprintf(Output,"\n");
 
    Stats.Packages++;
    
@@ -887,12 +868,11 @@ bool SourcesWriter::DoPackage(string FileName)
 // ContentsWriter::ContentsWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
-		    FTWScanner(Arch), Db(DB), Stats(Db.Stats)
+ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB, string const &Arch) :
+		    FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats)
 
 {
    SetExts(".deb");
-   Output = stdout;
 }
 									/*}}}*/
 // ContentsWriter::DoPackage - Process a single package			/*{{{*/
@@ -974,7 +954,7 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres
 // ReleaseWriter::ReleaseWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-ReleaseWriter::ReleaseWriter(string const &/*DB*/)
+ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput)
 {
    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
    {
@@ -996,7 +976,6 @@ ReleaseWriter::ReleaseWriter(string const &/*DB*/)
    }
    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 
-   Output = stdout;
    time_t const now = time(NULL);
 
    setlocale(LC_TIME, "C");
@@ -1040,7 +1019,8 @@ ReleaseWriter::ReleaseWriter(string const &/*DB*/)
       if (Value == "")
          continue;
 
-      fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
+      std::string const out = I->first + ": " + Value + "\n";
+      Output->Write(out.c_str(), out.length());
    }
 
    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
@@ -1087,29 +1067,35 @@ bool ReleaseWriter::DoPackage(string FileName)
 									/*}}}*/
 // ReleaseWriter::Finish - Output the checksums				/*{{{*/
 // ---------------------------------------------------------------------
-static void printChecksumTypeRecord(FILE * const Output, char const * const Type, map const &CheckSums)
+static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map const &CheckSums)
 {
-      fprintf(Output, "%s:\n", Type);
-      for(map::const_iterator I = CheckSums.begin();
-	  I != CheckSums.end(); ++I)
-      {
-	 HashString const * const hs = I->second.Hashes.find(Type);
-	 if (hs == NULL)
-	    continue;
-	 fprintf(Output, " %s %16llu %s\n",
-		 hs->HashValue().c_str(),
-		 (*I).second.size,
-		 (*I).first.c_str());
-      }
+   {
+      std::string out;
+      strprintf(out, "%s:\n", Type);
+      Output.Write(out.c_str(), out.length());
+   }
+   for(map::const_iterator I = CheckSums.begin();
+	 I != CheckSums.end(); ++I)
+   {
+      HashString const * const hs = I->second.Hashes.find(Type);
+      if (hs == NULL)
+	 continue;
+      std::string out;
+      strprintf(out, " %s %16llu %s\n",
+	    hs->HashValue().c_str(),
+	    (*I).second.size,
+	    (*I).first.c_str());
+      Output.Write(out.c_str(), out.length());
+   }
 }
 void ReleaseWriter::Finish()
 {
    if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
-      printChecksumTypeRecord(Output, "MD5Sum", CheckSums);
+      printChecksumTypeRecord(*Output, "MD5Sum", CheckSums);
    if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
-      printChecksumTypeRecord(Output, "SHA1", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA1", CheckSums);
    if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
-      printChecksumTypeRecord(Output, "SHA256", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA256", CheckSums);
    if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
-      printChecksumTypeRecord(Output, "SHA512", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA512", CheckSums);
 }
diff --git a/ftparchive/writer.h b/ftparchive/writer.h
index 226996475..0ba60db5e 100644
--- a/ftparchive/writer.h
+++ b/ftparchive/writer.h
@@ -42,10 +42,10 @@ class FTWScanner
    string Arch;
    const char *OriginalPath;
    bool ErrorPrinted;
-   
+
    // Stuff for the delinker
    bool NoLinkAct;
-   
+
    static FTWScanner *Owner;
    static int ScannerFTW(const char *File,const struct stat *sb,int Flag);
    static int ScannerFile(const char *File, bool const &ReadLink);
@@ -59,10 +59,11 @@ class FTWScanner
       {
 	 c1out << endl;
 	 ErrorPrinted = true;
-      }	 
+      }
    }
-   
+
    public:
+   FileFd *Output;
    unsigned int DoHashes;
 
    unsigned long DeLinkLimit;
@@ -76,8 +77,8 @@ class FTWScanner
    void AddPattern(char const *Pattern) { Patterns.push_back(Pattern); };
    void AddPatterns(std::vector const &patterns) { Patterns.insert(Patterns.end(), patterns.begin(), patterns.end()); };
    bool SetExts(string const &Vals);
-      
-   FTWScanner(string const &Arch = string());
+
+   FTWScanner(FileFd * const Output, string const &Arch = string());
    virtual ~FTWScanner() {};
 };
 
@@ -86,9 +87,9 @@ class MultiCompress;
 class TranslationWriter
 {
    MultiCompress *Comp;
-   FILE *Output;
    std::set Included;
    unsigned short RefCounter;
+   FileFd *Output;
 
    public:
    void IncreaseRefCounter() { ++RefCounter; };
@@ -97,7 +98,7 @@ class TranslationWriter
    bool DoPackage(string const &Pkg, string const &Desc, string const &MD5);
 
    TranslationWriter(string const &File, string const &TransCompress, mode_t const &Permissions);
-   TranslationWriter() : Comp(NULL), Output(NULL), RefCounter(0) {};
+   TranslationWriter() : Comp(NULL), RefCounter(0) {};
    ~TranslationWriter();
 };
 
@@ -105,7 +106,7 @@ class PackagesWriter : public FTWScanner
 {
    Override Over;
    CacheDB Db;
-      
+
    public:
 
    // Some flags
@@ -117,7 +118,6 @@ class PackagesWriter : public FTWScanner
    // General options
    string PathPrefix;
    string DirStrip;
-   FILE *Output;
    struct CacheDB::Stats &Stats;
    TranslationWriter *TransWriter;
 
@@ -126,7 +126,7 @@ class PackagesWriter : public FTWScanner
       {return Over.ReadExtraOverride(File);};
    virtual bool DoPackage(string FileName);
 
-   PackagesWriter(string const &DB,
+   PackagesWriter(FileFd * const Output, string const &DB,
                   string const &Overrides,
                   string const &ExtOverrides = "",
 		  string const &Arch = "");
@@ -136,25 +136,24 @@ class PackagesWriter : public FTWScanner
 class ContentsWriter : public FTWScanner
 {
    CacheDB Db;
-   
+
    GenContents Gen;
-   
+
    public:
 
    // General options
-   FILE *Output;
    struct CacheDB::Stats &Stats;
    string Prefix;
-   
+
    bool DoPackage(string FileName,string Package);
    virtual bool DoPackage(string FileName) 
              {return DoPackage(FileName,string());};
    bool ReadFromPkgs(string const &PkgFile,string const &PkgCompress);
 
-   void Finish() {Gen.Print(Output);};
+   void Finish() {Gen.Print(*Output);};
    inline bool ReadyDB(string const &DB) {return Db.ReadyDB(DB);};
-   
-   ContentsWriter(string const &DB, string const &Arch = string());
+
+   ContentsWriter(FileFd * const Output, string const &DB, string const &Arch = string());
    virtual ~ContentsWriter() {};
 };
 
@@ -165,21 +164,20 @@ class SourcesWriter : public FTWScanner
    Override SOver;
    char *Buffer;
    unsigned long long BufSize;
-   
+
    public:
 
    bool NoOverride;
    bool DoAlwaysStat;
-   
+
    // General options
    string PathPrefix;
    string DirStrip;
-   FILE *Output;
    struct CacheDB::Stats &Stats;
 
    virtual bool DoPackage(string FileName);
 
-   SourcesWriter(string const &DB,string const &BOverrides,string const &SOverrides,
+   SourcesWriter(FileFd * const Output, string const &DB,string const &BOverrides,string const &SOverrides,
 		 string const &ExtOverrides=string());
    virtual ~SourcesWriter() {free(Buffer);};
 };
@@ -187,11 +185,10 @@ class SourcesWriter : public FTWScanner
 class ReleaseWriter : public FTWScanner
 {
 public:
-   ReleaseWriter(string const &DB);
+   ReleaseWriter(FileFd * const Output, string const &DB);
    virtual bool DoPackage(string FileName);
    void Finish();
 
-   FILE *Output;
    // General options
    string PathPrefix;
    string DirStrip;
diff --git a/test/integration/test-apt-cli-show b/test/integration/test-apt-cli-show
index 5604620fd..43072cf03 100755
--- a/test/integration/test-apt-cli-show
+++ b/test/integration/test-apt-cli-show
@@ -25,11 +25,11 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 # note that we do not display Description-md5 with the "apt" cmd
 # and also show some additional fields that are calculated
 testsuccessequal "Package: foo
+Version: 1.0
 Priority: optional
 Section: other
-Installed-Size: 43.0 kB
 Maintainer: Joe Sixpack 
-Version: 1.0
+Installed-Size: 43.0 kB
 Download-Size: unknown
 APT-Manual-Installed: yes
 APT-Sources: file:$APTARCHIVE/ unstable/main i386 Packages
diff --git a/test/libapt/indexcopytosourcelist_test.cc b/test/libapt/indexcopytosourcelist_test.cc
index bec87601f..1b0427564 100644
--- a/test/libapt/indexcopytosourcelist_test.cc
+++ b/test/libapt/indexcopytosourcelist_test.cc
@@ -16,7 +16,7 @@ class NoCopy : public IndexCopy {
 	 return Path;
       }
       bool GetFile(std::string &/*Filename*/, unsigned long long &/*Size*/) { return false; }
-      bool RewriteEntry(FILE * /*Target*/, std::string /*File*/) { return false; }
+      bool RewriteEntry(FileFd & /*Target*/, std::string const &/*File*/) { return false; }
       const char *GetFileName() { return NULL; }
       const char *Type() { return NULL; }
 
diff --git a/test/libapt/tagsection_test.cc b/test/libapt/tagsection_test.cc
new file mode 100644
index 000000000..f250177af
--- /dev/null
+++ b/test/libapt/tagsection_test.cc
@@ -0,0 +1,270 @@
+#include 
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+
+#include "file-helpers.h"
+
+std::string packageValue = "aaaa";
+std::string typoValue = "aa\n"
+   " .\n"
+   " cc";
+std::string typoRawValue = "\n " + typoValue;
+std::string overrideValue = "1";
+/*
+   std::cerr << "FILECONTENT: »";
+   char buffer[3000];
+   while (fd.ReadLine(buffer, sizeof(buffer)))
+      std::cerr << buffer;
+   std::cerr << "«" << std::endl;;
+*/
+
+void setupTestcaseStart(FileFd &fd, pkgTagSection §ion, std::string &content)
+{
+   createTemporaryFile("writesection", fd, NULL, NULL);
+   content = "Package: " + packageValue + "\n"
+      "TypoA:\n " + typoValue + "\n"
+      "Override: " + overrideValue + "\n"
+      "Override-Backup: " + overrideValue + "\n"
+      "\n";
+   EXPECT_TRUE(section.Scan(content.c_str(), content.length(), true));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(typoRawValue, section.FindRawS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodified)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   EXPECT_TRUE(section.Write(fd));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrder)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrderReversed)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Override", "TypoA", "Package", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrderNotAll)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRename)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rename("TypoA", "TypoB"));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoB"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRemove)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Remove("TypoA"));
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", ""));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_FALSE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(2, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRewrite)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", "42"));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(42, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRename)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rename("TypoA", "TypoB"));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoB"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRemove)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Remove("TypoA"));
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", ""));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_FALSE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(2, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRewrite)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", "42"));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(42, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
-- 
cgit v1.2.3-70-g09d2


From dcbb364fc69e1108b3fea3adb12a7ba83d9af467 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 12 May 2015 00:30:16 +0200
Subject: detect 416 complete file in partial by expected hash

If we have the expected hashes we can check with them if the file we
have in partial we got a 416 for is the expected file. We detected this
with same-size before, but not every server sends a good Content-Range
header with a 416 response.
---
 methods/https.cc                              | 37 +++++++++++++++++++++------
 methods/https.h                               |  1 +
 methods/server.cc                             | 19 +++++++++++---
 test/integration/framework                    |  2 +-
 test/integration/test-apt-update-transactions |  1 +
 test/integration/test-partial-file-support    | 10 ++++++--
 test/interactive-helper/aptwebserver.cc       |  9 ++++---
 7 files changed, 62 insertions(+), 17 deletions(-)

(limited to 'test')

diff --git a/methods/https.cc b/methods/https.cc
index c6b75d9ad..712e9ee73 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -40,7 +40,9 @@ using namespace std;
 struct APT_HIDDEN CURLUserPointer {
    HttpsMethod * const https;
    HttpsMethod::FetchResult * const Res;
-   CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res) : https(https), Res(Res) {}
+   HttpsMethod::FetchItem const * const Itm;
+   CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res,
+	 HttpsMethod::FetchItem const * const Itm) : https(https), Res(Res), Itm(Itm) {}
 };
 
 size_t
@@ -61,13 +63,32 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
    {
       if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0)
 	 ;
-      else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize())
+      else if (me->https->Server->Result == 416)
       {
-         me->https->Server->Result = 200;
-	 me->https->Server->StartPos = me->https->Server->Size;
-	 // the actual size is not important for https as curl will deal with it
-	 // by itself and e.g. doesn't bother us with transport-encoding…
-	 me->https->Server->JunkSize = std::numeric_limits::max();
+	 bool partialHit = false;
+	 if (me->Itm->ExpectedHashes.usable() == true)
+	 {
+	    Hashes resultHashes(me->Itm->ExpectedHashes);
+	    FileFd file(me->Itm->DestFile, FileFd::ReadOnly);
+	    me->https->Server->Size = file.FileSize();
+	    me->https->Server->Date = file.ModificationTime();
+	    resultHashes.AddFD(file);
+	    HashStringList const hashList = resultHashes.GetHashStringList();
+	    partialHit = (me->Itm->ExpectedHashes == hashList);
+	 }
+	 else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize())
+	    partialHit = true;
+
+	 if (partialHit == true)
+	 {
+	    me->https->Server->Result = 200;
+	    me->https->Server->StartPos = me->https->Server->Size;
+	    // the actual size is not important for https as curl will deal with it
+	    // by itself and e.g. doesn't bother us with transport-encoding…
+	    me->https->Server->JunkSize = std::numeric_limits::max();
+	 }
+	 else
+	    me->https->Server->StartPos = 0;
       }
       else
 	 me->https->Server->StartPos = 0;
@@ -221,7 +242,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 
    FetchResult Res;
-   CURLUserPointer userp(this, &Res);
+   CURLUserPointer userp(this, &Res, Itm);
    // callbacks
    curl_easy_setopt(curl, CURLOPT_URL, static_cast(Uri).c_str());
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
diff --git a/methods/https.h b/methods/https.h
index 6e32e8d3d..57fc292ee 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -79,6 +79,7 @@ class HttpsMethod : public ServerMethod
    virtual bool Configuration(std::string Message);
    virtual ServerState * CreateServerState(URI uri);
    using pkgAcqMethod::FetchResult;
+   using pkgAcqMethod::FetchItem;
 
    HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL)
    {
diff --git a/methods/server.cc b/methods/server.cc
index 2116926b0..bd01c3e98 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -269,7 +269,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
       Res.LastModified = Queue->LastModified;
       return IMS_HIT;
    }
-   
+
    /* Redirect
     *
     * Note that it is only OK for us to treat all redirection the same
@@ -314,7 +314,20 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
       struct stat SBuf;
       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
       {
-	 if ((unsigned long long)SBuf.st_size == Server->Size)
+	 bool partialHit = false;
+	 if (Queue->ExpectedHashes.usable() == true)
+	 {
+	    Hashes resultHashes(Queue->ExpectedHashes);
+	    FileFd file(Queue->DestFile, FileFd::ReadOnly);
+	    Server->Size = file.FileSize();
+	    Server->Date = file.ModificationTime();
+	    resultHashes.AddFD(file);
+	    HashStringList const hashList = resultHashes.GetHashStringList();
+	    partialHit = (Queue->ExpectedHashes == hashList);
+	 }
+	 else if ((unsigned long long)SBuf.st_size == Server->Size)
+	    partialHit = true;
+	 if (partialHit == true)
 	 {
 	    // the file is completely downloaded, but was not moved
 	    if (Server->HaveContent == true)
@@ -351,7 +364,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
    // This is some sort of 2xx 'data follows' reply
    Res.LastModified = Server->Date;
    Res.Size = Server->Size;
-   
+
    // Open the file
    delete File;
    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
diff --git a/test/integration/framework b/test/integration/framework
index 03c188189..2a53e8365 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1128,7 +1128,7 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 downloadfile() {
 	local PROTO="${1%%:*}"
 	if ! apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
-		download-file "$1" "$2" 2>&1 ; then
+		download-file "$1" "$2" "$3" 2>&1 ; then
 		return 1
 	fi
 	# only if the file exists the download was successful
diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions
index f028ac0c7..67dd633f9 100755
--- a/test/integration/test-apt-update-transactions
+++ b/test/integration/test-apt-update-transactions
@@ -63,6 +63,7 @@ testsetup 'file'
 changetowebserver
 webserverconfig 'aptwebserver::support::modified-since' 'false' "$1"
 webserverconfig 'aptwebserver::support::last-modified' 'false' "$1"  # curl is clever and sees hits here also
+webserverconfig 'aptwebserver::support::range' 'false' "$1"
 
 testsetup 'http'
 
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 85046b3eb..c07af7bd0 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -17,8 +17,8 @@ DOWNLOADLOG='rootdir/tmp/testdownloadfile.log'
 
 testdownloadfile() {
 	rm -f "$DOWNLOADLOG"
-	msgtest "Testing download of file $2 with" "$1"
-	if ! downloadfile "$2" "$3" > "$DOWNLOADLOG"; then
+	msgtest "Testing download of file $2 with" "$1 $5"
+	if ! downloadfile "$2" "$3" "$5" > "$DOWNLOADLOG"; then
 		cat >&2 "$DOWNLOADLOG"
 		msgfail
 	else
@@ -78,6 +78,12 @@ followuprequest() {
 	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
 	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
 
+	webserverconfig 'aptwebserver::support::content-range' 'false'
+	copysource $TESTFILE 1M $DOWN
+	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' "SHA1:$(sha1sum "$TESTFILE" | cut -d' ' -f 1)"
+	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+	webserverconfig 'aptwebserver::support::content-range' 'true'
+
 	copysource $TESTFILE 1M $DOWN
 	copysource "${TESTFILE}2" 20 "${DOWN}2"
 	msgtest 'Testing download of files with' 'completely downloaded file + partial file'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 6a411e24e..c933060e7 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -745,9 +745,12 @@ static void * handleClient(void * voidclient)				/*{{{*/
 		     }
 		     else
 		     {
-			std::ostringstream contentrange;
-			contentrange << "Content-Range: bytes */" << filesize;
-			headers.push_back(contentrange.str());
+			if (_config->FindB("aptwebserver::support::content-range", true) == true)
+			{
+			   std::ostringstream contentrange;
+			   contentrange << "Content-Range: bytes */" << filesize;
+			   headers.push_back(contentrange.str());
+			}
 			sendError(client, 416, *m, sendContent, "", headers);
 			break;
 		     }
-- 
cgit v1.2.3-70-g09d2


From 8eafc759544298211cd0bfaa3919afc0fadd47d1 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Wed, 13 May 2015 16:09:12 +0200
Subject: detect Releasefile IMS hits even if the server doesn't

Not all servers we are talking to support If-Modified-Since and some are
not even sending Last-Modified for us, so in an effort to detect such
hits we run a hashsum check on the 'old' compared to the 'new' file, we
got the hashes for the 'new' already for "free" from the methods anyway
and hence just need to calculated the old ones.

This allows us to detect hits even with unsupported servers, which in
turn means we benefit from all the new hit behavior also here.
---
 apt-pkg/acquire-item.cc                        | 31 ++++++++++----
 apt-pkg/acquire-item.h                         |  2 +-
 apt-pkg/acquire-worker.cc                      |  8 ++--
 methods/https.cc                               |  2 +-
 test/integration/framework                     |  9 ++++
 test/integration/test-apt-update-expected-size |  1 +
 test/integration/test-apt-update-not-modified  | 58 +++++++++++++++++++++++++-
 test/integration/test-apt-update-rollback      |  1 +
 test/integration/test-apt-update-transactions  |  2 +
 9 files changed, 99 insertions(+), 15 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 1090912f5..78dace12c 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -253,7 +253,10 @@ bool pkgAcquire::Item::Rename(string From,string To)
    strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno),
 	 From.c_str(),To.c_str());
    Status = StatError;
-   ErrorText += S;
+   if (ErrorText.empty())
+      ErrorText = S;
+   else
+      ErrorText = ErrorText + ": " + S;
    return false;
 }
 									/*}}}*/
@@ -1794,7 +1797,7 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size,
 
    if(AuthPass == false)
    {
-      if(CheckDownloadDone(Message) == true)
+      if(CheckDownloadDone(Message, Hashes) == true)
       {
          // destfile will be modified to point to MetaIndexFile for the
          // gpgv method, so we need to save it here
@@ -1837,7 +1840,8 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
          Status = StatDone;
       } else {
          _error->Error("%s", downgrade_msg.c_str());
-         Rename(MetaIndexFile, MetaIndexFile+".FAILED");
+	 if (TransactionManager->IMSHit == false)
+	    Rename(MetaIndexFile, MetaIndexFile+".FAILED");
 	 Item::Failed("Message: " + downgrade_msg, Cnf);
          TransactionManager->AbortTransaction();
          return;
@@ -1922,12 +1926,12 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,	/*{{{*/
 {
    Item::Done(Message,Size,Hashes,Cfg);
 
-   if(CheckDownloadDone(Message))
+   if(CheckDownloadDone(Message, Hashes))
    {
       // we have a Release file, now download the Signature, all further
       // verify/queue for additional downloads will be done in the
       // pkgAcqMetaSig::Done() code
-      std::string MetaIndexFile = DestFile;
+      std::string const MetaIndexFile = DestFile;
       new pkgAcqMetaSig(Owner, TransactionManager, 
                         MetaIndexSigURI, MetaIndexSigURIDesc,
                         MetaIndexSigShortDesc, MetaIndexFile, IndexTargets, 
@@ -2008,7 +2012,7 @@ void pkgAcqMetaBase::QueueForSignatureVerify(const std::string &MetaIndexFile,
 }
 									/*}}}*/
 // pkgAcqMetaBase::CheckDownloadDone					/*{{{*/
-bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message)
+bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message, HashStringList const &Hashes)
 {
    // We have just finished downloading a Release file (it is not
    // verified yet)
@@ -2031,7 +2035,18 @@ bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message)
 
    // make sure to verify against the right file on I-M-S hit
    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
-   if(IMSHit)
+   if (IMSHit == false)
+   {
+      // detect IMS-Hits servers haven't detected by Hash comparison
+      std::string FinalFile = GetFinalFilename();
+      if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
+      {
+	 IMSHit = true;
+	 unlink(DestFile.c_str());
+      }
+   }
+
+   if(IMSHit == true)
    {
       // for simplicity, the transaction manager is always InRelease
       // even if it doesn't exist.
@@ -2273,7 +2288,7 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size,
 
    if(AuthPass == false)
    {
-      if(CheckDownloadDone(Message) == true)
+      if(CheckDownloadDone(Message, Hashes) == true)
          QueueForSignatureVerify(DestFile, DestFile);
       return;
    }
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 33a28671c..646de8416 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -429,7 +429,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     *  \param Message The message block received from the fetch
     *  subprocess.
     */
-   bool CheckDownloadDone(const std::string &Message);
+   bool CheckDownloadDone(const std::string &Message, HashStringList const &Hashes);
 
    /** \brief Queue the downloaded Signature for verification */
    void QueueForSignatureVerify(const std::string &MetaIndexFile,
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index bee01e620..9254e20a3 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -333,13 +333,12 @@ bool pkgAcquire::Worker::RunMessages()
 
 	    // see if there is a hash to verify
 	    HashStringList ReceivedHashes;
-	    HashStringList expectedHashes = Owner->HashSums();
-	    for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs)
+	    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 	    {
-	       std::string const tagname = hs->HashType() + "-Hash";
+	       std::string const tagname = std::string(*type) + "-Hash";
 	       std::string const hashsum = LookupTag(Message, tagname.c_str());
 	       if (hashsum.empty() == false)
-		  ReceivedHashes.push_back(HashString(hs->HashType(), hashsum));
+		  ReceivedHashes.push_back(HashString(*type, hashsum));
 	    }
 
 	    if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
@@ -349,6 +348,7 @@ bool pkgAcquire::Worker::RunMessages()
 	       for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs)
 		  std::clog <<  "\t- " << hs->toStr() << std::endl;
 	       std::clog << "ExpectedHash:" << endl;
+	       HashStringList expectedHashes = Owner->HashSums();
 	       for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs)
 		  std::clog <<  "\t- " << hs->toStr() << std::endl;
 	       std::clog << endl;
diff --git a/methods/https.cc b/methods/https.cc
index 712e9ee73..fa143439a 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -444,7 +444,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       char err[255];
       snprintf(err, sizeof(err) - 1, "HttpError%i", Server->Result);
       SetFailReason(err);
-      _error->Error("%s", err);
+      _error->Error("%i %s", Server->Result, Server->Code);
       // unlink, no need keep 401/404 page content in partial/
       unlink(File->Name().c_str());
       return false;
diff --git a/test/integration/framework b/test/integration/framework
index 2a53e8365..8c8936ead 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1005,6 +1005,15 @@ signreleasefiles() {
 	msgdone "info"
 }
 
+redatereleasefiles() {
+	local DATE="$(date -d "$1" '+%a, %d %b %Y %H:%M:%S %Z')"
+	for release in $(find aptarchive/ -name 'Release'); do
+		sed -i "s/^Date: .*$/Date: ${DATE}/" $release
+		touch -d "$DATE" $release
+	done
+	signreleasefiles "${2:-Joe Sixpack}"
+}
+
 webserverconfig() {
 	local WEBSERVER="${3:-http://localhost:8080}"
 	local NOCHECK=false
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 7efccaa57..55a5da848 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -26,6 +26,7 @@ test_inreleasetoobig() {
 }
 
 test_packagestoobig() {
+	redatereleasefiles '+1hour'
 	# append junk at the end of the Packages.gz/Packages
 	SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
 	find aptarchive/dists -name 'Packages*' | while read pkg; do
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
index 2dc56e76c..a67ecb760 100755
--- a/test/integration/test-apt-update-not-modified
+++ b/test/integration/test-apt-update-not-modified
@@ -12,7 +12,7 @@ insertpackage 'unstable' 'apt' 'all' '1.0'
 setupaptarchive --no-update
 
 methodtest() {
-	msgmsg 'Test with' "$1"
+	msgmsg 'Test InRelease with' "$1"
 	rm -rf rootdir/var/lib/apt/lists
 	# get our cache populated
 	testsuccess aptget update
@@ -36,6 +36,62 @@ Reading package lists..." aptget update
 Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
 Reading package lists..." aptget update
 	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+	webserverconfig 'aptwebserver::support::modified-since' 'false'
+	webserverconfig 'aptwebserver::support::last-modified' 'false'
+	testsuccessequal "Get:1 $1 unstable InRelease [$(stat -c '%s' 'aptarchive/dists/unstable/InRelease') B]
+Reading package lists..." aptget update
+	webserverconfig 'aptwebserver::support::modified-since' 'true'
+	webserverconfig 'aptwebserver::support::last-modified' 'true'
+
+	msgmsg 'Test Release.gpg with' "$1"
+	rm -rf rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists  aptarchive/dists.good
+	find aptarchive/dists -name 'InRelease' -delete
+	# get our cache populated
+	testsuccess aptget update
+	listcurrentlistsdirectory > listsdir.lst
+
+	# hit again with a good cache
+	testsuccessequal "Ign $1 unstable InRelease
+  404  Not Found
+Hit $1 unstable Release
+Hit $1 unstable Release.gpg
+Reading package lists..." aptget update
+	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+	# drop an architecture, which means the file should be gone now
+	configarchitecture 'i386'
+	sed '/_binary-amd64_Packages/ d' listsdir.lst > listsdir-without-amd64.lst
+	testsuccessequal "Ign $1 unstable InRelease
+  404  Not Found
+Hit $1 unstable Release
+Hit $1 unstable Release.gpg
+Reading package lists..." aptget update
+	testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)"
+
+	# readd arch so its downloaded again
+	configarchitecture 'amd64' 'i386'
+	testsuccessequal "Ign $1 unstable InRelease
+  404  Not Found
+Hit $1 unstable Release
+Hit $1 unstable Release.gpg
+Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Reading package lists..." aptget update
+	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+	webserverconfig 'aptwebserver::support::modified-since' 'false'
+	webserverconfig 'aptwebserver::support::last-modified' 'false'
+	testsuccessequal "Ign $1 unstable InRelease
+  404  Not Found
+Get:1 $1 unstable Release [$(stat -c '%s' 'aptarchive/dists/unstable/Release') B]
+Get:2 $1 unstable Release.gpg [$(stat -c '%s' 'aptarchive/dists/unstable/Release.gpg') B]
+Reading package lists..." aptget update
+	webserverconfig 'aptwebserver::support::modified-since' 'true'
+	webserverconfig 'aptwebserver::support::last-modified' 'true'
+
+	rm -rf aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
 }
 
 changetowebserver
diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback
index f4500b69d..29fe1ab56 100755
--- a/test/integration/test-apt-update-rollback
+++ b/test/integration/test-apt-update-rollback
@@ -158,6 +158,7 @@ test_inrelease_to_broken_gzip() {
     msgmsg "Test InRelease to broken gzip"
     start_with_good_inrelease
 
+    redatereleasefiles '+2hours'
     # append junk at the end of the compressed file
     echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz
     touch -d '+2min' $APTARCHIVE/dists/unstable/main/source/Sources.gz
diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions
index 67dd633f9..63b318056 100755
--- a/test/integration/test-apt-update-transactions
+++ b/test/integration/test-apt-update-transactions
@@ -47,6 +47,7 @@ testrun() {
 
 testsetup() {
 	msgmsg 'Test with no initial data over' "$1"
+	redatereleasefiles 'now'
 	rm -rf rootdir/var/lib/apt/lists
 	mkdir -p rootdir/var/lib/apt/lists/partial
 	listcurrentlistsdirectory > listsdir.lst
@@ -55,6 +56,7 @@ testsetup() {
 	msgmsg 'Test with initial data over' "$1"
 	testsuccess aptget update
 	listcurrentlistsdirectory > listsdir.lst
+	redatereleasefiles '+1hour'
 	testrun 'listsdir.lst'
 }
 
-- 
cgit v1.2.3-70-g09d2


From 6bf93605fdb8e858d3f0a79a124c1d39f760094d Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 18 May 2015 22:15:06 +0200
Subject: treat older Release files than we already have as an IMSHit

Valid-Until protects us from long-living downgrade attacks, but not all
repositories have it and an attacker could still use older but still
valid files to downgrade us. While this makes it sounds like a security
improvement now, its a bit theoretical at best as an attacker with
capabilities to pull this off could just as well always keep us days
(but in the valid period) behind and always knows which state we have,
as we tell him with the If-Modified-Since header. This is also why this
is 'silently' ignored and treated as an IMSHit rather than screamed at
the user as this can at best be an annoyance for attackers.

An error here would 'regularily' be encountered by users by out-of-sync
mirrors serving a single run (e.g. load balancer) or in two consecutive
runs on the other hand, so it would just help teaching people ignore it.

That said, most of the code churn is caused by enforcing this additional
requirement. Crisscross from InRelease to Release.gpg is e.g. very
unlikely in practice, but if we would ignore it an attacker could
sidestep it this way.
---
 apt-pkg/acquire-item.cc                        | 264 ++++++++++++++++++-------
 apt-pkg/acquire-item.h                         | 105 +++++-----
 apt-pkg/indexrecords.cc                        |  28 +--
 apt-pkg/indexrecords.h                         |   2 +
 test/integration/framework                     |   7 +-
 test/integration/test-apt-update-ims           |  12 +-
 test/integration/test-apt-update-nofallback    |   5 +-
 test/integration/test-apt-update-not-modified  |   4 -
 test/integration/test-apt-update-rollback      |   4 +-
 test/integration/test-releasefile-date-older   |  62 ++++++
 test/integration/test-releasefile-valid-until  |  37 ++--
 test/integration/test-releasefile-verification |  70 +++----
 12 files changed, 383 insertions(+), 217 deletions(-)
 create mode 100755 test/integration/test-releasefile-date-older

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 78dace12c..8155b9bfe 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1637,7 +1637,7 @@ pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire *Owner,
       HashStringList const &ExpectedHashes,
       pkgAcqMetaBase *TransactionManager)
 : Item(Owner, ExpectedHashes, TransactionManager),
-   MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets),
+   MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL), IndexTargets(IndexTargets),
    AuthPass(false), RealURI(RealURI), IMSHit(false)
 {
 }
@@ -1697,6 +1697,14 @@ void pkgAcqMetaBase::CommitTransaction()
    Transaction.clear();
 }
 									/*}}}*/
+bool pkgAcqMetaBase::TransactionState(TransactionStates const state)	/*{{{*/
+{
+   // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
+   if (TransactionManager->IMSHit == false)
+      return pkgAcquire::Item::TransactionState(state);
+   return true;
+}
+									/*}}}*/
 // AcqMetaBase::TransactionStageCopy - Stage a file for copying		/*{{{*/
 void pkgAcqMetaBase::TransactionStageCopy(Item *I,
                                           const std::string &From,
@@ -1715,15 +1723,15 @@ void pkgAcqMetaBase::TransactionStageRemoval(Item *I,
 }
 									/*}}}*/
 // AcqMetaBase::GenerateAuthWarning - Check gpg authentication error	/*{{{*/
-bool pkgAcqMetaBase::CheckStopAuthentication(const std::string &Message)
+bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message)
 {
    // FIXME: this entire function can do now that we disallow going to
    //        a unauthenticated state and can cleanly rollback
 
-   string const Final = GetFinalFilename();
+   string const Final = I->GetFinalFilename();
    if(FileExists(Final))
    {
-      Status = StatTransientNetworkError;
+      I->Status = StatTransientNetworkError;
       _error->Warning(_("An error occurred during the signature "
                         "verification. The repository is not updated "
                         "and the previous index files will be used. "
@@ -1737,7 +1745,7 @@ bool pkgAcqMetaBase::CheckStopAuthentication(const std::string &Message)
       _error->Error(_("GPG error: %s: %s"),
                     Desc.Description.c_str(),
                     LookupTag(Message,"Message").c_str());
-      Status = StatError;
+      I->Status = StatError;
       return true;
    } else {
       _error->Warning(_("GPG error: %s: %s"),
@@ -1751,17 +1759,13 @@ bool pkgAcqMetaBase::CheckStopAuthentication(const std::string &Message)
 									/*}}}*/
 // AcqMetaSig::AcqMetaSig - Constructor					/*{{{*/
 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,
-                             pkgAcqMetaBase *TransactionManager,
-			     string URI,string URIDesc,string ShortDesc,
-                             string MetaIndexFile,
-			     const vector* IndexTargets,
-			     indexRecords* MetaIndexParser) :
-   pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, URI,
-                  HashStringList(), TransactionManager),
-   MetaIndexFile(MetaIndexFile), URIDesc(URIDesc),
-   ShortDesc(ShortDesc)
+      pkgAcqMetaBase *TransactionManager,
+      string const &URI, string const &URIDesc,string const &ShortDesc,
+      pkgAcqMetaIndex * const MetaIndex) :
+   pkgAcquire::Item(Owner, HashStringList(), TransactionManager), MetaIndex(MetaIndex),
+   URIDesc(URIDesc), RealURI(URI)
 {
-   DestFile = GetPartialFileNameFromURI(RealURI);
+   DestFile = GetPartialFileNameFromURI(URI);
 
    // remove any partial downloaded sig-file in partial/.
    // it may confuse proxies and is too small to warrant a
@@ -1779,11 +1783,28 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,
    Desc.ShortDesc = ShortDesc;
    Desc.URI = URI;
 
-   QueueURI(Desc);
+   // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
+   // so we skip the download step and go instantly to verification
+   if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename()))
+   {
+      Complete = true;
+      Status = StatDone;
+      PartialFile = DestFile = GetFinalFilename();
+      MetaIndexFileSignature = DestFile;
+      MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
+   }
+   else
+      QueueURI(Desc);
 }
 									/*}}}*/
 pkgAcqMetaSig::~pkgAcqMetaSig()						/*{{{*/
 {
+}
+									/*}}}*/
+// pkgAcqMetaSig::GetFinalFilename - Return the full final file path	/*{{{*/
+std::string pkgAcqMetaSig::GetFinalFilename() const
+{
+   return GetFinalFileNameFromURI(RealURI);
 }
 									/*}}}*/
 // pkgAcqMetaSig::Done - The signature was downloaded/verified		/*{{{*/
@@ -1793,21 +1814,32 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size,
                          HashStringList const &Hashes,
 			 pkgAcquire::MethodConfig *Cfg)
 {
+   if (MetaIndexFileSignature.empty() == false)
+   {
+      DestFile = MetaIndexFileSignature;
+      MetaIndexFileSignature.clear();
+   }
    Item::Done(Message, Size, Hashes, Cfg);
 
-   if(AuthPass == false)
+   if(MetaIndex->AuthPass == false)
    {
-      if(CheckDownloadDone(Message, Hashes) == true)
+      if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
       {
-         // destfile will be modified to point to MetaIndexFile for the
-         // gpgv method, so we need to save it here
-         MetaIndexFileSignature = DestFile;
-         QueueForSignatureVerify(MetaIndexFile, MetaIndexFileSignature);
+	 // destfile will be modified to point to MetaIndexFile for the
+	 // gpgv method, so we need to save it here
+	 MetaIndexFileSignature = DestFile;
+	 MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
       }
       return;
    }
-   else if(CheckAuthDone(Message) == true)
-      TransactionManager->TransactionStageCopy(this, MetaIndexFileSignature, GetFinalFilename());
+   else if(MetaIndex->CheckAuthDone(Message) == true)
+   {
+      if (TransactionManager->IMSHit == false)
+      {
+	 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+	 TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
+      }
+   }
 }
 									/*}}}*/
 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
@@ -1815,20 +1847,18 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
    Item::Failed(Message,Cnf);
 
    // check if we need to fail at this point 
-   if (AuthPass == true && CheckStopAuthentication(Message))
+   if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
          return;
 
-   // FIXME: meh, this is not really elegant
-   string const Final = GetFinalFileNameFromURI(RealURI);
-   string const InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12,
-                                         "InRelease");
-   string const FinalInRelease = GetFinalFileNameFromURI(InReleaseURI);
+   string const FinalRelease = MetaIndex->GetFinalFilename();
+   string const FinalReleasegpg = GetFinalFilename();
+   string const FinalInRelease = TransactionManager->GetFinalFilename();
 
-   if (RealFileExists(Final) || RealFileExists(FinalInRelease))
+   if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
    {
       std::string downgrade_msg;
       strprintf(downgrade_msg, _("The repository '%s' is no longer signed."),
-                URIDesc.c_str());
+                MetaIndex->URIDesc.c_str());
       if(_config->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
       {
          // meh, the users wants to take risks (we still mark the packages
@@ -1841,7 +1871,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
       } else {
          _error->Error("%s", downgrade_msg.c_str());
 	 if (TransactionManager->IMSHit == false)
-	    Rename(MetaIndexFile, MetaIndexFile+".FAILED");
+	    Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
 	 Item::Failed("Message: " + downgrade_msg, Cnf);
          TransactionManager->AbortTransaction();
          return;
@@ -1850,23 +1880,44 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
    else
       _error->Warning(_("The data from '%s' is not signed. Packages "
 	       "from that repository can not be authenticated."),
-	    URIDesc.c_str());
+	    MetaIndex->URIDesc.c_str());
 
-   // this ensures that any file in the lists/ dir is removed by the
-   // transaction
-   DestFile = GetPartialFileNameFromURI(RealURI);
+   // ensures that a Release.gpg file in the lists/ is removed by the transaction
    TransactionManager->TransactionStageRemoval(this, DestFile);
 
    // only allow going further if the users explicitely wants it
-   if(AllowInsecureRepositories(MetaIndexParser, TransactionManager, this) == true)
+   if(AllowInsecureRepositories(MetaIndex->MetaIndexParser, TransactionManager, this) == true)
    {
+      if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
+      {
+	 // open the last Release if we have it
+	 if (TransactionManager->IMSHit == false)
+	 {
+	    MetaIndex->LastMetaIndexParser = new indexRecords;
+	    _error->PushToStack();
+	    if (RealFileExists(FinalInRelease))
+	       MetaIndex->LastMetaIndexParser->Load(FinalInRelease);
+	    else
+	       MetaIndex->LastMetaIndexParser->Load(FinalRelease);
+	    // its unlikely to happen, but if what we have is bad ignore it
+	    if (_error->PendingError())
+	    {
+	       delete MetaIndex->LastMetaIndexParser;
+	       MetaIndex->LastMetaIndexParser = NULL;
+	    }
+	    _error->RevertToStack();
+	 }
+      }
+
       // we parse the indexes here because at this point the user wanted
       // a repository that may potentially harm him
-      MetaIndexParser->Load(MetaIndexFile);
-      if (!VerifyVendor(Message))
+      MetaIndex->MetaIndexParser->Load(MetaIndex->DestFile);
+      if (MetaIndex->VerifyVendor(Message) == false)
 	 /* expired Release files are still a problem you need extra force for */;
       else
-	 QueueIndexes(true);
+	 MetaIndex->QueueIndexes(true);
+
+      TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
    }
 
    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
@@ -1926,18 +1977,14 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,	/*{{{*/
 {
    Item::Done(Message,Size,Hashes,Cfg);
 
-   if(CheckDownloadDone(Message, Hashes))
+   if(CheckDownloadDone(this, Message, Hashes))
    {
       // we have a Release file, now download the Signature, all further
       // verify/queue for additional downloads will be done in the
       // pkgAcqMetaSig::Done() code
-      std::string const MetaIndexFile = DestFile;
-      new pkgAcqMetaSig(Owner, TransactionManager, 
+      new pkgAcqMetaSig(Owner, TransactionManager,
                         MetaIndexSigURI, MetaIndexSigURIDesc,
-                        MetaIndexSigShortDesc, MetaIndexFile, IndexTargets, 
-                        MetaIndexParser);
-
-      TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+                        MetaIndexSigShortDesc, this);
    }
 }
 									/*}}}*/
@@ -1948,6 +1995,40 @@ bool pkgAcqMetaBase::CheckAuthDone(string Message)			/*{{{*/
    // perform additional verification of its contents, and use them
    // to verify the indexes we are about to download
 
+   if (TransactionManager->IMSHit == false)
+   {
+      // open the last (In)Release if we have it
+      std::string const FinalFile = GetFinalFilename();
+      std::string FinalRelease;
+      std::string FinalInRelease;
+      if (APT::String::Endswith(FinalFile, "InRelease"))
+      {
+	 FinalInRelease = FinalFile;
+	 FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release";
+      }
+      else
+      {
+	 FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
+	 FinalRelease = FinalFile;
+      }
+      if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
+      {
+	 LastMetaIndexParser = new indexRecords;
+	 _error->PushToStack();
+	 if (RealFileExists(FinalInRelease))
+	    LastMetaIndexParser->Load(FinalInRelease);
+	 else
+	    LastMetaIndexParser->Load(FinalRelease);
+	 // its unlikely to happen, but if what we have is bad ignore it
+	 if (_error->PendingError())
+	 {
+	    delete LastMetaIndexParser;
+	    LastMetaIndexParser = NULL;
+	 }
+	 _error->RevertToStack();
+      }
+   }
+
    if (!MetaIndexParser->Load(DestFile))
    {
       Status = StatAuthError;
@@ -2001,48 +2082,47 @@ std::string pkgAcqMetaBase::GetFinalFilename() const
 }
 									/*}}}*/
 // pkgAcqMetaBase::QueueForSignatureVerify				/*{{{*/
-void pkgAcqMetaBase::QueueForSignatureVerify(const std::string &MetaIndexFile,
-                                    const std::string &MetaIndexFileSignature)
+void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcquire::Item * const I, std::string const &File, std::string const &Signature)
 {
    AuthPass = true;
-   Desc.URI = "gpgv:" + MetaIndexFileSignature;
-   DestFile = MetaIndexFile;
-   QueueURI(Desc);
-   SetActiveSubprocess("gpgv");
+   I->Desc.URI = "gpgv:" + Signature;
+   I->DestFile = File;
+   QueueURI(I->Desc);
+   I->SetActiveSubprocess("gpgv");
 }
 									/*}}}*/
 // pkgAcqMetaBase::CheckDownloadDone					/*{{{*/
-bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message, HashStringList const &Hashes)
+bool pkgAcqMetaBase::CheckDownloadDone(pkgAcquire::Item * const I, const std::string &Message, HashStringList const &Hashes) const
 {
    // We have just finished downloading a Release file (it is not
    // verified yet)
 
-   string FileName = LookupTag(Message,"Filename");
+   string const FileName = LookupTag(Message,"Filename");
    if (FileName.empty() == true)
    {
-      Status = StatError;
-      ErrorText = "Method gave a blank filename";
+      I->Status = StatError;
+      I->ErrorText = "Method gave a blank filename";
       return false;
    }
 
-   if (FileName != DestFile)
+   if (FileName != I->DestFile)
    {
-      Local = true;
-      Desc.URI = "copy:" + FileName;
-      QueueURI(Desc);
+      I->Local = true;
+      I->Desc.URI = "copy:" + FileName;
+      I->QueueURI(I->Desc);
       return false;
    }
 
    // make sure to verify against the right file on I-M-S hit
-   IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
-   if (IMSHit == false)
+   bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false);
+   if (IMSHit == false && Hashes.usable())
    {
       // detect IMS-Hits servers haven't detected by Hash comparison
-      std::string FinalFile = GetFinalFilename();
+      std::string const FinalFile = I->GetFinalFilename();
       if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
       {
 	 IMSHit = true;
-	 unlink(DestFile.c_str());
+	 unlink(I->DestFile.c_str());
       }
    }
 
@@ -2052,11 +2132,11 @@ bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message, HashStringLis
       // even if it doesn't exist.
       if (TransactionManager != NULL)
 	 TransactionManager->IMSHit = true;
-      DestFile = GetFinalFilename();
+      I->PartialFile = I->DestFile = I->GetFinalFilename();
    }
 
    // set Item to complete as the remaining work is all local (verify etc)
-   Complete = true;
+   I->Complete = true;
 
    return true;
 }
@@ -2175,6 +2255,19 @@ bool pkgAcqMetaBase::VerifyVendor(string Message)			/*{{{*/
       }
    }
 
+   /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
+      as a prevention of downgrading us to older (still valid) files */
+   if (TransactionManager->IMSHit == false && LastMetaIndexParser != NULL &&
+	 LastMetaIndexParser->GetDate() > MetaIndexParser->GetDate())
+   {
+      TransactionManager->IMSHit = true;
+      unlink(DestFile.c_str());
+      PartialFile = DestFile = GetFinalFilename();
+      delete MetaIndexParser;
+      MetaIndexParser = LastMetaIndexParser;
+      LastMetaIndexParser = NULL;
+   }
+
    if (_config->FindB("Debug::pkgAcquire::Auth", false)) 
    {
       std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
@@ -2248,7 +2341,6 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,		/*{{{*/
 {
    // index targets + (worst case:) Release/Release.gpg
    ExpectedAdditionalItems = IndexTargets->size() + 2;
-
 }
 									/*}}}*/
 pkgAcqMetaClearSig::~pkgAcqMetaClearSig()				/*{{{*/
@@ -2268,14 +2360,25 @@ string pkgAcqMetaClearSig::Custom600Headers()
 }
 									/*}}}*/
 // pkgAcqMetaClearSig::Done - We got a file				/*{{{*/
-// ---------------------------------------------------------------------
+class APT_HIDDEN DummyItem : public pkgAcquire::Item
+{
+   std::string URI;
+   public:
+   virtual std::string DescURI() {return URI;};
+
+   DummyItem(pkgAcquire *Owner, std::string const &URI) : pkgAcquire::Item(Owner), URI(URI)
+   {
+      Status = StatDone;
+      DestFile = GetFinalFileNameFromURI(URI);
+   }
+};
 void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size,
                               HashStringList const &Hashes,
                               pkgAcquire::MethodConfig *Cnf)
 {
    Item::Done(Message, Size, Hashes, Cnf);
 
-   // if we expect a ClearTextSignature (InRelase), ensure that
+   // if we expect a ClearTextSignature (InRelease), ensure that
    // this is what we get and if not fail to queue a 
    // Release/Release.gpg, see #346386
    if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
@@ -2288,12 +2391,23 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size,
 
    if(AuthPass == false)
    {
-      if(CheckDownloadDone(Message, Hashes) == true)
-         QueueForSignatureVerify(DestFile, DestFile);
+      if(CheckDownloadDone(this, Message, Hashes) == true)
+         QueueForSignatureVerify(this, DestFile, DestFile);
       return;
    }
    else if(CheckAuthDone(Message) == true)
-      TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+   {
+      if (TransactionManager->IMSHit == false)
+	 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+      else if (RealFileExists(GetFinalFilename()) == false)
+      {
+	 // We got an InRelease file IMSHit, but we haven't one, which means
+	 // we had a valid Release/Release.gpg combo stepping in, which we have
+	 // to 'acquire' now to ensure list cleanup isn't removing them
+	 new DummyItem(Owner, MetaIndexURI);
+	 new DummyItem(Owner, MetaSigURI);
+      }
+   }
 }
 									/*}}}*/
 void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
@@ -2318,7 +2432,7 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*
    }
    else
    {
-      if(CheckStopAuthentication(Message))
+      if(CheckStopAuthentication(this, Message))
          return;
 
       _error->Warning(_("The data from '%s' is not signed. Packages "
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 646de8416..07c86f31b 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -394,6 +394,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
 
    /** \brief A package-system-specific parser for the meta-index file. */
    indexRecords *MetaIndexParser;
+   indexRecords *LastMetaIndexParser;
 
    /** \brief The index files which should be looked up in the meta-index
     *  and then downloaded.
@@ -429,11 +430,10 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     *  \param Message The message block received from the fetch
     *  subprocess.
     */
-   bool CheckDownloadDone(const std::string &Message, HashStringList const &Hashes);
+   bool CheckDownloadDone(pkgAcquire::Item * const I, const std::string &Message, HashStringList const &Hashes) const;
 
    /** \brief Queue the downloaded Signature for verification */
-   void QueueForSignatureVerify(const std::string &MetaIndexFile,
-                                const std::string &MetaIndexFileSignature);
+   void QueueForSignatureVerify(pkgAcquire::Item * const I, std::string const &File, std::string const &Signature);
 
 #if APT_PKG_ABI >= 413
    virtual std::string Custom600Headers() const;
@@ -453,7 +453,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
    bool CheckAuthDone(std::string Message);
 
    /** Check if the current item should fail at this point */
-   bool CheckStopAuthentication(const std::string &Message);
+   bool CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message);
 
    /** \brief Check that the release file is a release file for the
     *  correct distribution.
@@ -462,8 +462,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     */
    bool VerifyVendor(std::string Message);
 
-   /** \brief Get the full pathname of the final file for the current URI */
-   virtual std::string GetFinalFilename() const;
+   virtual bool TransactionState(TransactionStates const state);
 
  public:
    // This refers more to the Transaction-Manager than the actual file
@@ -487,6 +486,9 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     */
    void TransactionStageRemoval(Item *I, const std::string &FinalFile);
 
+   /** \brief Get the full pathname of the final file for the current URI */
+   virtual std::string GetFinalFilename() const;
+
    pkgAcqMetaBase(pkgAcquire *Owner,
                   const std::vector* IndexTargets,
                   indexRecords* MetaIndexParser,
@@ -495,50 +497,6 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
                   pkgAcqMetaBase *TransactionManager=NULL);
 };
 									/*}}}*/
-/** \brief An acquire item that downloads the detached signature	{{{
- *  of a meta-index (Release) file, then queues up the release
- *  file itself.
- *
- *  \todo Why protected members?
- *
- *  \sa pkgAcqMetaIndex
- */
-class APT_HIDDEN pkgAcqMetaSig : public pkgAcqMetaBase
-{
-   void *d;
-
-   protected:
-
-   /** \brief The file we need to verify */
-   std::string MetaIndexFile;
-
-   /** \brief The file we use to verify the MetaIndexFile with */
-   std::string MetaIndexFileSignature;
-
-   /** \brief Long URI description used in the acquire system */
-   std::string URIDesc;
-
-   /** \brief Short URI description used in the acquire system */
-   std::string ShortDesc;
-
-   public:
-
-   // Specialized action members
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-   virtual void Done(std::string Message,unsigned long long Size,
-                     HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
-
-   /** \brief Create a new pkgAcqMetaSig. */
-   pkgAcqMetaSig(pkgAcquire *Owner,
-                 pkgAcqMetaBase *TransactionManager,
-                 std::string URI,std::string URIDesc, std::string ShortDesc,
-                 std::string MetaIndexFile,
-		 const std::vector* IndexTargets,
-		 indexRecords* MetaIndexParser);
-   virtual ~pkgAcqMetaSig();
-};
-									/*}}}*/
 /** \brief An item that is responsible for downloading the meta-index	{{{
  *  file (i.e., Release) itself and verifying its signature.
  *
@@ -584,6 +542,53 @@ class APT_HIDDEN pkgAcqMetaIndex : public pkgAcqMetaBase
                    std::string MetaIndexSigURI, std::string MetaIndexSigURIDesc, std::string MetaIndexSigShortDesc,
 		   const std::vector* IndexTargets,
 		   indexRecords* MetaIndexParser);
+
+   friend class pkgAcqMetaSig;
+};
+									/*}}}*/
+/** \brief An acquire item that downloads the detached signature	{{{
+ *  of a meta-index (Release) file, then queues up the release
+ *  file itself.
+ *
+ *  \todo Why protected members?
+ *
+ *  \sa pkgAcqMetaIndex
+ */
+class APT_HIDDEN pkgAcqMetaSig : public pkgAcquire::Item
+{
+   void *d;
+
+   pkgAcqMetaIndex * const MetaIndex;
+
+   /** \brief The file we use to verify the MetaIndexFile with (not always set!) */
+   std::string MetaIndexFileSignature;
+
+   protected:
+
+   /** \brief Long URI description used in the acquire system */
+   std::string URIDesc;
+
+   /** \brief URI used to get the file */
+   std::string RealURI;
+
+   /** \brief Get the full pathname of the final file for the current URI */
+   virtual std::string GetFinalFilename() const;
+
+   public:
+   virtual std::string DescURI() {return RealURI;};
+
+   // Specialized action members
+   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
+   virtual void Done(std::string Message,unsigned long long Size,
+                     HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig *Cnf);
+
+   /** \brief Create a new pkgAcqMetaSig. */
+   pkgAcqMetaSig(pkgAcquire *Owner,
+	 pkgAcqMetaBase *TransactionManager,
+	 std::string const &URI,std::string const &URIDesc,
+	 std::string const &ShortDesc, pkgAcqMetaIndex * const MetaIndex);
+   virtual ~pkgAcqMetaSig();
 };
 									/*}}}*/
 /** \brief An item repsonsible for downloading clearsigned metaindexes	{{{*/
diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc
index d65266f64..de2617833 100644
--- a/apt-pkg/indexrecords.cc
+++ b/apt-pkg/indexrecords.cc
@@ -58,6 +58,11 @@ APT_PURE time_t indexRecords::GetValidUntil() const
    return this->ValidUntil;
 }
 
+APT_PURE time_t indexRecords::GetDate() const
+{
+   return this->Date;
+}
+
 APT_PURE indexRecords::checkSum *indexRecords::Lookup(const string MetaKey)
 {
    std::map::const_iterator sum = Entries.find(MetaKey);
@@ -133,9 +138,15 @@ bool indexRecords::Load(const string Filename)				/*{{{*/
       return false;
    }
 
-   string Label = Section.FindS("Label");
-   string StrDate = Section.FindS("Date");
-   string StrValidUntil = Section.FindS("Valid-Until");
+   string const StrDate = Section.FindS("Date");
+   if (RFC1123StrToTime(StrDate.c_str(), Date) == false)
+   {
+      strprintf(ErrorText, _("Invalid 'Date' entry in Release file %s"), Filename.c_str());
+      return false;
+   }
+
+   string const Label = Section.FindS("Label");
+   string const StrValidUntil = Section.FindS("Valid-Until");
 
    // if we have a Valid-Until header in the Release file, use it as default
    if (StrValidUntil.empty() == false)
@@ -158,20 +169,13 @@ bool indexRecords::Load(const string Filename)				/*{{{*/
       (MinAge == 0 || ValidUntil == 0)) // No user settings, use the one from the Release file
       return true;
 
-   time_t date;
-   if (RFC1123StrToTime(StrDate.c_str(), date) == false)
-   {
-      strprintf(ErrorText, _("Invalid 'Date' entry in Release file %s"), Filename.c_str());
-      return false;
-   }
-
    if (MinAge != 0 && ValidUntil != 0) {
-      time_t const min_date = date + MinAge;
+      time_t const min_date = Date + MinAge;
       if (ValidUntil < min_date)
 	 ValidUntil = min_date;
    }
    if (MaxAge != 0) {
-      time_t const max_date = date + MaxAge;
+      time_t const max_date = Date + MaxAge;
       if (ValidUntil == 0 || ValidUntil > max_date)
 	 ValidUntil = max_date;
    }
diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h
index 35e534c12..6ed5f0c2b 100644
--- a/apt-pkg/indexrecords.h
+++ b/apt-pkg/indexrecords.h
@@ -36,6 +36,7 @@ class indexRecords
    std::string Dist;
    std::string Suite;
    std::string ExpectedDist;
+   time_t Date;
    time_t ValidUntil;
    bool SupportsAcquireByHash;
 
@@ -62,6 +63,7 @@ class indexRecords
    std::string GetSuite() const;
    bool GetSupportsAcquireByHash() const;
    time_t GetValidUntil() const;
+   time_t GetDate() const;
    std::string GetExpectedDist() const;
 
    /** \brief check if source is marked as always trusted */
diff --git a/test/integration/framework b/test/integration/framework
index 8c8936ead..b253deb91 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1164,9 +1164,9 @@ testfileequal() {
 	shift
 	msgtest "Test for correctness of file" "$FILE"
 	if [ -z "$*" ]; then
-		echo -n "" | checkdiff $FILE - && msgpass || msgfail
+		echo -n "" | checkdiff - $FILE && msgpass || msgfail
 	else
-		echo "$*" | checkdiff $FILE - && msgpass || msgfail
+		echo "$*" | checkdiff - $FILE && msgpass || msgfail
 	fi
 }
 
@@ -1547,7 +1547,8 @@ aptautotest_aptcdrom_add() { aptautotest_aptget_update "$@"; }
 testaptautotestnodpkgwarning() {
 	local TESTCALL="$1"
 	while [ -n "$2" ]; do
-		if expr match "$2" '^-[a-z]*s' >/dev/null 2>&1; then return; fi
+		if expr match "$2" '^-[a-z]*s' >/dev/null 2>&1; then return; fi # simulation mode
+		if expr match "$2" '^-dy\?' >/dev/null 2>&1; then return; fi # download-only mode
 		shift
 	done
 	testfailure grep '^dpkg: warning:.*ignor.*' "${TMPWORKINGDIRECTORY}/rootdir/tmp-before/${TESTCALL}.output"
diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index f091bffaa..7385e701a 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -43,7 +43,7 @@ runtest() {
     testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
 
     # ensure that we still do a hash check for other files on ims hit of Release
-    if grep -q '^Hit .* \(InRelease\|Release.gpg\)$' expected.output ; then
+    if grep -q '^Hit .* InRelease$' expected.output || ! grep -q '^Ign .* Release\(\.gpg\)\?$' expected.output; then
 	    $TEST aptget update -o Debug::Acquire::gpgv=1
 	    cp rootdir/tmp/${TEST}.output goodsign.output
 	    testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
@@ -66,7 +66,6 @@ msgmsg 'Release/Release.gpg'
 EXPECT='Ign http://localhost:8080 unstable InRelease
   404  Not Found
 Hit http://localhost:8080 unstable Release
-Hit http://localhost:8080 unstable Release.gpg
 Reading package lists...'
 find aptarchive -name 'InRelease' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
@@ -81,7 +80,7 @@ Hit http://localhost:8080 unstable Release
 Ign http://localhost:8080 unstable Release.gpg
   404  Not Found
 Reading package lists...
-W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated."
+W: The data from 'http://localhost:8080 unstable Release' is not signed. Packages from that repository can not be authenticated."
 find aptarchive -name 'Release.gpg' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'warning'
@@ -108,8 +107,7 @@ msgmsg 'expired Release/Release.gpg'
 EXPECT='Ign http://localhost:8080 unstable InRelease
   404  Not Found
 Hit http://localhost:8080 unstable Release
-Hit http://localhost:8080 unstable Release.gpg
-E: Release file for http://localhost:8080/dists/unstable/Release.gpg is expired (invalid since). Updates for this repository will not be applied.'
+E: Release file for http://localhost:8080/dists/unstable/Release is expired (invalid since). Updates for this repository will not be applied.'
 find aptarchive -name 'InRelease' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'failure'
@@ -122,8 +120,8 @@ EXPECT="Ign http://localhost:8080 unstable InRelease
 Hit http://localhost:8080 unstable Release
 Ign http://localhost:8080 unstable Release.gpg
   404  Not Found
-W: The data from 'http://localhost:8080 unstable Release.gpg' is not signed. Packages from that repository can not be authenticated.
-E: Release file for http://localhost:8080/dists/unstable/InRelease is expired (invalid since). Updates for this repository will not be applied."
+W: The data from 'http://localhost:8080 unstable Release' is not signed. Packages from that repository can not be authenticated.
+E: Release file for http://localhost:8080/dists/unstable/Release is expired (invalid since). Updates for this repository will not be applied."
 find aptarchive -name 'Release.gpg' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'failure' 'warning'
diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback
index 71576de81..db4430ea3 100755
--- a/test/integration/test-apt-update-nofallback
+++ b/test/integration/test-apt-update-nofallback
@@ -8,6 +8,7 @@ set -e
 
 simulate_mitm_and_inject_evil_package()
 {
+    redatereleasefiles '+1 hour'
     rm -f $APTARCHIVE/dists/unstable/InRelease
     rm -f $APTARCHIVE/dists/unstable/Release.gpg
     inject_evil_package
@@ -31,7 +32,7 @@ EOF
 
 assert_update_is_refused_and_last_good_state_used()
 {
-    testfailureequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
+    testfailuremsg "E: The repository 'file: unstable Release' is no longer signed." aptget update
 
     assert_repo_is_intact
 }
@@ -193,7 +194,7 @@ test_release_gpg_to_invalid_release_release_gpg()
     echo "Some evil data" >>  $APTARCHIVE/dists/unstable/Release
     inject_evil_package
 
-    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release.gpg: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
+    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
 W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg  The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
index a67ecb760..b1d55c156 100755
--- a/test/integration/test-apt-update-not-modified
+++ b/test/integration/test-apt-update-not-modified
@@ -56,7 +56,6 @@ Reading package lists..." aptget update
 	testsuccessequal "Ign $1 unstable InRelease
   404  Not Found
 Hit $1 unstable Release
-Hit $1 unstable Release.gpg
 Reading package lists..." aptget update
 	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
 
@@ -66,7 +65,6 @@ Reading package lists..." aptget update
 	testsuccessequal "Ign $1 unstable InRelease
   404  Not Found
 Hit $1 unstable Release
-Hit $1 unstable Release.gpg
 Reading package lists..." aptget update
 	testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)"
 
@@ -75,7 +73,6 @@ Reading package lists..." aptget update
 	testsuccessequal "Ign $1 unstable InRelease
   404  Not Found
 Hit $1 unstable Release
-Hit $1 unstable Release.gpg
 Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
 Reading package lists..." aptget update
 	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
@@ -85,7 +82,6 @@ Reading package lists..." aptget update
 	testsuccessequal "Ign $1 unstable InRelease
   404  Not Found
 Get:1 $1 unstable Release [$(stat -c '%s' 'aptarchive/dists/unstable/Release') B]
-Get:2 $1 unstable Release.gpg [$(stat -c '%s' 'aptarchive/dists/unstable/Release.gpg') B]
 Reading package lists..." aptget update
 	webserverconfig 'aptwebserver::support::modified-since' 'true'
 	webserverconfig 'aptwebserver::support::last-modified' 'true'
diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback
index 29fe1ab56..b464a04a1 100755
--- a/test/integration/test-apt-update-rollback
+++ b/test/integration/test-apt-update-rollback
@@ -78,7 +78,7 @@ test_inrelease_to_valid_release() {
     rm $APTARCHIVE/dists/unstable/Release.gpg
 
     # update fails
-    testfailureequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq
+    testfailureequal "E: The repository 'file: unstable Release' is no longer signed." aptget update -qq
 
     # test that security downgrade was not successful
     testfileequal lists.before "$(listcurrentlistsdirectory)"
@@ -101,7 +101,7 @@ test_inrelease_to_release_reverts_all() {
     break_repository_sources_index '+1hour'
 
     # ensure error
-    testfailureequal "E: The repository 'file: unstable Release.gpg' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1
+    testfailureequal "E: The repository 'file: unstable Release' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1
 
     # ensure that the Packages file is also rolled back
     testfileequal lists.before "$(listcurrentlistsdirectory)"
diff --git a/test/integration/test-releasefile-date-older b/test/integration/test-releasefile-date-older
new file mode 100755
index 000000000..5cdc34fac
--- /dev/null
+++ b/test/integration/test-releasefile-date-older
@@ -0,0 +1,62 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'i386'
+
+insertpackage 'wheezy' 'apt' 'all' '0.8.15'
+
+setupaptarchive --no-update
+
+# we don't complain as the server could have just sent a 'Hit' here and this
+# 'downgrade attack' is usually performed by out-of-sync mirrors. Valid-Until
+# catches the 'real' downgrade attacks (expect that it finds stale mirrors).
+# Scaring users with an error here serves hence no point.
+
+msgmsg 'InRelease file is silently rejected if' 'new Date is before old Date'
+rm -rf rootdir/var/lib/apt/lists
+generatereleasefiles 'now' 'now + 7 days'
+signreleasefiles
+testsuccess aptget update
+listcurrentlistsdirectory > listsdir.lst
+redatereleasefiles 'now - 2 days'
+testsuccess aptget update
+testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+msgmsg 'Release.gpg file is silently rejected if' 'new Date is before old Date'
+rm -rf rootdir/var/lib/apt/lists
+generatereleasefiles 'now' 'now + 7 days'
+signreleasefiles
+find aptarchive -name 'InRelease' -delete
+testsuccess aptget update
+listcurrentlistsdirectory > listsdir.lst
+redatereleasefiles 'now - 2 days'
+find aptarchive -name 'InRelease' -delete
+testsuccess aptget update
+testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+msgmsg 'Crisscross InRelease/Release.gpg file is silently rejected if' 'new Date is before old Date'
+rm -rf rootdir/var/lib/apt/lists
+generatereleasefiles 'now' 'now + 7 days'
+signreleasefiles
+find aptarchive -name 'Release.gpg' -delete
+testsuccess aptget update
+listcurrentlistsdirectory > listsdir.lst
+redatereleasefiles 'now - 2 days'
+find aptarchive -name 'InRelease' -delete
+testsuccess aptget update
+testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+msgmsg 'Crisscross Release.gpg/InRelease file is silently rejected if' 'new Date is before old Date'
+rm -rf rootdir/var/lib/apt/lists
+generatereleasefiles 'now' 'now + 7 days'
+signreleasefiles
+find aptarchive -name 'InRelease' -delete
+testsuccess aptget update
+listcurrentlistsdirectory > listsdir.lst
+redatereleasefiles 'now - 2 days'
+find aptarchive -name 'Release.gpg' -delete
+testsuccess aptget update
+testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
diff --git a/test/integration/test-releasefile-valid-until b/test/integration/test-releasefile-valid-until
index 0d9a91254..e000abf5d 100755
--- a/test/integration/test-releasefile-valid-until
+++ b/test/integration/test-releasefile-valid-until
@@ -16,13 +16,12 @@ setupaptarchive --no-update
 
 runtest() {
 	local MSG="$1"
-	msgtest "$1" "$2"
+	msgtest "Release file is $MSG as it has" "$2"
 	rm -rf rootdir/var/lib/apt/lists
-	aptget clean
 	generatereleasefiles "$3" "$4"
 	signreleasefiles
 	shift 4
-	if expr match "$MSG" '.*accepted.*' >/dev/null; then
+	if [ "$MSG" = 'accepted' ]; then
 		testsuccess --nomsg aptget update "$@"
 		testfailure grep -q 'is expired' rootdir/tmp/testsuccess.output
 	else
@@ -31,19 +30,19 @@ runtest() {
 	fi
 }
 
-runtest 'Release file is accepted as it has' 'no Until' '' ''
-runtest 'Release file is accepted as it has' 'no Until and good Max-Valid' '' '' -o Acquire::Max-ValidTime=3600
-runtest 'Release file is rejected as it has' 'no Until, but bad Max-Valid' 'now - 2 days' '' -o Acquire::Max-ValidTime=3600
-runtest 'Release file is accepted as it has' 'good Until' 'now - 3 days' 'now + 1 day'
-runtest 'Release file is rejected as it has' 'bad Until' 'now - 7 days' 'now - 4 days'
-runtest 'Release file is rejected as it has' 'bad Until (ignore good Max-Valid)' 'now - 7 days' 'now - 4 days' -o Acquire::Max-ValidTime=1209600
-runtest 'Release file is rejected as it has' 'bad Max-Valid (bad Until)' 'now - 7 days' 'now - 4 days' -o Acquire::Max-ValidTime=86400
-runtest 'Release file is rejected as it has' 'bad Max-Valid (good Until)' 'now - 7 days' 'now + 4 days' -o Acquire::Max-ValidTime=86400
-runtest 'Release file is accepted as it has' 'good labeled Max-Valid' 'now - 7 days' 'now + 4 days' -o Acquire::Max-ValidTime=86400 -o Acquire::Max-ValidTime::Testcases=1209600
-runtest 'Release file is rejected as it has' 'bad labeled Max-Valid' 'now - 7 days' 'now + 4 days' -o Acquire::Max-ValidTime=1209600 -o Acquire::Max-ValidTime::Testcases=86400
-runtest 'Release file is accepted as it has' 'good Until (good Min-Valid, no Max-Valid)' 'now - 7 days' 'now + 1 days' -o Acquire::Min-ValidTime=1209600
-runtest 'Release file is accepted as it has' 'good Min-Valid (bad Until, no Max-Valid)' 'now - 7 days' 'now - 4 days' -o Acquire::Min-ValidTime=1209600
-runtest 'Release file is accepted as it has' 'good Min-Valid (bad Until, good Max-Valid) <' 'now - 7 days' 'now - 2 days' -o Acquire::Min-ValidTime=1209600 -o Acquire::Max-ValidTime=2419200
-runtest 'Release file is rejected as it has' 'bad Max-Valid (bad Until, good Min-Valid) >' 'now - 7 days' 'now - 2 days' -o Acquire::Max-ValidTime=12096 -o Acquire::Min-ValidTime=2419200
-runtest 'Release file is rejected as it has' 'bad Max-Valid (bad Until, bad Min-Valid) <' 'now - 7 days' 'now - 2 days' -o Acquire::Min-ValidTime=12096 -o Acquire::Max-ValidTime=241920
-runtest 'Release file is rejected as it has' 'bad Max-Valid (bad Until, bad Min-Valid) >' 'now - 7 days' 'now - 2 days' -o Acquire::Max-ValidTime=12096 -o Acquire::Min-ValidTime=241920
+runtest 'accepted' 'no Until' '' ''
+runtest 'accepted' 'no Until and good Max-Valid' '' '' -o Acquire::Max-ValidTime=3600
+runtest 'rejected' 'no Until, but bad Max-Valid' 'now - 2 days' '' -o Acquire::Max-ValidTime=3600
+runtest 'accepted' 'good Until' 'now - 3 days' 'now + 1 day'
+runtest 'rejected' 'bad Until' 'now - 7 days' 'now - 4 days'
+runtest 'rejected' 'bad Until (ignore good Max-Valid)' 'now - 7 days' 'now - 4 days' -o Acquire::Max-ValidTime=1209600
+runtest 'rejected' 'bad Max-Valid (bad Until)' 'now - 7 days' 'now - 4 days' -o Acquire::Max-ValidTime=86400
+runtest 'rejected' 'bad Max-Valid (good Until)' 'now - 7 days' 'now + 4 days' -o Acquire::Max-ValidTime=86400
+runtest 'accepted' 'good labeled Max-Valid' 'now - 7 days' 'now + 4 days' -o Acquire::Max-ValidTime=86400 -o Acquire::Max-ValidTime::Testcases=1209600
+runtest 'rejected' 'bad labeled Max-Valid' 'now - 7 days' 'now + 4 days' -o Acquire::Max-ValidTime=1209600 -o Acquire::Max-ValidTime::Testcases=86400
+runtest 'accepted' 'good Until (good Min-Valid, no Max-Valid)' 'now - 7 days' 'now + 1 days' -o Acquire::Min-ValidTime=1209600
+runtest 'accepted' 'good Min-Valid (bad Until, no Max-Valid)' 'now - 7 days' 'now - 4 days' -o Acquire::Min-ValidTime=1209600
+runtest 'accepted' 'good Min-Valid (bad Until, good Max-Valid) <' 'now - 7 days' 'now - 2 days' -o Acquire::Min-ValidTime=1209600 -o Acquire::Max-ValidTime=2419200
+runtest 'rejected' 'bad Max-Valid (bad Until, good Min-Valid) >' 'now - 7 days' 'now - 2 days' -o Acquire::Max-ValidTime=12096 -o Acquire::Min-ValidTime=2419200
+runtest 'rejected' 'bad Max-Valid (bad Until, bad Min-Valid) <' 'now - 7 days' 'now - 2 days' -o Acquire::Min-ValidTime=12096 -o Acquire::Max-ValidTime=241920
+runtest 'rejected' 'bad Max-Valid (bad Until, bad Min-Valid) >' 'now - 7 days' 'now - 2 days' -o Acquire::Max-ValidTime=12096 -o Acquire::Min-ValidTime=241920
diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification
index 363b7fe5b..469ed34d2 100755
--- a/test/integration/test-releasefile-verification
+++ b/test/integration/test-releasefile-verification
@@ -91,25 +91,9 @@ touch aptarchive/apt.deb
 
 PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')"
 
-updatesuccess() {
-	local LOG='update.log'
-	if aptget update >$LOG 2>&1 || grep -q -E '^(W|E): ' $LOG; then
-		msgpass
-	else
-		cat $LOG
-		msgfail
-	fi
-}
-
-updatefailure() {
-	local LOG='update.log'
-	aptget update >$LOG 2>&1 || true
-	if grep -q -E "$1" $LOG; then
-		msgpass
-	else
-		cat $LOG
-		msgfail
-	fi
+updatewithwarnings() {
+	testwarning aptget update
+	testsuccess grep -E "$1" rootdir/tmp/testwarning.output
 }
 
 runtest() {
@@ -117,8 +101,8 @@ runtest() {
 	rm -rf rootdir/var/lib/apt/lists
 	signreleasefiles 'Joe Sixpack'
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Cold archive signed by' 'Joe Sixpack'
-	updatesuccess
+	msgmsg 'Cold archive signed by' 'Joe Sixpack'
+	testsuccess aptget update
 	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
@@ -126,8 +110,8 @@ runtest() {
 	prepare ${PKGFILE}-new
 	signreleasefiles 'Joe Sixpack'
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Good warm archive signed by' 'Joe Sixpack'
-	updatesuccess
+	msgmsg 'Good warm archive signed by' 'Joe Sixpack'
+	testsuccess aptget update
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
@@ -137,8 +121,8 @@ runtest() {
 	cp keys/rexexpired.pub rootdir/etc/apt/trusted.gpg.d/rexexpired.gpg
 	signreleasefiles 'Rex Expired'
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Cold archive signed by' 'Rex Expired'
-	updatefailure '^W: .* KEYEXPIRED'
+	msgmsg 'Cold archive signed by' 'Rex Expired'
+	updatewithwarnings '^W: .* KEYEXPIRED'
 	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	failaptold
@@ -148,8 +132,8 @@ runtest() {
 	rm -rf rootdir/var/lib/apt/lists
 	signreleasefiles 'Marvin Paranoid'
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Cold archive signed by' 'Marvin Paranoid'
-	updatefailure '^W: .* NO_PUBKEY'
+	msgmsg 'Cold archive signed by' 'Marvin Paranoid'
+	updatewithwarnings '^W: .* NO_PUBKEY'
 	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	failaptold
@@ -162,8 +146,8 @@ runtest() {
 	done
 	signreleasefiles 'Joe Sixpack'
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Bad warm archive signed by' 'Joe Sixpack'
-	updatesuccess
+	msgmsg 'Bad warm archive signed by' 'Joe Sixpack'
+	testsuccess aptget update
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
@@ -173,8 +157,8 @@ runtest() {
 	rm -rf rootdir/var/lib/apt/lists
 	signreleasefiles 'Joe Sixpack'
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Cold archive signed by' 'Joe Sixpack'
-	updatesuccess
+	msgmsg 'Cold archive signed by' 'Joe Sixpack'
+	testsuccess aptget update
 	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
@@ -182,8 +166,8 @@ runtest() {
 	prepare ${PKGFILE}-new
 	signreleasefiles 'Marvin Paranoid'
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Good warm archive signed by' 'Marvin Paranoid'
-	updatefailure '^W: .* NO_PUBKEY'
+	msgmsg 'Good warm archive signed by' 'Marvin Paranoid'
+	updatewithwarnings '^W: .* NO_PUBKEY'
 	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
@@ -192,8 +176,8 @@ runtest() {
 	cp keys/rexexpired.pub rootdir/etc/apt/trusted.gpg.d/rexexpired.gpg
 	signreleasefiles 'Rex Expired'
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Good warm archive signed by' 'Rex Expired'
-	updatefailure '^W: .* KEYEXPIRED'
+	msgmsg 'Good warm archive signed by' 'Rex Expired'
+	updatewithwarnings '^W: .* KEYEXPIRED'
 	testsuccessequal "$(cat ${PKGFILE})
 " aptcache show apt
 	installaptold
@@ -202,8 +186,8 @@ runtest() {
 	prepare ${PKGFILE}-new
 	signreleasefiles
 	find aptarchive/ -name "$DELETEFILE" -delete
-	msgtest 'Good warm archive signed by' 'Joe Sixpack'
-	updatesuccess
+	msgmsg 'Good warm archive signed by' 'Joe Sixpack'
+	testsuccess aptget update
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	installaptnew
@@ -213,24 +197,24 @@ runtest2() {
 	prepare ${PKGFILE}
 	rm -rf rootdir/var/lib/apt/lists
 	signreleasefiles 'Joe Sixpack'
-	msgtest 'Cold archive signed by' 'Joe Sixpack'
-	updatesuccess
+	msgmsg 'Cold archive signed by' 'Joe Sixpack'
+	testsuccess aptget update
 
 	# New .deb but now an unsigned archive. For example MITM to circumvent
 	# package verification.
 	prepare ${PKGFILE}-new
 	find aptarchive/ -name InRelease -delete
 	find aptarchive/ -name Release.gpg -delete
-	msgtest 'Warm archive signed by' 'nobody'
-	updatesuccess
+	msgmsg 'Warm archive signed by' 'nobody'
+	updatewithwarnings 'W: .* no longer signed.'
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	failaptnew
 
 	# Unsigned archive from the beginning must also be detected.
 	rm -rf rootdir/var/lib/apt/lists
-	msgtest 'Cold archive signed by' 'nobody'
-	updatesuccess
+	msgmsg 'Cold archive signed by' 'nobody'
+	updatewithwarnings 'W: .* is not signed.'
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt
 	failaptnew
-- 
cgit v1.2.3-70-g09d2


From ceafe8a6edc815df2923ba892894617829e9d3c2 Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Fri, 22 May 2015 15:28:53 +0200
Subject: Fix endless loop in apt-get update that can cause disk fillup

The apt http code parses Content-Length and Content-Range. For
both requests the variable "Size" is used and the semantic for
this Size is the total file size. However Content-Length is not
the entire file size for partital file requests. For servers that
send the Content-Range header first and then the Content-Length
header this can lead to globbing of Size so that its less than
the real file size. This may lead to a subsequent passing of a
negative number into the CircleBuf which leads to a endless
loop that writes data.

Thanks to Anton Blanchard for the analysis and initial patch.

LP: #1445239
---
 methods/http.cc                         |  2 +-
 methods/server.cc                       | 20 +++++++++++++++-----
 methods/server.h                        |  3 ++-
 test/interactive-helper/aptwebserver.cc |  6 +++---
 4 files changed, 21 insertions(+), 10 deletions(-)

(limited to 'test')

diff --git a/methods/http.cc b/methods/http.cc
index 1b996db98..ad90c9891 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -443,7 +443,7 @@ bool HttpServerState::RunData(FileFd * const File)
       else if (JunkSize != 0)
 	 In.Limit(JunkSize);
       else
-	 In.Limit(Size - StartPos);
+	 In.Limit(DownloadSize);
       
       // Just transfer the whole block.
       do
diff --git a/methods/server.cc b/methods/server.cc
index e321e0230..ba0a8864b 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -164,15 +164,22 @@ bool ServerState::HeaderLine(string Line)
 	 Encoding = Stream;
       HaveContent = true;
 
-      unsigned long long * SizePtr = &Size;
+      unsigned long long * DownloadSizePtr = &DownloadSize;
       if (Result == 416)
-	 SizePtr = &JunkSize;
+	 DownloadSizePtr = &JunkSize;
 
-      *SizePtr = strtoull(Val.c_str(), NULL, 10);
-      if (*SizePtr >= std::numeric_limits::max())
+      *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
+      if (*DownloadSizePtr >= std::numeric_limits::max())
 	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
-      else if (*SizePtr == 0)
+      else if (*DownloadSizePtr == 0)
 	 HaveContent = false;
+
+      // On partial content (206) the Content-Length less than the real
+      // size, so do not set it here but leave that to the Content-Range
+      // header instead
+      if(Result != 206 && Size == 0)
+         Size = DownloadSize;
+
       return true;
    }
 
@@ -193,6 +200,9 @@ bool ServerState::HeaderLine(string Line)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
 	 return _error->Error(_("This HTTP server has broken range support"));
+
+      // figure out what we will download
+      DownloadSize = Size - StartPos;
       return true;
    }
 
diff --git a/methods/server.h b/methods/server.h
index 1b81e3549..ed3cb456a 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,7 +34,8 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
-   unsigned long long Size; // size of the usable content (aka: the file)
+   unsigned long long Size; // total size of the usable content (aka: the file)
+   unsigned long long DownloadSize; // size we actually download (can be smaller than Size if we have partial content)
    unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
    time_t Date;
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index cd52da692..9c67b67e4 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -654,13 +654,13 @@ static void * handleClient(void * voidclient)				/*{{{*/
 		     if (filesize > filestart)
 		     {
 			data.Skip(filestart);
-			std::ostringstream contentlength;
-			contentlength << "Content-Length: " << (filesize - filestart);
-			headers.push_back(contentlength.str());
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes " << filestart << "-"
 			   << filesize - 1 << "/" << filesize;
 			headers.push_back(contentrange.str());
+			std::ostringstream contentlength;
+			contentlength << "Content-Length: " << (filesize - filestart);
+			headers.push_back(contentlength.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
 			   sendFile(client, headers, data);
-- 
cgit v1.2.3-70-g09d2


From f4a91278c65b156c1addf27b0b05b029692959e8 Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Fri, 22 May 2015 16:05:05 +0200
Subject: Add regression test for LP: #1445239

Add a regression test that reproduced the hang of apt when a
partial file is present.

Git-Dch: ignore
---
 test/integration/test-bug-lp1445239-download-loop | 29 +++++++++++++++++++++++
 test/interactive-helper/aptwebserver.cc           |  2 ++
 2 files changed, 31 insertions(+)
 create mode 100755 test/integration/test-bug-lp1445239-download-loop

(limited to 'test')

diff --git a/test/integration/test-bug-lp1445239-download-loop b/test/integration/test-bug-lp1445239-download-loop
new file mode 100755
index 000000000..342e3c61e
--- /dev/null
+++ b/test/integration/test-bug-lp1445239-download-loop
@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# this is a regression test for LP: #1445239 where a partial download can
+# trigger an endless hang of the download method
+#
+
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+changetowebserver
+webserverconfig 'aptwebserver::support::range' 'true'
+
+TESTFILE='aptarchive/testfile'
+dd if=/dev/zero of=$TESTFILE bs=100k count=1 2>/dev/null
+
+DOWNLOADLOG='rootdir/tmp/testdownloadfile.log'
+
+TARGET=testfile-downloaded
+dd if=/dev/zero of=$TARGET bs=99k count=1 2>/dev/null
+if ! downloadfile http://localhost:8080/testfile "$TARGET" > "$DOWNLOADLOG"; then
+    cat >&2 "$DOWNLOADLOG"
+    msgfail
+else
+    msgpass
+fi
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 9c67b67e4..e02caa2b3 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -654,6 +654,8 @@ static void * handleClient(void * voidclient)				/*{{{*/
 		     if (filesize > filestart)
 		     {
 			data.Skip(filestart);
+                        // make sure to send content-range before conent-length
+                        // as regression test for LP: #1445239
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes " << filestart << "-"
 			   << filesize - 1 << "/" << filesize;
-- 
cgit v1.2.3-70-g09d2


From 58702f8563a443a7c6e66253b259c2488b877290 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 19 May 2015 10:40:55 +0200
Subject: don't try other compressions on hashsum mismatch
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If we e.g. fail on hash verification for Packages.xz its highly unlikely
that it will be any better with Packages.gz, so we just waste download
bandwidth and time. It also causes us always to fallback to the
uncompressed Packages file for which the error will finally be reported,
which in turn confuses users as the file usually doesn't exist on the
mirrors, so a bug in apt is suspected for even trying it…
---
 apt-pkg/acquire-item.cc                       | 33 +++++++++++++++++++--------
 test/integration/test-apt-update-not-modified | 23 +++++++++++++++++--
 test/integration/test-apt-update-rollback     |  4 ++--
 test/integration/test-apt-update-transactions |  8 +++++--
 4 files changed, 52 insertions(+), 16 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 8155b9bfe..cf89717c4 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -154,7 +154,18 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 	 return;
       }
 
-      Status = StatError;
+      switch (Status)
+      {
+	 case StatIdle:
+	 case StatFetching:
+	 case StatDone:
+	    Status = StatError;
+	    break;
+	 case StatAuthError:
+	 case StatError:
+	 case StatTransientNetworkError:
+	    break;
+      }
       Complete = false;
       Dequeue();
    }
@@ -167,7 +178,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       RenameOnError(MaximumSizeExceeded);
 
    // report mirror failure back to LP if we actually use a mirror
-   if(FailReason.size() != 0)
+   if(FailReason.empty() == false)
       ReportMirrorFailure(FailReason);
    else
       ReportMirrorFailure(ErrorText);
@@ -1403,17 +1414,19 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
    Item::Failed(Message,Cnf);
 
-   size_t const nextExt = CompressionExtensions.find(' ');
-   if (nextExt != std::string::npos)
+   // authorisation matches will not be fixed by other compression types
+   if (Status != StatAuthError)
    {
-      CompressionExtensions = CompressionExtensions.substr(nextExt+1);
-      Init(RealURI, Desc.Description, Desc.ShortDesc);
-      Status = StatIdle;
-      return;
+      size_t const nextExt = CompressionExtensions.find(' ');
+      if (nextExt != std::string::npos)
+      {
+	 CompressionExtensions = CompressionExtensions.substr(nextExt+1);
+	 Init(RealURI, Desc.Description, Desc.ShortDesc);
+	 Status = StatIdle;
+	 return;
+      }
    }
 
-   Item::Failed(Message,Cnf);
-
    if(Target->IsOptional() && ExpectedHashes.empty() && Stage == STAGE_DOWNLOAD)
       Status = StatDone;
    else
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
index b1d55c156..bac33d531 100755
--- a/test/integration/test-apt-update-not-modified
+++ b/test/integration/test-apt-update-not-modified
@@ -14,6 +14,7 @@ setupaptarchive --no-update
 methodtest() {
 	msgmsg 'Test InRelease with' "$1"
 	rm -rf rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists  aptarchive/dists.good
 	# get our cache populated
 	testsuccess aptget update
 	listcurrentlistsdirectory > listsdir.lst
@@ -30,8 +31,27 @@ Reading package lists..." aptget update
 Reading package lists..." aptget update
 	testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)"
 
-	# readd arch so its downloaded again
+	# readd arch so its downloaded again…
 	configarchitecture 'amd64' 'i386'
+	# … but oh noes, hashsum mismatch!
+	find aptarchive/dists/unstable/main/binary-amd64/ -type f -delete
+	cat >> aptarchive/dists/unstable/main/binary-amd64/Packages < "$1" < listsdir.lst
 	redatereleasefiles '+1hour'
 	testrun 'listsdir.lst'
-- 
cgit v1.2.3-70-g09d2


From 448c38bdcd72b52f11ec5f326f822cf57653f81c Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 6 Jun 2015 12:28:00 +0200
Subject: rework hashsum verification in the acquire system
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Having every item having its own code to verify the file(s) it handles
is an errorprune process and easy to break, especially if items move
through various stages (download, uncompress, patching, …). With a giant
rework we centralize (most of) the verification to have a better
enforcement rate and (hopefully) less chance for bugs, but it breaks the
ABI bigtime in exchange – and as we break it anyway, it is broken even
harder.

It shouldn't effect most frontends as they don't deal with the acquire
system at all or implement their own items, but some do and will need to
be patched (might be an opportunity to use apt on-board material).

The theory is simple: Items implement methods to decide if hashes need to
be checked (in this stage) and to return the expected hashes for this
item (in this stage). The verification itself is done in worker message
passing which has the benefit that a hashsum error is now a proper error
for the acquire system rather than a Done() which is later revised to a
Failed().
---
 apt-pkg/acquire-item.cc                            | 3750 ++++++++++----------
 apt-pkg/acquire-item.h                             |  677 ++--
 apt-pkg/acquire-method.cc                          |    5 +-
 apt-pkg/acquire-worker.cc                          |  189 +-
 apt-pkg/contrib/hashes.cc                          |   10 +
 apt-pkg/contrib/hashes.h                           |    9 +
 apt-pkg/deb/debindexfile.cc                        |    4 +-
 apt-pkg/deb/debmetaindex.cc                        |   78 +-
 apt-pkg/indexrecords.cc                            |    2 +-
 apt-pkg/pkgcache.h                                 |    7 +-
 methods/file.cc                                    |    6 +-
 test/integration/test-apt-get-source-authenticated |    2 +-
 test/integration/test-apt-sources-deb822           |   55 +-
 test/integration/test-apt-update-expected-size     |    2 +-
 test/integration/test-apt-update-file              |    4 +
 test/integration/test-apt-update-nofallback        |   11 +-
 test/integration/test-apt-update-not-modified      |   30 +-
 test/integration/test-apt-update-stale             |    9 +-
 .../test-bug-595691-empty-and-broken-archive-files |    2 +-
 .../test-ubuntu-bug-1098738-apt-get-source-md5sum  |    8 +
 test/libapt/acqprogress_test.cc                    |    6 +-
 test/libapt/hashsums_test.cc                       |    3 +
 22 files changed, 2411 insertions(+), 2458 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index cf89717c4..ec6ec6e84 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -107,36 +107,296 @@ static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser
 }
 									/*}}}*/
 
+// all ::HashesRequired and ::GetExpectedHashes implementations		/*{{{*/
+/* ::GetExpectedHashes is abstract and has to be implemented by all subclasses.
+   It is best to implement it as broadly as possible, while ::HashesRequired defaults
+   to true and should be as restrictive as possible for false cases. Note that if
+   a hash is returned by ::GetExpectedHashes it must match. Only if it doesn't
+   ::HashesRequired is called to evaluate if its okay to have no hashes. */
+APT_CONST bool pkgAcqTransactionItem::HashesRequired() const
+{
+   /* signed repositories obviously have a parser and good hashes.
+      unsigned repositories, too, as even if we can't trust them for security,
+      we can at least trust them for integrity of the download itself.
+      Only repositories without a Release file can (obviously) not have
+      hashes – and they are very uncommon and strongly discouraged */
+   return TransactionManager->MetaIndexParser != NULL;
+}
+HashStringList pkgAcqTransactionItem::GetExpectedHashes() const
+{
+   return GetExpectedHashesFor(GetMetaKey());
+}
+
+APT_CONST bool pkgAcqMetaBase::HashesRequired() const
+{
+   // Release and co have no hashes 'by design'.
+   return false;
+}
+HashStringList pkgAcqMetaBase::GetExpectedHashes() const
+{
+   return HashStringList();
+}
+
+APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
+{
+   /* FIXME: We have only hashes for uncompressed pdiffs.
+      rred uncompresses them on the fly while parsing.
+      In StateFetchDiff state we also uncompress on the fly for hash check.
+      Hashes are checked while searching for (next) patch to apply. */
+   return false;
+}
+HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
+{
+   return HashStringList();
+}
+
+APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
+{
+   /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
+      we can check the rred result after all patches are applied as
+      we know the expected result rather than potentially apply more patches */
+   return State == StateApplyDiff;
+}
+HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
+{
+   if (State == StateApplyDiff)
+      return GetExpectedHashesFor(Target->MetaKey);
+   return HashStringList();
+}
+
+APT_CONST bool pkgAcqArchive::HashesRequired() const
+{
+   return LocalSource == false;
+}
+HashStringList pkgAcqArchive::GetExpectedHashes() const
+{
+   // figured out while parsing the records
+   return ExpectedHashes;
+}
+
+APT_CONST bool pkgAcqFile::HashesRequired() const
+{
+   // supplied as parameter at creation time, so the caller decides
+   return ExpectedHashes.usable();
+}
+HashStringList pkgAcqFile::GetExpectedHashes() const
+{
+   return ExpectedHashes;
+}
+									/*}}}*/
+// Acquire::Item::QueueURI and specialisations from child classes	/*{{{*/
+bool pkgAcquire::Item::QueueURI(pkgAcquire::ItemDesc &Item)
+{
+   Owner->Enqueue(Item);
+   return true;
+}
+/* The idea here is that an item isn't queued if it exists on disk and the
+   transition manager was a hit as this means that the files it contains
+   the checksums for can't be updated either (or they are and we are asking
+   for a hashsum mismatch to happen which helps nobody) */
+bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
+{
+   std::string const FinalFile = GetFinalFilename();
+   if (TransactionManager != NULL && TransactionManager->IMSHit == true &&
+	 FileExists(FinalFile) == true)
+   {
+      PartialFile = DestFile = FinalFile;
+      Status = StatDone;
+      return false;
+   }
+   return pkgAcquire::Item::QueueURI(Item);
+}
+/* The transition manager InRelease itself (or its older sisters-in-law
+   Release & Release.gpg) is always queued as this allows us to rerun gpgv
+   on it to verify that we aren't stalled with old files */
+bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item)
+{
+   return pkgAcquire::Item::QueueURI(Item);
+}
+/* the Diff/Index needs to queue also the up-to-date complete index file
+   to ensure that the list cleaner isn't eating it */
+bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item)
+{
+   if (pkgAcqTransactionItem::QueueURI(Item) == true)
+      return true;
+   QueueOnIMSHit();
+   return false;
+}
+									/*}}}*/
+// Acquire::Item::GetFinalFilename and specialisations for child classes	/*{{{*/
+std::string pkgAcquire::Item::GetFinalFilename() const
+{
+   return GetFinalFileNameFromURI(Desc.URI);
+}
+std::string pkgAcqDiffIndex::GetFinalFilename() const
+{
+   // the logic we inherent from pkgAcqBaseIndex isn't what we need here
+   return pkgAcquire::Item::GetFinalFilename();
+}
+std::string pkgAcqIndex::GetFinalFilename() const
+{
+   std::string const FinalFile = GetFinalFileNameFromURI(Target->URI);
+   return GetCompressedFileName(Target->URI, FinalFile, CurrentCompressionExtension);
+}
+std::string pkgAcqMetaSig::GetFinalFilename() const
+{
+   return GetFinalFileNameFromURI(Target->URI);
+}
+std::string pkgAcqBaseIndex::GetFinalFilename() const
+{
+   return GetFinalFileNameFromURI(Target->URI);
+}
+std::string pkgAcqMetaBase::GetFinalFilename() const
+{
+   return GetFinalFileNameFromURI(DataTarget.URI);
+}
+std::string pkgAcqArchive::GetFinalFilename() const
+{
+   return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
+}
+									/*}}}*/
+// pkgAcqTransactionItem::GetMetaKey and specialisations for child classes	/*{{{*/
+std::string pkgAcqTransactionItem::GetMetaKey() const
+{
+   return Target->MetaKey;
+}
+std::string pkgAcqIndex::GetMetaKey() const
+{
+   if (Stage == STAGE_DECOMPRESS_AND_VERIFY || CurrentCompressionExtension == "uncompressed")
+      return Target->MetaKey;
+   return Target->MetaKey + "." + CurrentCompressionExtension;
+}
+std::string pkgAcqDiffIndex::GetMetaKey() const
+{
+   return Target->MetaKey + ".diff/Index";
+}
+									/*}}}*/
+//pkgAcqTransactionItem::TransactionState and specialisations for child classes	/*{{{*/
+bool pkgAcqTransactionItem::TransactionState(TransactionStates const state)
+{
+   bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
+   switch(state)
+   {
+      case TransactionAbort:
+	 if(Debug == true)
+	    std::clog << "  Cancel: " << DestFile << std::endl;
+	 if (Status == pkgAcquire::Item::StatIdle)
+	 {
+	    Status = pkgAcquire::Item::StatDone;
+	    Dequeue();
+	 }
+	 break;
+      case TransactionCommit:
+	 if(PartialFile != "")
+	 {
+	    if(Debug == true)
+	       std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl;
+
+	    Rename(PartialFile, DestFile);
+	 } else {
+	    if(Debug == true)
+	       std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
+	    unlink(DestFile.c_str());
+	 }
+	 break;
+   }
+   return true;
+}
+bool pkgAcqMetaBase::TransactionState(TransactionStates const state)
+{
+   // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
+   if (TransactionManager->IMSHit == false)
+      return pkgAcqTransactionItem::TransactionState(state);
+   return true;
+}
+bool pkgAcqIndex::TransactionState(TransactionStates const state)
+{
+   if (pkgAcqTransactionItem::TransactionState(state) == false)
+      return false;
+
+   switch (state)
+   {
+      case TransactionAbort:
+	 if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
+	 {
+	    // keep the compressed file, but drop the decompressed
+	    EraseFileName.clear();
+	    if (PartialFile.empty() == false && flExtension(PartialFile) == "decomp")
+	       unlink(PartialFile.c_str());
+	 }
+	 break;
+      case TransactionCommit:
+	 if (EraseFileName.empty() == false)
+	    unlink(EraseFileName.c_str());
+	 break;
+   }
+   return true;
+}
+bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)
+{
+   if (pkgAcqTransactionItem::TransactionState(state) == false)
+      return false;
+
+   switch (state)
+   {
+      case TransactionCommit:
+	 break;
+      case TransactionAbort:
+	 std::string const Partial = GetPartialFileNameFromURI(Target->URI);
+	 unlink(Partial.c_str());
+	 break;
+   }
+
+   return true;
+}
+									/*}}}*/
 
 // Acquire::Item::Item - Constructor					/*{{{*/
 APT_IGNORE_DEPRECATED_PUSH
-pkgAcquire::Item::Item(pkgAcquire *Owner,
-                       HashStringList const &ExpectedHashes,
-                       pkgAcqMetaBase *TransactionManager)
-   : Owner(Owner), FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false),
-     Local(false), QueueCounter(0), TransactionManager(TransactionManager),
-     ExpectedAdditionalItems(0), ExpectedHashes(ExpectedHashes)
+pkgAcquire::Item::Item(pkgAcquire * const Owner) :
+   FileSize(0), PartialSize(0), Mode(0), Complete(false), Local(false),
+    QueueCounter(0), ExpectedAdditionalItems(0), Owner(Owner)
 {
    Owner->Add(this);
    Status = StatIdle;
-   if(TransactionManager != NULL)
-      TransactionManager->Add(this);
 }
 APT_IGNORE_DEPRECATED_POP
 									/*}}}*/
 // Acquire::Item::~Item - Destructor					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 pkgAcquire::Item::~Item()
 {
    Owner->Remove(this);
 }
 									/*}}}*/
+std::string pkgAcquire::Item::Custom600Headers() const			/*{{{*/
+{
+   return std::string();
+}
+									/*}}}*/
+std::string pkgAcquire::Item::ShortDesc() const				/*{{{*/
+{
+   return DescURI();
+}
+									/*}}}*/
+APT_CONST void pkgAcquire::Item::Finished()				/*{{{*/
+{
+}
+									/*}}}*/
+APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const		/*{{{*/
+{
+   return Owner;
+}
+									/*}}}*/
+APT_CONST bool pkgAcquire::Item::IsTrusted() const			/*{{{*/
+{
+   return false;
+}
+									/*}}}*/
 // Acquire::Item::Failed - Item failed to download			/*{{{*/
 // ---------------------------------------------------------------------
 /* We return to an idle state if there are still other queues that could
    fetch this object */
-void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
+void pkgAcquire::Item::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
 {
    if(ErrorText.empty())
       ErrorText = LookupTag(Message,"Message");
@@ -169,59 +429,28 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       Complete = false;
       Dequeue();
    }
-   else
-      Status = StatIdle;
 
-   // check fail reason
    string const FailReason = LookupTag(Message, "FailReason");
-   if(FailReason == "MaximumSizeExceeded")
+   if (FailReason == "MaximumSizeExceeded")
       RenameOnError(MaximumSizeExceeded);
+   else if (Status == StatAuthError)
+      RenameOnError(HashSumMismatch);
 
    // report mirror failure back to LP if we actually use a mirror
-   if(FailReason.empty() == false)
+   if (FailReason.empty() == false)
       ReportMirrorFailure(FailReason);
    else
       ReportMirrorFailure(ErrorText);
-}
-									/*}}}*/
-bool pkgAcquire::Item::TransactionState(TransactionStates const state)	/*{{{*/
-{
-   bool const Debug = _config->FindB("Debug::Acquire::Transaction", false);
-   switch(state)
-   {
-      case TransactionAbort:
-	 if(Debug == true)
-	    std::clog << "  Cancel: " << DestFile << std::endl;
-	 if (Status == pkgAcquire::Item::StatIdle)
-	 {
-	    Status = pkgAcquire::Item::StatDone;
-	    Dequeue();
-	 }
-	 break;
-      case TransactionCommit:
-	 if(PartialFile != "")
-	 {
-	    if(Debug == true)
-	       std::clog << "mv " << PartialFile << " -> "<< DestFile << " # " << DescURI() << std::endl;
 
-	    Rename(PartialFile, DestFile);
-	 } else {
-	    if(Debug == true)
-	       std::clog << "rm " << DestFile << " # " << DescURI() << std::endl;
-	    unlink(DestFile.c_str());
-	 }
-	 // mark that this transaction is finished
-	 TransactionManager = 0;
-	 break;
-   }
-   return true;
+   if (QueueCounter > 1)
+      Status = StatIdle;
 }
 									/*}}}*/
 // Acquire::Item::Start - Item has begun to download			/*{{{*/
 // ---------------------------------------------------------------------
-/* Stash status and the file size. Note that setting Complete means 
+/* Stash status and the file size. Note that setting Complete means
    sub-phases of the acquire process such as decompresion are operating */
-void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
+void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const Size)
 {
    Status = StatFetching;
    ErrorText.clear();
@@ -230,22 +459,24 @@ void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 }
 									/*}}}*/
 // Acquire::Item::Done - Item downloaded OK				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/,
-			    pkgAcquire::MethodConfig * /*Cnf*/)
+void pkgAcquire::Item::Done(string const &Message, HashStringList const &Hashes,
+			    pkgAcquire::MethodConfig const * const /*Cnf*/)
 {
    // We just downloaded something..
    string FileName = LookupTag(Message,"Filename");
    UsedMirror = LookupTag(Message,"UsedMirror");
-   if (Complete == false && !Local && FileName == DestFile)
+   unsigned long long const downloadedSize = Hashes.FileSize();
+   if (downloadedSize != 0)
    {
-      if (Owner->Log != 0)
-	 Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str()));
-   }
+      if (Complete == false && !Local && FileName == DestFile)
+      {
+	 if (Owner->Log != 0)
+	    Owner->Log->Fetched(Hashes.FileSize(),atoi(LookupTag(Message,"Resume-Point","0").c_str()));
+      }
 
-   if (FileSize == 0)
-      FileSize= Size;
+      if (FileSize == 0)
+	 FileSize= downloadedSize;
+   }
    Status = StatDone;
    ErrorText = string();
    Owner->Dequeue(this);
@@ -255,7 +486,7 @@ void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringLis
 // ---------------------------------------------------------------------
 /* This helper function is used by a lot of item methods as their final
    step */
-bool pkgAcquire::Item::Rename(string From,string To)
+bool pkgAcquire::Item::Rename(string const &From,string const &To)
 {
    if (From == To || rename(From.c_str(),To.c_str()) == 0)
       return true;
@@ -271,76 +502,40 @@ bool pkgAcquire::Item::Rename(string From,string To)
    return false;
 }
 									/*}}}*/
-// Acquire::Item::QueueURI and specialisations from child classes	/*{{{*/
-/* The idea here is that an item isn't queued if it exists on disk and the
-   transition manager was a hit as this means that the files it contains
-   the checksums for can't be updated either (or they are and we are asking
-   for a hashsum mismatch to happen which helps nobody) */
-bool pkgAcquire::Item::QueueURI(ItemDesc &Item)
+void pkgAcquire::Item::Dequeue()					/*{{{*/
 {
-   std::string const FinalFile = GetFinalFilename();
-   if (TransactionManager != NULL && TransactionManager->IMSHit == true &&
-	 FileExists(FinalFile) == true)
-   {
-      PartialFile = DestFile = FinalFile;
-      Status = StatDone;
-      return false;
-   }
-
-   Owner->Enqueue(Item);
-   return true;
+   Owner->Dequeue(this);
 }
-/* The transition manager InRelease itself (or its older sisters-in-law
-   Release & Release.gpg) is always queued as this allows us to rerun gpgv
-   on it to verify that we aren't stalled with old files */
-bool pkgAcqMetaBase::QueueURI(pkgAcquire::ItemDesc &Item)
-{
-   Owner->Enqueue(Item);
-   return true;
-}
-/* the Diff/Index needs to queue also the up-to-date complete index file
-   to ensure that the list cleaner isn't eating it */
-bool pkgAcqDiffIndex::QueueURI(pkgAcquire::ItemDesc &Item)
-{
-   if (pkgAcquire::Item::QueueURI(Item) == true)
-      return true;
-   QueueOnIMSHit();
-   return false;
-}
-									/*}}}*/
-void pkgAcquire::Item::Dequeue()					/*{{{*/
-{
-   Owner->Dequeue(this);
-}
-									/*}}}*/
-bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
+									/*}}}*/
+bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 {
    if (RealFileExists(DestFile))
       Rename(DestFile, DestFile + ".FAILED");
 
+   std::string errtext;
    switch (error)
    {
       case HashSumMismatch:
-	 ErrorText = _("Hash Sum mismatch");
+	 errtext = _("Hash Sum mismatch");
 	 Status = StatAuthError;
 	 ReportMirrorFailure("HashChecksumFailure");
 	 break;
       case SizeMismatch:
-	 ErrorText = _("Size mismatch");
+	 errtext = _("Size mismatch");
 	 Status = StatAuthError;
 	 ReportMirrorFailure("SizeFailure");
 	 break;
       case InvalidFormat:
-	 ErrorText = _("Invalid file format");
+	 errtext = _("Invalid file format");
 	 Status = StatError;
 	 // do not report as usually its not the mirrors fault, but Portal/Proxy
 	 break;
       case SignatureError:
-	 ErrorText = _("Signature error");
+	 errtext = _("Signature error");
 	 Status = StatError;
 	 break;
       case NotClearsigned:
-	 ErrorText = _("Does not start with a cleartext signature");
+	 errtext = _("Does not start with a cleartext signature");
 	 Status = StatError;
 	 break;
       case MaximumSizeExceeded:
@@ -351,6 +546,8 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const
 	 // no handling here, done by callers
 	 break;
    }
+   if (ErrorText.empty())
+      ErrorText = errtext;
    return false;
 }
 									/*}}}*/
@@ -360,15 +557,8 @@ void pkgAcquire::Item::SetActiveSubprocess(const std::string &subprocess)/*{{{*/
       APT_IGNORE_DEPRECATED(Mode = ActiveSubprocess.c_str();)
 }
 									/*}}}*/
-// Acquire::Item::GetFinalFilename - Return the full final file path	/*{{{*/
-std::string pkgAcquire::Item::GetFinalFilename() const
-{
-   return GetFinalFileNameFromURI(Desc.URI);
-}
-									/*}}}*/
 // Acquire::Item::ReportMirrorFailure					/*{{{*/
-// ---------------------------------------------------------------------
-void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
+void pkgAcquire::Item::ReportMirrorFailure(string const &FailCode)
 {
    // we only act if a mirror was used at all
    if(UsedMirror.empty())
@@ -411,2083 +601,1897 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
    }
 }
 									/*}}}*/
-// AcqDiffIndex::AcqDiffIndex - Constructor				/*{{{*/
-// ---------------------------------------------------------------------
-/* Get the DiffIndex file first and see if there are patches available
- * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
- * patches. If anything goes wrong in that process, it will fall back to
- * the original packages file
- */
-pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner,
-                                 pkgAcqMetaBase *TransactionManager,
-                                 IndexTarget const * const Target,
-				 HashStringList const &ExpectedHashes,
-                                 indexRecords *MetaIndexParser)
-   : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, 
-                     MetaIndexParser)
+std::string pkgAcquire::Item::HashSum() const				/*{{{*/
 {
-   
-   Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
-
-   RealURI = Target->URI;
-   Desc.Owner = this;
-   Desc.Description = Target->Description + ".diff/Index";
-   Desc.ShortDesc = Target->ShortDesc;
-   Desc.URI = Target->URI + ".diff/Index";
-
-   DestFile = GetPartialFileNameFromURI(Desc.URI);
+   HashStringList const hashes = GetExpectedHashes();
+   HashString const * const hs = hashes.find(NULL);
+   return hs != NULL ? hs->toStr() : "";
+}
+									/*}}}*/
 
-   if(Debug)
-      std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
+pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner,	/*{{{*/
+      pkgAcqMetaBase * const TransactionManager, IndexTarget const * const Target) :
+   pkgAcquire::Item(Owner), Target(Target), TransactionManager(TransactionManager)
+{
+   if (TransactionManager != this)
+      TransactionManager->Add(this);
+}
+									/*}}}*/
+pkgAcqTransactionItem::~pkgAcqTransactionItem()				/*{{{*/
+{
+}
+									/*}}}*/
+HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const MetaKey) const	/*{{{*/
+{
+   if (TransactionManager->MetaIndexParser == NULL)
+      return HashStringList();
+   indexRecords::checkSum * const R = TransactionManager->MetaIndexParser->Lookup(MetaKey);
+   if (R == NULL)
+      return HashStringList();
+   return R->Hashes;
+}
+									/*}}}*/
 
-   // look for the current package file
-   CurrentPackagesFile = GetFinalFileNameFromURI(RealURI);
+// AcqMetaBase - Constructor						/*{{{*/
+pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner,
+      pkgAcqMetaBase * const TransactionManager,
+      std::vector const * const IndexTargets,
+      IndexTarget const &DataTarget,
+      indexRecords * const MetaIndexParser)
+: pkgAcqTransactionItem(Owner, TransactionManager, NULL), DataTarget(DataTarget),
+   MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL), IndexTargets(IndexTargets),
+   AuthPass(false), IMSHit(false)
+{
+}
+									/*}}}*/
+// AcqMetaBase::Add - Add a item to the current Transaction		/*{{{*/
+void pkgAcqMetaBase::Add(pkgAcqTransactionItem * const I)
+{
+   Transaction.push_back(I);
+}
+									/*}}}*/
+// AcqMetaBase::AbortTransaction - Abort the current Transaction	/*{{{*/
+void pkgAcqMetaBase::AbortTransaction()
+{
+   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+      std::clog << "AbortTransaction: " << TransactionManager << std::endl;
 
-   // FIXME: this file:/ check is a hack to prevent fetching
-   //        from local sources. this is really silly, and
-   //        should be fixed cleanly as soon as possible
-   if(!FileExists(CurrentPackagesFile) || 
-      Desc.URI.substr(0,strlen("file:/")) == "file:/")
+   // ensure the toplevel is in error state too
+   for (std::vector::iterator I = Transaction.begin();
+        I != Transaction.end(); ++I)
    {
-      // we don't have a pkg file or we don't want to queue
-      Failed("No index file, local or canceld by user", NULL);
-      return;
+      (*I)->TransactionState(TransactionAbort);
    }
+   Transaction.clear();
+}
+									/*}}}*/
+// AcqMetaBase::TransactionHasError - Check for errors in Transaction	/*{{{*/
+APT_PURE bool pkgAcqMetaBase::TransactionHasError() const
+{
+   for (std::vector::const_iterator I = Transaction.begin();
+        I != Transaction.end(); ++I)
+   {
+      switch((*I)->Status) {
+	 case StatDone: break;
+	 case StatIdle: break;
+	 case StatAuthError: return true;
+	 case StatError: return true;
+	 case StatTransientNetworkError: return true;
+	 case StatFetching: break;
+      }
+   }
+   return false;
+}
+									/*}}}*/
+// AcqMetaBase::CommitTransaction - Commit a transaction		/*{{{*/
+void pkgAcqMetaBase::CommitTransaction()
+{
+   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+      std::clog << "CommitTransaction: " << this << std::endl;
 
-   if(Debug)
-      std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
-	 << CurrentPackagesFile << std::endl;
-
-   QueueURI(Desc);
-
+   // move new files into place *and* remove files that are not
+   // part of the transaction but are still on disk
+   for (std::vector::iterator I = Transaction.begin();
+        I != Transaction.end(); ++I)
+   {
+      (*I)->TransactionState(TransactionCommit);
+   }
+   Transaction.clear();
 }
 									/*}}}*/
-// Acquire::Item::GetFinalFilename - Return the full final file path	/*{{{*/
-std::string pkgAcqDiffIndex::GetFinalFilename() const
+// AcqMetaBase::TransactionStageCopy - Stage a file for copying		/*{{{*/
+void pkgAcqMetaBase::TransactionStageCopy(pkgAcqTransactionItem * const I,
+                                          const std::string &From,
+                                          const std::string &To)
 {
-   // the logic we inherent from pkgAcqBaseIndex isn't what we need here
-   return pkgAcquire::Item::GetFinalFilename();
+   I->PartialFile = From;
+   I->DestFile = To;
 }
 									/*}}}*/
-// AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
+// AcqMetaBase::TransactionStageRemoval - Stage a file for removal	/*{{{*/
+void pkgAcqMetaBase::TransactionStageRemoval(pkgAcqTransactionItem * const I,
+                                             const std::string &FinalFile)
+{
+   I->PartialFile = "";
+   I->DestFile = FinalFile;
+}
+									/*}}}*/
+// AcqMetaBase::GenerateAuthWarning - Check gpg authentication error	/*{{{*/
+bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message)
+{
+   // FIXME: this entire function can do now that we disallow going to
+   //        a unauthenticated state and can cleanly rollback
+
+   string const Final = I->GetFinalFilename();
+   if(FileExists(Final))
+   {
+      I->Status = StatTransientNetworkError;
+      _error->Warning(_("An error occurred during the signature "
+                        "verification. The repository is not updated "
+                        "and the previous index files will be used. "
+                        "GPG error: %s: %s\n"),
+                      Desc.Description.c_str(),
+                      LookupTag(Message,"Message").c_str());
+      RunScripts("APT::Update::Auth-Failure");
+      return true;
+   } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
+      /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
+      _error->Error(_("GPG error: %s: %s"),
+                    Desc.Description.c_str(),
+                    LookupTag(Message,"Message").c_str());
+      I->Status = StatError;
+      return true;
+   } else {
+      _error->Warning(_("GPG error: %s: %s"),
+                      Desc.Description.c_str(),
+                      LookupTag(Message,"Message").c_str());
+   }
+   // gpgv method failed
+   ReportMirrorFailure("GPGFailure");
+   return false;
+}
+									/*}}}*/
+// AcqMetaBase::Custom600Headers - Get header for AcqMetaBase		/*{{{*/
 // ---------------------------------------------------------------------
-/* The only header we use is the last-modified header. */
-#if APT_PKG_ABI >= 413
-string pkgAcqDiffIndex::Custom600Headers() const
-#else
-string pkgAcqDiffIndex::Custom600Headers()
-#endif
+string pkgAcqMetaBase::Custom600Headers() const
 {
-   string const Final = GetFinalFilename();
+   std::string Header = "\nIndex-File: true";
+   std::string MaximumSize;
+   strprintf(MaximumSize, "\nMaximum-Size: %i",
+             _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
+   Header += MaximumSize;
 
-   if(Debug)
-      std::clog << "Custom600Header-IMS: " << Final << std::endl;
+   string const FinalFile = GetFinalFilename();
 
    struct stat Buf;
-   if (stat(Final.c_str(),&Buf) != 0)
-      return "\nIndex-File: true";
-   
-   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+   if (stat(FinalFile.c_str(),&Buf) == 0)
+      Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+
+   return Header;
 }
 									/*}}}*/
-void pkgAcqDiffIndex::QueueOnIMSHit() const				/*{{{*/
+// AcqMetaBase::QueueForSignatureVerify					/*{{{*/
+void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature)
 {
-   // list cleanup needs to know that this file as well as the already
-   // present index is ours, so we create an empty diff to save it for us
-   new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
-	 ExpectedHashes, MetaIndexParser);
+   AuthPass = true;
+   I->Desc.URI = "gpgv:" + Signature;
+   I->DestFile = File;
+   QueueURI(I->Desc);
+   I->SetActiveSubprocess("gpgv");
 }
 									/*}}}*/
-bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
+// AcqMetaBase::CheckDownloadDone					/*{{{*/
+bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const
 {
-   // failing here is fine: our caller will take care of trying to
-   // get the complete file if patching fails
-   if(Debug)
-      std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
-	 << std::endl;
-
-   FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
-   pkgTagFile TF(&Fd);
-   if (_error->PendingError() == true)
-      return false;
-
-   pkgTagSection Tags;
-   if(unlikely(TF.Step(Tags) == false))
-      return false;
-
-   HashStringList ServerHashes;
-   unsigned long long ServerSize = 0;
+   // We have just finished downloading a Release file (it is not
+   // verified yet)
 
-   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+   string const FileName = LookupTag(Message,"Filename");
+   if (FileName.empty() == true)
    {
-      std::string tagname = *type;
-      tagname.append("-Current");
-      std::string const tmp = Tags.FindS(tagname.c_str());
-      if (tmp.empty() == true)
-	 continue;
-
-      string hash;
-      unsigned long long size;
-      std::stringstream ss(tmp);
-      ss >> hash >> size;
-      if (unlikely(hash.empty() == true))
-	 continue;
-      if (unlikely(ServerSize != 0 && ServerSize != size))
-	 continue;
-      ServerHashes.push_back(HashString(*type, hash));
-      ServerSize = size;
+      I->Status = StatError;
+      I->ErrorText = "Method gave a blank filename";
+      return false;
    }
 
-   if (ServerHashes.usable() == false)
+   if (FileName != I->DestFile)
    {
-      if (Debug == true)
-	 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Did not find a good hashsum in the index" << std::endl;
+      I->Local = true;
+      I->Desc.URI = "copy:" + FileName;
+      I->QueueURI(I->Desc);
       return false;
    }
 
-   if (ServerHashes != HashSums())
+   // make sure to verify against the right file on I-M-S hit
+   bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false);
+   if (IMSHit == false && Hashes.usable())
    {
-      if (Debug == true)
+      // detect IMS-Hits servers haven't detected by Hash comparison
+      std::string const FinalFile = I->GetFinalFilename();
+      if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
       {
-	 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
-         printHashSumComparision(CurrentPackagesFile, ServerHashes, HashSums());
+	 IMSHit = true;
+	 unlink(I->DestFile.c_str());
       }
-      return false;
    }
 
-   if (ServerHashes.VerifyFile(CurrentPackagesFile) == true)
+   if(IMSHit == true)
    {
-      // we have the same sha1 as the server so we are done here
-      if(Debug)
-	 std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
-      QueueOnIMSHit();
-      return true;
+      // for simplicity, the transaction manager is always InRelease
+      // even if it doesn't exist.
+      if (TransactionManager != NULL)
+	 TransactionManager->IMSHit = true;
+      I->PartialFile = I->DestFile = I->GetFinalFilename();
    }
 
-   FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
-   Hashes LocalHashesCalc;
-   LocalHashesCalc.AddFD(fd);
-   HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
+   // set Item to complete as the remaining work is all local (verify etc)
+   I->Complete = true;
 
-   if(Debug)
-      std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
-	 << fd.Name() << " " << fd.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
+   return true;
+}
+									/*}}}*/
+bool pkgAcqMetaBase::CheckAuthDone(string const &Message)		/*{{{*/
+{
+   // At this point, the gpgv method has succeeded, so there is a
+   // valid signature from a key in the trusted keyring.  We
+   // perform additional verification of its contents, and use them
+   // to verify the indexes we are about to download
 
-   // parse all of (provided) history
-   vector available_patches;
-   bool firstAcceptedHashes = true;
-   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+   if (TransactionManager->IMSHit == false)
    {
-      if (LocalHashes.find(*type) == NULL)
-	 continue;
-
-      std::string tagname = *type;
-      tagname.append("-History");
-      std::string const tmp = Tags.FindS(tagname.c_str());
-      if (tmp.empty() == true)
-	 continue;
-
-      string hash, filename;
-      unsigned long long size;
-      std::stringstream ss(tmp);
-
-      while (ss >> hash >> size >> filename)
+      // open the last (In)Release if we have it
+      std::string const FinalFile = GetFinalFilename();
+      std::string FinalRelease;
+      std::string FinalInRelease;
+      if (APT::String::Endswith(FinalFile, "InRelease"))
       {
-	 if (unlikely(hash.empty() == true || filename.empty() == true))
-	    continue;
-
-	 // see if we have a record for this file already
-	 std::vector::iterator cur = available_patches.begin();
-	 for (; cur != available_patches.end(); ++cur)
-	 {
-	    if (cur->file != filename || unlikely(cur->result_size != size))
-	       continue;
-	    cur->result_hashes.push_back(HashString(*type, hash));
-	    break;
-	 }
-	 if (cur != available_patches.end())
-	    continue;
-	 if (firstAcceptedHashes == true)
-	 {
-	    DiffInfo next;
-	    next.file = filename;
-	    next.result_hashes.push_back(HashString(*type, hash));
-	    next.result_size = size;
-	    next.patch_size = 0;
-	    available_patches.push_back(next);
-	 }
+	 FinalInRelease = FinalFile;
+	 FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release";
+      }
+      else
+      {
+	 FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
+	 FinalRelease = FinalFile;
+      }
+      if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
+      {
+	 TransactionManager->LastMetaIndexParser = new indexRecords;
+	 _error->PushToStack();
+	 if (RealFileExists(FinalInRelease))
+	    TransactionManager->LastMetaIndexParser->Load(FinalInRelease);
 	 else
+	    TransactionManager->LastMetaIndexParser->Load(FinalRelease);
+	 // its unlikely to happen, but if what we have is bad ignore it
+	 if (_error->PendingError())
 	 {
-	    if (Debug == true)
-	       std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
-		  << " wasn't in the list for the first parsed hash! (history)" << std::endl;
-	    break;
+	    delete TransactionManager->LastMetaIndexParser;
+	    TransactionManager->LastMetaIndexParser = NULL;
 	 }
+	 _error->RevertToStack();
       }
-      firstAcceptedHashes = false;
    }
 
-   if (unlikely(available_patches.empty() == true))
+   if (TransactionManager->MetaIndexParser->Load(DestFile) == false)
    {
-      if (Debug)
-	 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
-	    << "Couldn't find any patches for the patch series." << std::endl;
+      Status = StatAuthError;
+      ErrorText = TransactionManager->MetaIndexParser->ErrorText;
       return false;
    }
 
-   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+   if (!VerifyVendor(Message))
    {
-      if (LocalHashes.find(*type) == NULL)
-	 continue;
+      Status = StatAuthError;
+      return false;
+   }
 
-      std::string tagname = *type;
-      tagname.append("-Patches");
-      std::string const tmp = Tags.FindS(tagname.c_str());
-      if (tmp.empty() == true)
-	 continue;
+   if (_config->FindB("Debug::pkgAcquire::Auth", false))
+      std::cerr << "Signature verification succeeded: "
+                << DestFile << std::endl;
 
-      string hash, filename;
-      unsigned long long size;
-      std::stringstream ss(tmp);
+   // Download further indexes with verification
+   QueueIndexes(true);
 
-      while (ss >> hash >> size >> filename)
+   return true;
+}
+									/*}}}*/
+void pkgAcqMetaBase::QueueIndexes(bool const verify)			/*{{{*/
+{
+   // at this point the real Items are loaded in the fetcher
+   ExpectedAdditionalItems = 0;
+
+   vector ::const_iterator Target;
+   for (Target = IndexTargets->begin();
+        Target != IndexTargets->end();
+        ++Target)
+   {
+      if (verify == true && TransactionManager->MetaIndexParser->Exists((*Target)->MetaKey) == false)
       {
-	 if (unlikely(hash.empty() == true || filename.empty() == true))
+	 // optional target that we do not have in the Release file are skipped
+	 if ((*Target)->IsOptional())
 	    continue;
 
-	 // see if we have a record for this file already
-	 std::vector::iterator cur = available_patches.begin();
-	 for (; cur != available_patches.end(); ++cur)
-	 {
-	    if (cur->file != filename)
-	       continue;
-	    if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
-	       continue;
-	    cur->patch_hashes.push_back(HashString(*type, hash));
-	    cur->patch_size = size;
-	    break;
-	 }
-	 if (cur != available_patches.end())
-	       continue;
-	 if (Debug == true)
-	    std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
-	       << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
-	 break;
+         Status = StatAuthError;
+         strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
+         return;
       }
+
+      /* Queue the Index file (Packages, Sources, Translation-$foo
+         (either diff or full packages files, depending
+         on the users option) - we also check if the PDiff Index file is listed
+         in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
+         instead, but passing the required info to it is to much hassle */
+      if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
+          TransactionManager->MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
+         new pkgAcqDiffIndex(Owner, TransactionManager, *Target);
+      else
+         new pkgAcqIndex(Owner, TransactionManager, *Target);
    }
+}
+									/*}}}*/
+bool pkgAcqMetaBase::VerifyVendor(string const &Message)		/*{{{*/
+{
+   string::size_type pos;
 
-   bool foundStart = false;
-   for (std::vector::iterator cur = available_patches.begin();
-	 cur != available_patches.end(); ++cur)
+   // check for missing sigs (that where not fatal because otherwise we had
+   // bombed earlier)
+   string missingkeys;
+   string msg = _("There is no public key available for the "
+		  "following key IDs:\n");
+   pos = Message.find("NO_PUBKEY ");
+   if (pos != std::string::npos)
    {
-      if (LocalHashes != cur->result_hashes)
-	 continue;
-
-      available_patches.erase(available_patches.begin(), cur);
-      foundStart = true;
-      break;
+      string::size_type start = pos+strlen("NO_PUBKEY ");
+      string Fingerprint = Message.substr(start, Message.find("\n")-start);
+      missingkeys += (Fingerprint);
    }
+   if(!missingkeys.empty())
+      _error->Warning("%s", (msg + missingkeys).c_str());
 
-   if (foundStart == false || unlikely(available_patches.empty() == true))
+   string Transformed = TransactionManager->MetaIndexParser->GetExpectedDist();
+
+   if (Transformed == "../project/experimental")
    {
-      if (Debug)
-	 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
-	    << "Couldn't find the start of the patch series." << std::endl;
-      return false;
+      Transformed = "experimental";
    }
 
-   // patching with too many files is rather slow compared to a fast download
-   unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
-   if (fileLimit != 0 && fileLimit < available_patches.size())
+   pos = Transformed.rfind('/');
+   if (pos != string::npos)
    {
-      if (Debug)
-	 std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
-	    << ") so fallback to complete download" << std::endl;
-      return false;
+      Transformed = Transformed.substr(0, pos);
    }
 
-   // calculate the size of all patches we have to get
-   // note that all sizes are uncompressed, while we download compressed files
-   unsigned long long patchesSize = 0;
-   for (std::vector::const_iterator cur = available_patches.begin();
-	 cur != available_patches.end(); ++cur)
-      patchesSize += cur->patch_size;
-   unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
-   if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
+   if (Transformed == ".")
    {
-      if (Debug)
-	 std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
-	    << ") so fallback to complete download" << std::endl;
-      return false;
+      Transformed = "";
    }
 
-   // we have something, queue the diffs
-   string::size_type const last_space = Description.rfind(" ");
-   if(last_space != string::npos)
-      Description.erase(last_space, Description.size()-last_space);
+   if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
+       TransactionManager->MetaIndexParser->GetValidUntil() > 0) {
+      time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil();
+      if (invalid_since > 0)
+      {
+	 std::string errmsg;
+	 strprintf(errmsg,
+	       // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
+	       // the time since then the file is invalid - formated in the same way as in
+	       // the download progress display (e.g. 7d 3h 42min 1s)
+	       _("Release file for %s is expired (invalid since %s). "
+		  "Updates for this repository will not be applied."),
+	       DataTarget.URI.c_str(), TimeToStr(invalid_since).c_str());
+	 if (ErrorText.empty())
+	    ErrorText = errmsg;
+	 return _error->Error("%s", errmsg.c_str());
+      }
+   }
 
-   /* decide if we should download patches one by one or in one go:
-      The first is good if the server merges patches, but many don't so client
-      based merging can be attempt in which case the second is better.
-      "bad things" will happen if patches are merged on the server,
-      but client side merging is attempt as well */
-   bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
-   if (pdiff_merge == true)
+   /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
+      as a prevention of downgrading us to older (still valid) files */
+   if (TransactionManager->IMSHit == false && TransactionManager->LastMetaIndexParser != NULL &&
+	 TransactionManager->LastMetaIndexParser->GetDate() > TransactionManager->MetaIndexParser->GetDate())
    {
-      // reprepro adds this flag if it has merged patches on the server
-      std::string const precedence = Tags.FindS("X-Patch-Precedence");
-      pdiff_merge = (precedence != "merged");
+      TransactionManager->IMSHit = true;
+      unlink(DestFile.c_str());
+      PartialFile = DestFile = GetFinalFilename();
+      delete TransactionManager->MetaIndexParser;
+      TransactionManager->MetaIndexParser = TransactionManager->LastMetaIndexParser;
+      TransactionManager->LastMetaIndexParser = NULL;
    }
 
-   if (pdiff_merge == false)
+   if (_config->FindB("Debug::pkgAcquire::Auth", false)) 
    {
-      new pkgAcqIndexDiffs(Owner, TransactionManager, Target, ExpectedHashes, 
-                           MetaIndexParser, available_patches);
+      std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetDist() << std::endl;
+      std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl;
+      std::cerr << "Transformed Dist: " << Transformed << std::endl;
    }
-   else
+
+   if (TransactionManager->MetaIndexParser->CheckDist(Transformed) == false)
    {
-      std::vector *diffs = new std::vector(available_patches.size());
-      for(size_t i = 0; i < available_patches.size(); ++i)
-	 (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
-               Target,
-	       ExpectedHashes,
-	       MetaIndexParser,
-	       available_patches[i],
-	       diffs);
+      // This might become fatal one day
+//       Status = StatAuthError;
+//       ErrorText = "Conflicting distribution; expected "
+//          + MetaIndexParser->GetExpectedDist() + " but got "
+//          + MetaIndexParser->GetDist();
+//       return false;
+      if (!Transformed.empty())
+      {
+         _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
+                         Desc.Description.c_str(),
+                         Transformed.c_str(),
+                         TransactionManager->MetaIndexParser->GetDist().c_str());
+      }
    }
 
-   Complete = false;
-   Status = StatDone;
-   Dequeue();
    return true;
 }
 									/*}}}*/
-void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
-{
-   Item::Failed(Message,Cnf);
-   Status = StatDone;
-
-   if(Debug)
-      std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
-		<< "Falling back to normal index file acquire" << std::endl;
 
-   new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
+pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,	/*{{{*/
+      IndexTarget const &ClearsignedTarget,
+      IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget,
+      const vector* const IndexTargets,
+      indexRecords * const MetaIndexParser) :
+   pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget, IndexTargets, MetaIndexParser),
+   ClearsignedTarget(ClearsignedTarget),
+   DetachedDataTarget(DetachedDataTarget), DetachedSigTarget(DetachedSigTarget)
+{
+   // index targets + (worst case:) Release/Release.gpg
+   ExpectedAdditionalItems = IndexTargets->size() + 2;
+   TransactionManager->Add(this);
 }
 									/*}}}*/
-bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)	/*{{{*/
+pkgAcqMetaClearSig::~pkgAcqMetaClearSig()				/*{{{*/
 {
-   if (pkgAcquire::Item::TransactionState(state) == false)
-      return false;
-
-   switch (state)
-   {
-      case TransactionCommit:
-	 break;
-      case TransactionAbort:
-	 std::string const Partial = GetPartialFileNameFromURI(RealURI);
-	 unlink(Partial.c_str());
-	 break;
-   }
-
-   return true;
 }
 									/*}}}*/
-void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,	/*{{{*/
-			   pkgAcquire::MethodConfig *Cnf)
+// pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers	/*{{{*/
+string pkgAcqMetaClearSig::Custom600Headers() const
 {
-   if(Debug)
-      std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
-
-   Item::Done(Message, Size, Hashes, Cnf);
+   string Header = pkgAcqMetaBase::Custom600Headers();
+   Header += "\nFail-Ignore: true";
+   return Header;
+}
+									/*}}}*/
+// pkgAcqMetaClearSig::Done - We got a file				/*{{{*/
+class APT_HIDDEN DummyItem : public pkgAcquire::Item
+{
+   IndexTarget const * const Target;
+   public:
+   virtual std::string DescURI() const {return Target->URI;};
+   virtual HashStringList GetExpectedHashes()  const {return HashStringList();};
 
-   // verify the index target
-   if(Target && Target->MetaKey != "" && MetaIndexParser && Hashes.usable())
+   DummyItem(pkgAcquire * const Owner, IndexTarget const * const Target) :
+      pkgAcquire::Item(Owner), Target(Target)
    {
-      std::string IndexMetaKey  = Target->MetaKey + ".diff/Index";
-      indexRecords::checkSum *Record = MetaIndexParser->Lookup(IndexMetaKey);
-      if(Record && Record->Hashes.usable() && Hashes != Record->Hashes)
-      {
-         RenameOnError(HashSumMismatch);
-         printHashSumComparision(RealURI, Record->Hashes, Hashes);
-         Failed(Message, Cnf);
-         return;
-      }
-
+      Status = StatDone;
+      DestFile = GetFinalFileNameFromURI(Target->URI);
    }
+};
+void pkgAcqMetaClearSig::Done(std::string const &Message,
+                              HashStringList const &Hashes,
+                              pkgAcquire::MethodConfig const * const Cnf)
+{
+   Item::Done(Message, Hashes, Cnf);
 
-   string const FinalFile = GetFinalFilename();
-   if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
-      DestFile = FinalFile;
-
-   if(ParseDiffIndex(DestFile) == false)
+   // if we expect a ClearTextSignature (InRelease), ensure that
+   // this is what we get and if not fail to queue a 
+   // Release/Release.gpg, see #346386
+   if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
    {
-      Failed("Message: Couldn't parse pdiff index", Cnf);
-      // queue for final move - this should happen even if we fail
-      // while parsing (e.g. on sizelimit) and download the complete file.
-      TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
+      pkgAcquire::Item::Failed(Message, Cnf);
+      RenameOnError(NotClearsigned);
+      TransactionManager->AbortTransaction();
       return;
    }
 
-   TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
-
-   Complete = true;
-   Status = StatDone;
-   Dequeue();
-
-   return;
+   if(AuthPass == false)
+   {
+      if(CheckDownloadDone(this, Message, Hashes) == true)
+         QueueForSignatureVerify(this, DestFile, DestFile);
+      return;
+   }
+   else if(CheckAuthDone(Message) == true)
+   {
+      if (TransactionManager->IMSHit == false)
+	 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+      else if (RealFileExists(GetFinalFilename()) == false)
+      {
+	 // We got an InRelease file IMSHit, but we haven't one, which means
+	 // we had a valid Release/Release.gpg combo stepping in, which we have
+	 // to 'acquire' now to ensure list cleanup isn't removing them
+	 new DummyItem(Owner, &DetachedDataTarget);
+	 new DummyItem(Owner, &DetachedSigTarget);
+      }
+   }
 }
 									/*}}}*/
-// AcqIndexDiffs::AcqIndexDiffs - Constructor				/*{{{*/
-// ---------------------------------------------------------------------
-/* The package diff is added to the queue. one object is constructed
- * for each diff and the index
- */
-pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
-                                   pkgAcqMetaBase *TransactionManager,
-                                   struct IndexTarget const * const Target,
-                                   HashStringList const &ExpectedHashes,
-                                   indexRecords *MetaIndexParser,
-				   vector diffs)
-   : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser),
-     available_patches(diffs)
+void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf) /*{{{*/
 {
-   DestFile = GetPartialFileNameFromURI(Target->URI);
-
-   Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
+   Item::Failed(Message, Cnf);
 
-   RealURI = Target->URI;
-   Desc.Owner = this;
-   Description = Target->Description;
-   Desc.ShortDesc = Target->ShortDesc;
+   // we failed, we will not get additional items from this method
+   ExpectedAdditionalItems = 0;
 
-   if(available_patches.empty() == true)
+   if (AuthPass == false)
    {
-      // we are done (yeah!), check hashes against the final file
-      DestFile = GetFinalFileNameFromURI(Target->URI);
-      Finish(true);
+      // Queue the 'old' InRelease file for removal if we try Release.gpg
+      // as otherwise the file will stay around and gives a false-auth
+      // impression (CVE-2012-0214)
+      TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
+      Status = StatDone;
+
+      new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget, IndexTargets, TransactionManager->MetaIndexParser);
    }
    else
    {
-      // patching needs to be bootstrapped with the 'old' version
-      std::string const PartialFile = GetPartialFileNameFromURI(RealURI);
-      if (RealFileExists(PartialFile) == false)
+      if(CheckStopAuthentication(this, Message))
+         return;
+
+      _error->Warning(_("The data from '%s' is not signed. Packages "
+                        "from that repository can not be authenticated."),
+                      ClearsignedTarget.Description.c_str());
+
+      // No Release file was present, or verification failed, so fall
+      // back to queueing Packages files without verification
+      // only allow going further if the users explicitely wants it
+      if(AllowInsecureRepositories(TransactionManager->MetaIndexParser, TransactionManager, this) == true)
       {
-	 if (symlink(GetFinalFilename().c_str(), PartialFile.c_str()) != 0)
+	 Status = StatDone;
+
+	 /* InRelease files become Release files, otherwise
+	  * they would be considered as trusted later on */
+	 string const FinalRelease = GetFinalFileNameFromURI(DetachedDataTarget.URI);
+	 string const PartialRelease = GetPartialFileNameFromURI(DetachedDataTarget.URI);
+	 string const FinalReleasegpg = GetFinalFileNameFromURI(DetachedSigTarget.URI);
+	 string const FinalInRelease = GetFinalFilename();
+	 Rename(DestFile, PartialRelease);
+	 TransactionManager->TransactionStageCopy(this, PartialRelease, FinalRelease);
+
+	 if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
 	 {
-	    Failed("Link creation of " + PartialFile + " to " + GetFinalFilename() + " failed", NULL);
-	    return;
+	    // open the last Release if we have it
+	    if (TransactionManager->IMSHit == false)
+	    {
+	       TransactionManager->LastMetaIndexParser = new indexRecords;
+	       _error->PushToStack();
+	       if (RealFileExists(FinalInRelease))
+		  TransactionManager->LastMetaIndexParser->Load(FinalInRelease);
+	       else
+		  TransactionManager->LastMetaIndexParser->Load(FinalRelease);
+	       // its unlikely to happen, but if what we have is bad ignore it
+	       if (_error->PendingError())
+	       {
+		  delete TransactionManager->LastMetaIndexParser;
+		  TransactionManager->LastMetaIndexParser = NULL;
+	       }
+	       _error->RevertToStack();
+	    }
 	 }
-      }
 
-      // get the next diff
-      State = StateFetchDiff;
-      QueueNextDiff();
+	 // we parse the indexes here because at this point the user wanted
+	 // a repository that may potentially harm him
+	 if (TransactionManager->MetaIndexParser->Load(PartialRelease) == false || VerifyVendor(Message) == false)
+	    /* expired Release files are still a problem you need extra force for */;
+	 else
+	    QueueIndexes(true);
+      }
    }
 }
 									/*}}}*/
-void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
-{
-   Item::Failed(Message,Cnf);
-   Status = StatDone;
 
-   if(Debug)
-      std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
-		<< "Falling back to normal index file acquire" << std::endl;
-   DestFile = GetPartialFileNameFromURI(Target->URI);
-   RenameOnError(PDiffError);
-   new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
-   Finish();
-}
-									/*}}}*/
-// Finish - helper that cleans the item out of the fetcher queue	/*{{{*/
-void pkgAcqIndexDiffs::Finish(bool allDone)
+pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner,		/*{{{*/
+                                 pkgAcqMetaBase * const TransactionManager,
+				 IndexTarget const &DataTarget,
+				 IndexTarget const &DetachedSigTarget,
+				 vector const * const IndexTargets,
+				 indexRecords * const MetaIndexParser) :
+   pkgAcqMetaBase(Owner, TransactionManager, IndexTargets, DataTarget, MetaIndexParser),
+   DetachedSigTarget(DetachedSigTarget)
 {
-   if(Debug)
-      std::clog << "pkgAcqIndexDiffs::Finish(): " 
-                << allDone << " "
-                << Desc.URI << std::endl;
-
-   // we restore the original name, this is required, otherwise
-   // the file will be cleaned
-   if(allDone) 
-   {
-      if(HashSums().usable() && !HashSums().VerifyFile(DestFile))
-      {
-	 RenameOnError(HashSumMismatch);
-	 Dequeue();
-	 return;
-      }
+   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+      std::clog << "New pkgAcqMetaIndex with TransactionManager "
+                << this->TransactionManager << std::endl;
 
-      TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+   DestFile = GetPartialFileNameFromURI(DataTarget.URI);
 
-      // this is for the "real" finish
-      Complete = true;
-      Status = StatDone;
-      Dequeue();
-      if(Debug)
-	 std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
-      return;
-   }
+   // Create the item
+   Desc.Description = DataTarget.Description;
+   Desc.Owner = this;
+   Desc.ShortDesc = DataTarget.ShortDesc;
+   Desc.URI = DataTarget.URI;
 
-   if(Debug)
-      std::clog << "Finishing: " << Desc.URI << std::endl;
-   Complete = false;
-   Status = StatDone;
-   Dequeue();
-   return;
+   // we expect more item
+   ExpectedAdditionalItems = IndexTargets->size();
+   QueueURI(Desc);
 }
 									/*}}}*/
-bool pkgAcqIndexDiffs::QueueNextDiff()					/*{{{*/
+void pkgAcqMetaIndex::Done(string const &Message,			/*{{{*/
+                           HashStringList const &Hashes,
+			   pkgAcquire::MethodConfig const * const Cfg)
 {
-   // calc sha1 of the just patched file
-   std::string const FinalFile = GetPartialFileNameFromURI(RealURI);
+   Item::Done(Message,Hashes,Cfg);
 
-   if(!FileExists(FinalFile))
+   if(CheckDownloadDone(this, Message, Hashes))
    {
-      Failed("Message: No FinalFile " + FinalFile + " available", NULL);
-      return false;
+      // we have a Release file, now download the Signature, all further
+      // verify/queue for additional downloads will be done in the
+      // pkgAcqMetaSig::Done() code
+      new pkgAcqMetaSig(Owner, TransactionManager, &DetachedSigTarget, this);
    }
+}
+									/*}}}*/
+// pkgAcqMetaIndex::Failed - no Release file present			/*{{{*/
+void pkgAcqMetaIndex::Failed(string const &Message,
+                             pkgAcquire::MethodConfig const * const Cnf)
+{
+   pkgAcquire::Item::Failed(Message, Cnf);
+   Status = StatDone;
 
-   FileFd fd(FinalFile, FileFd::ReadOnly);
-   Hashes LocalHashesCalc;
-   LocalHashesCalc.AddFD(fd);
-   HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
-
-   if(Debug)
-      std::clog << "QueueNextDiff: " << FinalFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
+   _error->Warning(_("The repository '%s' does not have a Release file. "
+                     "This is deprecated, please contact the owner of the "
+                     "repository."), DataTarget.Description.c_str());
 
-   if (unlikely(LocalHashes.usable() == false || ExpectedHashes.usable() == false))
+   // No Release file was present so fall
+   // back to queueing Packages files without verification
+   // only allow going further if the users explicitely wants it
+   if(AllowInsecureRepositories(TransactionManager->MetaIndexParser, TransactionManager, this) == true)
    {
-      Failed("Local/Expected hashes are not usable", NULL);
-      return false;
-   }
-
+      // ensure old Release files are removed
+      TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
+      delete TransactionManager->MetaIndexParser;
+      TransactionManager->MetaIndexParser = NULL;
 
-   // final file reached before all patches are applied
-   if(LocalHashes == ExpectedHashes)
-   {
-      Finish(true);
-      return true;
-   }
-
-   // remove all patches until the next matching patch is found
-   // this requires the Index file to be ordered
-   for(vector::iterator I = available_patches.begin();
-       available_patches.empty() == false &&
-	  I != available_patches.end() &&
-	  I->result_hashes != LocalHashes;
-       ++I)
-   {
-      available_patches.erase(I);
+      // queue without any kind of hashsum support
+      QueueIndexes(false);
    }
+}
+									/*}}}*/
+void pkgAcqMetaIndex::Finished()					/*{{{*/
+{
+   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+      std::clog << "Finished: " << DestFile <TransactionHasError() == false)
+      TransactionManager->CommitTransaction();
+}
+									/*}}}*/
+std::string pkgAcqMetaIndex::DescURI() const				/*{{{*/
+{
+   return DataTarget.URI;
+}
+									/*}}}*/
 
-   // error checking and falling back if no patch was found
-   if(available_patches.empty() == true)
-   {
-      Failed("No patches left to reach target", NULL);
-      return false;
-   }
+// AcqMetaSig::AcqMetaSig - Constructor					/*{{{*/
+pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner,
+      pkgAcqMetaBase * const TransactionManager,
+      IndexTarget const * const Target,
+      pkgAcqMetaIndex * const MetaIndex) :
+   pkgAcqTransactionItem(Owner, TransactionManager, Target), MetaIndex(MetaIndex)
+{
+   DestFile = GetPartialFileNameFromURI(Target->URI);
 
-   // queue the right diff
-   Desc.URI = RealURI + ".diff/" + available_patches[0].file + ".gz";
-   Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
-   DestFile = GetPartialFileNameFromURI(RealURI + ".diff/" + available_patches[0].file);
+   // remove any partial downloaded sig-file in partial/.
+   // it may confuse proxies and is too small to warrant a
+   // partial download anyway
+   unlink(DestFile.c_str());
 
-   if(Debug)
-      std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
+   // set the TransactionManager
+   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+      std::clog << "New pkgAcqMetaSig with TransactionManager "
+                << TransactionManager << std::endl;
 
-   QueueURI(Desc);
+   // Create the item
+   Desc.Description = Target->Description;
+   Desc.Owner = this;
+   Desc.ShortDesc = Target->ShortDesc;
+   Desc.URI = Target->URI;
 
-   return true;
+   // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
+   // so we skip the download step and go instantly to verification
+   if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename()))
+   {
+      Complete = true;
+      Status = StatDone;
+      PartialFile = DestFile = GetFinalFilename();
+      MetaIndexFileSignature = DestFile;
+      MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
+   }
+   else
+      QueueURI(Desc);
 }
 									/*}}}*/
-void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes,	/*{{{*/
-			    pkgAcquire::MethodConfig *Cnf)
+pkgAcqMetaSig::~pkgAcqMetaSig()						/*{{{*/
 {
-   if(Debug)
-      std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
-
-   Item::Done(Message, Size, Hashes, Cnf);
-
-   // FIXME: verify this download too before feeding it to rred
-   std::string const FinalFile = GetPartialFileNameFromURI(RealURI);
-
-   // success in downloading a diff, enter ApplyDiff state
-   if(State == StateFetchDiff)
+}
+									/*}}}*/
+// AcqMetaSig::Done - The signature was downloaded/verified		/*{{{*/
+void pkgAcqMetaSig::Done(string const &Message, HashStringList const &Hashes,
+			 pkgAcquire::MethodConfig const * const Cfg)
+{
+   if (MetaIndexFileSignature.empty() == false)
    {
-      FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Gzip);
-      class Hashes LocalHashesCalc;
-      LocalHashesCalc.AddFD(fd);
-      HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
+      DestFile = MetaIndexFileSignature;
+      MetaIndexFileSignature.clear();
+   }
+   Item::Done(Message, Hashes, Cfg);
 
-      if (fd.Size() != available_patches[0].patch_size ||
-	    available_patches[0].patch_hashes != LocalHashes)
+   if(MetaIndex->AuthPass == false)
+   {
+      if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
       {
-	 // patchfiles are dated, so bad indicates a bad download, so kill it
-	 unlink(DestFile.c_str());
-	 Failed("Patch has Size/Hashsum mismatch", NULL);
-	 return;
+	 // destfile will be modified to point to MetaIndexFile for the
+	 // gpgv method, so we need to save it here
+	 MetaIndexFileSignature = DestFile;
+	 MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
+      }
+      return;
+   }
+   else if(MetaIndex->CheckAuthDone(Message) == true)
+   {
+      if (TransactionManager->IMSHit == false)
+      {
+	 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+	 TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
       }
+   }
+}
+									/*}}}*/
+void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
+{
+   Item::Failed(Message,Cnf);
 
-      // rred excepts the patch as $FinalFile.ed
-      Rename(DestFile,FinalFile+".ed");
+   // check if we need to fail at this point
+   if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
+         return;
 
-      if(Debug)
-	 std::clog << "Sending to rred method: " << FinalFile << std::endl;
+   string const FinalRelease = MetaIndex->GetFinalFilename();
+   string const FinalReleasegpg = GetFinalFilename();
+   string const FinalInRelease = TransactionManager->GetFinalFilename();
 
-      State = StateApplyDiff;
-      Local = true;
-      Desc.URI = "rred:" + FinalFile;
-      QueueURI(Desc);
-      SetActiveSubprocess("rred");
-      return;
-   } 
+   if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
+   {
+      std::string downgrade_msg;
+      strprintf(downgrade_msg, _("The repository '%s' is no longer signed."),
+                MetaIndex->DataTarget.Description.c_str());
+      if(_config->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
+      {
+         // meh, the users wants to take risks (we still mark the packages
+         // from this repository as unauthenticated)
+         _error->Warning("%s", downgrade_msg.c_str());
+         _error->Warning(_("This is normally not allowed, but the option "
+                           "Acquire::AllowDowngradeToInsecureRepositories was "
+                           "given to override it."));
+         Status = StatDone;
+      } else {
+         _error->Error("%s", downgrade_msg.c_str());
+	 if (TransactionManager->IMSHit == false)
+	    Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
+	 Item::Failed("Message: " + downgrade_msg, Cnf);
+         TransactionManager->AbortTransaction();
+         return;
+      }
+   }
+   else
+      _error->Warning(_("The data from '%s' is not signed. Packages "
+	       "from that repository can not be authenticated."),
+	    MetaIndex->DataTarget.Description.c_str());
 
+   // ensures that a Release.gpg file in the lists/ is removed by the transaction
+   TransactionManager->TransactionStageRemoval(this, DestFile);
 
-   // success in download/apply a diff, queue next (if needed)
-   if(State == StateApplyDiff)
+   // only allow going further if the users explicitely wants it
+   if(AllowInsecureRepositories(TransactionManager->MetaIndexParser, TransactionManager, this) == true)
    {
-      // remove the just applied patch
-      available_patches.erase(available_patches.begin());
-      unlink((FinalFile + ".ed").c_str());
-
-      // move into place
-      if(Debug) 
+      if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
       {
-	 std::clog << "Moving patched file in place: " << std::endl
-		   << DestFile << " -> " << FinalFile << std::endl;
+	 // open the last Release if we have it
+	 if (TransactionManager->IMSHit == false)
+	 {
+	    TransactionManager->LastMetaIndexParser = new indexRecords;
+	    _error->PushToStack();
+	    if (RealFileExists(FinalInRelease))
+	       TransactionManager->LastMetaIndexParser->Load(FinalInRelease);
+	    else
+	       TransactionManager->LastMetaIndexParser->Load(FinalRelease);
+	    // its unlikely to happen, but if what we have is bad ignore it
+	    if (_error->PendingError())
+	    {
+	       delete TransactionManager->LastMetaIndexParser;
+	       TransactionManager->LastMetaIndexParser = NULL;
+	    }
+	    _error->RevertToStack();
+	 }
       }
-      Rename(DestFile,FinalFile);
-      chmod(FinalFile.c_str(),0644);
 
-      // see if there is more to download
-      if(available_patches.empty() == false) {
-	 new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
-			      ExpectedHashes, MetaIndexParser,
-			      available_patches);
-	 return Finish();
-      } else 
-         // update
-         DestFile = FinalFile;
-	 return Finish(true);
+      // we parse the indexes here because at this point the user wanted
+      // a repository that may potentially harm him
+      if (TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile) == false || MetaIndex->VerifyVendor(Message) == false)
+	 /* expired Release files are still a problem you need extra force for */;
+      else
+	 MetaIndex->QueueIndexes(true);
+
+      TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
+   }
+
+   // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
+   if (Cnf->LocalOnly == true ||
+       StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
+   {
+      // Ignore this
+      Status = StatDone;
    }
 }
 									/*}}}*/
-// AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor			/*{{{*/
-pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
-                                             pkgAcqMetaBase *TransactionManager,
-                                             struct IndexTarget const * const Target,
-                                             HashStringList const &ExpectedHashes,
-                                             indexRecords *MetaIndexParser,
-                                             DiffInfo const &patch,
-                                             std::vector const * const allPatches)
-  : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser),
-     patch(patch), allPatches(allPatches), State(StateFetchDiff)
+
+
+// AcqBaseIndex - Constructor						/*{{{*/
+pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner,
+      pkgAcqMetaBase * const TransactionManager,
+      IndexTarget const * const Target)
+: pkgAcqTransactionItem(Owner, TransactionManager, Target)
+{
+}
+									/*}}}*/
+
+// AcqDiffIndex::AcqDiffIndex - Constructor				/*{{{*/
+// ---------------------------------------------------------------------
+/* Get the DiffIndex file first and see if there are patches available
+ * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
+ * patches. If anything goes wrong in that process, it will fall back to
+ * the original packages file
+ */
+pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
+                                 pkgAcqMetaBase * const TransactionManager,
+                                 IndexTarget const * const Target)
+   : pkgAcqBaseIndex(Owner, TransactionManager, Target)
 {
    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
-   RealURI = Target->URI;
    Desc.Owner = this;
-   Description = Target->Description;
+   Desc.Description = Target->Description + ".diff/Index";
    Desc.ShortDesc = Target->ShortDesc;
+   Desc.URI = Target->URI + ".diff/Index";
 
-   Desc.URI = RealURI + ".diff/" + patch.file + ".gz";
-   Desc.Description = Description + " " + patch.file + string(".pdiff");
+   DestFile = GetPartialFileNameFromURI(Desc.URI);
+
+   if(Debug)
+      std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
 
-   DestFile = GetPartialFileNameFromURI(RealURI + ".diff/" + patch.file);
+   // look for the current package file
+   CurrentPackagesFile = GetFinalFileNameFromURI(Target->URI);
+
+   // FIXME: this file:/ check is a hack to prevent fetching
+   //        from local sources. this is really silly, and
+   //        should be fixed cleanly as soon as possible
+   if(!FileExists(CurrentPackagesFile) || 
+      Desc.URI.substr(0,strlen("file:/")) == "file:/")
+   {
+      // we don't have a pkg file or we don't want to queue
+      Failed("No index file, local or canceld by user", NULL);
+      return;
+   }
 
    if(Debug)
-      std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
+      std::clog << "pkgAcqDiffIndex::pkgAcqDiffIndex(): "
+	 << CurrentPackagesFile << std::endl;
 
    QueueURI(Desc);
+
 }
 									/*}}}*/
-void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
+// AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
+// ---------------------------------------------------------------------
+/* The only header we use is the last-modified header. */
+string pkgAcqDiffIndex::Custom600Headers() const
 {
-   if(Debug)
-      std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
-
-   Item::Failed(Message,Cnf);
-   Status = StatDone;
+   string const Final = GetFinalFilename();
 
-   // check if we are the first to fail, otherwise we are done here
-   State = StateDoneDiff;
-   for (std::vector::const_iterator I = allPatches->begin();
-	 I != allPatches->end(); ++I)
-      if ((*I)->State == StateErrorDiff)
-	 return;
+   if(Debug)
+      std::clog << "Custom600Header-IMS: " << Final << std::endl;
 
-   // first failure means we should fallback
-   State = StateErrorDiff;
-   if (Debug)
-      std::clog << "Falling back to normal index file acquire" << std::endl;
-   DestFile = GetPartialFileNameFromURI(Target->URI);
-   RenameOnError(PDiffError);
-   new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
+   struct stat Buf;
+   if (stat(Final.c_str(),&Buf) != 0)
+      return "\nIndex-File: true";
+   
+   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+}
+									/*}}}*/
+void pkgAcqDiffIndex::QueueOnIMSHit() const				/*{{{*/
+{
+   // list cleanup needs to know that this file as well as the already
+   // present index is ours, so we create an empty diff to save it for us
+   new pkgAcqIndexDiffs(Owner, TransactionManager, Target);
 }
 									/*}}}*/
-void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes,	/*{{{*/
-			    pkgAcquire::MethodConfig *Cnf)
+bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 {
+   // failing here is fine: our caller will take care of trying to
+   // get the complete file if patching fails
    if(Debug)
-      std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
-
-   Item::Done(Message,Size,Hashes,Cnf);
-
-   // FIXME: verify download before feeding it to rred
-   string const FinalFile = GetPartialFileNameFromURI(RealURI);
-
-   if (State == StateFetchDiff)
-   {
-      FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Gzip);
-      class Hashes LocalHashesCalc;
-      LocalHashesCalc.AddFD(fd);
-      HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
-
-      if (fd.Size() != patch.patch_size || patch.patch_hashes != LocalHashes)
-      {
-	 // patchfiles are dated, so bad indicates a bad download, so kill it
-	 unlink(DestFile.c_str());
-	 Failed("Patch has Size/Hashsum mismatch", NULL);
-	 return;
-      }
+      std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
+	 << std::endl;
 
-      // rred expects the patch as $FinalFile.ed.$patchname.gz
-      Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
+   FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
+   pkgTagFile TF(&Fd);
+   if (_error->PendingError() == true)
+      return false;
 
-      // check if this is the last completed diff
-      State = StateDoneDiff;
-      for (std::vector::const_iterator I = allPatches->begin();
-	    I != allPatches->end(); ++I)
-	 if ((*I)->State != StateDoneDiff)
-	 {
-	    if(Debug)
-	       std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
-	    return;
-	 }
+   pkgTagSection Tags;
+   if(unlikely(TF.Step(Tags) == false))
+      return false;
 
-      // this is the last completed diff, so we are ready to apply now
-      State = StateApplyDiff;
+   HashStringList ServerHashes;
+   unsigned long long ServerSize = 0;
 
-      // patching needs to be bootstrapped with the 'old' version
-      if (symlink(GetFinalFilename().c_str(), FinalFile.c_str()) != 0)
-      {
-	 Failed("Link creation of " + FinalFile + " to " + GetFinalFilename() + " failed", NULL);
-	 return;
-      }
+   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+   {
+      std::string tagname = *type;
+      tagname.append("-Current");
+      std::string const tmp = Tags.FindS(tagname.c_str());
+      if (tmp.empty() == true)
+	 continue;
 
-      if(Debug)
-	 std::clog << "Sending to rred method: " << FinalFile << std::endl;
+      string hash;
+      unsigned long long size;
+      std::stringstream ss(tmp);
+      ss >> hash >> size;
+      if (unlikely(hash.empty() == true))
+	 continue;
+      if (unlikely(ServerSize != 0 && ServerSize != size))
+	 continue;
+      ServerHashes.push_back(HashString(*type, hash));
+      ServerSize = size;
+   }
 
-      Local = true;
-      Desc.URI = "rred:" + FinalFile;
-      QueueURI(Desc);
-      SetActiveSubprocess("rred");
-      return;
+   if (ServerHashes.usable() == false)
+   {
+      if (Debug == true)
+	 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Did not find a good hashsum in the index" << std::endl;
+      return false;
    }
-   // success in download/apply all diffs, clean up
-   else if (State == StateApplyDiff)
+
+   HashStringList const TargetFileHashes = GetExpectedHashesFor(Target->MetaKey);
+   if (TargetFileHashes.usable() == false || ServerHashes != TargetFileHashes)
    {
-      // see if we really got the expected file
-      if(ExpectedHashes.usable() && ExpectedHashes != Hashes)
+      if (Debug == true)
       {
-	 RenameOnError(HashSumMismatch);
-	 return;
+	 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": Index has different hashes than parser, probably older, so fail pdiffing" << std::endl;
+         printHashSumComparision(CurrentPackagesFile, ServerHashes, TargetFileHashes);
       }
+      return false;
+   }
 
-      // move the result into place
-      std::string const Final = GetFinalFilename();
+   if (ServerHashes.VerifyFile(CurrentPackagesFile) == true)
+   {
+      // we have the same sha1 as the server so we are done here
       if(Debug)
-	 std::clog << "Queue patched file in place: " << std::endl
-		   << DestFile << " -> " << Final << std::endl;
+	 std::clog << "pkgAcqDiffIndex: Package file " << CurrentPackagesFile << " is up-to-date" << std::endl;
+      QueueOnIMSHit();
+      return true;
+   }
 
-      // queue for copy by the transaction manager
-      TransactionManager->TransactionStageCopy(this, DestFile, Final);
+   FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
+   Hashes LocalHashesCalc;
+   LocalHashesCalc.AddFD(fd);
+   HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
 
-      // ensure the ed's are gone regardless of list-cleanup
-      for (std::vector::const_iterator I = allPatches->begin();
-	    I != allPatches->end(); ++I)
-      {
-	 std::string const PartialFile = GetPartialFileNameFromURI(RealURI);
-	 std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz";
-	 unlink(patch.c_str());
-      }
-      unlink(FinalFile.c_str());
+   if(Debug)
+      std::clog << "Server-Current: " << ServerHashes.find(NULL)->toStr() << " and we start at "
+	 << fd.Name() << " " << fd.FileSize() << " " << LocalHashes.find(NULL)->toStr() << std::endl;
 
-      // all set and done
-      Complete = true;
-      if(Debug)
-	 std::clog << "allDone: " << DestFile << "\n" << std::endl;
-   }
-}
-									/*}}}*/
-// AcqBaseIndex - Constructor						/*{{{*/
-pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire *Owner,
-      pkgAcqMetaBase *TransactionManager,
-      struct IndexTarget const * const Target,
-      HashStringList const &ExpectedHashes,
-      indexRecords *MetaIndexParser)
-: Item(Owner, ExpectedHashes, TransactionManager), Target(Target),
-   MetaIndexParser(MetaIndexParser)
-{
-}
-									/*}}}*/
-// AcqBaseIndex::VerifyHashByMetaKey - verify hash for the given metakey /*{{{*/
-bool pkgAcqBaseIndex::VerifyHashByMetaKey(HashStringList const &Hashes)
-{
-   if(MetaKey != "" && Hashes.usable())
+   // parse all of (provided) history
+   vector available_patches;
+   bool firstAcceptedHashes = true;
+   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
    {
-      indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
-      if(Record && Record->Hashes.usable() && Hashes != Record->Hashes)
-      {
-         printHashSumComparision(RealURI, Record->Hashes, Hashes);
-         return false;
-      }
-   }
-   return true;
-}
-									/*}}}*/
-// AcqBaseIndex::GetFinalFilename - Return the full final file path	/*{{{*/
-std::string pkgAcqBaseIndex::GetFinalFilename() const
-{
-   return GetFinalFileNameFromURI(RealURI);
-}
-									/*}}}*/
-// AcqIndex::AcqIndex - Constructor					/*{{{*/
-// ---------------------------------------------------------------------
-/* The package file is added to the queue and a second class is
-   instantiated to fetch the revision file */
-pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
-			 string URI,string URIDesc,string ShortDesc,
-			 HashStringList const  &ExpectedHash)
-   : pkgAcqBaseIndex(Owner, 0, NULL, ExpectedHash, NULL)
-{
-   RealURI = URI;
-
-   AutoSelectCompression();
-   Init(URI, URIDesc, ShortDesc);
+      if (LocalHashes.find(*type) == NULL)
+	 continue;
 
-   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-      std::clog << "New pkgIndex with TransactionManager "
-                << TransactionManager << std::endl;
-}
-									/*}}}*/
-// AcqIndex::AcqIndex - Constructor					/*{{{*/
-pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
-                         pkgAcqMetaBase *TransactionManager,
-                         IndexTarget const *Target,
-			 HashStringList const &ExpectedHash,
-                         indexRecords *MetaIndexParser)
-   : pkgAcqBaseIndex(Owner, TransactionManager, Target, ExpectedHash,
-                     MetaIndexParser)
-{
-   RealURI = Target->URI;
+      std::string tagname = *type;
+      tagname.append("-History");
+      std::string const tmp = Tags.FindS(tagname.c_str());
+      if (tmp.empty() == true)
+	 continue;
 
-   // autoselect the compression method
-   AutoSelectCompression();
-   Init(Target->URI, Target->Description, Target->ShortDesc);
+      string hash, filename;
+      unsigned long long size;
+      std::stringstream ss(tmp);
 
-   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-      std::clog << "New pkgIndex with TransactionManager "
-                << TransactionManager << std::endl;
-}
-									/*}}}*/
-// AcqIndex::AutoSelectCompression - Select compression			/*{{{*/
-void pkgAcqIndex::AutoSelectCompression()
-{
-   std::vector types = APT::Configuration::getCompressionTypes();
-   CompressionExtensions = "";
-   if (ExpectedHashes.usable())
-   {
-      for (std::vector::const_iterator t = types.begin();
-           t != types.end(); ++t)
+      while (ss >> hash >> size >> filename)
       {
-         std::string CompressedMetaKey = string(Target->MetaKey).append(".").append(*t);
-         if (*t == "uncompressed" ||
-             MetaIndexParser->Exists(CompressedMetaKey) == true)
-            CompressionExtensions.append(*t).append(" ");
+	 if (unlikely(hash.empty() == true || filename.empty() == true))
+	    continue;
+
+	 // see if we have a record for this file already
+	 std::vector::iterator cur = available_patches.begin();
+	 for (; cur != available_patches.end(); ++cur)
+	 {
+	    if (cur->file != filename || unlikely(cur->result_size != size))
+	       continue;
+	    cur->result_hashes.push_back(HashString(*type, hash));
+	    break;
+	 }
+	 if (cur != available_patches.end())
+	    continue;
+	 if (firstAcceptedHashes == true)
+	 {
+	    DiffInfo next;
+	    next.file = filename;
+	    next.result_hashes.push_back(HashString(*type, hash));
+	    next.result_size = size;
+	    next.patch_size = 0;
+	    available_patches.push_back(next);
+	 }
+	 else
+	 {
+	    if (Debug == true)
+	       std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+		  << " wasn't in the list for the first parsed hash! (history)" << std::endl;
+	    break;
+	 }
       }
+      firstAcceptedHashes = false;
    }
-   else
+
+   if (unlikely(available_patches.empty() == true))
    {
-      for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t)
-	 CompressionExtensions.append(*t).append(" ");
+      if (Debug)
+	 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
+	    << "Couldn't find any patches for the patch series." << std::endl;
+      return false;
    }
-   if (CompressionExtensions.empty() == false)
-      CompressionExtensions.erase(CompressionExtensions.end()-1);
-}
-									/*}}}*/
-// AcqIndex::Init - defered Constructor					/*{{{*/
-void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
-                       string const &ShortDesc)
-{
-   Stage = STAGE_DOWNLOAD;
-
-   DestFile = GetPartialFileNameFromURI(URI);
 
-   CurrentCompressionExtension = CompressionExtensions.substr(0, CompressionExtensions.find(' '));
-   if (CurrentCompressionExtension == "uncompressed")
+   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
    {
-      Desc.URI = URI;
-      if(Target)
-         MetaKey = string(Target->MetaKey);
-   }
-   else
-   {
-      Desc.URI = URI + '.' + CurrentCompressionExtension;
-      DestFile = DestFile + '.' + CurrentCompressionExtension;
-      if(Target)
-         MetaKey = string(Target->MetaKey) + '.' + CurrentCompressionExtension;
-   }
-
-   // load the filesize
-   if(MetaIndexParser)
-   {
-      indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
-      if(Record)
-         FileSize = Record->Size;
-
-      InitByHashIfNeeded(MetaKey);
-   }
-
-   Desc.Description = URIDesc;
-   Desc.Owner = this;
-   Desc.ShortDesc = ShortDesc;
-
-   QueueURI(Desc);
-}
-									/*}}}*/
-// AcqIndex::AdjustForByHash - modify URI for by-hash support		/*{{{*/
-void pkgAcqIndex::InitByHashIfNeeded(const std::string MetaKey)
-{
-   // TODO:
-   //  - (maybe?) add support for by-hash into the sources.list as flag
-   //  - make apt-ftparchive generate the hashes (and expire?)
-   std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
-   if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
-      _config->FindB(HostKnob, false) == true ||
-      MetaIndexParser->GetSupportsAcquireByHash())
-   {
-      indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey);
-      if(Record)
-      {
-         // FIXME: should we really use the best hash here? or a fixed one?
-         const HashString *TargetHash = Record->Hashes.find("");
-         std::string ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
-         size_t trailing_slash = Desc.URI.find_last_of("/");
-         Desc.URI = Desc.URI.replace(
-            trailing_slash,
-            Desc.URI.substr(trailing_slash+1).size()+1,
-            ByHash);
-      } else {
-         _error->Warning(
-            "Fetching ByHash requested but can not find record for %s",
-            MetaKey.c_str());
-      }
-   }
-}
-									/*}}}*/
-// AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
-// ---------------------------------------------------------------------
-/* The only header we use is the last-modified header. */
-#if APT_PKG_ABI >= 413
-string pkgAcqIndex::Custom600Headers() const
-#else
-string pkgAcqIndex::Custom600Headers()
-#endif
-{
-   string Final = GetFinalFilename();
-
-   string msg = "\nIndex-File: true";
-   struct stat Buf;
-   if (stat(Final.c_str(),&Buf) == 0)
-      msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+      if (LocalHashes.find(*type) == NULL)
+	 continue;
 
-   if(Target->IsOptional())
-      msg += "\nFail-Ignore: true";
+      std::string tagname = *type;
+      tagname.append("-Patches");
+      std::string const tmp = Tags.FindS(tagname.c_str());
+      if (tmp.empty() == true)
+	 continue;
 
-   return msg;
-}
-									/*}}}*/
-// pkgAcqIndex::Failed - getting the indexfile failed			/*{{{*/
-void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
-{
-   Item::Failed(Message,Cnf);
+      string hash, filename;
+      unsigned long long size;
+      std::stringstream ss(tmp);
 
-   // authorisation matches will not be fixed by other compression types
-   if (Status != StatAuthError)
-   {
-      size_t const nextExt = CompressionExtensions.find(' ');
-      if (nextExt != std::string::npos)
+      while (ss >> hash >> size >> filename)
       {
-	 CompressionExtensions = CompressionExtensions.substr(nextExt+1);
-	 Init(RealURI, Desc.Description, Desc.ShortDesc);
-	 Status = StatIdle;
-	 return;
-      }
-   }
-
-   if(Target->IsOptional() && ExpectedHashes.empty() && Stage == STAGE_DOWNLOAD)
-      Status = StatDone;
-   else
-      TransactionManager->AbortTransaction();
-}
-									/*}}}*/
-bool pkgAcqIndex::TransactionState(TransactionStates const state)	/*{{{*/
-{
-   if (pkgAcquire::Item::TransactionState(state) == false)
-      return false;
+	 if (unlikely(hash.empty() == true || filename.empty() == true))
+	    continue;
 
-   switch (state)
-   {
-      case TransactionAbort:
-	 if (Stage == STAGE_DECOMPRESS_AND_VERIFY)
+	 // see if we have a record for this file already
+	 std::vector::iterator cur = available_patches.begin();
+	 for (; cur != available_patches.end(); ++cur)
 	 {
-	    // keep the compressed file, but drop the decompressed
-	    EraseFileName.clear();
-	    if (PartialFile.empty() == false && flExtension(PartialFile) == "decomp")
-	       unlink(PartialFile.c_str());
+	    if (cur->file != filename)
+	       continue;
+	    if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
+	       continue;
+	    cur->patch_hashes.push_back(HashString(*type, hash));
+	    cur->patch_size = size;
+	    break;
 	 }
+	 if (cur != available_patches.end())
+	       continue;
+	 if (Debug == true)
+	    std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+	       << " wasn't in the list for the first parsed hash! (patches)" << std::endl;
 	 break;
-      case TransactionCommit:
-	 if (EraseFileName.empty() == false)
-	    unlink(EraseFileName.c_str());
-	 break;
-   }
-   return true;
-}
-									/*}}}*/
-// pkgAcqIndex::GetFinalFilename - Return the full final file path	/*{{{*/
-std::string pkgAcqIndex::GetFinalFilename() const
-{
-   std::string const FinalFile = GetFinalFileNameFromURI(RealURI);
-   return GetCompressedFileName(RealURI, FinalFile, CurrentCompressionExtension);
-}
-									/*}}}*/
-// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit		/*{{{*/
-void pkgAcqIndex::ReverifyAfterIMS()
-{
-   // update destfile to *not* include the compression extension when doing
-   // a reverify (as its uncompressed on disk already)
-   DestFile = GetCompressedFileName(RealURI, GetPartialFileNameFromURI(RealURI), CurrentCompressionExtension);
-
-   // copy FinalFile into partial/ so that we check the hash again
-   string FinalFile = GetFinalFilename();
-   Stage = STAGE_DECOMPRESS_AND_VERIFY;
-   Desc.URI = "copy:" + FinalFile;
-   QueueURI(Desc);
-}
-									/*}}}*/
-// AcqIndex::ValidateFile - Validate the content of the downloaded file	/*{{{*/
-bool pkgAcqIndex::ValidateFile(const std::string &FileName)
-{
-   // FIXME: this can go away once we only ever download stuff that
-   //        has a valid hash and we never do GET based probing
-   // FIXME2: this also leaks debian-isms into the code and should go therefore
-
-   /* Always validate the index file for correctness (all indexes must
-    * have a Package field) (LP: #346386) (Closes: #627642) 
-    */
-   FileFd fd(FileName, FileFd::ReadOnly, FileFd::Extension);
-   // Only test for correctness if the content of the file is not empty
-   // (empty is ok)
-   if (fd.Size() > 0)
-   {
-      pkgTagSection sec;
-      pkgTagFile tag(&fd);
-      
-      // all our current indexes have a field 'Package' in each section
-      if (_error->PendingError() == true ||
-          tag.Step(sec) == false ||
-          sec.Exists("Package") == false)
-         return false;
-   }
-   return true;
-}
-									/*}}}*/
-// AcqIndex::Done - Finished a fetch					/*{{{*/
-// ---------------------------------------------------------------------
-/* This goes through a number of states.. On the initial fetch the
-   method could possibly return an alternate filename which points
-   to the uncompressed version of the file. If this is so the file
-   is copied into the partial directory. In all other cases the file
-   is decompressed with a compressed uri. */
-void pkgAcqIndex::Done(string Message,
-                       unsigned long long Size,
-                       HashStringList const &Hashes,
-		       pkgAcquire::MethodConfig *Cfg)
-{
-   Item::Done(Message,Size,Hashes,Cfg);
-
-   switch(Stage) 
-   {
-      case STAGE_DOWNLOAD:
-         StageDownloadDone(Message, Hashes, Cfg);
-         break;
-      case STAGE_DECOMPRESS_AND_VERIFY:
-         StageDecompressDone(Message, Hashes, Cfg);
-         break;
-   }
-}
-									/*}}}*/
-// AcqIndex::StageDownloadDone - Queue for decompress and verify	/*{{{*/
-void pkgAcqIndex::StageDownloadDone(string Message,
-                                    HashStringList const &Hashes,
-                                    pkgAcquire::MethodConfig *Cfg)
-{
-   // First check if the calculcated Hash of the (compressed) downloaded
-   // file matches the hash we have in the MetaIndexRecords for this file
-   if(VerifyHashByMetaKey(Hashes) == false)
-   {
-      RenameOnError(HashSumMismatch);
-      Failed(Message, Cfg);
-      return;
-   }
-
-   Complete = true;
-
-   // Handle the unzipd case
-   string FileName = LookupTag(Message,"Alt-Filename");
-   if (FileName.empty() == false)
-   {
-      Stage = STAGE_DECOMPRESS_AND_VERIFY;
-      Local = true;
-      DestFile += ".decomp";
-      Desc.URI = "copy:" + FileName;
-      QueueURI(Desc);
-      SetActiveSubprocess("copy");
-      return;
-   }
-
-   FileName = LookupTag(Message,"Filename");
-   if (FileName.empty() == true)
-   {
-      Status = StatError;
-      ErrorText = "Method gave a blank filename";
-   }
-
-   // Methods like e.g. "file:" will give us a (compressed) FileName that is
-   // not the "DestFile" we set, in this case we uncompress from the local file
-   if (FileName != DestFile)
-      Local = true;
-   else
-      EraseFileName = FileName;
-
-   // we need to verify the file against the current Release file again
-   // on if-modfied-since hit to avoid a stale attack against us
-   if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
-   {
-      // The files timestamp matches, reverify by copy into partial/
-      EraseFileName = "";
-      ReverifyAfterIMS();
-      return;
-   }
-
-   // If we have compressed indexes enabled, queue for hash verification
-   if (_config->FindB("Acquire::GzipIndexes",false))
-   {
-      DestFile = GetPartialFileNameFromURI(RealURI + '.' + CurrentCompressionExtension);
-      EraseFileName = "";
-      Stage = STAGE_DECOMPRESS_AND_VERIFY;
-      Desc.URI = "copy:" + FileName;
-      QueueURI(Desc);
-      SetActiveSubprocess("copy");
-      return;
-    }
-
-   // get the binary name for your used compression type
-   string decompProg;
-   if(CurrentCompressionExtension == "uncompressed")
-      decompProg = "copy";
-   else
-      decompProg = _config->Find(string("Acquire::CompressionTypes::").append(CurrentCompressionExtension),"");
-   if(decompProg.empty() == true)
-   {
-      _error->Error("Unsupported extension: %s", CurrentCompressionExtension.c_str());
-      return;
-   }
-
-   // queue uri for the next stage
-   Stage = STAGE_DECOMPRESS_AND_VERIFY;
-   DestFile += ".decomp";
-   Desc.URI = decompProg + ":" + FileName;
-   QueueURI(Desc);
-   SetActiveSubprocess(decompProg);
-}
-									/*}}}*/
-// pkgAcqIndex::StageDecompressDone - Final verification		/*{{{*/
-void pkgAcqIndex::StageDecompressDone(string Message,
-                                      HashStringList const &Hashes,
-                                      pkgAcquire::MethodConfig *Cfg)
-{
-   if (ExpectedHashes.usable() && ExpectedHashes != Hashes)
-   {
-      Desc.URI = RealURI;
-      RenameOnError(HashSumMismatch);
-      printHashSumComparision(RealURI, ExpectedHashes, Hashes);
-      Failed(Message, Cfg);
-      return;
-   }
-
-   if(!ValidateFile(DestFile))
-   {
-      RenameOnError(InvalidFormat);
-      Failed(Message, Cfg);
-      return;
-   }
-
-   // Done, queue for rename on transaction finished
-   TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
-
-   return;
-}
-									/*}}}*/
-// AcqMetaBase - Constructor						/*{{{*/
-pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire *Owner,
-      const std::vector* IndexTargets,
-      indexRecords* MetaIndexParser,
-      std::string const &RealURI,
-      HashStringList const &ExpectedHashes,
-      pkgAcqMetaBase *TransactionManager)
-: Item(Owner, ExpectedHashes, TransactionManager),
-   MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL), IndexTargets(IndexTargets),
-   AuthPass(false), RealURI(RealURI), IMSHit(false)
-{
-}
-									/*}}}*/
-// AcqMetaBase::Add - Add a item to the current Transaction		/*{{{*/
-void pkgAcqMetaBase::Add(Item *I)
-{
-   Transaction.push_back(I);
-}
-									/*}}}*/
-// AcqMetaBase::AbortTransaction - Abort the current Transaction	/*{{{*/
-void pkgAcqMetaBase::AbortTransaction()
-{
-   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-      std::clog << "AbortTransaction: " << TransactionManager << std::endl;
-
-   // ensure the toplevel is in error state too
-   for (std::vector::iterator I = Transaction.begin();
-        I != Transaction.end(); ++I)
-   {
-      (*I)->TransactionState(TransactionAbort);
-   }
-   Transaction.clear();
-}
-									/*}}}*/
-// AcqMetaBase::TransactionHasError - Check for errors in Transaction	/*{{{*/
-bool pkgAcqMetaBase::TransactionHasError()
-{
-   for (pkgAcquire::ItemIterator I = Transaction.begin();
-        I != Transaction.end(); ++I)
-   {
-      switch((*I)->Status) {
-	 case StatDone: break;
-	 case StatIdle: break;
-	 case StatAuthError: return true;
-	 case StatError: return true;
-	 case StatTransientNetworkError: return true;
-	 case StatFetching: break;
       }
    }
-   return false;
-}
-									/*}}}*/
-// AcqMetaBase::CommitTransaction - Commit a transaction		/*{{{*/
-void pkgAcqMetaBase::CommitTransaction()
-{
-   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-      std::clog << "CommitTransaction: " << this << std::endl;
-
-   // move new files into place *and* remove files that are not
-   // part of the transaction but are still on disk
-   for (std::vector::iterator I = Transaction.begin();
-        I != Transaction.end(); ++I)
-   {
-      (*I)->TransactionState(TransactionCommit);
-   }
-   Transaction.clear();
-}
-									/*}}}*/
-bool pkgAcqMetaBase::TransactionState(TransactionStates const state)	/*{{{*/
-{
-   // Do not remove InRelease on IMSHit of Release.gpg [yes, this is very edgecasey]
-   if (TransactionManager->IMSHit == false)
-      return pkgAcquire::Item::TransactionState(state);
-   return true;
-}
-									/*}}}*/
-// AcqMetaBase::TransactionStageCopy - Stage a file for copying		/*{{{*/
-void pkgAcqMetaBase::TransactionStageCopy(Item *I,
-                                          const std::string &From,
-                                          const std::string &To)
-{
-   I->PartialFile = From;
-   I->DestFile = To;
-}
-									/*}}}*/
-// AcqMetaBase::TransactionStageRemoval - Stage a file for removal	/*{{{*/
-void pkgAcqMetaBase::TransactionStageRemoval(Item *I,
-                                             const std::string &FinalFile)
-{
-   I->PartialFile = "";
-   I->DestFile = FinalFile;
-}
-									/*}}}*/
-// AcqMetaBase::GenerateAuthWarning - Check gpg authentication error	/*{{{*/
-bool pkgAcqMetaBase::CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message)
-{
-   // FIXME: this entire function can do now that we disallow going to
-   //        a unauthenticated state and can cleanly rollback
 
-   string const Final = I->GetFinalFilename();
-   if(FileExists(Final))
-   {
-      I->Status = StatTransientNetworkError;
-      _error->Warning(_("An error occurred during the signature "
-                        "verification. The repository is not updated "
-                        "and the previous index files will be used. "
-                        "GPG error: %s: %s\n"),
-                      Desc.Description.c_str(),
-                      LookupTag(Message,"Message").c_str());
-      RunScripts("APT::Update::Auth-Failure");
-      return true;
-   } else if (LookupTag(Message,"Message").find("NODATA") != string::npos) {
-      /* Invalid signature file, reject (LP: #346386) (Closes: #627642) */
-      _error->Error(_("GPG error: %s: %s"),
-                    Desc.Description.c_str(),
-                    LookupTag(Message,"Message").c_str());
-      I->Status = StatError;
-      return true;
-   } else {
-      _error->Warning(_("GPG error: %s: %s"),
-                      Desc.Description.c_str(),
-                      LookupTag(Message,"Message").c_str());
+   bool foundStart = false;
+   for (std::vector::iterator cur = available_patches.begin();
+	 cur != available_patches.end(); ++cur)
+   {
+      if (LocalHashes != cur->result_hashes)
+	 continue;
+
+      available_patches.erase(available_patches.begin(), cur);
+      foundStart = true;
+      break;
    }
-   // gpgv method failed
-   ReportMirrorFailure("GPGFailure");
-   return false;
-}
-									/*}}}*/
-// AcqMetaSig::AcqMetaSig - Constructor					/*{{{*/
-pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner,
-      pkgAcqMetaBase *TransactionManager,
-      string const &URI, string const &URIDesc,string const &ShortDesc,
-      pkgAcqMetaIndex * const MetaIndex) :
-   pkgAcquire::Item(Owner, HashStringList(), TransactionManager), MetaIndex(MetaIndex),
-   URIDesc(URIDesc), RealURI(URI)
-{
-   DestFile = GetPartialFileNameFromURI(URI);
 
-   // remove any partial downloaded sig-file in partial/.
-   // it may confuse proxies and is too small to warrant a
-   // partial download anyway
-   unlink(DestFile.c_str());
+   if (foundStart == false || unlikely(available_patches.empty() == true))
+   {
+      if (Debug)
+	 std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": "
+	    << "Couldn't find the start of the patch series." << std::endl;
+      return false;
+   }
 
-   // set the TransactionManager
-   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-      std::clog << "New pkgAcqMetaSig with TransactionManager "
-                << TransactionManager << std::endl;
+   // patching with too many files is rather slow compared to a fast download
+   unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
+   if (fileLimit != 0 && fileLimit < available_patches.size())
+   {
+      if (Debug)
+	 std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
+	    << ") so fallback to complete download" << std::endl;
+      return false;
+   }
 
-   // Create the item
-   Desc.Description = URIDesc;
-   Desc.Owner = this;
-   Desc.ShortDesc = ShortDesc;
-   Desc.URI = URI;
+   // calculate the size of all patches we have to get
+   // note that all sizes are uncompressed, while we download compressed files
+   unsigned long long patchesSize = 0;
+   for (std::vector::const_iterator cur = available_patches.begin();
+	 cur != available_patches.end(); ++cur)
+      patchesSize += cur->patch_size;
+   unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
+   if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
+   {
+      if (Debug)
+	 std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
+	    << ") so fallback to complete download" << std::endl;
+      return false;
+   }
 
-   // If we got a hit for Release, we will get one for Release.gpg too (or obscure errors),
-   // so we skip the download step and go instantly to verification
-   if (TransactionManager->IMSHit == true && RealFileExists(GetFinalFilename()))
+   // we have something, queue the diffs
+   string::size_type const last_space = Description.rfind(" ");
+   if(last_space != string::npos)
+      Description.erase(last_space, Description.size()-last_space);
+
+   /* decide if we should download patches one by one or in one go:
+      The first is good if the server merges patches, but many don't so client
+      based merging can be attempt in which case the second is better.
+      "bad things" will happen if patches are merged on the server,
+      but client side merging is attempt as well */
+   bool pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
+   if (pdiff_merge == true)
    {
-      Complete = true;
-      Status = StatDone;
-      PartialFile = DestFile = GetFinalFilename();
-      MetaIndexFileSignature = DestFile;
-      MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
+      // reprepro adds this flag if it has merged patches on the server
+      std::string const precedence = Tags.FindS("X-Patch-Precedence");
+      pdiff_merge = (precedence != "merged");
    }
+
+   if (pdiff_merge == false)
+      new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches);
    else
-      QueueURI(Desc);
-}
-									/*}}}*/
-pkgAcqMetaSig::~pkgAcqMetaSig()						/*{{{*/
-{
+   {
+      std::vector *diffs = new std::vector(available_patches.size());
+      for(size_t i = 0; i < available_patches.size(); ++i)
+	 (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager,
+               Target,
+	       available_patches[i],
+	       diffs);
+   }
+
+   Complete = false;
+   Status = StatDone;
+   Dequeue();
+   return true;
 }
 									/*}}}*/
-// pkgAcqMetaSig::GetFinalFilename - Return the full final file path	/*{{{*/
-std::string pkgAcqMetaSig::GetFinalFilename() const
+void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 {
-   return GetFinalFileNameFromURI(RealURI);
+   Item::Failed(Message,Cnf);
+   Status = StatDone;
+
+   if(Debug)
+      std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
+		<< "Falling back to normal index file acquire" << std::endl;
+
+   new pkgAcqIndex(Owner, TransactionManager, Target);
 }
 									/*}}}*/
-// pkgAcqMetaSig::Done - The signature was downloaded/verified		/*{{{*/
-// ---------------------------------------------------------------------
-/* The only header we use is the last-modified header. */
-void pkgAcqMetaSig::Done(string Message,unsigned long long Size,
-                         HashStringList const &Hashes,
-			 pkgAcquire::MethodConfig *Cfg)
+void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes,	/*{{{*/
+			   pkgAcquire::MethodConfig const * const Cnf)
 {
-   if (MetaIndexFileSignature.empty() == false)
-   {
-      DestFile = MetaIndexFileSignature;
-      MetaIndexFileSignature.clear();
-   }
-   Item::Done(Message, Size, Hashes, Cfg);
+   if(Debug)
+      std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl;
 
-   if(MetaIndex->AuthPass == false)
+   Item::Done(Message, Hashes, Cnf);
+
+   string const FinalFile = GetFinalFilename();
+   if(StringToBool(LookupTag(Message,"IMS-Hit"),false))
+      DestFile = FinalFile;
+
+   if(ParseDiffIndex(DestFile) == false)
    {
-      if(MetaIndex->CheckDownloadDone(this, Message, Hashes) == true)
-      {
-	 // destfile will be modified to point to MetaIndexFile for the
-	 // gpgv method, so we need to save it here
-	 MetaIndexFileSignature = DestFile;
-	 MetaIndex->QueueForSignatureVerify(this, MetaIndex->DestFile, DestFile);
-      }
+      Failed("Message: Couldn't parse pdiff index", Cnf);
+      // queue for final move - this should happen even if we fail
+      // while parsing (e.g. on sizelimit) and download the complete file.
+      TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
       return;
    }
-   else if(MetaIndex->CheckAuthDone(Message) == true)
-   {
-      if (TransactionManager->IMSHit == false)
-      {
-	 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
-	 TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
-      }
-   }
+
+   TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
+
+   Complete = true;
+   Status = StatDone;
+   Dequeue();
+
+   return;
 }
 									/*}}}*/
-void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
+
+// AcqIndexDiffs::AcqIndexDiffs - Constructor				/*{{{*/
+// ---------------------------------------------------------------------
+/* The package diff is added to the queue. one object is constructed
+ * for each diff and the index
+ */
+pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire * const Owner,
+                                   pkgAcqMetaBase * const TransactionManager,
+                                   IndexTarget const * const Target,
+				   vector const &diffs)
+   : pkgAcqBaseIndex(Owner, TransactionManager, Target),
+     available_patches(diffs)
 {
-   Item::Failed(Message,Cnf);
+   DestFile = GetPartialFileNameFromURI(Target->URI);
 
-   // check if we need to fail at this point 
-   if (MetaIndex->AuthPass == true && MetaIndex->CheckStopAuthentication(this, Message))
-         return;
+   Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
-   string const FinalRelease = MetaIndex->GetFinalFilename();
-   string const FinalReleasegpg = GetFinalFilename();
-   string const FinalInRelease = TransactionManager->GetFinalFilename();
+   Desc.Owner = this;
+   Description = Target->Description;
+   Desc.ShortDesc = Target->ShortDesc;
 
-   if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
+   if(available_patches.empty() == true)
    {
-      std::string downgrade_msg;
-      strprintf(downgrade_msg, _("The repository '%s' is no longer signed."),
-                MetaIndex->URIDesc.c_str());
-      if(_config->FindB("Acquire::AllowDowngradeToInsecureRepositories"))
-      {
-         // meh, the users wants to take risks (we still mark the packages
-         // from this repository as unauthenticated)
-         _error->Warning("%s", downgrade_msg.c_str());
-         _error->Warning(_("This is normally not allowed, but the option "
-                           "Acquire::AllowDowngradeToInsecureRepositories was "
-                           "given to override it."));
-         Status = StatDone;
-      } else {
-         _error->Error("%s", downgrade_msg.c_str());
-	 if (TransactionManager->IMSHit == false)
-	    Rename(MetaIndex->DestFile, MetaIndex->DestFile + ".FAILED");
-	 Item::Failed("Message: " + downgrade_msg, Cnf);
-         TransactionManager->AbortTransaction();
-         return;
-      }
+      // we are done (yeah!), check hashes against the final file
+      DestFile = GetFinalFileNameFromURI(Target->URI);
+      Finish(true);
    }
    else
-      _error->Warning(_("The data from '%s' is not signed. Packages "
-	       "from that repository can not be authenticated."),
-	    MetaIndex->URIDesc.c_str());
-
-   // ensures that a Release.gpg file in the lists/ is removed by the transaction
-   TransactionManager->TransactionStageRemoval(this, DestFile);
-
-   // only allow going further if the users explicitely wants it
-   if(AllowInsecureRepositories(MetaIndex->MetaIndexParser, TransactionManager, this) == true)
    {
-      if (RealFileExists(FinalReleasegpg) || RealFileExists(FinalInRelease))
+      // patching needs to be bootstrapped with the 'old' version
+      std::string const PartialFile = GetPartialFileNameFromURI(Target->URI);
+      if (RealFileExists(PartialFile) == false)
       {
-	 // open the last Release if we have it
-	 if (TransactionManager->IMSHit == false)
+	 if (symlink(GetFinalFilename().c_str(), PartialFile.c_str()) != 0)
 	 {
-	    MetaIndex->LastMetaIndexParser = new indexRecords;
-	    _error->PushToStack();
-	    if (RealFileExists(FinalInRelease))
-	       MetaIndex->LastMetaIndexParser->Load(FinalInRelease);
-	    else
-	       MetaIndex->LastMetaIndexParser->Load(FinalRelease);
-	    // its unlikely to happen, but if what we have is bad ignore it
-	    if (_error->PendingError())
-	    {
-	       delete MetaIndex->LastMetaIndexParser;
-	       MetaIndex->LastMetaIndexParser = NULL;
-	    }
-	    _error->RevertToStack();
+	    Failed("Link creation of " + PartialFile + " to " + GetFinalFilename() + " failed", NULL);
+	    return;
 	 }
       }
 
-      // we parse the indexes here because at this point the user wanted
-      // a repository that may potentially harm him
-      MetaIndex->MetaIndexParser->Load(MetaIndex->DestFile);
-      if (MetaIndex->VerifyVendor(Message) == false)
-	 /* expired Release files are still a problem you need extra force for */;
-      else
-	 MetaIndex->QueueIndexes(true);
-
-      TransactionManager->TransactionStageCopy(MetaIndex, MetaIndex->DestFile, MetaIndex->GetFinalFilename());
+      // get the next diff
+      State = StateFetchDiff;
+      QueueNextDiff();
    }
+}
+									/*}}}*/
+void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
+{
+   Item::Failed(Message,Cnf);
+   Status = StatDone;
 
-   // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
-   if (Cnf->LocalOnly == true ||
-       StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
+		<< "Falling back to normal index file acquire" << std::endl;
+   DestFile = GetPartialFileNameFromURI(Target->URI);
+   RenameOnError(PDiffError);
+   new pkgAcqIndex(Owner, TransactionManager, Target);
+   Finish();
+}
+									/*}}}*/
+// Finish - helper that cleans the item out of the fetcher queue	/*{{{*/
+void pkgAcqIndexDiffs::Finish(bool allDone)
+{
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::Finish(): " 
+                << allDone << " "
+                << Desc.URI << std::endl;
+
+   // we restore the original name, this is required, otherwise
+   // the file will be cleaned
+   if(allDone)
    {
-      // Ignore this
+      TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+
+      // this is for the "real" finish
+      Complete = true;
       Status = StatDone;
+      Dequeue();
+      if(Debug)
+	 std::clog << "\n\nallDone: " << DestFile << "\n" << std::endl;
+      return;
    }
+
+   if(Debug)
+      std::clog << "Finishing: " << Desc.URI << std::endl;
+   Complete = false;
+   Status = StatDone;
+   Dequeue();
+   return;
 }
 									/*}}}*/
-pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner,			/*{{{*/
-                                 pkgAcqMetaBase *TransactionManager,
-				 string URI,string URIDesc,string ShortDesc,
-                                 string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc,
-				 const vector* IndexTargets,
-				 indexRecords* MetaIndexParser) :
-   pkgAcqMetaBase(Owner, IndexTargets, MetaIndexParser, URI, HashStringList(),
-                  TransactionManager), 
-   URIDesc(URIDesc), ShortDesc(ShortDesc),
-   MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc),
-   MetaIndexSigShortDesc(MetaIndexSigShortDesc)
+bool pkgAcqIndexDiffs::QueueNextDiff()					/*{{{*/
 {
-   if(TransactionManager == NULL)
+   // calc sha1 of the just patched file
+   std::string const FinalFile = GetPartialFileNameFromURI(Target->URI);
+
+   if(!FileExists(FinalFile))
    {
-      this->TransactionManager = this;
-      this->TransactionManager->Add(this);
+      Failed("Message: No FinalFile " + FinalFile + " available", NULL);
+      return false;
    }
 
-   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-      std::clog << "New pkgAcqMetaIndex with TransactionManager "
-                << this->TransactionManager << std::endl;
-
-
-   Init(URIDesc, ShortDesc);
-}
-									/*}}}*/
-// pkgAcqMetaIndex::Init - Delayed constructor				/*{{{*/
-void pkgAcqMetaIndex::Init(std::string URIDesc, std::string ShortDesc)
-{
-   DestFile = GetPartialFileNameFromURI(RealURI);
-
-   // Create the item
-   Desc.Description = URIDesc;
-   Desc.Owner = this;
-   Desc.ShortDesc = ShortDesc;
-   Desc.URI = RealURI;
+   FileFd fd(FinalFile, FileFd::ReadOnly);
+   Hashes LocalHashesCalc;
+   LocalHashesCalc.AddFD(fd);
+   HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
 
-   // we expect more item
-   ExpectedAdditionalItems = IndexTargets->size();
-   QueueURI(Desc);
-}
-									/*}}}*/
-void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,	/*{{{*/
-                           HashStringList const &Hashes,
-			   pkgAcquire::MethodConfig *Cfg)
-{
-   Item::Done(Message,Size,Hashes,Cfg);
+   if(Debug)
+      std::clog << "QueueNextDiff: " << FinalFile << " (" << LocalHashes.find(NULL)->toStr() << ")" << std::endl;
 
-   if(CheckDownloadDone(this, Message, Hashes))
+   HashStringList const TargetFileHashes = GetExpectedHashesFor(Target->MetaKey);
+   if (unlikely(LocalHashes.usable() == false || TargetFileHashes.usable() == false))
    {
-      // we have a Release file, now download the Signature, all further
-      // verify/queue for additional downloads will be done in the
-      // pkgAcqMetaSig::Done() code
-      new pkgAcqMetaSig(Owner, TransactionManager,
-                        MetaIndexSigURI, MetaIndexSigURIDesc,
-                        MetaIndexSigShortDesc, this);
+      Failed("Local/Expected hashes are not usable", NULL);
+      return false;
    }
-}
-									/*}}}*/
-bool pkgAcqMetaBase::CheckAuthDone(string Message)			/*{{{*/
-{
-   // At this point, the gpgv method has succeeded, so there is a
-   // valid signature from a key in the trusted keyring.  We
-   // perform additional verification of its contents, and use them
-   // to verify the indexes we are about to download
 
-   if (TransactionManager->IMSHit == false)
+
+   // final file reached before all patches are applied
+   if(LocalHashes == TargetFileHashes)
    {
-      // open the last (In)Release if we have it
-      std::string const FinalFile = GetFinalFilename();
-      std::string FinalRelease;
-      std::string FinalInRelease;
-      if (APT::String::Endswith(FinalFile, "InRelease"))
-      {
-	 FinalInRelease = FinalFile;
-	 FinalRelease = FinalFile.substr(0, FinalFile.length() - strlen("InRelease")) + "Release";
-      }
-      else
-      {
-	 FinalInRelease = FinalFile.substr(0, FinalFile.length() - strlen("Release")) + "InRelease";
-	 FinalRelease = FinalFile;
-      }
-      if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease))
-      {
-	 LastMetaIndexParser = new indexRecords;
-	 _error->PushToStack();
-	 if (RealFileExists(FinalInRelease))
-	    LastMetaIndexParser->Load(FinalInRelease);
-	 else
-	    LastMetaIndexParser->Load(FinalRelease);
-	 // its unlikely to happen, but if what we have is bad ignore it
-	 if (_error->PendingError())
-	 {
-	    delete LastMetaIndexParser;
-	    LastMetaIndexParser = NULL;
-	 }
-	 _error->RevertToStack();
-      }
+      Finish(true);
+      return true;
    }
 
-   if (!MetaIndexParser->Load(DestFile))
+   // remove all patches until the next matching patch is found
+   // this requires the Index file to be ordered
+   for(vector::iterator I = available_patches.begin();
+       available_patches.empty() == false &&
+	  I != available_patches.end() &&
+	  I->result_hashes != LocalHashes;
+       ++I)
    {
-      Status = StatAuthError;
-      ErrorText = MetaIndexParser->ErrorText;
-      return false;
+      available_patches.erase(I);
    }
 
-   if (!VerifyVendor(Message))
+   // error checking and falling back if no patch was found
+   if(available_patches.empty() == true)
    {
-      Status = StatAuthError;
+      Failed("No patches left to reach target", NULL);
       return false;
    }
 
-   if (_config->FindB("Debug::pkgAcquire::Auth", false))
-      std::cerr << "Signature verification succeeded: "
-                << DestFile << std::endl;
+   // queue the right diff
+   Desc.URI = Target->URI + ".diff/" + available_patches[0].file + ".gz";
+   Desc.Description = Description + " " + available_patches[0].file + string(".pdiff");
+   DestFile = GetPartialFileNameFromURI(Target->URI + ".diff/" + available_patches[0].file);
 
-   // Download further indexes with verification
-   QueueIndexes(true);
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::QueueNextDiff(): " << Desc.URI << std::endl;
+
+   QueueURI(Desc);
 
    return true;
 }
 									/*}}}*/
-// pkgAcqMetaBase::Custom600Headers - Get header for AcqMetaBase	/*{{{*/
-// ---------------------------------------------------------------------
-#if APT_PKG_ABI >= 413
-string pkgAcqMetaBase::Custom600Headers() const
-#else
-string pkgAcqMetaBase::Custom600Headers()
-#endif
+void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes,	/*{{{*/
+			    pkgAcquire::MethodConfig const * const Cnf)
 {
-   std::string Header = "\nIndex-File: true";
-   std::string MaximumSize;
-   strprintf(MaximumSize, "\nMaximum-Size: %i",
-             _config->FindI("Acquire::MaxReleaseFileSize", 10*1000*1000));
-   Header += MaximumSize;
-
-   string const FinalFile = GetFinalFilename();
+   if(Debug)
+      std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl;
 
-   struct stat Buf;
-   if (stat(FinalFile.c_str(),&Buf) == 0)
-      Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+   Item::Done(Message, Hashes, Cnf);
 
-   return Header;
-}
-									/*}}}*/
-// pkgAcqMetaBase::GetFinalFilename - Return the full final file path	/*{{{*/
-std::string pkgAcqMetaBase::GetFinalFilename() const
-{
-   return GetFinalFileNameFromURI(RealURI);
-}
-									/*}}}*/
-// pkgAcqMetaBase::QueueForSignatureVerify				/*{{{*/
-void pkgAcqMetaBase::QueueForSignatureVerify(pkgAcquire::Item * const I, std::string const &File, std::string const &Signature)
-{
-   AuthPass = true;
-   I->Desc.URI = "gpgv:" + Signature;
-   I->DestFile = File;
-   QueueURI(I->Desc);
-   I->SetActiveSubprocess("gpgv");
-}
-									/*}}}*/
-// pkgAcqMetaBase::CheckDownloadDone					/*{{{*/
-bool pkgAcqMetaBase::CheckDownloadDone(pkgAcquire::Item * const I, const std::string &Message, HashStringList const &Hashes) const
-{
-   // We have just finished downloading a Release file (it is not
-   // verified yet)
+   // FIXME: verify this download too before feeding it to rred
+   std::string const FinalFile = GetPartialFileNameFromURI(Target->URI);
 
-   string const FileName = LookupTag(Message,"Filename");
-   if (FileName.empty() == true)
+   // success in downloading a diff, enter ApplyDiff state
+   if(State == StateFetchDiff)
    {
-      I->Status = StatError;
-      I->ErrorText = "Method gave a blank filename";
-      return false;
-   }
+      FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Gzip);
+      class Hashes LocalHashesCalc;
+      LocalHashesCalc.AddFD(fd);
+      HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
 
-   if (FileName != I->DestFile)
-   {
-      I->Local = true;
-      I->Desc.URI = "copy:" + FileName;
-      I->QueueURI(I->Desc);
-      return false;
-   }
+      if (fd.Size() != available_patches[0].patch_size ||
+	    available_patches[0].patch_hashes != LocalHashes)
+      {
+	 // patchfiles are dated, so bad indicates a bad download, so kill it
+	 unlink(DestFile.c_str());
+	 Failed("Patch has Size/Hashsum mismatch", NULL);
+	 return;
+      }
 
-   // make sure to verify against the right file on I-M-S hit
-   bool IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"), false);
-   if (IMSHit == false && Hashes.usable())
+      // rred excepts the patch as $FinalFile.ed
+      Rename(DestFile,FinalFile+".ed");
+
+      if(Debug)
+	 std::clog << "Sending to rred method: " << FinalFile << std::endl;
+
+      State = StateApplyDiff;
+      Local = true;
+      Desc.URI = "rred:" + FinalFile;
+      QueueURI(Desc);
+      SetActiveSubprocess("rred");
+      return;
+   } 
+
+
+   // success in download/apply a diff, queue next (if needed)
+   if(State == StateApplyDiff)
    {
-      // detect IMS-Hits servers haven't detected by Hash comparison
-      std::string const FinalFile = I->GetFinalFilename();
-      if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
+      // remove the just applied patch
+      available_patches.erase(available_patches.begin());
+      unlink((FinalFile + ".ed").c_str());
+
+      // move into place
+      if(Debug) 
       {
-	 IMSHit = true;
-	 unlink(I->DestFile.c_str());
+	 std::clog << "Moving patched file in place: " << std::endl
+		   << DestFile << " -> " << FinalFile << std::endl;
       }
-   }
+      Rename(DestFile,FinalFile);
+      chmod(FinalFile.c_str(),0644);
 
-   if(IMSHit == true)
-   {
-      // for simplicity, the transaction manager is always InRelease
-      // even if it doesn't exist.
-      if (TransactionManager != NULL)
-	 TransactionManager->IMSHit = true;
-      I->PartialFile = I->DestFile = I->GetFinalFilename();
+      // see if there is more to download
+      if(available_patches.empty() == false) {
+	 new pkgAcqIndexDiffs(Owner, TransactionManager, Target,
+			      available_patches);
+	 return Finish();
+      } else 
+         // update
+         DestFile = FinalFile;
+	 return Finish(true);
    }
+}
+									/*}}}*/
+
+// AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor			/*{{{*/
+pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner,
+                                             pkgAcqMetaBase * const TransactionManager,
+                                             IndexTarget const * const Target,
+                                             DiffInfo const &patch,
+                                             std::vector const * const allPatches)
+  : pkgAcqBaseIndex(Owner, TransactionManager, Target),
+     patch(patch), allPatches(allPatches), State(StateFetchDiff)
+{
+   Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
+
+   Desc.Owner = this;
+   Description = Target->Description;
+   Desc.ShortDesc = Target->ShortDesc;
+
+   Desc.URI = Target->URI + ".diff/" + patch.file + ".gz";
+   Desc.Description = Description + " " + patch.file + string(".pdiff");
+
+   DestFile = GetPartialFileNameFromURI(Target->URI + ".diff/" + patch.file);
+
+   if(Debug)
+      std::clog << "pkgAcqIndexMergeDiffs: " << Desc.URI << std::endl;
+
+   QueueURI(Desc);
+}
+									/*}}}*/
+void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
+{
+   if(Debug)
+      std::clog << "pkgAcqIndexMergeDiffs failed: " << Desc.URI << " with " << Message << std::endl;
 
-   // set Item to complete as the remaining work is all local (verify etc)
-   I->Complete = true;
+   Item::Failed(Message,Cnf);
+   Status = StatDone;
 
-   return true;
+   // check if we are the first to fail, otherwise we are done here
+   State = StateDoneDiff;
+   for (std::vector::const_iterator I = allPatches->begin();
+	 I != allPatches->end(); ++I)
+      if ((*I)->State == StateErrorDiff)
+	 return;
+
+   // first failure means we should fallback
+   State = StateErrorDiff;
+   if (Debug)
+      std::clog << "Falling back to normal index file acquire" << std::endl;
+   DestFile = GetPartialFileNameFromURI(Target->URI);
+   RenameOnError(PDiffError);
+   new pkgAcqIndex(Owner, TransactionManager, Target);
 }
 									/*}}}*/
-void pkgAcqMetaBase::QueueIndexes(bool verify)				/*{{{*/
+void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Hashes,	/*{{{*/
+			    pkgAcquire::MethodConfig const * const Cnf)
 {
-   // at this point the real Items are loaded in the fetcher
-   ExpectedAdditionalItems = 0;
+   if(Debug)
+      std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl;
 
-   vector ::const_iterator Target;
-   for (Target = IndexTargets->begin();
-        Target != IndexTargets->end();
-        ++Target)
-   {
-      HashStringList ExpectedIndexHashes;
-      const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
+   Item::Done(Message, Hashes, Cnf);
 
-      // optional target that we do not have in the Release file are 
-      // skipped
-      if (verify == true && Record == NULL && (*Target)->IsOptional())
-         continue;
+   // FIXME: verify download before feeding it to rred
+   string const FinalFile = GetPartialFileNameFromURI(Target->URI);
+
+   if (State == StateFetchDiff)
+   {
+      FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Gzip);
+      class Hashes LocalHashesCalc;
+      LocalHashesCalc.AddFD(fd);
+      HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
 
-      // targets without a hash record are a error when verify is required
-      if (verify == true && Record == NULL)
+      if (fd.Size() != patch.patch_size || patch.patch_hashes != LocalHashes)
       {
-         Status = StatAuthError;
-         strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
-         return;
+	 // patchfiles are dated, so bad indicates a bad download, so kill it
+	 unlink(DestFile.c_str());
+	 Failed("Patch has Size/Hashsum mismatch", NULL);
+	 return;
       }
 
-      if (Record)
-         ExpectedIndexHashes = Record->Hashes;
-      
-      if (_config->FindB("Debug::pkgAcquire::Auth", false))
-      {
-         std::cerr << "Queueing: " << (*Target)->URI << std::endl
-                   << "Expected Hash:" << std::endl;
-         for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs)
-            std::cerr <<  "\t- " << hs->toStr() << std::endl;
-         std::cerr << "For: " << ((Record == NULL) ? "" : Record->MetaKeyFilename) << std::endl;
+      // rred expects the patch as $FinalFile.ed.$patchname.gz
+      Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
+
+      // check if this is the last completed diff
+      State = StateDoneDiff;
+      for (std::vector::const_iterator I = allPatches->begin();
+	    I != allPatches->end(); ++I)
+	 if ((*I)->State != StateDoneDiff)
+	 {
+	    if(Debug)
+	       std::clog << "Not the last done diff in the batch: " << Desc.URI << std::endl;
+	    return;
+	 }
+
+      // this is the last completed diff, so we are ready to apply now
+      State = StateApplyDiff;
 
+      // patching needs to be bootstrapped with the 'old' version
+      if (symlink(GetFinalFilename().c_str(), FinalFile.c_str()) != 0)
+      {
+	 Failed("Link creation of " + FinalFile + " to " + GetFinalFilename() + " failed", NULL);
+	 return;
       }
-      if (verify == true && ExpectedIndexHashes.empty() == true)
+
+      if(Debug)
+	 std::clog << "Sending to rred method: " << FinalFile << std::endl;
+
+      Local = true;
+      Desc.URI = "rred:" + FinalFile;
+      QueueURI(Desc);
+      SetActiveSubprocess("rred");
+      return;
+   }
+   // success in download/apply all diffs, clean up
+   else if (State == StateApplyDiff)
+   {
+      // move the result into place
+      std::string const Final = GetFinalFilename();
+      if(Debug)
+	 std::clog << "Queue patched file in place: " << std::endl
+		   << DestFile << " -> " << Final << std::endl;
+
+      // queue for copy by the transaction manager
+      TransactionManager->TransactionStageCopy(this, DestFile, Final);
+
+      // ensure the ed's are gone regardless of list-cleanup
+      for (std::vector::const_iterator I = allPatches->begin();
+	    I != allPatches->end(); ++I)
       {
-         Status = StatAuthError;
-         strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
-         return;
+	 std::string const PartialFile = GetPartialFileNameFromURI(Target->URI);
+	 std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz";
+	 unlink(patch.c_str());
       }
+      unlink(FinalFile.c_str());
 
-      /* Queue the Index file (Packages, Sources, Translation-$foo
-         (either diff or full packages files, depending
-         on the users option) - we also check if the PDiff Index file is listed
-         in the Meta-Index file. Ideal would be if pkgAcqDiffIndex would test this
-         instead, but passing the required info to it is to much hassle */
-      if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false ||
-          MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true))
-         new pkgAcqDiffIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser);
-      else
-         new pkgAcqIndex(Owner, TransactionManager, *Target, ExpectedIndexHashes, MetaIndexParser);
+      // all set and done
+      Complete = true;
+      if(Debug)
+	 std::clog << "allDone: " << DestFile << "\n" << std::endl;
    }
 }
 									/*}}}*/
-bool pkgAcqMetaBase::VerifyVendor(string Message)			/*{{{*/
+
+// AcqIndex::AcqIndex - Constructor					/*{{{*/
+pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
+                         pkgAcqMetaBase * const TransactionManager,
+                         IndexTarget const * const Target)
+   : pkgAcqBaseIndex(Owner, TransactionManager, Target)
 {
-   string::size_type pos;
+   // autoselect the compression method
+   AutoSelectCompression();
+   Init(Target->URI, Target->Description, Target->ShortDesc);
 
-   // check for missing sigs (that where not fatal because otherwise we had
-   // bombed earlier)
-   string missingkeys;
-   string msg = _("There is no public key available for the "
-		  "following key IDs:\n");
-   pos = Message.find("NO_PUBKEY ");
-   if (pos != std::string::npos)
+   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
+      std::clog << "New pkgIndex with TransactionManager "
+                << TransactionManager << std::endl;
+}
+									/*}}}*/
+// AcqIndex::AutoSelectCompression - Select compression			/*{{{*/
+void pkgAcqIndex::AutoSelectCompression()
+{
+   std::vector types = APT::Configuration::getCompressionTypes();
+   CompressionExtensions = "";
+   if (TransactionManager->MetaIndexParser != NULL && TransactionManager->MetaIndexParser->Exists(Target->MetaKey))
    {
-      string::size_type start = pos+strlen("NO_PUBKEY ");
-      string Fingerprint = Message.substr(start, Message.find("\n")-start);
-      missingkeys += (Fingerprint);
+      for (std::vector::const_iterator t = types.begin();
+           t != types.end(); ++t)
+      {
+         std::string CompressedMetaKey = string(Target->MetaKey).append(".").append(*t);
+         if (*t == "uncompressed" ||
+             TransactionManager->MetaIndexParser->Exists(CompressedMetaKey) == true)
+            CompressionExtensions.append(*t).append(" ");
+      }
    }
-   if(!missingkeys.empty())
-      _error->Warning("%s", (msg + missingkeys).c_str());
-
-   string Transformed = MetaIndexParser->GetExpectedDist();
-
-   if (Transformed == "../project/experimental")
+   else
    {
-      Transformed = "experimental";
+      for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t)
+	 CompressionExtensions.append(*t).append(" ");
    }
+   if (CompressionExtensions.empty() == false)
+      CompressionExtensions.erase(CompressionExtensions.end()-1);
+}
+									/*}}}*/
+// AcqIndex::Init - defered Constructor					/*{{{*/
+void pkgAcqIndex::Init(string const &URI, string const &URIDesc,
+                       string const &ShortDesc)
+{
+   Stage = STAGE_DOWNLOAD;
 
-   pos = Transformed.rfind('/');
-   if (pos != string::npos)
-   {
-      Transformed = Transformed.substr(0, pos);
-   }
+   DestFile = GetPartialFileNameFromURI(URI);
 
-   if (Transformed == ".")
+   size_t const nextExt = CompressionExtensions.find(' ');
+   if (nextExt == std::string::npos)
    {
-      Transformed = "";
+      CurrentCompressionExtension = CompressionExtensions;
+      CompressionExtensions.clear();
    }
-
-   if (_config->FindB("Acquire::Check-Valid-Until", true) == true &&
-       MetaIndexParser->GetValidUntil() > 0) {
-      time_t const invalid_since = time(NULL) - MetaIndexParser->GetValidUntil();
-      if (invalid_since > 0)
-      {
-	 std::string errmsg;
-	 strprintf(errmsg,
-	       // TRANSLATOR: The first %s is the URL of the bad Release file, the second is
-	       // the time since then the file is invalid - formated in the same way as in
-	       // the download progress display (e.g. 7d 3h 42min 1s)
-	       _("Release file for %s is expired (invalid since %s). "
-		  "Updates for this repository will not be applied."),
-	       RealURI.c_str(), TimeToStr(invalid_since).c_str());
-	 if (ErrorText.empty())
-	    ErrorText = errmsg;
-	 return _error->Error("%s", errmsg.c_str());
-      }
+   else
+   {
+      CurrentCompressionExtension = CompressionExtensions.substr(0, nextExt);
+      CompressionExtensions = CompressionExtensions.substr(nextExt+1);
    }
 
-   /* Did we get a file older than what we have? This is a last minute IMS hit and doubles
-      as a prevention of downgrading us to older (still valid) files */
-   if (TransactionManager->IMSHit == false && LastMetaIndexParser != NULL &&
-	 LastMetaIndexParser->GetDate() > MetaIndexParser->GetDate())
+   if (CurrentCompressionExtension == "uncompressed")
    {
-      TransactionManager->IMSHit = true;
-      unlink(DestFile.c_str());
-      PartialFile = DestFile = GetFinalFilename();
-      delete MetaIndexParser;
-      MetaIndexParser = LastMetaIndexParser;
-      LastMetaIndexParser = NULL;
+      Desc.URI = URI;
    }
-
-   if (_config->FindB("Debug::pkgAcquire::Auth", false)) 
+   else if (unlikely(CurrentCompressionExtension.empty()))
+      return;
+   else
    {
-      std::cerr << "Got Codename: " << MetaIndexParser->GetDist() << std::endl;
-      std::cerr << "Expecting Dist: " << MetaIndexParser->GetExpectedDist() << std::endl;
-      std::cerr << "Transformed Dist: " << Transformed << std::endl;
+      Desc.URI = URI + '.' + CurrentCompressionExtension;
+      DestFile = DestFile + '.' + CurrentCompressionExtension;
    }
 
-   if (MetaIndexParser->CheckDist(Transformed) == false)
+   if(TransactionManager->MetaIndexParser != NULL)
+      InitByHashIfNeeded();
+
+   Desc.Description = URIDesc;
+   Desc.Owner = this;
+   Desc.ShortDesc = ShortDesc;
+
+   QueueURI(Desc);
+}
+									/*}}}*/
+// AcqIndex::AdjustForByHash - modify URI for by-hash support		/*{{{*/
+void pkgAcqIndex::InitByHashIfNeeded()
+{
+   // TODO:
+   //  - (maybe?) add support for by-hash into the sources.list as flag
+   //  - make apt-ftparchive generate the hashes (and expire?)
+   std::string HostKnob = "APT::Acquire::" + ::URI(Desc.URI).Host + "::By-Hash";
+   if(_config->FindB("APT::Acquire::By-Hash", false) == true ||
+      _config->FindB(HostKnob, false) == true ||
+      TransactionManager->MetaIndexParser->GetSupportsAcquireByHash())
    {
-      // This might become fatal one day
-//       Status = StatAuthError;
-//       ErrorText = "Conflicting distribution; expected "
-//          + MetaIndexParser->GetExpectedDist() + " but got "
-//          + MetaIndexParser->GetDist();
-//       return false;
-      if (!Transformed.empty())
+      HashStringList const Hashes = GetExpectedHashes();
+      if(Hashes.usable())
       {
-         _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"),
-                         Desc.Description.c_str(),
-                         Transformed.c_str(),
-                         MetaIndexParser->GetDist().c_str());
+         // FIXME: should we really use the best hash here? or a fixed one?
+         HashString const * const TargetHash = Hashes.find("");
+         std::string const ByHash = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
+         size_t const trailing_slash = Desc.URI.find_last_of("/");
+         Desc.URI = Desc.URI.replace(
+            trailing_slash,
+            Desc.URI.substr(trailing_slash+1).size()+1,
+            ByHash);
+      } else {
+         _error->Warning(
+            "Fetching ByHash requested but can not find record for %s",
+            GetMetaKey().c_str());
       }
    }
-
-   return true;
 }
 									/*}}}*/
-// pkgAcqMetaIndex::Failed - no Release file present			/*{{{*/
-void pkgAcqMetaIndex::Failed(string Message,
-                             pkgAcquire::MethodConfig * Cnf)
+// AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
+// ---------------------------------------------------------------------
+/* The only header we use is the last-modified header. */
+string pkgAcqIndex::Custom600Headers() const
 {
-   pkgAcquire::Item::Failed(Message, Cnf);
-   Status = StatDone;
-
-   _error->Warning(_("The repository '%s' does not have a Release file. "
-                     "This is deprecated, please contact the owner of the "
-                     "repository."), URIDesc.c_str());
+   string Final = GetFinalFilename();
 
-   // No Release file was present so fall
-   // back to queueing Packages files without verification
-   // only allow going further if the users explicitely wants it
-   if(AllowInsecureRepositories(MetaIndexParser, TransactionManager, this) == true)
-   {
-      // Done, queue for rename on transaction finished
-      if (FileExists(DestFile)) 
-         TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
+   string msg = "\nIndex-File: true";
+   struct stat Buf;
+   if (stat(Final.c_str(),&Buf) == 0)
+      msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 
-      // queue without any kind of hashsum support
-      QueueIndexes(false);
-   }
+   if(Target->IsOptional())
+      msg += "\nFail-Ignore: true";
+
+   return msg;
 }
 									/*}}}*/
-void pkgAcqMetaIndex::Finished()					/*{{{*/
+// AcqIndex::Failed - getting the indexfile failed			/*{{{*/
+void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
 {
-   if(_config->FindB("Debug::Acquire::Transaction", false) == true)
-      std::clog << "Finished: " << DestFile <TransactionHasError() == false)
-      TransactionManager->CommitTransaction();
+   Item::Failed(Message,Cnf);
+
+   // authorisation matches will not be fixed by other compression types
+   if (Status != StatAuthError)
+   {
+      if (CompressionExtensions.empty() == false)
+      {
+	 Init(Target->URI, Desc.Description, Desc.ShortDesc);
+	 Status = StatIdle;
+	 return;
+      }
+   }
+
+   if(Target->IsOptional() && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
+      Status = StatDone;
+   else
+      TransactionManager->AbortTransaction();
 }
 									/*}}}*/
-pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,		/*{{{*/
-		string const &URI, string const &URIDesc, string const &ShortDesc,
-		string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
-		string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
-		const vector* IndexTargets,
-		indexRecords* MetaIndexParser) :
-   pkgAcqMetaIndex(Owner, NULL, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser),
-       MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
-       MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
+// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit		/*{{{*/
+void pkgAcqIndex::ReverifyAfterIMS()
 {
-   // index targets + (worst case:) Release/Release.gpg
-   ExpectedAdditionalItems = IndexTargets->size() + 2;
+   // update destfile to *not* include the compression extension when doing
+   // a reverify (as its uncompressed on disk already)
+   DestFile = GetCompressedFileName(Target->URI, GetPartialFileNameFromURI(Target->URI), CurrentCompressionExtension);
+
+   // copy FinalFile into partial/ so that we check the hash again
+   string FinalFile = GetFinalFilename();
+   Stage = STAGE_DECOMPRESS_AND_VERIFY;
+   Desc.URI = "copy:" + FinalFile;
+   QueueURI(Desc);
 }
 									/*}}}*/
-pkgAcqMetaClearSig::~pkgAcqMetaClearSig()				/*{{{*/
+// AcqIndex::ValidateFile - Validate the content of the downloaded file	/*{{{*/
+bool pkgAcqIndex::ValidateFile(const std::string &FileName)
 {
+   // FIXME: this can go away once we only ever download stuff that
+   //        has a valid hash and we never do GET based probing
+   // FIXME2: this also leaks debian-isms into the code and should go therefore
+
+   /* Always validate the index file for correctness (all indexes must
+    * have a Package field) (LP: #346386) (Closes: #627642) 
+    */
+   FileFd fd(FileName, FileFd::ReadOnly, FileFd::Extension);
+   // Only test for correctness if the content of the file is not empty
+   // (empty is ok)
+   if (fd.Size() > 0)
+   {
+      pkgTagSection sec;
+      pkgTagFile tag(&fd);
+      
+      // all our current indexes have a field 'Package' in each section
+      if (_error->PendingError() == true ||
+          tag.Step(sec) == false ||
+          sec.Exists("Package") == false)
+         return false;
+   }
+   return true;
 }
 									/*}}}*/
-// pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers	/*{{{*/
-#if APT_PKG_ABI >= 413
-string pkgAcqMetaClearSig::Custom600Headers() const
-#else
-string pkgAcqMetaClearSig::Custom600Headers()
-#endif
+// AcqIndex::Done - Finished a fetch					/*{{{*/
+// ---------------------------------------------------------------------
+/* This goes through a number of states.. On the initial fetch the
+   method could possibly return an alternate filename which points
+   to the uncompressed version of the file. If this is so the file
+   is copied into the partial directory. In all other cases the file
+   is decompressed with a compressed uri. */
+void pkgAcqIndex::Done(string const &Message,
+                       HashStringList const &Hashes,
+		       pkgAcquire::MethodConfig const * const Cfg)
 {
-   string Header = pkgAcqMetaBase::Custom600Headers();
-   Header += "\nFail-Ignore: true";
-   return Header;
+   Item::Done(Message,Hashes,Cfg);
+
+   switch(Stage) 
+   {
+      case STAGE_DOWNLOAD:
+         StageDownloadDone(Message, Hashes, Cfg);
+         break;
+      case STAGE_DECOMPRESS_AND_VERIFY:
+         StageDecompressDone(Message, Hashes, Cfg);
+         break;
+   }
 }
 									/*}}}*/
-// pkgAcqMetaClearSig::Done - We got a file				/*{{{*/
-class APT_HIDDEN DummyItem : public pkgAcquire::Item
+// AcqIndex::StageDownloadDone - Queue for decompress and verify	/*{{{*/
+void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const &,
+                                    pkgAcquire::MethodConfig const * const)
 {
-   std::string URI;
-   public:
-   virtual std::string DescURI() {return URI;};
+   Complete = true;
 
-   DummyItem(pkgAcquire *Owner, std::string const &URI) : pkgAcquire::Item(Owner), URI(URI)
+   // Handle the unzipd case
+   string FileName = LookupTag(Message,"Alt-Filename");
+   if (FileName.empty() == false)
    {
-      Status = StatDone;
-      DestFile = GetFinalFileNameFromURI(URI);
+      Stage = STAGE_DECOMPRESS_AND_VERIFY;
+      Local = true;
+      DestFile += ".decomp";
+      Desc.URI = "copy:" + FileName;
+      QueueURI(Desc);
+      SetActiveSubprocess("copy");
+      return;
    }
-};
-void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size,
-                              HashStringList const &Hashes,
-                              pkgAcquire::MethodConfig *Cnf)
-{
-   Item::Done(Message, Size, Hashes, Cnf);
 
-   // if we expect a ClearTextSignature (InRelease), ensure that
-   // this is what we get and if not fail to queue a 
-   // Release/Release.gpg, see #346386
-   if (FileExists(DestFile) && !StartsWithGPGClearTextSignature(DestFile))
+   FileName = LookupTag(Message,"Filename");
+   if (FileName.empty() == true)
    {
-      pkgAcquire::Item::Failed(Message, Cnf);
-      RenameOnError(NotClearsigned);
-      TransactionManager->AbortTransaction();
-      return;
+      Status = StatError;
+      ErrorText = "Method gave a blank filename";
    }
 
-   if(AuthPass == false)
+   // Methods like e.g. "file:" will give us a (compressed) FileName that is
+   // not the "DestFile" we set, in this case we uncompress from the local file
+   if (FileName != DestFile)
+      Local = true;
+   else
+      EraseFileName = FileName;
+
+   // we need to verify the file against the current Release file again
+   // on if-modfied-since hit to avoid a stale attack against us
+   if(StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
    {
-      if(CheckDownloadDone(this, Message, Hashes) == true)
-         QueueForSignatureVerify(this, DestFile, DestFile);
+      // The files timestamp matches, reverify by copy into partial/
+      EraseFileName = "";
+      ReverifyAfterIMS();
       return;
    }
-   else if(CheckAuthDone(Message) == true)
+
+   // If we have compressed indexes enabled, queue for hash verification
+   if (_config->FindB("Acquire::GzipIndexes",false))
    {
-      if (TransactionManager->IMSHit == false)
-	 TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
-      else if (RealFileExists(GetFinalFilename()) == false)
-      {
-	 // We got an InRelease file IMSHit, but we haven't one, which means
-	 // we had a valid Release/Release.gpg combo stepping in, which we have
-	 // to 'acquire' now to ensure list cleanup isn't removing them
-	 new DummyItem(Owner, MetaIndexURI);
-	 new DummyItem(Owner, MetaSigURI);
-      }
+      DestFile = GetPartialFileNameFromURI(Target->URI + '.' + CurrentCompressionExtension);
+      EraseFileName = "";
+      Stage = STAGE_DECOMPRESS_AND_VERIFY;
+      Desc.URI = "copy:" + FileName;
+      QueueURI(Desc);
+      SetActiveSubprocess("copy");
+      return;
+    }
+
+   // get the binary name for your used compression type
+   string decompProg;
+   if(CurrentCompressionExtension == "uncompressed")
+      decompProg = "copy";
+   else
+      decompProg = _config->Find(string("Acquire::CompressionTypes::").append(CurrentCompressionExtension),"");
+   if(decompProg.empty() == true)
+   {
+      _error->Error("Unsupported extension: %s", CurrentCompressionExtension.c_str());
+      return;
    }
+
+   // queue uri for the next stage
+   Stage = STAGE_DECOMPRESS_AND_VERIFY;
+   DestFile += ".decomp";
+   Desc.URI = decompProg + ":" + FileName;
+   QueueURI(Desc);
+   SetActiveSubprocess(decompProg);
 }
 									/*}}}*/
-void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
+// AcqIndex::StageDecompressDone - Final verification			/*{{{*/
+void pkgAcqIndex::StageDecompressDone(string const &Message,
+                                      HashStringList const &,
+                                      pkgAcquire::MethodConfig const * const Cfg)
 {
-   Item::Failed(Message, Cnf);
-
-   // we failed, we will not get additional items from this method
-   ExpectedAdditionalItems = 0;
-
-   if (AuthPass == false)
+   if(!ValidateFile(DestFile))
    {
-      // Queue the 'old' InRelease file for removal if we try Release.gpg
-      // as otherwise the file will stay around and gives a false-auth
-      // impression (CVE-2012-0214)
-      TransactionManager->TransactionStageRemoval(this, GetFinalFilename());
-      Status = StatDone;
-
-      new pkgAcqMetaIndex(Owner, TransactionManager,
-			MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
-			MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
-			IndexTargets, MetaIndexParser);
+      RenameOnError(InvalidFormat);
+      Failed(Message, Cfg);
+      return;
    }
-   else
-   {
-      if(CheckStopAuthentication(this, Message))
-         return;
 
-      _error->Warning(_("The data from '%s' is not signed. Packages "
-                        "from that repository can not be authenticated."),
-                      URIDesc.c_str());
-
-      // No Release file was present, or verification failed, so fall
-      // back to queueing Packages files without verification
-      // only allow going further if the users explicitely wants it
-      if(AllowInsecureRepositories(MetaIndexParser, TransactionManager, this) == true)
-      {
-	 Status = StatDone;
+   // Done, queue for rename on transaction finished
+   TransactionManager->TransactionStageCopy(this, DestFile, GetFinalFilename());
 
-         /* Always move the meta index, even if gpgv failed. This ensures
-          * that PackageFile objects are correctly filled in */
-         if (FileExists(DestFile))
-         {
-            string FinalFile = GetFinalFilename();
-            /* InRelease files become Release files, otherwise
-             * they would be considered as trusted later on */
-            RealURI = RealURI.replace(RealURI.rfind("InRelease"), 9,
-                                      "Release");
-            FinalFile = FinalFile.replace(FinalFile.rfind("InRelease"), 9,
-                                          "Release");
-
-            // Done, queue for rename on transaction finished
-            TransactionManager->TransactionStageCopy(this, DestFile, FinalFile);
-         }
-         QueueIndexes(false);
-      }
-   }
+   return;
 }
 									/*}}}*/
+
+
 // AcqArchive::AcqArchive - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* This just sets up the initial fetch environment and queues the first
    possibilitiy */
-pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
-			     pkgRecords *Recs,pkgCache::VerIterator const &Version,
+pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources,
+			     pkgRecords * const Recs,pkgCache::VerIterator const &Version,
 			     string &StoreFilename) :
-               Item(Owner, HashStringList()), Version(Version), Sources(Sources), Recs(Recs), 
-               StoreFilename(StoreFilename), Vf(Version.FileList()), 
+               Item(Owner), LocalSource(false), Version(Version), Sources(Sources), Recs(Recs),
+               StoreFilename(StoreFilename), Vf(Version.FileList()),
 	       Trusted(false)
 {
    Retries = _config->FindI("Acquire::Retries",0);
@@ -2573,15 +2577,17 @@ bool pkgAcqArchive::QueueNext()
 {
    for (; Vf.end() == false; ++Vf)
    {
+      pkgCache::PkgFileIterator const PkgF = Vf.File();
       // Ignore not source sources
-      if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
+      if ((PkgF->Flags & pkgCache::Flag::NotSource) != 0)
 	 continue;
 
       // Try to cross match against the source list
       pkgIndexFile *Index;
-      if (Sources->FindIndex(Vf.File(),Index) == false)
+      if (Sources->FindIndex(PkgF, Index) == false)
 	    continue;
-      
+      LocalSource = (PkgF->Flags & pkgCache::Flag::LocalSource) == pkgCache::Flag::LocalSource;
+
       // only try to get a trusted package from another source if that source
       // is also trusted
       if(Trusted && !Index->IsTrusted()) 
@@ -2681,25 +2687,10 @@ bool pkgAcqArchive::QueueNext()
 // AcqArchive::Done - Finished fetching					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes,
-			 pkgAcquire::MethodConfig *Cfg)
+void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
+			 pkgAcquire::MethodConfig const * const Cfg)
 {
-   Item::Done(Message, Size, CalcHashes, Cfg);
-   
-   // Check the size
-   if (Size != Version->Size)
-   {
-      RenameOnError(SizeMismatch);
-      return;
-   }
-
-   // FIXME: could this empty() check impose *any* sort of security issue?
-   if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
-   {
-      RenameOnError(HashSumMismatch);
-      printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
-      return;
-   }
+   Item::Done(Message, Hashes, Cfg);
 
    // Grab the output filename
    string FileName = LookupTag(Message,"Filename");
@@ -2726,23 +2717,17 @@ void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList
    Complete = true;
 }
 									/*}}}*/
-// Acquire::Item::GetFinalFilename - Return the full final file path	/*{{{*/
-std::string pkgAcqArchive::GetFinalFilename() const
-{
-   return _config->FindDir("Dir::Cache::Archives") + flNotDir(StoreFilename);
-}
-									/*}}}*/
 // AcqArchive::Failed - Failure handler					/*{{{*/
 // ---------------------------------------------------------------------
 /* Here we try other sources */
-void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
+void pkgAcqArchive::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
 {
    Item::Failed(Message,Cnf);
 
-   /* We don't really want to retry on failed media swaps, this prevents 
+   /* We don't really want to retry on failed media swaps, this prevents
       that. An interesting observation is that permanent failures are not
       recorded. */
-   if (Cnf->Removable == true && 
+   if (Cnf->Removable == true &&
        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
    {
       // Vf = Version.FileList();
@@ -2770,21 +2755,12 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
    }
 }
 									/*}}}*/
-// AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/
-// ---------------------------------------------------------------------
-#if APT_PKG_ABI >= 413
-APT_PURE bool pkgAcqArchive::IsTrusted() const
-#else
-APT_PURE bool pkgAcqArchive::IsTrusted()
-#endif
+APT_PURE bool pkgAcqArchive::IsTrusted() const				/*{{{*/
 {
    return Trusted;
 }
 									/*}}}*/
-// AcqArchive::Finished - Fetching has finished, tidy up		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-void pkgAcqArchive::Finished()
+void pkgAcqArchive::Finished()						/*{{{*/
 {
    if (Status == pkgAcquire::Item::StatDone &&
        Complete == true)
@@ -2792,17 +2768,26 @@ void pkgAcqArchive::Finished()
    StoreFilename = string();
 }
 									/*}}}*/
+std::string pkgAcqArchive::DescURI() const				/*{{{*/
+{
+   return Desc.URI;
+}
+									/*}}}*/
+std::string pkgAcqArchive::ShortDesc() const				/*{{{*/
+{
+   return Desc.ShortDesc;
+}
+									/*}}}*/
+
 // AcqFile::pkgAcqFile - Constructor					/*{{{*/
-// ---------------------------------------------------------------------
-/* The file is added to the queue */
-pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes,
-		       unsigned long long Size,string Dsc,string ShortDesc,
+pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes,
+		       unsigned long long const Size,string const &Dsc,string const &ShortDesc,
 		       const string &DestDir, const string &DestFilename,
-                       bool IsIndexFile) :
-                       Item(Owner, Hashes), IsIndexFile(IsIndexFile)
+                       bool const IsIndexFile) :
+                       Item(Owner), IsIndexFile(IsIndexFile), ExpectedHashes(Hashes)
 {
    Retries = _config->FindI("Acquire::Retries",0);
-   
+
    if(!DestFilename.empty())
       DestFile = DestFilename;
    else if(!DestDir.empty())
@@ -2817,7 +2802,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashe
 
    // Set the short description to the archive component
    Desc.ShortDesc = ShortDesc;
-      
+
    // Get the transfer sizes
    FileSize = Size;
    struct stat Buf;
@@ -2834,21 +2819,11 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashe
 }
 									/*}}}*/
 // AcqFile::Done - Item downloaded OK					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes,
-		      pkgAcquire::MethodConfig *Cnf)
+void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
+		      pkgAcquire::MethodConfig const * const Cnf)
 {
-   Item::Done(Message,Size,CalcHashes,Cnf);
+   Item::Done(Message,CalcHashes,Cnf);
 
-   // Check the hash
-   if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes)
-   {
-      RenameOnError(HashSumMismatch);
-      printHashSumComparision(DestFile, ExpectedHashes, CalcHashes);
-      return;
-   }
-   
    string FileName = LookupTag(Message,"Filename");
    if (FileName.empty() == true)
    {
@@ -2858,11 +2833,11 @@ void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList cons
    }
 
    Complete = true;
-   
+
    // The files timestamp matches
    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
       return;
-   
+
    // We have to copy it into place
    if (FileName != DestFile)
    {
@@ -2874,7 +2849,7 @@ void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList cons
 	 QueueURI(Desc);
 	 return;
       }
-      
+
       // Erase the file if it is a symlink so we can overwrite it
       struct stat St;
       if (lstat(DestFile.c_str(),&St) == 0)
@@ -2882,7 +2857,7 @@ void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList cons
 	 if (S_ISLNK(St.st_mode) != 0)
 	    unlink(DestFile.c_str());
       }
-      
+
       // Symlink the file
       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
       {
@@ -2894,14 +2869,14 @@ void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList cons
 	 ErrorText = msg.str();
 	 Status = StatError;
 	 Complete = false;
-      }      
+      }
    }
 }
 									/*}}}*/
 // AcqFile::Failed - Failure handler					/*{{{*/
 // ---------------------------------------------------------------------
 /* Here we try other sources */
-void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
+void pkgAcqFile::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
 {
    Item::Failed(Message,Cnf);
 
@@ -2918,14 +2893,7 @@ void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 
 }
 									/*}}}*/
-// AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
-// ---------------------------------------------------------------------
-/* The only header we use is the last-modified header. */
-#if APT_PKG_ABI >= 413
-string pkgAcqFile::Custom600Headers() const
-#else
-string pkgAcqFile::Custom600Headers()
-#endif
+string pkgAcqFile::Custom600Headers() const				/*{{{*/
 {
    if (IsIndexFile)
       return "\nIndex-File: true";
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 07c86f31b..97d5ea1dd 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -6,15 +6,15 @@
    Acquire Item - Item to acquire
 
    When an item is instantiated it will add it self to the local list in
-   the Owner Acquire class. Derived classes will then call QueueURI to 
-   register all the URI's they wish to fetch at the initial moment.   
-   
+   the Owner Acquire class. Derived classes will then call QueueURI to
+   register all the URI's they wish to fetch at the initial moment.
+
    Three item classes are provided to provide functionality for
    downloading of Index, Translation and Packages files.
-   
+
    A Archive class is provided for downloading .deb files. It does Hash
    checking and source location as well as a retry algorithm.
-   
+
    ##################################################################### */
 									/*}}}*/
 #ifndef PKGLIB_ACQUIRE_ITEM_H
@@ -49,7 +49,48 @@ class pkgSourceList;
 class IndexTarget;
 class pkgAcqMetaBase;
 
-/** \brief Represents the process by which a pkgAcquire object should	{{{
+class APT_HIDDEN IndexTarget						/*{{{*/
+/** \brief Information about an index file. */
+{
+   public:
+   /** \brief A URI from which the index file can be downloaded. */
+   std::string const URI;
+
+   /** \brief A description of the index file. */
+   std::string const Description;
+
+   /** \brief A shorter description of the index file. */
+   std::string const ShortDesc;
+
+   /** \brief The key by which this index file should be
+    *  looked up within the meta signature file.
+    */
+   std::string const MetaKey;
+
+   virtual bool IsOptional() const {
+      return false;
+   }
+
+   IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
+	 std::string const &LongDesc, std::string const &URI) :
+      URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey) {}
+};
+									/*}}}*/
+class APT_HIDDEN OptionalIndexTarget : public IndexTarget		/*{{{*/
+/** \brief Information about an optional index file. */
+{
+   public:
+   virtual bool IsOptional() const {
+      return true;
+   }
+
+   OptionalIndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
+	 std::string const &LongDesc, std::string const &URI) :
+      IndexTarget(MetaKey, ShortDesc, LongDesc, URI) {}
+};
+									/*}}}*/
+class pkgAcquire::Item : public WeakPointable				/*{{{*/
+/** \brief Represents the process by which a pkgAcquire object should
  *  retrieve a file or a collection of files.
  *
  *  By convention, Item subclasses should insert themselves into the
@@ -61,46 +102,7 @@ class pkgAcqMetaBase;
  *
  *  \see pkgAcquire
  */
-class pkgAcquire::Item : public WeakPointable
-{  
-   friend class pkgAcqMetaBase;
-
-   void *d;
-
-   protected:
-   
-   /** \brief The acquire object with which this item is associated. */
-   pkgAcquire *Owner;
-
-   /** \brief Insert this item into its owner's queue.
-    *
-    *  The method is designed to check if the request would end
-    *  in an IMSHit and if it determines that it would, it isn't
-    *  queueing the Item and instead sets it to completion instantly.
-    *
-    *  \param Item Metadata about this item (its URI and
-    *  description).
-    *  \return true if the item was inserted, false if IMSHit was detected
-    */
-   virtual bool QueueURI(ItemDesc &Item);
-
-   /** \brief Remove this item from its owner's queue. */
-   void Dequeue();
-
-   /** \brief Rename a file without modifying its timestamp.
-    *
-    *  Many item methods call this as their final action.
-    *
-    *  \param From The file to be renamed.
-    *
-    *  \param To The new name of \a From.  If \a To exists it will be
-    *  overwritten.
-    */
-   bool Rename(std::string From,std::string To);
-
-   /** \brief Get the full pathname of the final file for the current URI */
-   virtual std::string GetFinalFilename() const;
-
+{
    public:
 
    /** \brief The current status of this item. */
@@ -125,7 +127,7 @@ class pkgAcquire::Item : public WeakPointable
 	*/
        StatAuthError,
 
-       /** \brief The item was could not be downloaded because of 
+       /** \brief The item was could not be downloaded because of
 	*  a transient network error (e.g. network down)
 	*/
        StatTransientNetworkError,
@@ -153,11 +155,11 @@ class pkgAcquire::Item : public WeakPointable
    std::string ActiveSubprocess;
 
    /** \brief A client-supplied unique identifier.
-    * 
+    *
     *  This field is initalized to 0; it is meant to be filled in by
     *  clients that wish to use it to uniquely identify items.
     *
-    *  \todo it's unused in apt itself
+    *  APT progress reporting will store an ID there as shown in "Get:42 …"
     */
    unsigned long ID;
 
@@ -173,6 +175,7 @@ class pkgAcquire::Item : public WeakPointable
     *  download progress indicator's overall statistics.
     */
    bool Local;
+
    std::string UsedMirror;
 
    /** \brief The number of fetch queues into which this item has been
@@ -185,9 +188,6 @@ class pkgAcquire::Item : public WeakPointable
     */
    unsigned int QueueCounter;
 
-   /** \brief TransactionManager */
-   pkgAcqMetaBase *TransactionManager;
-
    /** \brief The number of additional fetch items that are expected
     *  once this item is done.
     *
@@ -197,15 +197,12 @@ class pkgAcquire::Item : public WeakPointable
     *  progress.
     */
    unsigned int ExpectedAdditionalItems;
-   
+
    /** \brief The name of the file into which the retrieved object
     *  will be written.
     */
    std::string DestFile;
 
-   /** \brief storge name until a transaction is finished */
-   std::string PartialFile;
-
    /** \brief Invoked by the acquire worker when the object couldn't
     *  be fetched.
     *
@@ -219,7 +216,7 @@ class pkgAcquire::Item : public WeakPointable
     *
     *  \sa pkgAcqMethod
     */
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
 
    /** \brief Invoked by the acquire worker when the object was
     *  fetched successfully.
@@ -234,25 +231,24 @@ class pkgAcquire::Item : public WeakPointable
     *
     *  \param Message Data from the acquire method.  Use LookupTag()
     *  to parse it.
-    *  \param Size The size of the object that was fetched.
     *  \param Hashes The HashSums of the object that was fetched.
     *  \param Cnf The method via which the object was fetched.
     *
     *  \sa pkgAcqMethod
     */
-   virtual void Done(std::string Message, unsigned long long Size, HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
 
    /** \brief Invoked when the worker starts to fetch this object.
     *
     *  \param Message RFC822-formatted data from the worker process.
     *  Use LookupTag() to parse it.
     *
-    *  \param Size The size of the object being fetched.
+    *  \param Hashes The expected hashes of the object being fetched.
     *
     *  \sa pkgAcqMethod
     */
-   virtual void Start(std::string Message,unsigned long long Size);
+   virtual void Start(std::string const &Message, unsigned long long const Size);
 
    /** \brief Custom headers to be sent to the fetch process.
     *
@@ -262,61 +258,55 @@ class pkgAcquire::Item : public WeakPointable
     *  line, so they should (if nonempty) have a leading newline and
     *  no trailing newline.
     */
-#if APT_PKG_ABI >= 413
-   virtual std::string Custom600Headers() const {return std::string();};
-#else
-   virtual std::string Custom600Headers() {return std::string();};
-#endif
+   virtual std::string Custom600Headers() const;
 
    /** \brief A "descriptive" URI-like string.
     *
     *  \return a URI that should be used to describe what is being fetched.
     */
-   virtual std::string DescURI() = 0;
+   virtual std::string DescURI() const = 0;
    /** \brief Short item description.
     *
     *  \return a brief description of the object being fetched.
     */
-   virtual std::string ShortDesc() {return DescURI();}
+   virtual std::string ShortDesc() const;
 
    /** \brief Invoked by the worker when the download is completely done. */
-   virtual void Finished() {};
-   
-   /** \brief HashSums
+   virtual void Finished();
+
+   /** \return HashSums the DestFile is supposed to have in this stage */
+   virtual HashStringList GetExpectedHashes() const = 0;
+   /** \return the 'best' hash for display proposes like --print-uris */
+   std::string HashSum() const;
+
+   /** \return if having no hashes is a hard failure or not
     *
-    *  \return the HashSums of this object, if applicable; otherwise, an
-    *  empty list.
+    * Idealy this is always \b true for every subclass, but thanks to
+    * historical grow we don't have hashes for all files in all cases
+    * in all steps, so it is slightly more complicated than it should be.
     */
-   HashStringList HashSums() const {return ExpectedHashes;};
-   std::string HashSum() const {HashStringList const hashes = HashSums(); HashString const * const hs = hashes.find(NULL); return hs != NULL ? hs->toStr() : ""; };
+   virtual bool HashesRequired() const { return true; }
 
    /** \return the acquire process with which this item is associated. */
-   pkgAcquire *GetOwner() const {return Owner;};
-#if APT_PKG_ABI < 413
-   pkgAcquire *GetOwner() {return Owner;};
-#endif
+   pkgAcquire *GetOwner() const;
 
    /** \return \b true if this object is being fetched from a trusted source. */
-#if APT_PKG_ABI >= 413
-   virtual bool IsTrusted() const {return false;};
-#else
-   virtual bool IsTrusted() {return false;};
-#endif
-   
+   virtual bool IsTrusted() const;
+
    /** \brief Report mirror problem
-    * 
+    *
     *  This allows reporting mirror failures back to a centralized
     *  server. The apt-report-mirror-failure script is called for this
-    * 
+    *
     *  \param FailCode A short failure string that is send
     */
-   void ReportMirrorFailure(std::string FailCode);
+   void ReportMirrorFailure(std::string const &FailCode);
 
    /** \brief Set the name of the current active subprocess
     *
     *  See also #ActiveSubprocess
     */
-   void SetActiveSubprocess(const std::string &subprocess);
+   void SetActiveSubprocess(std::string const &subprocess);
 
    /** \brief Initialize an item.
     *
@@ -325,11 +315,8 @@ class pkgAcquire::Item : public WeakPointable
     *  manually invoke QueueURI() to do so).
     *
     *  \param Owner The new owner of this item.
-    *  \param ExpectedHashes of the file represented by this item
     */
-   Item(pkgAcquire *Owner,
-        HashStringList const &ExpectedHashes=HashStringList(),
-        pkgAcqMetaBase *TransactionManager=NULL);
+   Item(pkgAcquire * const Owner);
 
    /** \brief Remove this item from its owner's queue by invoking
     *  pkgAcquire::Remove.
@@ -337,6 +324,11 @@ class pkgAcquire::Item : public WeakPointable
    virtual ~Item();
 
    protected:
+   /** \brief The acquire object with which this item is associated. */
+   pkgAcquire * const Owner;
+
+   /** \brief The item that is currently being downloaded. */
+   pkgAcquire::ItemDesc Desc;
 
    enum RenameOnErrorState {
       HashSumMismatch,
@@ -354,63 +346,99 @@ class pkgAcquire::Item : public WeakPointable
     */
    bool RenameOnError(RenameOnErrorState const state);
 
+   /** \brief Insert this item into its owner's queue.
+    *
+    *  The method is designed to check if the request would end
+    *  in an IMSHit and if it determines that it would, it isn't
+    *  queueing the Item and instead sets it to completion instantly.
+    *
+    *  \param Item Metadata about this item (its URI and
+    *  description).
+    *  \return true if the item was inserted, false if IMSHit was detected
+    */
+   virtual bool QueueURI(ItemDesc &Item);
+
+   /** \brief Remove this item from its owner's queue. */
+   void Dequeue();
+
+   /** \brief Rename a file without modifying its timestamp.
+    *
+    *  Many item methods call this as their final action.
+    *
+    *  \param From The file to be renamed.
+    *
+    *  \param To The new name of \a From.  If \a To exists it will be
+    *  overwritten. If \a From and \a To are equal nothing happens.
+    */
+   bool Rename(std::string const &From, std::string const &To);
+
+   /** \brief Get the full pathname of the final file for the current URI */
+   virtual std::string GetFinalFilename() const;
+
+   private:
+   void *d;
+
+   friend class pkgAcqMetaBase;
+};
+									/*}}}*/
+class APT_HIDDEN pkgAcqTransactionItem: public pkgAcquire::Item		/*{{{*/
+/** \brief baseclass for the indexes files to manage them all together */
+{
+   protected:
+   IndexTarget const * const Target;
+   HashStringList GetExpectedHashesFor(std::string const MetaKey) const;
+
+   bool QueueURI(pkgAcquire::ItemDesc &Item);
+
+   public:
+   /** \brief storge name until a transaction is finished */
+   std::string PartialFile;
+
+   /** \brief TransactionManager */
+   pkgAcqMetaBase * const TransactionManager;
+
    enum TransactionStates {
       TransactionCommit,
       TransactionAbort,
    };
    virtual bool TransactionState(TransactionStates const state);
 
-   /** \brief The HashSums of the item is supposed to have than done */
-   HashStringList ExpectedHashes;
+   virtual std::string DescURI() const { return Target->URI; }
+   virtual HashStringList GetExpectedHashes() const;
+   virtual std::string GetMetaKey() const;
+   virtual bool HashesRequired() const;
 
-   /** \brief The item that is currently being downloaded. */
-   pkgAcquire::ItemDesc Desc;
-};
-									/*}}}*/
-/** \brief Information about an index patch (aka diff). */		/*{{{*/
-struct APT_HIDDEN DiffInfo {
-   /** The filename of the diff. */
-   std::string file;
 
-   /** The hashes of the diff */
-   HashStringList result_hashes;
-
-   /** The hashes of the file after the diff is applied */
-   HashStringList patch_hashes;
+   pkgAcqTransactionItem(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, IndexTarget const * const Target);
+   virtual ~pkgAcqTransactionItem();
 
-   /** The size of the file after the diff is applied */
-   unsigned long long result_size;
-
-   /** The size of the diff itself */
-   unsigned long long patch_size;
+   friend class pkgAcqMetaBase;
 };
 									/*}}}*/
-class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
+class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem		/*{{{*/
+/** \brief the manager of a transaction */
 {
    void *d;
 
  protected:
-   std::vector Transaction;
+   std::vector Transaction;
 
+   IndexTarget const DataTarget;
+ public:
    /** \brief A package-system-specific parser for the meta-index file. */
    indexRecords *MetaIndexParser;
    indexRecords *LastMetaIndexParser;
+ protected:
 
    /** \brief The index files which should be looked up in the meta-index
     *  and then downloaded.
     */
-   const std::vector* IndexTargets;
+   const std::vector* const IndexTargets;
 
    /** \brief If \b true, the index's signature is currently being verified.
     */
    bool AuthPass;
 
-   /** \brief The URI of the signature file.  Unlike Desc.URI, this is
-    *  never modified; it is used to determine the file that is being
-    *  downloaded.
-    */
-   std::string RealURI;
-
    /** \brief Starts downloading the individual index files.
     *
     *  \param verify If \b true, only indices whose expected hashsum
@@ -419,7 +447,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     *  #StatAuthError if there is a mismatch).  If verify is \b false,
     *  no hashsum checking will be performed.
     */
-   void QueueIndexes(bool verify);
+   void QueueIndexes(bool const verify);
 
    /** \brief Called when a file is finished being retrieved.
     *
@@ -430,16 +458,12 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     *  \param Message The message block received from the fetch
     *  subprocess.
     */
-   bool CheckDownloadDone(pkgAcquire::Item * const I, const std::string &Message, HashStringList const &Hashes) const;
+   bool CheckDownloadDone(pkgAcqTransactionItem * const I, const std::string &Message, HashStringList const &Hashes) const;
 
    /** \brief Queue the downloaded Signature for verification */
-   void QueueForSignatureVerify(pkgAcquire::Item * const I, std::string const &File, std::string const &Signature);
+   void QueueForSignatureVerify(pkgAcqTransactionItem * const I, std::string const &File, std::string const &Signature);
 
-#if APT_PKG_ABI >= 413
    virtual std::string Custom600Headers() const;
-#else
-   virtual std::string Custom600Headers();
-#endif
 
    /** \brief Called when authentication succeeded.
     *
@@ -450,7 +474,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     *  \param Message The message block received from the fetch
     *  subprocess.
     */
-   bool CheckAuthDone(std::string Message);
+   bool CheckAuthDone(std::string const &Message);
 
    /** Check if the current item should fail at this point */
    bool CheckStopAuthentication(pkgAcquire::Item * const I, const std::string &Message);
@@ -460,7 +484,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     *
     *  \return \b true if no fatal errors were encountered.
     */
-   bool VerifyVendor(std::string Message);
+   bool VerifyVendor(std::string const &Message);
 
    virtual bool TransactionState(TransactionStates const state);
 
@@ -468,33 +492,32 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
    // This refers more to the Transaction-Manager than the actual file
    bool IMSHit;
 
-   virtual std::string DescURI() {return RealURI; };
    virtual bool QueueURI(pkgAcquire::ItemDesc &Item);
+   virtual HashStringList GetExpectedHashes() const;
+   virtual bool HashesRequired() const;
 
    // transaction code
-   void Add(Item *I);
+   void Add(pkgAcqTransactionItem * const I);
    void AbortTransaction();
-   bool TransactionHasError() APT_PURE;
+   bool TransactionHasError() const;
    void CommitTransaction();
 
    /** \brief Stage (queue) a copy action when the transaction is committed
     */
-   void TransactionStageCopy(Item *I,
-                             const std::string &From, 
+   void TransactionStageCopy(pkgAcqTransactionItem * const I,
+                             const std::string &From,
                              const std::string &To);
    /** \brief Stage (queue) a removal action when the transaction is committed
     */
-   void TransactionStageRemoval(Item *I, const std::string &FinalFile);
+   void TransactionStageRemoval(pkgAcqTransactionItem * const I, const std::string &FinalFile);
 
    /** \brief Get the full pathname of the final file for the current URI */
    virtual std::string GetFinalFilename() const;
 
-   pkgAcqMetaBase(pkgAcquire *Owner,
-                  const std::vector* IndexTargets,
-                  indexRecords* MetaIndexParser,
-		  std::string const &RealURI,
-                  HashStringList const &ExpectedHashes=HashStringList(),
-                  pkgAcqMetaBase *TransactionManager=NULL);
+   pkgAcqMetaBase(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
+		  std::vector const * const IndexTargets,
+		  IndexTarget const &DataTarget,
+		  indexRecords* const MetaIndexParser);
 };
 									/*}}}*/
 /** \brief An item that is responsible for downloading the meta-index	{{{
@@ -512,36 +535,24 @@ class APT_HIDDEN pkgAcqMetaIndex : public pkgAcqMetaBase
    void *d;
 
    protected:
-   std::string URIDesc;
-   std::string ShortDesc;
-
-   /** \brief The URI of the meta-index file for the detached signature */
-   std::string MetaIndexSigURI;
-
-   /** \brief A "URI-style" description of the meta-index file */
-   std::string MetaIndexSigURIDesc;
-
-   /** \brief A brief description of the meta-index file */
-   std::string MetaIndexSigShortDesc;
+   IndexTarget const DetachedSigTarget;
 
    /** \brief delayed constructor */
-   void Init(std::string URIDesc, std::string ShortDesc);
-   
+   void Init(std::string const &URIDesc, std::string const &ShortDesc);
+
    public:
+   virtual std::string DescURI() const;
 
    // Specialized action members
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-   virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
    virtual void Finished();
 
    /** \brief Create a new pkgAcqMetaIndex. */
-   pkgAcqMetaIndex(pkgAcquire *Owner,
-                   pkgAcqMetaBase *TransactionManager,
-		   std::string URI,std::string URIDesc, std::string ShortDesc,
-                   std::string MetaIndexSigURI, std::string MetaIndexSigURIDesc, std::string MetaIndexSigShortDesc,
-		   const std::vector* IndexTargets,
-		   indexRecords* MetaIndexParser);
+   pkgAcqMetaIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
+		   IndexTarget const &DataTarget, IndexTarget const &DetachedSigTarget,
+		   const std::vector* const IndexTargets, indexRecords * const MetaIndexParser);
 
    friend class pkgAcqMetaSig;
 };
@@ -554,7 +565,7 @@ class APT_HIDDEN pkgAcqMetaIndex : public pkgAcqMetaBase
  *
  *  \sa pkgAcqMetaIndex
  */
-class APT_HIDDEN pkgAcqMetaSig : public pkgAcquire::Item
+class APT_HIDDEN pkgAcqMetaSig : public pkgAcqTransactionItem
 {
    void *d;
 
@@ -565,29 +576,20 @@ class APT_HIDDEN pkgAcqMetaSig : public pkgAcquire::Item
 
    protected:
 
-   /** \brief Long URI description used in the acquire system */
-   std::string URIDesc;
-
-   /** \brief URI used to get the file */
-   std::string RealURI;
-
    /** \brief Get the full pathname of the final file for the current URI */
    virtual std::string GetFinalFilename() const;
 
    public:
-   virtual std::string DescURI() {return RealURI;};
+   virtual bool HashesRequired() const { return false; }
 
    // Specialized action members
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-   virtual void Done(std::string Message,unsigned long long Size,
-                     HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
 
    /** \brief Create a new pkgAcqMetaSig. */
-   pkgAcqMetaSig(pkgAcquire *Owner,
-	 pkgAcqMetaBase *TransactionManager,
-	 std::string const &URI,std::string const &URIDesc,
-	 std::string const &ShortDesc, pkgAcqMetaIndex * const MetaIndex);
+   pkgAcqMetaSig(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, IndexTarget const * const Target,
+	 pkgAcqMetaIndex * const MetaIndex);
    virtual ~pkgAcqMetaSig();
 };
 									/*}}}*/
@@ -596,78 +598,37 @@ class APT_HIDDEN pkgAcqMetaClearSig : public pkgAcqMetaIndex
 {
    void *d;
 
-   /** \brief The URI of the meta-index file for the detached signature */
-   std::string MetaIndexURI;
-
-   /** \brief A "URI-style" description of the meta-index file */
-   std::string MetaIndexURIDesc;
-
-   /** \brief A brief description of the meta-index file */
-   std::string MetaIndexShortDesc;
-
-   /** \brief The URI of the detached meta-signature file if the clearsigned one failed. */
-   std::string MetaSigURI;
-
-   /** \brief A "URI-style" description of the meta-signature file */
-   std::string MetaSigURIDesc;
-
-   /** \brief A brief description of the meta-signature file */
-   std::string MetaSigShortDesc;
+   IndexTarget const ClearsignedTarget;
+   IndexTarget const DetachedDataTarget;
+   IndexTarget const DetachedSigTarget;
 
 public:
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-#if APT_PKG_ABI >= 413
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
    virtual std::string Custom600Headers() const;
-#else
-   virtual std::string Custom600Headers();
-#endif
-   virtual void Done(std::string Message,unsigned long long Size,
-                     HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
 
    /** \brief Create a new pkgAcqMetaClearSig. */
-   pkgAcqMetaClearSig(pkgAcquire *Owner,
-		std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc,
-		std::string const &MetaIndexURI, std::string const &MetaIndexURIDesc, std::string const &MetaIndexShortDesc,
-		std::string const &MetaSigURI, std::string const &MetaSigURIDesc, std::string const &MetaSigShortDesc,
-		const std::vector* IndexTargets,
-		indexRecords* MetaIndexParser);
+   pkgAcqMetaClearSig(pkgAcquire * const Owner,
+		IndexTarget const &ClearsignedTarget,
+		IndexTarget const &DetachedDataTarget,
+		IndexTarget const &DetachedSigTarget,
+		std::vector const * const IndexTargets,
+		indexRecords * const MetaIndexParser);
    virtual ~pkgAcqMetaClearSig();
 };
 									/*}}}*/
-/** \brief Common base class for all classes that deal with fetching 	{{{
-           indexes
- */
-class pkgAcqBaseIndex : public pkgAcquire::Item
+/** \brief Common base class for all classes that deal with fetching indexes	{{{*/
+class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem
 {
    void *d;
 
- protected:
-   /** \brief Pointer to the IndexTarget data
-    */
-   const struct IndexTarget * Target;
-
-   /** \brief Pointer to the indexRecords parser */
-   indexRecords *MetaIndexParser;
-
-   /** \brief The MetaIndex Key */
-   std::string MetaKey;
-
-   /** \brief The URI of the index file to recreate at our end (either
-    *  by downloading it or by applying partial patches).
-    */
-   std::string RealURI;
-
-   bool VerifyHashByMetaKey(HashStringList const &Hashes);
-
+ public:
    /** \brief Get the full pathname of the final file for the current URI */
    virtual std::string GetFinalFilename() const;
 
-   pkgAcqBaseIndex(pkgAcquire *Owner,
-                   pkgAcqMetaBase *TransactionManager,
-                   struct IndexTarget const * const Target,
-                   HashStringList const &ExpectedHashes,
-                   indexRecords *MetaIndexParser);
+   pkgAcqBaseIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
+                   IndexTarget const * const Target);
 };
 									/*}}}*/
 /** \brief An item that is responsible for fetching an index file of	{{{
@@ -705,15 +666,12 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
    virtual bool TransactionState(TransactionStates const state);
  public:
    // Specialized action members
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-   virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
-   virtual std::string DescURI() {return RealURI + "Index";};
-#if APT_PKG_ABI >= 413
+   virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string DescURI() const {return Target->URI + "Index";};
    virtual std::string Custom600Headers() const;
-#else
-   virtual std::string Custom600Headers();
-#endif
+   virtual std::string GetMetaKey() const;
 
    /** \brief Parse the Index file for a set of Packages diffs.
     *
@@ -725,7 +683,7 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
     *  \return \b true if the Index file was successfully parsed, \b
     *  false otherwise.
     */
-   bool ParseDiffIndex(std::string IndexDiffFile);
+   bool ParseDiffIndex(std::string const &IndexDiffFile);
 
    /** \brief Create a new pkgAcqDiffIndex.
     *
@@ -736,18 +694,30 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
     *  \param URIDesc A long description of the list file to download.
     *
     *  \param ShortDesc A short description of the list file to download.
-    *
-    *  \param ExpectedHashes The list file's hashsums which are expected.
     */
-   pkgAcqDiffIndex(pkgAcquire *Owner,
-                   pkgAcqMetaBase *TransactionManager,
-                   struct IndexTarget const * const Target,
-                   HashStringList const &ExpectedHashes,
-                   indexRecords *MetaIndexParser);
+   pkgAcqDiffIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
+                   IndexTarget const * const Target);
  private:
    APT_HIDDEN void QueueOnIMSHit() const;
 };
 									/*}}}*/
+struct APT_HIDDEN DiffInfo {						/*{{{*/
+   /** The filename of the diff. */
+   std::string file;
+
+   /** The hashes of the diff */
+   HashStringList result_hashes;
+
+   /** The hashes of the file after the diff is applied */
+   HashStringList patch_hashes;
+
+   /** The size of the file after the diff is applied */
+   unsigned long long result_size;
+
+   /** The size of the diff itself */
+   unsigned long long patch_size;
+};
+									/*}}}*/
 /** \brief An item that is responsible for fetching client-merge patches {{{
  *  that need to be applied to a given package index file.
  *
@@ -801,10 +771,12 @@ class APT_HIDDEN pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex
     *  This method will fall back to downloading the whole index file
     *  outright; its arguments are ignored.
     */
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-   virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes,
-	 pkgAcquire::MethodConfig *Cnf);
-   virtual std::string DescURI() {return RealURI + "Index";};
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+	 pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string DescURI() const {return Target->URI + "Index";};
+   virtual HashStringList GetExpectedHashes() const;
+   virtual bool HashesRequired() const;
 
    /** \brief Create an index merge-diff item.
     *
@@ -817,22 +789,15 @@ class APT_HIDDEN pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex
     *
     *  \param ShortDesc A brief description of this item.
     *
-    *  \param ExpectedHashes The expected md5sum of the completely
-    *  reconstructed package index file; the index file will be tested
-    *  against this value when it is entirely reconstructed.
-    *
     *  \param patch contains infos about the patch this item is supposed
     *  to download which were read from the index
     *
     *  \param allPatches contains all related items so that each item can
     *  check if it was the last one to complete the download step
     */
-   pkgAcqIndexMergeDiffs(pkgAcquire *Owner,
-                         pkgAcqMetaBase *TransactionManager,
-                         struct IndexTarget const * const Target,
-                         HashStringList const &ExpectedHash,
-                         indexRecords *MetaIndexParser,
-                         DiffInfo const &patch, 
+   pkgAcqIndexMergeDiffs(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
+                         IndexTarget const * const Target,
+                         DiffInfo const &patch,
                          std::vector const * const allPatches);
 };
 									/*}}}*/
@@ -875,7 +840,7 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex
     *  \param allDone If \b true, the file was entirely reconstructed,
     *  and its md5sum is verified. 
     */
-   APT_HIDDEN void Finish(bool allDone=false);
+   APT_HIDDEN void Finish(bool const allDone=false);
 
    protected:
 
@@ -905,26 +870,25 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex
 
 	 /** \brief The diff is currently being fetched. */
 	 StateFetchDiff,
-	 
-	 /** \brief The diff is currently being uncompressed. */
-	 StateUnzipDiff, // FIXME: No longer used
 
 	 /** \brief The diff is currently being applied. */
 	 StateApplyDiff
    } State;
 
    public:
-   
+
    /** \brief Called when the patch file failed to be downloaded.
     *
     *  This method will fall back to downloading the whole index file
     *  outright; its arguments are ignored.
     */
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
 
-   virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
-   virtual std::string DescURI() {return RealURI + "IndexDiffs";};
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string DescURI() const {return Target->URI + "IndexDiffs";};
+   virtual HashStringList GetExpectedHashes() const;
+   virtual bool HashesRequired() const;
 
    /** \brief Create an index diff item.
     *
@@ -940,20 +904,13 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex
     *
     *  \param ShortDesc A brief description of this item.
     *
-    *  \param ExpectedHashes The expected hashsums of the completely
-    *  reconstructed package index file; the index file will be tested
-    *  against this value when it is entirely reconstructed.
-    *
     *  \param diffs The remaining diffs from the index of diffs.  They
     *  should be ordered so that each diff appears before any diff
     *  that depends on it.
     */
-   pkgAcqIndexDiffs(pkgAcquire *Owner,
-                    pkgAcqMetaBase *TransactionManager,
-                    struct IndexTarget const * const Target,
-                    HashStringList const &ExpectedHash,
-                    indexRecords *MetaIndexParser,
-		    std::vector diffs=std::vector());
+   pkgAcqIndexDiffs(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
+                    IndexTarget const * const Target,
+		    std::vector const &diffs=std::vector());
 };
 									/*}}}*/
 /** \brief An acquire item that is responsible for fetching an index	{{{
@@ -981,16 +938,16 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
    AllStages Stage;
 
    /** \brief Handle what needs to be done when the download is done */
-   void StageDownloadDone(std::string Message,
+   void StageDownloadDone(std::string const &Message,
                           HashStringList const &Hashes,
-                          pkgAcquire::MethodConfig *Cfg);
+                          pkgAcquire::MethodConfig const * const Cfg);
 
    /** \brief Handle what needs to be done when the decompression/copy is
     *         done 
     */
-   void StageDecompressDone(std::string Message,
+   void StageDecompressDone(std::string const &Message,
                             HashStringList const &Hashes,
-                            pkgAcquire::MethodConfig *Cfg);
+                            pkgAcquire::MethodConfig const * const Cfg);
 
    /** \brief If \b set, this partially downloaded file will be
     *  removed when the download completes.
@@ -1006,7 +963,7 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
    std::string CurrentCompressionExtension;
 
    /** \brief Do the changes needed to fetch via AptByHash (if needed) */
-   void InitByHashIfNeeded(const std::string MetaKey);
+   void InitByHashIfNeeded();
 
    /** \brief Auto select the right compression to use */
    void AutoSelectCompression();
@@ -1015,7 +972,7 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
    void ReverifyAfterIMS();
 
    /** \brief Validate the downloaded index file */
-   bool ValidateFile(const std::string &FileName);
+   bool ValidateFile(std::string const &FileName);
 
    /** \brief Get the full pathname of the final file for the current URI */
    virtual std::string GetFinalFilename() const;
@@ -1024,82 +981,20 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
 
    public:
    // Specialized action members
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-   virtual void Done(std::string Message,unsigned long long Size, 
-                     HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
-#if APT_PKG_ABI >= 413
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
    virtual std::string Custom600Headers() const;
-#else
-   virtual std::string Custom600Headers();
-#endif
-   virtual std::string DescURI() {return Desc.URI;};
+   virtual std::string DescURI() const {return Desc.URI;};
+   virtual std::string GetMetaKey() const;
+
+   pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
+               IndexTarget const * const Target);
 
-   /** \brief Create a pkgAcqIndex.
-    *
-    *  \param Owner The pkgAcquire object with which this item is
-    *  associated.
-    *
-    *  \param URI The URI of the index file that is to be downloaded.
-    *
-    *  \param URIDesc A "URI-style" description of this index file.
-    *
-    *  \param ShortDesc A brief description of this index file.
-    *
-    *  \param ExpectedHashes The expected hashsum of this index file.
-    *
-    *  \param compressExt The compression-related extension with which
-    *  this index file should be downloaded, or "" to autodetect
-    *  Compression types can be set with config Acquire::CompressionTypes,
-    *  default is ".lzma" or ".bz2" (if the needed binaries are present)
-    *  fallback is ".gz" or none.
-    */
-   pkgAcqIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc,
-	       std::string ShortDesc, HashStringList const &ExpectedHashes);
-   pkgAcqIndex(pkgAcquire *Owner, pkgAcqMetaBase *TransactionManager,
-               IndexTarget const * const Target,
-               HashStringList const &ExpectedHash,
-               indexRecords *MetaIndexParser);
-               
    void Init(std::string const &URI, std::string const &URIDesc,
              std::string const &ShortDesc);
 };
 									/*}}}*/
-/** \brief Information about an index file. */				/*{{{*/
-class APT_HIDDEN IndexTarget
-{
-   void *d;
-
- public:
-   /** \brief A URI from which the index file can be downloaded. */
-   std::string URI;
-
-   /** \brief A description of the index file. */
-   std::string Description;
-
-   /** \brief A shorter description of the index file. */
-   std::string ShortDesc;
-
-   /** \brief The key by which this index file should be
-    *  looked up within the meta signature file.
-    */
-   std::string MetaKey;
-
-   virtual bool IsOptional() const {
-      return false;
-   }
-};
-									/*}}}*/
-/** \brief Information about an optional index file. */			/*{{{*/
-class APT_HIDDEN OptionalIndexTarget : public IndexTarget
-{
-   void *d;
-
-   virtual bool IsOptional() const {
-      return true;
-   }
-};
-									/*}}}*/
 /** \brief An item that is responsible for fetching a package file.	{{{
  *
  *  If the package file already exists in the cache, nothing will be
@@ -1109,6 +1004,9 @@ class pkgAcqArchive : public pkgAcquire::Item
 {
    void *d;
 
+   bool LocalSource;
+   HashStringList ExpectedHashes;
+
    protected:
    /** \brief The package version being fetched. */
    pkgCache::VerIterator Version;
@@ -1141,7 +1039,7 @@ class pkgAcqArchive : public pkgAcquire::Item
    /** \brief \b true if this version file is being downloaded from a
     *  trusted source.
     */
-   bool Trusted; 
+   bool Trusted;
 
    /** \brief Queue up the next available file for this version. */
    bool QueueNext();
@@ -1151,17 +1049,15 @@ class pkgAcqArchive : public pkgAcquire::Item
 
    public:
 
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-   virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig *Cnf);
-   virtual std::string DescURI() {return Desc.URI;};
-   virtual std::string ShortDesc() {return Desc.ShortDesc;};
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string DescURI() const;
+   virtual std::string ShortDesc() const;
    virtual void Finished();
-#if APT_PKG_ABI >= 413
    virtual bool IsTrusted() const;
-#else
-   virtual bool IsTrusted();
-#endif
+   virtual HashStringList GetExpectedHashes() const;
+   virtual bool HashesRequired() const;
 
    /** \brief Create a new pkgAcqArchive.
     *
@@ -1181,8 +1077,8 @@ class pkgAcqArchive : public pkgAcquire::Item
     *  basename in the constructor, and filled in with a fully
     *  qualified filename once the download finishes.
     */
-   pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources,
-		 pkgRecords *Recs,pkgCache::VerIterator const &Version,
+   pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources,
+		 pkgRecords * const Recs,pkgCache::VerIterator const &Version,
 		 std::string &StoreFilename);
 };
 									/*}}}*/
@@ -1200,22 +1096,21 @@ class pkgAcqFile : public pkgAcquire::Item
     *  Acquire::Retries.
     */
    unsigned int Retries;
-   
+
    /** \brief Should this file be considered a index file */
    bool IsIndexFile;
 
+   HashStringList const ExpectedHashes;
    public:
-   
+   virtual HashStringList GetExpectedHashes() const;
+   virtual bool HashesRequired() const;
+
    // Specialized action members
-   virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf);
-   virtual void Done(std::string Message,unsigned long long Size, HashStringList const &CalcHashes,
-		     pkgAcquire::MethodConfig *Cnf);
-   virtual std::string DescURI() {return Desc.URI;};
-#if APT_PKG_ABI >= 413
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &CalcHashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string DescURI() const {return Desc.URI;};
    virtual std::string Custom600Headers() const;
-#else
-   virtual std::string Custom600Headers();
-#endif
 
    /** \brief Create a new pkgAcqFile object.
     *
@@ -1248,10 +1143,10 @@ class pkgAcqFile : public pkgAcquire::Item
     * is the absolute name to which the file should be downloaded.
     */
 
-   pkgAcqFile(pkgAcquire *Owner, std::string URI, HashStringList const &Hashes, unsigned long long Size,
-	      std::string Desc, std::string ShortDesc,
-	      const std::string &DestDir="", const std::string &DestFilename="",
-	      bool IsIndexFile=false);
+   pkgAcqFile(pkgAcquire * const Owner, std::string const &URI, HashStringList const &Hashes, unsigned long long const Size,
+	      std::string const &Desc, std::string const &ShortDesc,
+	      std::string const &DestDir="", std::string const &DestFilename="",
+	      bool const IsIndexFile=false);
 };
 									/*}}}*/
 /** @} */
diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc
index c29ef469e..b77096efd 100644
--- a/apt-pkg/acquire-method.cc
+++ b/apt-pkg/acquire-method.cc
@@ -376,7 +376,10 @@ int pkgAcqMethod::Run(bool Single)
 		  Tmp->ExpectedHashes.push_back(HashString(*t, hash));
 	    }
             char *End;
-            Tmp->MaximumSize = strtoll(LookupTag(Message, "Maximum-Size", "0").c_str(), &End, 10);
+	    if (Tmp->ExpectedHashes.FileSize() > 0)
+	       Tmp->MaximumSize = Tmp->ExpectedHashes.FileSize();
+	    else
+	       Tmp->MaximumSize = strtoll(LookupTag(Message, "Maximum-Size", "0").c_str(), &End, 10);
 	    Tmp->Next = 0;
 	    
 	    // Append it to the list
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 9254e20a3..099a1f87d 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -55,8 +55,8 @@ pkgAcquire::Worker::Worker(Queue *Q,MethodConfig *Cnf,
    CurrentItem = 0;
    TotalSize = 0;
    CurrentSize = 0;
-   
-   Construct();   
+
+   Construct();
 }
 									/*}}}*/
 // Worker::Worker - Constructor for method config startup		/*{{{*/
@@ -70,8 +70,8 @@ pkgAcquire::Worker::Worker(MethodConfig *Cnf)
    CurrentItem = 0;
    TotalSize = 0;
    CurrentSize = 0;
-   
-   Construct();   
+
+   Construct();
 }
 									/*}}}*/
 // Worker::Construct - Constructor helper				/*{{{*/
@@ -136,7 +136,7 @@ bool pkgAcquire::Worker::Start()
    }
    for (int I = 0; I != 4; I++)
       SetCloseExec(Pipes[I],true);
-   
+
    // Fork off the process
    Process = ExecFork();
    if (Process == 0)
@@ -145,9 +145,9 @@ bool pkgAcquire::Worker::Start()
       dup2(Pipes[1],STDOUT_FILENO);
       dup2(Pipes[2],STDIN_FILENO);
       SetCloseExec(STDOUT_FILENO,false);
-      SetCloseExec(STDIN_FILENO,false);      
+      SetCloseExec(STDIN_FILENO,false);
       SetCloseExec(STDERR_FILENO,false);
-      
+
       const char *Args[2];
       Args[0] = Method.c_str();
       Args[1] = 0;
@@ -165,7 +165,7 @@ bool pkgAcquire::Worker::Start()
    close(Pipes[2]);
    OutReady = false;
    InReady = true;
-   
+
    // Read the configuration data
    if (WaitFd(InFd) == false ||
        ReadMessages() == false)
@@ -174,7 +174,7 @@ bool pkgAcquire::Worker::Start()
    RunMessages();
    if (OwnerQ != 0)
       SendConfiguration();
-   
+
    return true;
 }
 									/*}}}*/
@@ -201,7 +201,7 @@ bool pkgAcquire::Worker::RunMessages()
 
       if (Debug == true)
 	 clog << " <- " << Access << ':' << QuoteString(Message,"\n") << endl;
-      
+
       // Fetch the message number
       char *End;
       int Number = strtol(Message.c_str(),&End,10);
@@ -215,15 +215,15 @@ bool pkgAcquire::Worker::RunMessages()
 
       // update used mirror
       string UsedMirror = LookupTag(Message,"UsedMirror", "");
-      if (!UsedMirror.empty() && 
+      if (!UsedMirror.empty() &&
           Itm && 
-          Itm->Description.find(" ") != string::npos) 
+          Itm->Description.find(" ") != string::npos)
       {
          Itm->Description.replace(0, Itm->Description.find(" "), UsedMirror);
          // FIXME: will we need this as well?
          //Itm->ShortDesc = UsedMirror;
       }
-      
+
       // Determine the message number and dispatch
       switch (Number)
       {
@@ -232,18 +232,18 @@ bool pkgAcquire::Worker::RunMessages()
 	 if (Capabilities(Message) == false)
 	    return _error->Error("Unable to process Capabilities message from %s",Access.c_str());
 	 break;
-	 
+
 	 // 101 Log
 	 case 101:
 	 if (Debug == true)
 	    clog << " <- (log) " << LookupTag(Message,"Message") << endl;
 	 break;
-	 
+
 	 // 102 Status
 	 case 102:
 	 Status = LookupTag(Message,"Message");
 	 break;
-	    
+
          // 103 Redirect
          case 103:
          {
@@ -252,7 +252,7 @@ bool pkgAcquire::Worker::RunMessages()
                _error->Error("Method gave invalid 103 Redirect message");
                break;
             }
- 
+
             string NewURI = LookupTag(Message,"New-URI",URI.c_str());
             Itm->URI = NewURI;
 
@@ -272,7 +272,7 @@ bool pkgAcquire::Worker::RunMessages()
 	       Log->Done(Desc);
             break;
          }
-   
+
 	 // 200 URI Start
 	 case 200:
 	 {
@@ -281,23 +281,23 @@ bool pkgAcquire::Worker::RunMessages()
 	       _error->Error("Method gave invalid 200 URI Start message");
 	       break;
 	    }
-	    
+
 	    CurrentItem = Itm;
 	    CurrentSize = 0;
 	    TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
 	    ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10);
-	    Itm->Owner->Start(Message,strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10));
+	    Itm->Owner->Start(Message, TotalSize);
 
 	    // Display update before completion
 	    if (Log != 0 && Log->MorePulses == true)
 	       Log->Pulse(Itm->Owner->GetOwner());
-	    
+
 	    if (Log != 0)
 	       Log->Fetch(*Itm);
 
 	    break;
 	 }
-	 
+
 	 // 201 URI Done
 	 case 201:
 	 {
@@ -306,7 +306,7 @@ bool pkgAcquire::Worker::RunMessages()
 	       _error->Error("Method gave invalid 201 URI Done message");
 	       break;
 	    }
-	    
+
 	    pkgAcquire::Item *Owner = Itm->Owner;
 	    pkgAcquire::ItemDesc Desc = *Itm;
 
@@ -316,22 +316,11 @@ bool pkgAcquire::Worker::RunMessages()
 	    // Display update before completion
 	    if (Log != 0 && Log->MorePulses == true)
 	       Log->Pulse(Owner->GetOwner());
-	    
+
 	    OwnerQ->ItemDone(Itm);
-	    unsigned long long const ServerSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
-            bool isHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) ||
-                         StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false);
-            // Using the https method the server might return 200, but the
-            // If-Modified-Since condition is not satsified, libcurl will
-            // discard the download. In this case, however, TotalSize will be
-            // set to the actual size of the file, while ServerSize will be set
-            // to 0. Therefore, if the item is marked as a hit and the
-            // downloaded size (ServerSize) is 0, we ignore TotalSize.
-	    if (TotalSize != 0 && (!isHit || ServerSize != 0) && ServerSize != TotalSize)
-	       _error->Warning("Size of file %s is not what the server reported %s %llu",
-			       Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize);
-
-	    // see if there is a hash to verify
+
+	    HashStringList const ExpectedHashes = Owner->GetExpectedHashes();
+	    // see if we got hashes to verify
 	    HashStringList ReceivedHashes;
 	    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 	    {
@@ -340,6 +329,18 @@ bool pkgAcquire::Worker::RunMessages()
 	       if (hashsum.empty() == false)
 		  ReceivedHashes.push_back(HashString(*type, hashsum));
 	    }
+	    // not all methods always sent Hashes our way
+	    if (ExpectedHashes.usable() == true && ReceivedHashes.usable() == false)
+	    {
+	       std::string const filename = LookupTag(Message, "Filename", Owner->DestFile.c_str());
+	       if (filename.empty() == false && RealFileExists(filename))
+	       {
+		  Hashes calc(ExpectedHashes);
+		  FileFd file(filename, FileFd::ReadOnly, FileFd::None);
+		  calc.AddFD(file);
+		  ReceivedHashes = calc.GetHashStringList();
+	       }
+	    }
 
 	    if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
 	    {
@@ -348,30 +349,66 @@ bool pkgAcquire::Worker::RunMessages()
 	       for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs)
 		  std::clog <<  "\t- " << hs->toStr() << std::endl;
 	       std::clog << "ExpectedHash:" << endl;
-	       HashStringList expectedHashes = Owner->HashSums();
-	       for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs)
+	       for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
 		  std::clog <<  "\t- " << hs->toStr() << std::endl;
 	       std::clog << endl;
 	    }
-	    Owner->Done(Message, ServerSize, ReceivedHashes, Config);
-	    ItemDone();
 
-	    // Log that we are done
-	    if (Log != 0)
+	    // decide if what we got is what we expected
+	    bool consideredOkay = false;
+	    bool const isIMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) ||
+	       StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false);
+	    if (ExpectedHashes.usable())
 	    {
-	       if (isHit)
+	       if (ReceivedHashes.usable() == false)
 	       {
-		  /* Hide 'hits' for local only sources - we also manage to
-		     hide gets */
-		  if (Config->LocalOnly == false)
-		     Log->IMSHit(Desc);
-	       }	       
+		  /* IMS-Hits can't be checked here as we will have uncompressed file,
+		     but the hashes for the compressed file. What we have was good through
+		     so all we have to ensure later is that we are not stalled. */
+		  consideredOkay = isIMSHit;
+	       }
+	       else if (ReceivedHashes == ExpectedHashes)
+		  consideredOkay = true;
 	       else
-		  Log->Done(Desc);
+		  consideredOkay = false;
+
+	    }
+	    else if (Owner->HashesRequired() == true)
+	       consideredOkay = false;
+	    else
+	       consideredOkay = true;
+
+	    if (consideredOkay == true)
+	    {
+	       Owner->Done(Message, ReceivedHashes, Config);
+	       ItemDone();
+
+	       // Log that we are done
+	       if (Log != 0)
+	       {
+		  if (isIMSHit)
+		  {
+		     /* Hide 'hits' for local only sources - we also manage to
+			hide gets */
+		     if (Config->LocalOnly == false)
+			Log->IMSHit(Desc);
+		  }
+		  else
+		     Log->Done(Desc);
+	       }
+	    }
+	    else
+	    {
+	       Owner->Status = pkgAcquire::Item::StatAuthError;
+	       Owner->Failed(Message,Config);
+	       ItemDone();
+
+	       if (Log != 0)
+		  Log->Fail(Desc);
 	    }
 	    break;
-	 }	 
-	 
+	 }
+
 	 // 400 URI Failure
 	 case 400:
 	 {
@@ -408,18 +445,18 @@ bool pkgAcquire::Worker::RunMessages()
 	       Log->Fail(Desc);
 
 	    break;
-	 }	 
-	 
+	 }
+
 	 // 401 General Failure
 	 case 401:
 	 _error->Error("Method %s General failure: %s",Access.c_str(),LookupTag(Message,"Message").c_str());
 	 break;
-	 
+
 	 // 403 Media Change
 	 case 403:
-	 MediaChange(Message); 
+	 MediaChange(Message);
 	 break;
-      }      
+      }
    }
    return true;
 }
@@ -432,7 +469,7 @@ bool pkgAcquire::Worker::Capabilities(string Message)
 {
    if (Config == 0)
       return true;
-   
+
    Config->Version = LookupTag(Message,"Version");
    Config->SingleInstance = StringToBool(LookupTag(Message,"Single-Instance"),false);
    Config->Pipeline = StringToBool(LookupTag(Message,"Pipeline"),false);
@@ -447,13 +484,13 @@ bool pkgAcquire::Worker::Capabilities(string Message)
       clog << "Configured access method " << Config->Access << endl;
       clog << "Version:" << Config->Version <<
 	      " SingleInstance:" << Config->SingleInstance <<
-	      " Pipeline:" << Config->Pipeline << 
-	      " SendConfig:" << Config->SendConfig << 
-	      " LocalOnly: " << Config->LocalOnly << 
-	      " NeedsCleanup: " << Config->NeedsCleanup << 
+	      " Pipeline:" << Config->Pipeline <<
+	      " SendConfig:" << Config->SendConfig <<
+	      " LocalOnly: " << Config->LocalOnly <<
+	      " NeedsCleanup: " << Config->NeedsCleanup <<
 	      " Removable: " << Config->Removable << endl;
    }
-   
+
    return true;
 }
 									/*}}}*/
@@ -463,10 +500,10 @@ bool pkgAcquire::Worker::Capabilities(string Message)
 bool pkgAcquire::Worker::MediaChange(string Message)
 {
    int status_fd = _config->FindI("APT::Status-Fd",-1);
-   if(status_fd > 0) 
+   if(status_fd > 0)
    {
       string Media = LookupTag(Message,"Media");
-      string Drive = LookupTag(Message,"Drive"); 
+      string Drive = LookupTag(Message,"Drive");
       ostringstream msg,status;
       ioprintf(msg,_("Please insert the disc labeled: "
 		     "'%s' "
@@ -536,12 +573,12 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
 {
    if (OutFd == -1)
       return false;
-   
+
    string Message = "600 URI Acquire\n";
    Message.reserve(300);
    Message += "URI: " + Item->URI;
    Message += "\nFilename: " + Item->Owner->DestFile;
-   HashStringList const hsl = Item->Owner->HashSums();
+   HashStringList const hsl = Item->Owner->GetExpectedHashes();
    for (HashStringList::const_iterator hs = hsl.begin(); hs != hsl.end(); ++hs)
       Message += "\nExpected-" + hs->HashType() + ": " + hs->HashValue();
    if(Item->Owner->FileSize > 0)
@@ -564,7 +601,7 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
       clog << " -> " << Access << ':' << QuoteString(Message,"\n") << endl;
    OutQueue += Message;
    OutReady = true;
-   
+
    return true;
 }
 									/*}}}*/
@@ -586,7 +623,7 @@ bool pkgAcquire::Worker::OutFdReady()
    OutQueue.erase(0,Res);
    if (OutQueue.empty() == true)
       OutReady = false;
-   
+
    return true;
 }
 									/*}}}*/
@@ -608,7 +645,7 @@ bool pkgAcquire::Worker::InFdReady()
 bool pkgAcquire::Worker::MethodFailure()
 {
    _error->Error("Method %s has died unexpectedly!",Access.c_str());
-   
+
    // do not reap the child here to show meaningfull error to the user
    ExecWait(Process,Access.c_str(),false);
    Process = -1;
@@ -620,26 +657,22 @@ bool pkgAcquire::Worker::MethodFailure()
    InReady = false;
    OutQueue = string();
    MessageQueue.erase(MessageQueue.begin(),MessageQueue.end());
-   
+
    return false;
 }
 									/*}}}*/
-// Worker::Pulse - Called periodically 					/*{{{*/
+// Worker::Pulse - Called periodically					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
 void pkgAcquire::Worker::Pulse()
 {
    if (CurrentItem == 0)
       return;
- 
+
    struct stat Buf;
    if (stat(CurrentItem->Owner->DestFile.c_str(),&Buf) != 0)
       return;
    CurrentSize = Buf.st_size;
-   
-   // Hmm? Should not happen...
-   if (CurrentSize > TotalSize && TotalSize != 0)
-      TotalSize = CurrentSize;
 }
 									/*}}}*/
 // Worker::ItemDone - Called when the current item is finished		/*{{{*/
diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index 0fa443b4a..11a7e479b 100644
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 									/*}}}*/
@@ -178,6 +179,15 @@ HashString const * HashStringList::find(char const * const type) const /*{{{*/
    return NULL;
 }
 									/*}}}*/
+unsigned long long HashStringList::FileSize() const			/*{{{*/
+{
+   HashString const * const hsf = find("Checksum-FileSize");
+   if (hsf == NULL)
+      return 0;
+   std::string const hv = hsf->HashValue();
+   return strtoull(hv.c_str(), NULL, 10);
+}
+									/*}}}*/
 bool HashStringList::supported(char const * const type)			/*{{{*/
 {
    for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t)
diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h
index ac13c8ace..176ce4faa 100644
--- a/apt-pkg/contrib/hashes.h
+++ b/apt-pkg/contrib/hashes.h
@@ -87,6 +87,15 @@ class HashStringList
     */
    HashString const * find(char const * const type) const;
    HashString const * find(std::string const &type) const { return find(type.c_str()); }
+
+   /** finds the filesize hash and returns it as number
+    *
+    * @return beware: if the size isn't known we return \b 0 here,
+    * just like we would do for an empty file. If that is a problem
+    * for you have to get the size manually out of the list.
+    */
+   unsigned long long FileSize() const;
+
    /** check if the given hash type is supported
     *
     * @param type to check
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index d672b4fd8..185248619 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -742,13 +742,13 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const
 
    // and give it to the list parser
    debDebFileParser Parser(DebControl, DebFile);
-   if(Gen.SelectFile(DebFile, "local", *this) == false)
+   if(Gen.SelectFile(DebFile, "local", *this, pkgCache::Flag::LocalSource) == false)
       return _error->Error("Problem with SelectFile %s", DebFile.c_str());
 
    pkgCache::PkgFileIterator File = Gen.GetCurFile();
    File->Size = DebControl->Size();
    File->mtime = DebControl->ModificationTime();
-   
+
    if (Gen.MergeList(Parser) == false)
       return _error->Error("Problem with MergeLister for %s", DebFile.c_str());
 
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index aa2db8149..eb5e78e3b 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -192,11 +192,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const {
 		vector const SectionEntries = src->second;
 		for (vector::const_iterator I = SectionEntries.begin();
 		     I != SectionEntries.end(); ++I) {
-			IndexTarget * Target = new IndexTarget();
-			Target->ShortDesc = "Sources";
-			Target->MetaKey = SourceIndexURISuffix(Target->ShortDesc.c_str(), (*I)->Section);
-			Target->URI = SourceIndexURI(Target->ShortDesc.c_str(), (*I)->Section);
-			Target->Description = Info (Target->ShortDesc.c_str(), (*I)->Section);
+			char const * const ShortDesc = "Sources";
+			IndexTarget * const Target = new IndexTarget(
+			      SourceIndexURISuffix(ShortDesc, (*I)->Section),
+			      ShortDesc,
+			      Info(ShortDesc, (*I)->Section),
+			      SourceIndexURI(ShortDesc, (*I)->Section)
+			      );
 			IndexTargets->push_back (Target);
 		}
 	}
@@ -212,11 +214,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const {
 			continue;
 		for (vector ::const_iterator I = a->second.begin();
 		     I != a->second.end(); ++I) {
-			IndexTarget * Target = new IndexTarget();
-			Target->ShortDesc = "Packages";
-			Target->MetaKey = IndexURISuffix(Target->ShortDesc.c_str(), (*I)->Section, a->first);
-			Target->URI = IndexURI(Target->ShortDesc.c_str(), (*I)->Section, a->first);
-			Target->Description = Info (Target->ShortDesc.c_str(), (*I)->Section, a->first);
+			char const * const ShortDesc = "Packages";
+			IndexTarget * const Target = new IndexTarget(
+			      IndexURISuffix(ShortDesc, (*I)->Section, a->first),
+			      ShortDesc,
+			      Info (ShortDesc, (*I)->Section, a->first),
+			      IndexURI(ShortDesc, (*I)->Section, a->first)
+			      );
 			IndexTargets->push_back (Target);
 			sections.insert((*I)->Section);
 		}
@@ -235,11 +239,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const {
 	     s != sections.end(); ++s) {
 		for (std::vector::const_iterator l = lang.begin();
 		     l != lang.end(); ++l) {
-			IndexTarget * Target = new OptionalIndexTarget();
-			Target->ShortDesc = "Translation-" + *l;
-			Target->MetaKey = TranslationIndexURISuffix(l->c_str(), *s);
-			Target->URI = TranslationIndexURI(l->c_str(), *s);
-			Target->Description = Info (Target->ShortDesc.c_str(), *s);
+			std::string const ShortDesc = "Translation-" + *l;
+			IndexTarget * const Target = new OptionalIndexTarget(
+			      TranslationIndexURISuffix(l->c_str(), *s),
+			      ShortDesc,
+			      Info (ShortDesc.c_str(), *s),
+			      TranslationIndexURI(l->c_str(), *s)
+			      );
 			IndexTargets->push_back(Target);
 		}
 	}
@@ -249,8 +255,6 @@ vector * debReleaseIndex::ComputeIndexTargets() const {
 									/*}}}*/
 bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 {
-   bool const tryInRelease = _config->FindB("Acquire::TryInRelease", true);
-
    indexRecords * const iR = new indexRecords(Dist);
    if (Trusted == ALWAYS_TRUSTED)
       iR->SetTrusted(true);
@@ -258,37 +262,17 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
       iR->SetTrusted(false);
 
    // special case for --print-uris
-   if (GetAll) {
-      vector  *targets = ComputeIndexTargets();
-      for (vector ::const_iterator Target = targets->begin(); Target != targets->end(); ++Target) {
-	 new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
-			 (*Target)->ShortDesc, HashStringList());
-      }
-      delete targets;
-
-      // this is normally created in pkgAcqMetaSig, but if we run
-      // in --print-uris mode, we add it here
-      if (tryInRelease == false)
-	 new pkgAcqMetaIndex(Owner, NULL,
-                             MetaIndexURI("Release"),
-                             MetaIndexInfo("Release"), "Release",
-                             MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg",
-                             ComputeIndexTargets(),
-                             iR);
+   vector  const * const targets = ComputeIndexTargets();
+#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X))
+   pkgAcqMetaBase * const TransactionManager = new pkgAcqMetaClearSig(Owner,
+	 APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"),
+	 targets, iR);
+#undef APT_TARGET
+   if (GetAll)
+   {
+      for (vector ::const_iterator Target = targets->begin(); Target != targets->end(); ++Target)
+	 new pkgAcqIndex(Owner, TransactionManager, *Target);
    }
-   if (tryInRelease == true)
-      new pkgAcqMetaClearSig(Owner, 
-            MetaIndexURI("InRelease"), MetaIndexInfo("InRelease"), "InRelease",
-	    MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
-	    MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg",
-	    ComputeIndexTargets(),
-	    iR);
-   else
-      new pkgAcqMetaIndex(Owner, NULL,
-          MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
-          MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg",
-          ComputeIndexTargets(),
-          iR);
 
    return true;
 }
diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc
index de2617833..c26868cac 100644
--- a/apt-pkg/indexrecords.cc
+++ b/apt-pkg/indexrecords.cc
@@ -73,7 +73,7 @@ APT_PURE indexRecords::checkSum *indexRecords::Lookup(const string MetaKey)
 
 APT_PURE bool indexRecords::Exists(string const &MetaKey) const
 {
-   return Entries.count(MetaKey) == 1;
+   return Entries.find(MetaKey) != Entries.end();
 }
 
 bool indexRecords::Load(const string Filename)				/*{{{*/
diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h
index 2ba23c5c0..b4d56611a 100644
--- a/apt-pkg/pkgcache.h
+++ b/apt-pkg/pkgcache.h
@@ -190,7 +190,12 @@ class pkgCache								/*{{{*/
    struct Flag
    {
       enum PkgFlags {Auto=(1<<0),Essential=(1<<3),Important=(1<<4)};
-      enum PkgFFlags {NotSource=(1<<0),NotAutomatic=(1<<1),ButAutomaticUpgrades=(1<<2)};
+      enum PkgFFlags {
+	 NotSource=(1<<0), /*!< packages can't be fetched from here, e.g. dpkg/status file */
+	 NotAutomatic=(1<<1), /*!< archive has a default pin of 1 */
+	 ButAutomaticUpgrades=(1<<2), /*!< (together with the previous) archive has a default pin of 100 */
+	 LocalSource=(1<<3), /*!< local sources can't and will not be verified by hashes */
+      };
    };
    
    protected:
diff --git a/methods/file.cc b/methods/file.cc
index 043ab04b8..353e54bd5 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -57,7 +57,11 @@ bool FileMethod::Fetch(FetchItem *Itm)
       Res.LastModified = Buf.st_mtime;
       Res.IMSHit = false;
       if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
-	 Res.IMSHit = true;
+      {
+	 unsigned long long const filesize = Itm->ExpectedHashes.FileSize();
+	 if (filesize != 0 && filesize == Res.Size)
+	    Res.IMSHit = true;
+      }
    }
 
    // See if the uncompressed file exists and reuse it
diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated
index 685bc566b..da63f7cb3 100755
--- a/test/integration/test-apt-get-source-authenticated
+++ b/test/integration/test-apt-get-source-authenticated
@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # Regression test for debian bug #749795. Ensure that we fail with
-# a error if apt-get source foo will download a source that comes
+# an error if apt-get source foo will download a source that comes
 # from a unauthenticated repository
 #
 set -e
diff --git a/test/integration/test-apt-sources-deb822 b/test/integration/test-apt-sources-deb822
index d8b2334ad..51fe7bcfe 100755
--- a/test/integration/test-apt-sources-deb822
+++ b/test/integration/test-apt-sources-deb822
@@ -23,46 +23,45 @@ Description: summay
 
 msgtest 'Test sources.list' 'old style'
 echo "deb http://ftp.debian.org/debian stable main" > $SOURCES
-testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
-'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris
+testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris
 
 msgtest 'Test sources.list' 'simple deb822'
 echo "$BASE"  > $SOURCES
-testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
-'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris
-
+testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris
 
 msgtest 'Test deb822 with' 'two entries'
 # Two entries
 echo "$BASE" > $SOURCES
 echo "" >> $SOURCES
 echo "$BASE" | sed  s/stable/unstable/  >> $SOURCES
-testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
+testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 
 'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 
-'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 "  aptget update --print-uris
+'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris
 
 # two suite entries
 msgtest 'Test deb822 with' 'two Suite entries'
 echo "$BASE"  | sed -e "s/stable/stable unstable/" > $SOURCES
-testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
+testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 
 'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 
-'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 "  aptget update --print-uris 
+'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris
 
 msgtest 'Test deb822' 'architecture option'
 echo "$BASE" > $SOURCES
 echo "Architectures: amd64 armel" >> $SOURCES
-testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0 
+testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0 
 'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0 
-'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris
 
 
 msgtest 'Test old-style sources.list file which has' 'malformed dist'
@@ -85,20 +84,20 @@ testempty aptget update --print-uris
 # multiple URIs
 msgtest 'Test deb822 sources.list file which has' 'Multiple URIs work'
 echo "$BASE"  | sed -e 's#http://ftp.debian.org/debian#http://ftp.debian.org/debian http://ftp.de.debian.org/debian#' > $SOURCES
-testequal --nomsg  "'http://ftp.de.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.de.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
+testequal --nomsg  "'http://ftp.de.debian.org/debian/dists/stable/InRelease' ftp.de.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.de.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.de.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
 'http://ftp.de.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.de.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
-'http://ftp.de.debian.org/debian/dists/stable/InRelease' ftp.de.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
 'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
-'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris 
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris
 
 # multiple Type in one field
 msgtest 'Test deb822 sources.list file which has' 'Multiple Types work'
 echo "$BASE"  | sed -e 's#Types: deb#Types: deb deb-src#' > $SOURCES
-testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/source/Sources.bz2' ftp.debian.org_debian_dists_stable_main_source_Sources 0 
+testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/stable/main/source/Sources.bz2' ftp.debian.org_debian_dists_stable_main_source_Sources 0 
 'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
-'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
-'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 "  aptget update --print-uris 
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris
 
 # a Suite
 msgtest 'Test deb822 sources.list file which has' 'a exact path and no sections'
@@ -107,6 +106,6 @@ Types: deb
 URIs: http://emacs.naquadah.org
 Suites: stable/
 EOF
-testequal --nomsg "'http://emacs.naquadah.org/stable/Packages.bz2' emacs.naquadah.org_stable_Packages 0 
-'http://emacs.naquadah.org/stable/en.bz2' emacs.naquadah.org_stable_en 0 
-'http://emacs.naquadah.org/stable/InRelease' emacs.naquadah.org_stable_InRelease 0 "  aptget update --print-uris 
+testequal --nomsg "'http://emacs.naquadah.org/stable/InRelease' emacs.naquadah.org_stable_InRelease 0 
+'http://emacs.naquadah.org/stable/Packages.bz2' emacs.naquadah.org_stable_Packages 0 
+'http://emacs.naquadah.org/stable/en.bz2' emacs.naquadah.org_stable_en 0 " aptget update --print-uris
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 55a5da848..55bba8188 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -35,7 +35,7 @@ test_packagestoobig() {
 	done
 	NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
 	testfailuremsg "W: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages  Writing more data than expected ($NEW_SIZE > $SIZE)
-E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0 -o Debug::Acquire::Transaction=0
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::Transaction=0
 }
 
 methodtest() {
diff --git a/test/integration/test-apt-update-file b/test/integration/test-apt-update-file
index 1ecf9a38a..665f94fa5 100755
--- a/test/integration/test-apt-update-file
+++ b/test/integration/test-apt-update-file
@@ -22,6 +22,10 @@ addtrap 'prefix' 'chmod 750 aptarchive/dists/unstable/main/binary-amd64;'
 chmod 550 aptarchive/dists/unstable/main/binary-amd64
 
 testsuccess aptget update
+
+# the release files aren't an IMS-hit, but the indexes are
+redatereleasefiles '+1 hour'
+
 testsuccess aptget update -o Debug::pkgAcquire::Auth=1
 cp -a rootdir/tmp/testsuccess.output rootdir/tmp/update.output
 
diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback
index db4430ea3..f132bcf8e 100755
--- a/test/integration/test-apt-update-nofallback
+++ b/test/integration/test-apt-update-nofallback
@@ -28,6 +28,7 @@ Description: an autogenerated evil package
 EOF
     # avoid ims hit
     touch -d '+1hour' aptarchive/dists/unstable/main/binary-i386/Packages
+    compressfile aptarchive/dists/unstable/main/binary-i386/Packages
 }
 
 assert_update_is_refused_and_last_good_state_used()
@@ -87,16 +88,16 @@ test_from_inrelease_to_unsigned_with_override()
 {
     # setup archive with InRelease file
     setupaptarchive_with_lists_clean
-    # FIXME: is not what the server reported 4104 4106
-    testsuccess aptget update #-o Debug::pkgAcquire::Worker=1
+    testsuccess aptget update
 
     # simulate moving to a unsigned but otherwise valid repo
     simulate_mitm_and_inject_evil_package
-    generatereleasefiles
+    generatereleasefiles '+2 hours'
+    find $APTARCHIVE -name '*Packages*' -exec touch -d '+2 hours' {} \;
 
     # and ensure we can update to it (with enough force) 
     testwarning aptget update --allow-insecure-repositories \
-        -o Acquire::AllowDowngradeToInsecureRepositories=1
+        -o Acquire::AllowDowngradeToInsecureRepositories=1 -o Debug::pkgAcquire::Worker=1 -o Debug::pkgAcquire::Auth=1
     # but that the individual packages are still considered untrusted
     testfailureequal "WARNING: The following packages cannot be authenticated!
   evil
@@ -167,7 +168,7 @@ test_inrelease_to_invalid_inrelease()
     listcurrentlistsdirectory > lists.before
 
     # now remove InRelease and subvert Release do no longer verify
-    sed -i 's/Codename.*/Codename: evil!'/ $APTARCHIVE/dists/unstable/InRelease
+    sed -i 's/^Codename:.*/Codename: evil!/' $APTARCHIVE/dists/unstable/InRelease
     inject_evil_package
 
     testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
index bac33d531..a490f00de 100755
--- a/test/integration/test-apt-update-not-modified
+++ b/test/integration/test-apt-update-not-modified
@@ -43,7 +43,9 @@ Version: 1
 EOF
 	compressfile aptarchive/dists/unstable/main/binary-amd64/Packages
 	testfailureequal "Hit $1 unstable InRelease
-Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists.good/unstable/main/binary-amd64/Packages.gz') B]
+Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Err $1 unstable/main amd64 Packages
+  Hash Sum mismatch
 W: Failed to fetch $1/dists/unstable/main/binary-amd64/Packages.gz  Hash Sum mismatch
 
 E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update
@@ -87,8 +89,32 @@ Hit $1 unstable Release
 Reading package lists..." aptget update
 	testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)"
 
-	# readd arch so its downloaded again
+	# readd arch so its downloaded again…
 	configarchitecture 'amd64' 'i386'
+	# … but oh noes, hashsum mismatch!
+	find aptarchive/dists/unstable/main/binary-amd64/ -type f -delete
+	cat >> aptarchive/dists/unstable/main/binary-amd64/Packages < rootdir/etc/apt/apt.conf.d/00nolanguages
-testsuccess aptget update
+testsuccess aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
 listcurrentlistsdirectory > lists.before
 
 # insert new version
@@ -26,7 +26,7 @@ mkdir aptarchive/dists/unstable/main/binary-i386/saved
 cp -p aptarchive/dists/unstable/main/binary-i386/Packages* \
      aptarchive/dists/unstable/main/binary-i386/saved
 insertpackage 'unstable' 'foo' 'all' '2.0'
-
+touch -d '+1 hour' aptarchive/dists/unstable/main/binary-i386/Packages
 compressfile aptarchive/dists/unstable/main/binary-i386/Packages
 # ensure that we do not get a I-M-S hit for the Release file
 
@@ -39,7 +39,6 @@ cp -p aptarchive/dists/unstable/main/binary-i386/saved/Packages* \
      aptarchive/dists/unstable/main/binary-i386/
 
 # ensure this raises an error
-testfailureequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages  Hash Sum mismatch
-
-E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+testfailuremsg "W: Failed to fetch copy:$(readlink -f ./rootdir)/var/lib/apt/lists/localhost:8080_dists_unstable_main_binary-i386_Packages  Hash Sum mismatch
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
 testfileequal lists.before "$(listcurrentlistsdirectory)"
diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index bca07268c..486b8ba02 100755
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -12,7 +12,7 @@ setupflataptarchive
 
 testaptgetupdate() {
 	rm -rf rootdir/var/lib/apt
-	aptget update 2>> testaptgetupdate.diff >> testaptgetupdate.diff || true
+	aptget update >testaptgetupdate.diff 2>&1 || true
 	sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
 	GIVEN="$1"
 	shift
diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
index ec74a750b..555d8fcaa 100755
--- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
+++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
@@ -176,7 +176,11 @@ testmismatch() {
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080/  $1 1.0 (dsc) [3 B]
+Err http://localhost:8080/  $1 1.0 (dsc)
+  Hash Sum mismatch
 Get:2 http://localhost:8080/  $1 1.0 (tar) [3 B]
+Err http://localhost:8080/  $1 1.0 (tar)
+  Hash Sum mismatch
 E: Failed to fetch http://localhost:8080/${1}_1.0.dsc  Hash Sum mismatch
 
 E: Failed to fetch http://localhost:8080/${1}_1.0.tar.gz  Hash Sum mismatch
@@ -238,6 +242,8 @@ Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080/  pkg-mixed-sha1-bad 1.0 (tar) [3 B]
 Get:2 http://localhost:8080/  pkg-mixed-sha1-bad 1.0 (dsc) [3 B]
+Err http://localhost:8080/  pkg-mixed-sha1-bad 1.0 (dsc)
+  Hash Sum mismatch
 E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc  Hash Sum mismatch
 
 E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha1-bad
@@ -247,6 +253,8 @@ testfailureequal 'Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080/  pkg-mixed-sha2-bad 1.0 (tar) [3 B]
+Err http://localhost:8080/  pkg-mixed-sha2-bad 1.0 (tar)
+  Hash Sum mismatch
 Get:2 http://localhost:8080/  pkg-mixed-sha2-bad 1.0 (dsc) [3 B]
 E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz  Hash Sum mismatch
 
diff --git a/test/libapt/acqprogress_test.cc b/test/libapt/acqprogress_test.cc
index 288e05aca..c634733d4 100644
--- a/test/libapt/acqprogress_test.cc
+++ b/test/libapt/acqprogress_test.cc
@@ -1,4 +1,5 @@
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -10,9 +11,10 @@
 class TestItem: public pkgAcquire::Item
 {
 public:
-   TestItem(pkgAcquire * const Acq) : pkgAcquire::Item(Acq, "", NULL) {}
+   TestItem(pkgAcquire * const Acq) : pkgAcquire::Item(Acq) {}
 
-   virtual std::string DescURI() { return ""; }
+   virtual std::string DescURI() const { return ""; }
+   virtual HashStringList GetExpectedHashes() const { return HashStringList(); }
 
 };
 
diff --git a/test/libapt/hashsums_test.cc b/test/libapt/hashsums_test.cc
index edcd8a11a..63c63ecd3 100644
--- a/test/libapt/hashsums_test.cc
+++ b/test/libapt/hashsums_test.cc
@@ -306,6 +306,7 @@ TEST(HashSumsTest, HashStringList)
    EXPECT_EQ(NULL, list.find(NULL));
    EXPECT_EQ(NULL, list.find(""));
    EXPECT_EQ(NULL, list.find("MD5Sum"));
+   EXPECT_EQ(0, list.FileSize());
 
    // empty lists aren't equal
    HashStringList list2;
@@ -316,6 +317,8 @@ TEST(HashSumsTest, HashStringList)
    list.push_back(HashString("Checksum-FileSize", "29"));
    EXPECT_FALSE(list.empty());
    EXPECT_FALSE(list.usable());
+   EXPECT_EQ(1, list.size());
+   EXPECT_EQ(29, list.FileSize());
 
    Hashes hashes;
    hashes.Add("The quick brown fox jumps over the lazy dog");
-- 
cgit v1.2.3-70-g09d2


From 3679515479136179e0d95325a6559fcc6d0af7f8 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 6 Jun 2015 19:16:45 +0200
Subject: check patch hashes in rred worker instead of in the handler
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

rred is responsible for unpacking and reading the patch files in one go,
but we currently only have hashes for the uncompressed patch files, so
the handler read the entire patch file before dispatching it to the
worker which would read it again – both with an implicit uncompress.
Worse, while the workers operate in parallel the handler is the central
orchestration unit, so having it busy with work means the workers do
(potentially) nothing.

This means rred is working with 'untrusted' data, which is bad. Yet,
having the unpack in the handler meant that the untrusted uncompress was
done as root which isn't better either. Now, we have it at least
contained in a binary which we can harden a bit better. In the long run,
we want hashes for the compressed patch files through to be safe.
---
 apt-pkg/acquire-item.cc           | 95 +++++++++++++++++++++++----------------
 apt-pkg/acquire-item.h            |  2 +
 apt-pkg/acquire-method.cc         | 14 +++---
 apt-pkg/acquire-method.h          |  5 ++-
 methods/rred.cc                   | 62 ++++++++++++++++++++-----
 test/integration/test-pdiff-usage |  2 +
 6 files changed, 121 insertions(+), 59 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index ec6ec6e84..7b69ee993 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -95,6 +95,19 @@ static std::string GetCompressedFileName(std::string const &URI, std::string con
    return Name;
 }
 									/*}}}*/
+static std::string GetMergeDiffsPatchFileName(std::string const &Final, std::string const &Patch)/*{{{*/
+{
+   // rred expects the patch as $FinalFile.ed.$patchname.gz
+   return Final + ".ed." + Patch + ".gz";
+}
+									/*}}}*/
+static std::string GetDiffsPatchFileName(std::string const &Final)	/*{{{*/
+{
+   // rred expects the patch as $FinalFile.ed
+   return Final + ".ed";
+}
+									/*}}}*/
+
 static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser, pkgAcqMetaBase * const TransactionManager, pkgAcquire::Item * const I) /*{{{*/
 {
    if(MetaIndexParser->IsAlwaysTrusted() || _config->FindB("Acquire::AllowInsecureRepositories") == true)
@@ -1860,6 +1873,9 @@ void pkgAcqIndexDiffs::Failed(string const &Message,pkgAcquire::MethodConfig con
 		<< "Falling back to normal index file acquire" << std::endl;
    DestFile = GetPartialFileNameFromURI(Target->URI);
    RenameOnError(PDiffError);
+   std::string const patchname = GetDiffsPatchFileName(DestFile);
+   if (RealFileExists(patchname))
+      rename(patchname.c_str(), std::string(patchname + ".FAILED").c_str());
    new pkgAcqIndex(Owner, TransactionManager, Target);
    Finish();
 }
@@ -1968,28 +1984,13 @@ void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes,
 
    Item::Done(Message, Hashes, Cnf);
 
-   // FIXME: verify this download too before feeding it to rred
    std::string const FinalFile = GetPartialFileNameFromURI(Target->URI);
+   std::string const PatchFile = GetDiffsPatchFileName(FinalFile);
 
    // success in downloading a diff, enter ApplyDiff state
    if(State == StateFetchDiff)
    {
-      FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Gzip);
-      class Hashes LocalHashesCalc;
-      LocalHashesCalc.AddFD(fd);
-      HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
-
-      if (fd.Size() != available_patches[0].patch_size ||
-	    available_patches[0].patch_hashes != LocalHashes)
-      {
-	 // patchfiles are dated, so bad indicates a bad download, so kill it
-	 unlink(DestFile.c_str());
-	 Failed("Patch has Size/Hashsum mismatch", NULL);
-	 return;
-      }
-
-      // rred excepts the patch as $FinalFile.ed
-      Rename(DestFile,FinalFile+".ed");
+      Rename(DestFile, PatchFile);
 
       if(Debug)
 	 std::clog << "Sending to rred method: " << FinalFile << std::endl;
@@ -2000,18 +2001,17 @@ void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes,
       QueueURI(Desc);
       SetActiveSubprocess("rred");
       return;
-   } 
-
+   }
 
    // success in download/apply a diff, queue next (if needed)
    if(State == StateApplyDiff)
    {
       // remove the just applied patch
       available_patches.erase(available_patches.begin());
-      unlink((FinalFile + ".ed").c_str());
+      unlink(PatchFile.c_str());
 
       // move into place
-      if(Debug) 
+      if(Debug)
       {
 	 std::clog << "Moving patched file in place: " << std::endl
 		   << DestFile << " -> " << FinalFile << std::endl;
@@ -2031,6 +2031,18 @@ void pkgAcqIndexDiffs::Done(string const &Message, HashStringList const &Hashes,
    }
 }
 									/*}}}*/
+std::string pkgAcqIndexDiffs::Custom600Headers() const			/*{{{*/
+{
+   if(State != StateApplyDiff)
+      return pkgAcqBaseIndex::Custom600Headers();
+   std::ostringstream patchhashes;
+   HashStringList const ExpectedHashes = available_patches[0].patch_hashes;
+   for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
+      patchhashes <<  "\nPatch-0-" << hs->HashType() << "-Hash: " << hs->HashValue();
+   patchhashes << pkgAcqBaseIndex::Custom600Headers();
+   return patchhashes.str();
+}
+									/*}}}*/
 
 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor			/*{{{*/
 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner,
@@ -2079,6 +2091,9 @@ void pkgAcqIndexMergeDiffs::Failed(string const &Message,pkgAcquire::MethodConfi
       std::clog << "Falling back to normal index file acquire" << std::endl;
    DestFile = GetPartialFileNameFromURI(Target->URI);
    RenameOnError(PDiffError);
+   std::string const patchname = GetMergeDiffsPatchFileName(DestFile, patch.file);
+   if (RealFileExists(patchname))
+      rename(patchname.c_str(), std::string(patchname + ".FAILED").c_str());
    new pkgAcqIndex(Owner, TransactionManager, Target);
 }
 									/*}}}*/
@@ -2090,26 +2105,10 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha
 
    Item::Done(Message, Hashes, Cnf);
 
-   // FIXME: verify download before feeding it to rred
    string const FinalFile = GetPartialFileNameFromURI(Target->URI);
-
    if (State == StateFetchDiff)
    {
-      FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Gzip);
-      class Hashes LocalHashesCalc;
-      LocalHashesCalc.AddFD(fd);
-      HashStringList const LocalHashes = LocalHashesCalc.GetHashStringList();
-
-      if (fd.Size() != patch.patch_size || patch.patch_hashes != LocalHashes)
-      {
-	 // patchfiles are dated, so bad indicates a bad download, so kill it
-	 unlink(DestFile.c_str());
-	 Failed("Patch has Size/Hashsum mismatch", NULL);
-	 return;
-      }
-
-      // rred expects the patch as $FinalFile.ed.$patchname.gz
-      Rename(DestFile, FinalFile + ".ed." + patch.file + ".gz");
+      Rename(DestFile, GetMergeDiffsPatchFileName(FinalFile, patch.file));
 
       // check if this is the last completed diff
       State = StateDoneDiff;
@@ -2158,7 +2157,7 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha
 	    I != allPatches->end(); ++I)
       {
 	 std::string const PartialFile = GetPartialFileNameFromURI(Target->URI);
-	 std::string patch = PartialFile + ".ed." + (*I)->patch.file + ".gz";
+	 std::string const patch = GetMergeDiffsPatchFileName(PartialFile, (*I)->patch.file);
 	 unlink(patch.c_str());
       }
       unlink(FinalFile.c_str());
@@ -2170,6 +2169,24 @@ void pkgAcqIndexMergeDiffs::Done(string const &Message, HashStringList const &Ha
    }
 }
 									/*}}}*/
+std::string pkgAcqIndexMergeDiffs::Custom600Headers() const		/*{{{*/
+{
+   if(State != StateApplyDiff)
+      return pkgAcqBaseIndex::Custom600Headers();
+   std::ostringstream patchhashes;
+   unsigned int seen_patches = 0;
+   for (std::vector::const_iterator I = allPatches->begin();
+	 I != allPatches->end(); ++I)
+   {
+      HashStringList const ExpectedHashes = (*I)->patch.patch_hashes;
+      for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
+	 patchhashes <<  "\nPatch-" << seen_patches << "-" << hs->HashType() << "-Hash: " << hs->HashValue();
+      ++seen_patches;
+   }
+   patchhashes << pkgAcqBaseIndex::Custom600Headers();
+   return patchhashes.str();
+}
+									/*}}}*/
 
 // AcqIndex::AcqIndex - Constructor					/*{{{*/
 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 97d5ea1dd..f24af1aec 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -774,6 +774,7 @@ class APT_HIDDEN pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex
    virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
    virtual void Done(std::string const &Message, HashStringList const &Hashes,
 	 pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string Custom600Headers() const;
    virtual std::string DescURI() const {return Target->URI + "Index";};
    virtual HashStringList GetExpectedHashes() const;
    virtual bool HashesRequired() const;
@@ -886,6 +887,7 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex
 
    virtual void Done(std::string const &Message, HashStringList const &Hashes,
 		     pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string Custom600Headers() const;
    virtual std::string DescURI() const {return Target->URI + "IndexDiffs";};
    virtual HashStringList GetExpectedHashes() const;
    virtual bool HashesRequired() const;
diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc
index b77096efd..a8fc75f8e 100644
--- a/apt-pkg/acquire-method.cc
+++ b/apt-pkg/acquire-method.cc
@@ -388,14 +388,14 @@ int pkgAcqMethod::Run(bool Single)
 	    *I = Tmp;
 	    if (QueueBack == 0)
 	       QueueBack = Tmp;
-	    
+
 	    // Notify that this item is to be fetched.
-	    if (Fetch(Tmp) == false)
+	    if (URIAcquire(Message, Tmp) == false)
 	       Fail();
-	    
-	    break;					     
-	 }   
-      }      
+
+	    break;
+	 }
+      }
    }
 
    Exit();
@@ -403,8 +403,6 @@ int pkgAcqMethod::Run(bool Single)
 }
 									/*}}}*/
 // AcqMethod::PrintStatus - privately really send a log/status message	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void pkgAcqMethod::PrintStatus(char const * const header, const char* Format,
 			       va_list &args) const
 {
diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h
index 399454892..6480eb4b5 100644
--- a/apt-pkg/acquire-method.h
+++ b/apt-pkg/acquire-method.h
@@ -76,11 +76,12 @@ class pkgAcqMethod
    std::string FailReason;
    std::string UsedMirror;
    std::string IP;
-   
+
    // Handlers for messages
    virtual bool Configuration(std::string Message);
    virtual bool Fetch(FetchItem * /*Item*/) {return true;};
-   
+   virtual bool URIAcquire(std::string const &/*Message*/, FetchItem *Itm) { return Fetch(Itm); };
+
    // Outgoing messages
    void Fail(bool Transient = false);
    inline void Fail(const char *Why, bool Transient = false) {Fail(std::string(Why),Transient);};
diff --git a/methods/rred.cc b/methods/rred.cc
index 554ac99b4..3da33c126 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -388,7 +388,7 @@ class Patch {
 
    public:
 
-   void read_diff(FileFd &f)
+   void read_diff(FileFd &f, Hashes * const h)
    {
       char buffer[BLOCK_SIZE];
       bool cmdwanted = true;
@@ -396,6 +396,8 @@ class Patch {
       Change ch(0);
       while(f.ReadLine(buffer, sizeof(buffer)))
       {
+	 if (h != NULL)
+	    h->Add(buffer);
 	 if (cmdwanted) {
 	    char *m, *c;
 	    size_t s, e;
@@ -519,8 +521,29 @@ class RredMethod : public pkgAcqMethod {
    private:
       bool Debug;
 
+      struct PDiffFile {
+	 std::string FileName;
+	 HashStringList ExpectedHashes;
+	 PDiffFile(std::string const &FileName, HashStringList const &ExpectedHashes) :
+	    FileName(FileName), ExpectedHashes(ExpectedHashes) {}
+      };
+
+      HashStringList ReadExpectedHashesForPatch(unsigned int const patch, std::string const &Message)
+      {
+	 HashStringList ExpectedHashes;
+	 for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+	 {
+	    std::string tagname;
+	    strprintf(tagname, "Patch-%d-%s-Hash", patch, *type);
+	    std::string const hashsum = LookupTag(Message, tagname.c_str());
+	    if (hashsum.empty() == false)
+	       ExpectedHashes.push_back(HashString(*type, hashsum));
+	 }
+	 return ExpectedHashes;
+      }
+
    protected:
-      virtual bool Fetch(FetchItem *Itm) {
+      virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) {
 	 Debug = _config->FindB("Debug::pkgAcquire::RRed", false);
 	 URI Get = Itm->Uri;
 	 std::string Path = Get.Host + Get.Path; // rred:/path - no host
@@ -534,11 +557,17 @@ class RredMethod : public pkgAcqMethod {
 	 } else
 	    URIStart(Res);
 
-	 std::vector patchpaths;
+	 std::vector patchfiles;
 	 Patch patch;
 
 	 if (FileExists(Path + ".ed") == true)
-	    patchpaths.push_back(Path + ".ed");
+	 {
+	    HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(0, Message);
+	    std::string const FileName = Path + ".ed";
+	    if (ExpectedHashes.usable() == false)
+	       return _error->Error("No hashes found for uncompressed patch: %s", FileName.c_str());
+	    patchfiles.push_back(PDiffFile(FileName, ExpectedHashes));
+	 }
 	 else
 	 {
 	    _error->PushToStack();
@@ -546,18 +575,27 @@ class RredMethod : public pkgAcqMethod {
 	    _error->RevertToStack();
 
 	    std::string const baseName = Path + ".ed.";
+	    unsigned int seen_patches = 0;
 	    for (std::vector::const_iterator p = patches.begin();
 		  p != patches.end(); ++p)
+	    {
 	       if (p->compare(0, baseName.length(), baseName) == 0)
-		  patchpaths.push_back(*p);
+	       {
+		  HashStringList const ExpectedHashes = ReadExpectedHashesForPatch(seen_patches, Message);
+		  if (ExpectedHashes.usable() == false)
+		     return _error->Error("No hashes found for uncompressed patch %d: %s", seen_patches, p->c_str());
+		  patchfiles.push_back(PDiffFile(*p, ExpectedHashes));
+		  ++seen_patches;
+	       }
+	    }
 	 }
 
 	 std::string patch_name;
-	 for (std::vector::iterator I = patchpaths.begin();
-	       I != patchpaths.end();
+	 for (std::vector::iterator I = patchfiles.begin();
+	       I != patchfiles.end();
 	       ++I)
 	 {
-	    patch_name = *I;
+	    patch_name = I->FileName;
 	    if (Debug == true)
 	       std::clog << "Patching " << Path << " with " << patch_name
 		  << std::endl;
@@ -569,8 +607,12 @@ class RredMethod : public pkgAcqMethod {
 	       _error->DumpErrors(std::cerr);
 	       abort();
 	    }
-	    patch.read_diff(p);
+	    Hashes patch_hash(I->ExpectedHashes);
+	    patch.read_diff(p, &patch_hash);
 	    p.Close();
+	    HashStringList const hsl = patch_hash.GetHashStringList();
+	    if (hsl != I->ExpectedHashes)
+	       return _error->Error("Patch %s doesn't have the expected hashsum", patch_name.c_str());
 	 }
 
 	 if (Debug == true)
@@ -643,7 +685,7 @@ int main(int argc, char **argv)
 	 _error->DumpErrors(std::cerr);
 	 exit(1);
       }
-      patch.read_diff(p);
+      patch.read_diff(p, NULL);
    }
 
    if (just_diff) {
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 7d72a6944..73df61895 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -170,6 +170,8 @@ SHA256-Patches:
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	testsuccess aptget update "$@"
+	cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output
+	testsuccess grep 'have the expected hashsum' rootdir/tmp/aptgetupdate.output
 	testnopackage oldstuff
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
-- 
cgit v1.2.3-70-g09d2


From 6d3e5bd8e08564c5eb12ecd869de5bd71e25f59d Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sun, 7 Jun 2015 02:17:15 +0200
Subject: add more parsing error checking for rred

The rred parser is very accepting regarding 'invalid' files. Given that
we can't trust the input it might be a bit too relaxed. In any case,
checking for more errors can't hurt given that we support only a very
specific subset of ed commands.
---
 methods/rred.cc                   |  70 +++++++++-----
 test/integration/test-method-rred | 194 ++++++++++++++++++++++++++++++++++++++
 test/integration/test-pdiff-usage |   3 +-
 3 files changed, 245 insertions(+), 22 deletions(-)
 create mode 100755 test/integration/test-method-rred

(limited to 'test')

diff --git a/methods/rred.cc b/methods/rred.cc
index 3da33c126..81ecf8553 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -21,6 +21,7 @@
 #include 
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -35,7 +36,7 @@ class MemBlock {
    char *start;
    size_t size;
    char *free;
-   struct MemBlock *next;
+   MemBlock *next;
 
    MemBlock(size_t size) : size(size), next(NULL)
    {
@@ -116,7 +117,7 @@ struct Change {
    size_t add_len; /* bytes */
    char *add;
 
-   Change(int off)
+   Change(size_t off)
    {
       offset = off;
       del_cnt = add_cnt = add_len = 0;
@@ -388,30 +389,37 @@ class Patch {
 
    public:
 
-   void read_diff(FileFd &f, Hashes * const h)
+   bool read_diff(FileFd &f, Hashes * const h)
    {
       char buffer[BLOCK_SIZE];
       bool cmdwanted = true;
 
-      Change ch(0);
-      while(f.ReadLine(buffer, sizeof(buffer)))
-      {
+      Change ch(std::numeric_limits::max());
+      if (f.ReadLine(buffer, sizeof(buffer)) == NULL)
+	 return _error->Error("Reading first line of patchfile %s failed", f.Name().c_str());
+      do {
 	 if (h != NULL)
 	    h->Add(buffer);
 	 if (cmdwanted) {
 	    char *m, *c;
 	    size_t s, e;
-	    s = strtol(buffer, &m, 10);
-	    if (m == buffer) {
-	       s = e = ch.offset + ch.add_cnt;
-	       c = buffer;
-	    } else if (*m == ',') {
-	       m++;
+	    errno = 0;
+	    s = strtoul(buffer, &m, 10);
+	    if (unlikely(m == buffer || s == ULONG_MAX || errno != 0))
+	       return _error->Error("Parsing patchfile %s failed: Expected an effected line start", f.Name().c_str());
+	    else if (*m == ',') {
+	       ++m;
 	       e = strtol(m, &c, 10);
+	       if (unlikely(m == c || e == ULONG_MAX || errno != 0))
+		  return _error->Error("Parsing patchfile %s failed: Expected an effected line end", f.Name().c_str());
+	       if (unlikely(e < s))
+		  return _error->Error("Parsing patchfile %s failed: Effected lines end %lu is before start %lu", f.Name().c_str(), e, s);
 	    } else {
 	       e = s;
 	       c = m;
 	    }
+	    if (s > ch.offset)
+	       return _error->Error("Parsing patchfile %s failed: Effected line is after previous effected line", f.Name().c_str());
 	    switch(*c) {
 	       case 'a':
 		  cmdwanted = false;
@@ -422,6 +430,8 @@ class Patch {
 		  ch.del_cnt = 0;
 		  break;
 	       case 'c':
+		  if (unlikely(s == 0))
+		     return _error->Error("Parsing patchfile %s failed: Change command can't effect line zero", f.Name().c_str());
 		  cmdwanted = false;
 		  ch.add = NULL;
 		  ch.add_cnt = 0;
@@ -430,6 +440,8 @@ class Patch {
 		  ch.del_cnt = e - s + 1;
 		  break;
 	       case 'd':
+		  if (unlikely(s == 0))
+		     return _error->Error("Parsing patchfile %s failed: Delete command can't effect line zero", f.Name().c_str());
 		  ch.offset = s - 1;
 		  ch.del_cnt = e - s + 1;
 		  ch.add = NULL;
@@ -437,9 +449,11 @@ class Patch {
 		  ch.add_len = 0;
 		  filechanges.add_change(ch);
 		  break;
+	       default:
+		  return _error->Error("Parsing patchfile %s failed: Unknown command", f.Name().c_str());
 	    }
 	 } else { /* !cmdwanted */
-	    if (buffer[0] == '.' && buffer[1] == '\n') {
+	    if (strcmp(buffer, ".\n") == 0) {
 	       cmdwanted = true;
 	       filechanges.add_change(ch);
 	    } else {
@@ -465,7 +479,8 @@ class Patch {
 	       }
 	    }
 	 }
-      }
+      } while(f.ReadLine(buffer, sizeof(buffer)));
+      return true;
    }
 
    void write_diff(FILE *f)
@@ -601,14 +616,14 @@ class RredMethod : public pkgAcqMethod {
 		  << std::endl;
 
 	    FileFd p;
+	    Hashes patch_hash(I->ExpectedHashes);
 	    // all patches are compressed, even if the name doesn't reflect it
-	    if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false) {
-	       std::cerr << "Could not open patch file " << patch_name << std::endl;
+	    if (p.Open(patch_name, FileFd::ReadOnly, FileFd::Gzip) == false ||
+		  patch.read_diff(p, &patch_hash) == false)
+	    {
 	       _error->DumpErrors(std::cerr);
-	       abort();
+	       return false;
 	    }
-	    Hashes patch_hash(I->ExpectedHashes);
-	    patch.read_diff(p, &patch_hash);
 	    p.Close();
 	    HashStringList const hsl = patch_hash.GetHashStringList();
 	    if (hsl != I->ExpectedHashes)
@@ -624,7 +639,6 @@ class RredMethod : public pkgAcqMethod {
 	 FILE *out = fopen(Itm->DestFile.c_str(), "w");
 
 	 Hashes hash(Itm->ExpectedHashes);
-
 	 patch.apply_against_file(out, inp, &hash);
 
 	 fclose(out);
@@ -657,6 +671,16 @@ class RredMethod : public pkgAcqMethod {
 	 return true;
       }
 
+      bool Configuration(std::string Message)
+      {
+	 if (pkgAcqMethod::Configuration(Message) == false)
+	    return false;
+
+	 DropPrivsOrDie();
+
+	 return true;
+      }
+
    public:
       RredMethod() : pkgAcqMethod("2.0",SingleInstance | SendConfig), Debug(false) {}
 };
@@ -685,7 +709,11 @@ int main(int argc, char **argv)
 	 _error->DumpErrors(std::cerr);
 	 exit(1);
       }
-      patch.read_diff(p, NULL);
+      if (patch.read_diff(p, NULL) == false)
+      {
+	 _error->DumpErrors(std::cerr);
+	 exit(2);
+      }
    }
 
    if (just_diff) {
diff --git a/test/integration/test-method-rred b/test/integration/test-method-rred
new file mode 100755
index 000000000..a8de3ea28
--- /dev/null
+++ b/test/integration/test-method-rred
@@ -0,0 +1,194 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'i386'
+
+echo 'Package: coolstuff
+Version: 0.8.15
+Description: collection of stuff
+ A lot, too much to iterate all, but at least this:
+ - stuff
+ - more stuff
+ - even more stuff
+ .
+ And a cow.
+
+Package: oldstuff
+Version: 0-1
+Description: collection of outdated stuff
+ A lot, but of no use nowadays, but at least this:
+ - stuff
+ - more stuff
+ - even more stuff
+ .
+ And a dog.' > Packages
+
+testrred() {
+	msgtest "$1" "$2"
+	if [ -z "$3" ]; then
+		echo -n '' > Packages.ed
+	else
+		echo "$3" > Packages.ed
+	fi
+	rred() {
+		cat Packages | runapt "${METHODSDIR}/rred" "$@"
+	}
+	testsuccessequal "$4" --nomsg rred -f Packages.ed
+}
+
+testrred 'Remove' 'first line' '1d' "$(tail -n +2 ./Packages)"
+testrred 'Remove' 'empty line' '10d' "$(head -n 9 ./Packages)
+$(tail -n 9 ./Packages)"
+testrred 'Remove' 'line in a paragraph' '5d' "$(head -n 4 ./Packages)
+$(tail -n 14 ./Packages)"
+testrred 'Remove' 'last line' '19d' "$(head -n -1 ./Packages)"
+testrred 'Remove' 'multiple single lines' '17d
+7d' "$(sed -e '/^ - even more stuff$/ d' ./Packages)"
+testrred 'Remove' 'first paragraph' '1,10d' "$(tail -n 9 ./Packages)"
+testrred 'Remove' 'a few lines in the middle' '5,14d' "$(head -n 4 ./Packages)
+$(tail -n 5 ./Packages)"
+testrred 'Remove' 'second paragraph' '10,19d' "$(head -n 9 ./Packages)"
+testrred 'Mass Remove' 'all stuff lines' '15,17d
+13d
+11d
+5,7d
+3d
+1d' "$(sed '/stuff/ d' ./Packages)"
+
+testrred 'Single line add' 'first line' '0a
+Format: 3.0 (native)
+.' "Format: 3.0 (native)
+$(cat ./Packages)"
+testrred 'Single line add' 'last line' '19a
+Multi-Arch: foreign
+.' "$(cat ./Packages)
+Multi-Arch: foreign"
+testrred 'Single line add' 'middle' '9a
+Multi-Arch: foreign
+.' "$(head -n 9 ./Packages)
+Multi-Arch: foreign
+$(tail -n 10 ./Packages)"
+
+testrred 'Multi line add' 'first line' '0a
+Format: 3.0 (native)
+Source: apt
+.' "Format: 3.0 (native)
+Source: apt
+$(cat ./Packages)"
+testrred 'Multi line add' 'last line' '19a
+Multi-Arch: foreign
+Homepage: https://debian.org
+.' "$(cat ./Packages)
+Multi-Arch: foreign
+Homepage: https://debian.org"
+testrred 'Multi line add' 'middle' '9a
+Multi-Arch: foreign
+Homepage: https://debian.org
+.' "$(head -n 9 ./Packages)
+Multi-Arch: foreign
+Homepage: https://debian.org
+$(tail -n 10 ./Packages)"
+
+testrred 'Single line change' 'first line' '1c
+Package: supercoolstuff
+.' "Package: supercoolstuff
+$(tail -n +2 ./Packages)"
+testrred 'Single line change' 'in the middle' '9c
+ And a super cow.
+.' "$(head -n 8 ./Packages)
+ And a super cow.
+$(tail -n 10 ./Packages)"
+testrred 'Single line change' 'an empty line' '10c
+
+.' "$(head -n 9 ./Packages)
+
+$(tail -n 9 ./Packages)"
+testrred 'Single line change' 'a spacy line' '10c
+   	  
+.' "$(head -n 9 ./Packages)
+   	  
+$(tail -n 9 ./Packages)"
+testrred 'Single line change' 'last line' '19c
+ And a cat.
+.' "$(head -n -1 ./Packages)
+ And a cat."
+
+testrred 'Multi line change' 'exchange' '5,7c
+ - good stuff
+ - more good stuff
+ - even more good stuff
+.' "$(head -n 4 ./Packages)
+ - good stuff
+ - more good stuff
+ - even more good stuff
+$(tail -n 12 ./Packages)"
+testrred 'Multi line change' 'less' '5,7c
+ - good stuff
+ - more good stuff
+.' "$(head -n 4 ./Packages)
+ - good stuff
+ - more good stuff
+$(tail -n 12 ./Packages)"
+testrred 'Multi line change' 'more' '5,7c
+ - good stuff
+ - more good stuff
+ - even more good stuff
+ - bonus good stuff
+.' "$(head -n 4 ./Packages)
+ - good stuff
+ - more good stuff
+ - even more good stuff
+ - bonus good stuff
+$(tail -n 12 ./Packages)"
+
+failrred() {
+	msgtest 'Failure caused by' "$1"
+	echo "$2" > Packages.ed
+	rred() {
+		cat Packages | runapt "${METHODSDIR}/rred" "$@"
+	}
+	testfailure --nomsg rred -f Packages.ed
+}
+
+failrred 'Bogus content' '
+'
+
+# not a problem per-se, but we want our parser to be really strict
+failrred 'Empty patch file' ''
+failrred 'Empty line patch file' '
+'
+failrred 'Empty line before command' '
+1d'
+failrred 'Empty line after command' '1d
+'
+failrred 'Empty line between commands' '17d
+
+7d'
+failrred 'Empty spaces lines before command' '   	   
+1d'
+failrred 'Empty spaces lines after command' '1d
+  	   '
+failrred 'Empty spaces lines between commands' '17d
+  	   
+7d'
+
+# the line before the first one can't be deleted/changed
+failrred 'zero line delete' '0d'
+failrred 'zero line change' '0c
+Package: supercoolstuff
+.'
+# and this makes no sense at all
+failrred 'negative line delete' '-1d'
+failrred 'negative line change' '-1c
+Package: supercoolstuff
+.'
+failrred 'negative line add' '-1a
+Package: supercoolstuff
+.'
+failrred 'Wrong order of commands' '7d
+17d'
+failrred 'End before start' '7,6d'
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 73df61895..7a9f6496b 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -165,7 +165,8 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
-	echo 'I am Mallory and I change files' >> $PATCHFILE
+	# needs to look like a valid command, otherwise the parser will fail before hashes are checked
+	echo '1d' >> $PATCHFILE
 	cat $PATCHFILE | gzip > ${PATCHFILE}.gz
 	generatereleasefiles '+1hour'
 	signreleasefiles
-- 
cgit v1.2.3-70-g09d2


From fec55559451dc1c440c4770ee3faa116522fc59c Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sun, 7 Jun 2015 09:35:37 +0200
Subject: fix download-file using testcases to run as root

Git-Dch: Ignore
---
 test/integration/test-apt-download-progress |  7 ++++---
 test/integration/test-apt-helper            | 26 +++++++++++++-------------
 2 files changed, 17 insertions(+), 16 deletions(-)

(limited to 'test')

diff --git a/test/integration/test-apt-download-progress b/test/integration/test-apt-download-progress
index 65c438e8f..7caeca971 100755
--- a/test/integration/test-apt-download-progress
+++ b/test/integration/test-apt-download-progress
@@ -24,18 +24,19 @@ assertprogress() {
 # actually report progress - but not too big to ensure its not delaying the
 # test too much
 TESTFILE=testfile.big
-testsuccess dd if=/dev/zero of=./aptarchive/$TESTFILE bs=800k count=1 
+testsuccess dd if=/dev/zero of=./aptarchive/$TESTFILE bs=800k count=1
 
 OPT='-o APT::Status-Fd=3 -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 -o Debug::Acquire::https=1'
 
 msgtest 'download progress works via' 'http'
 exec 3> apt-progress.log
-testsuccess --nomsg apthelper download-file "http://localhost:8080/$TESTFILE" http-$TESTFILE $OPT -o Acquire::http::Dl-Limit=800
+testsuccess --nomsg apthelper download-file "http://localhost:8080/$TESTFILE" ./downloaded/http-$TESTFILE $OPT -o Acquire::http::Dl-Limit=800
 assertprogress apt-progress.log
 
 msgtest 'download progress works via' 'https'
 exec 3> apt-progress.log
-testsuccess --nomsg apthelper download-file "https://localhost:4433/$TESTFILE" https-$TESTFILE $OPT -o Acquire::https::Dl-Limit=800
+testsuccess --nomsg apthelper download-file "https://localhost:4433/$TESTFILE" ./downloaded/https-$TESTFILE $OPT -o Acquire::https::Dl-Limit=800
+assertprogress apt-progress.log
 
 # cleanup
 rm -f apt-progress*.log
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index 431210797..00d859ad5 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -14,32 +14,32 @@ test_apt_helper_download() {
     echo 'bar' > aptarchive/foo2
 
     msgtest 'apt-file download-file md5sum'
-    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00
-    testfileequal foo2 'foo'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00
+    testfileequal ./downloaded/foo2 'foo'
 
     msgtest 'apt-file download-file sha1'
-    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
-    testfileequal foo1 'foo'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
+    testfileequal ./downloaded/foo1 'foo'
 
     msgtest 'apt-file download-file sha256'
-    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c
-    testfileequal foo3 'foo'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c
+    testfileequal ./downloaded/foo3 'foo'
 
     msgtest 'apt-file download-file no-hash'
-    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo4
-    testfileequal foo4 'foo'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo4
+    testfileequal ./downloaded/foo4 'foo'
     
     msgtest 'apt-file download-file wrong hash'
-    testfailure --nomsg apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc
+    testfailure --nomsg apthelper -qq download-file http://localhost:8080/foo ./downloaded/foo5 MD5Sum:aabbcc
     testfileequal rootdir/tmp/testfailure.output 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
 
 E: Download Failed'
-    testfileequal foo5.FAILED 'foo'
+    testfileequal ./downloaded/foo5.FAILED 'foo'
 
     msgtest 'apt-file download-file md5sum sha1'
-    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo6 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 http://localhost:8080/foo2 foo7 SHA1:e242ed3bffccdf271b7fbaf34ed72d089537b42f
-    testfileequal foo6 'foo'
-    testfileequal foo7 'bar'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo6 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 http://localhost:8080/foo2 ./downloaded/foo7 SHA1:e242ed3bffccdf271b7fbaf34ed72d089537b42f
+    testfileequal ./downloaded/foo6 'foo'
+    testfileequal ./downloaded/foo7 'bar'
 }
 
 test_apt_helper_detect_proxy() {
-- 
cgit v1.2.3-70-g09d2


From 4f51fd8636592a96aecf17c8bf4cfdb3ea2207cc Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 8 Jun 2015 00:06:41 +0200
Subject: support hashes for compressed pdiff files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

At the moment we only have hashes for the uncompressed pdiff files, but
via the new '$HASH-Download' field in the .diff/Index hashes can be
provided for the .gz compressed pdiff file, which apt will pick up now
and use to verify the download. Now, we "just" need a buy in from the
creators of repositories…
---
 apt-pkg/acquire-item.cc           | 72 ++++++++++++++++++++++++++++++++-------
 apt-pkg/acquire-item.h            | 11 +++---
 apt-pkg/contrib/hashes.cc         | 11 ++++--
 apt-pkg/contrib/hashes.h          |  7 ++++
 apt-pkg/indexrecords.cc           |  4 +--
 methods/rred.cc                   |  2 +-
 test/integration/test-pdiff-usage | 46 +++++++++++++++++++++++--
 7 files changed, 125 insertions(+), 28 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 7b69ee993..a3f47242f 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -152,14 +152,18 @@ HashStringList pkgAcqMetaBase::GetExpectedHashes() const
 
 APT_CONST bool pkgAcqIndexDiffs::HashesRequired() const
 {
-   /* FIXME: We have only hashes for uncompressed pdiffs.
-      rred uncompresses them on the fly while parsing.
-      In StateFetchDiff state we also uncompress on the fly for hash check.
-      Hashes are checked while searching for (next) patch to apply. */
+   /* We don't always have the diff of the downloaded pdiff file.
+      What we have for sure is hashes for the uncompressed file,
+      but rred uncompresses them on the fly while parsing, so not handled here.
+      Hashes are (also) checked while searching for (next) patch to apply. */
+   if (State == StateFetchDiff)
+      return available_patches[0].download_hashes.empty() == false;
    return false;
 }
 HashStringList pkgAcqIndexDiffs::GetExpectedHashes() const
 {
+   if (State == StateFetchDiff)
+      return available_patches[0].download_hashes;
    return HashStringList();
 }
 
@@ -168,11 +172,15 @@ APT_CONST bool pkgAcqIndexMergeDiffs::HashesRequired() const
    /* @see #pkgAcqIndexDiffs::HashesRequired, with the difference that
       we can check the rred result after all patches are applied as
       we know the expected result rather than potentially apply more patches */
+   if (State == StateFetchDiff)
+      return patch.download_hashes.empty() == false;
    return State == StateApplyDiff;
 }
 HashStringList pkgAcqIndexMergeDiffs::GetExpectedHashes() const
 {
-   if (State == StateApplyDiff)
+   if (State == StateFetchDiff)
+      return patch.download_hashes;
+   else if (State == StateApplyDiff)
       return GetExpectedHashesFor(Target->MetaKey);
    return HashStringList();
 }
@@ -1618,7 +1626,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 	 std::vector::iterator cur = available_patches.begin();
 	 for (; cur != available_patches.end(); ++cur)
 	 {
-	    if (cur->file != filename || unlikely(cur->result_size != size))
+	    if (cur->file != filename)
 	       continue;
 	    cur->result_hashes.push_back(HashString(*type, hash));
 	    break;
@@ -1630,8 +1638,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 	    DiffInfo next;
 	    next.file = filename;
 	    next.result_hashes.push_back(HashString(*type, hash));
-	    next.result_size = size;
-	    next.patch_size = 0;
+	    next.result_hashes.FileSize(size);
 	    available_patches.push_back(next);
 	 }
 	 else
@@ -1679,10 +1686,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 	 {
 	    if (cur->file != filename)
 	       continue;
-	    if (unlikely(cur->patch_size != 0 && cur->patch_size != size))
-	       continue;
+	    if (cur->patch_hashes.empty())
+	       cur->patch_hashes.FileSize(size);
 	    cur->patch_hashes.push_back(HashString(*type, hash));
-	    cur->patch_size = size;
 	    break;
 	 }
 	 if (cur != available_patches.end())
@@ -1694,6 +1700,48 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       }
    }
 
+   for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
+   {
+      std::string tagname = *type;
+      tagname.append("-Download");
+      std::string const tmp = Tags.FindS(tagname.c_str());
+      if (tmp.empty() == true)
+	 continue;
+
+      string hash, filename;
+      unsigned long long size;
+      std::stringstream ss(tmp);
+
+      // FIXME: all of pdiff supports only .gz compressed patches
+      while (ss >> hash >> size >> filename)
+      {
+	 if (unlikely(hash.empty() == true || filename.empty() == true))
+	    continue;
+	 if (unlikely(APT::String::Endswith(filename, ".gz") == false))
+	    continue;
+	 filename.erase(filename.length() - 3);
+
+	 // see if we have a record for this file already
+	 std::vector::iterator cur = available_patches.begin();
+	 for (; cur != available_patches.end(); ++cur)
+	 {
+	    if (cur->file != filename)
+	       continue;
+	    if (cur->download_hashes.empty())
+	       cur->download_hashes.FileSize(size);
+	    cur->download_hashes.push_back(HashString(*type, hash));
+	    break;
+	 }
+	 if (cur != available_patches.end())
+	       continue;
+	 if (Debug == true)
+	    std::clog << "pkgAcqDiffIndex: " << IndexDiffFile << ": File " << filename
+	       << " wasn't in the list for the first parsed hash! (download)" << std::endl;
+	 break;
+      }
+   }
+
+
    bool foundStart = false;
    for (std::vector::iterator cur = available_patches.begin();
 	 cur != available_patches.end(); ++cur)
@@ -1729,7 +1777,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
    unsigned long long patchesSize = 0;
    for (std::vector::const_iterator cur = available_patches.begin();
 	 cur != available_patches.end(); ++cur)
-      patchesSize += cur->patch_size;
+      patchesSize += cur->patch_hashes.FileSize();
    unsigned long long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
    if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
    {
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index f24af1aec..910e4131b 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -705,17 +705,14 @@ struct APT_HIDDEN DiffInfo {						/*{{{*/
    /** The filename of the diff. */
    std::string file;
 
-   /** The hashes of the diff */
+   /** The hashes of the file after the diff is applied */
    HashStringList result_hashes;
 
-   /** The hashes of the file after the diff is applied */
+   /** The hashes of the diff */
    HashStringList patch_hashes;
 
-   /** The size of the file after the diff is applied */
-   unsigned long long result_size;
-
-   /** The size of the diff itself */
-   unsigned long long patch_size;
+   /** The hashes of the compressed diff */
+   HashStringList download_hashes;
 };
 									/*}}}*/
 /** \brief An item that is responsible for fetching client-merge patches {{{
diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index 11a7e479b..46cf0ba08 100644
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -188,6 +188,13 @@ unsigned long long HashStringList::FileSize() const			/*{{{*/
    return strtoull(hv.c_str(), NULL, 10);
 }
 									/*}}}*/
+bool HashStringList::FileSize(unsigned long long const Size)		/*{{{*/
+{
+   std::string size;
+   strprintf(size, "%llu", Size);
+   return push_back(HashString("Checksum-FileSize", size));
+}
+									/*}}}*/
 bool HashStringList::supported(char const * const type)			/*{{{*/
 {
    for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t)
@@ -361,9 +368,7 @@ APT_IGNORE_DEPRECATED_PUSH
    if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
       hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
 APT_IGNORE_DEPRECATED_POP
-   std::string SizeStr;
-   strprintf(SizeStr, "%llu", d->FileSize);
-   hashes.push_back(HashString("Checksum-FileSize", SizeStr));
+   hashes.FileSize(d->FileSize);
    return hashes;
 }
 APT_IGNORE_DEPRECATED_PUSH
diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h
index 176ce4faa..e8d84da9e 100644
--- a/apt-pkg/contrib/hashes.h
+++ b/apt-pkg/contrib/hashes.h
@@ -96,6 +96,13 @@ class HashStringList
     */
    unsigned long long FileSize() const;
 
+   /** sets the filesize hash
+    *
+    * @param Size of the file
+    * @return @see #push_back
+    */
+   bool FileSize(unsigned long long const Size);
+
    /** check if the given hash type is supported
     *
     * @param type to check
diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc
index c26868cac..7e6da9558 100644
--- a/apt-pkg/indexrecords.cc
+++ b/apt-pkg/indexrecords.cc
@@ -121,9 +121,7 @@ bool indexRecords::Load(const string Filename)				/*{{{*/
             indexRecords::checkSum *Sum = new indexRecords::checkSum;
             Sum->MetaKeyFilename = Name;
             Sum->Size = Size;
-	    std::string SizeStr;
-	    strprintf(SizeStr, "%llu", Size);
-	    Sum->Hashes.push_back(HashString("Checksum-FileSize", SizeStr));
+	    Sum->Hashes.FileSize(Size);
             APT_IGNORE_DEPRECATED(Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash);)
             Entries[Name] = Sum;
          }
diff --git a/methods/rred.cc b/methods/rred.cc
index 81ecf8553..12cf2b4a5 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -627,7 +627,7 @@ class RredMethod : public pkgAcqMethod {
 	    p.Close();
 	    HashStringList const hsl = patch_hash.GetHashStringList();
 	    if (hsl != I->ExpectedHashes)
-	       return _error->Error("Patch %s doesn't have the expected hashsum", patch_name.c_str());
+	       return _error->Error("Hash Sum mismatch for uncompressed patch %s", patch_name.c_str());
 	 }
 
 	 if (Debug == true)
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 7a9f6496b..3295d5497 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -42,6 +42,8 @@ wasmergeused() {
 
 testrun() {
 	msgmsg "Testcase: setup the base with: $*"
+	local DOWNLOADHASH=true
+	if [ "$1" = 'nohash' ]; then DOWNLOADHASH=false; shift; fi
 	find aptarchive -name 'Packages*' -type f -delete
 	cp ${PKGFILE} aptarchive/Packages
 	compressfile 'aptarchive/Packages'
@@ -76,6 +78,15 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+	if $DOWNLOADHASH; then
+		echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+	fi
+
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	find aptarchive -name 'Packages*' -type f -delete
@@ -131,6 +142,17 @@ SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)
  $(sha256sum ${PATCHFILE2} | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}) $(basename ${PATCHFILE2})" > $PATCHINDEX
+	if $DOWNLOADHASH; then
+		echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+ $(sha1sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+ $(sha256sum ${PATCHFILE2}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE2}.gz) $(basename ${PATCHFILE2}.gz)" >> $PATCHINDEX
+	fi
+
 	generatereleasefiles '+2hour'
 	signreleasefiles
 	cp -a aptarchive/Packages Packages-future
@@ -150,6 +172,7 @@ SHA256-Patches:
 	mkdir -p aptarchive/Packages.diff
 	PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
 	diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true
+	cat $PATCHFILE | gzip > ${PATCHFILE}.gz
 	PATCHINDEX='aptarchive/Packages.diff/Index'
 	echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new)
 SHA1-History:
@@ -165,14 +188,22 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+	if $DOWNLOADHASH; then
+		echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz) $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+	fi
 	# needs to look like a valid command, otherwise the parser will fail before hashes are checked
-	echo '1d' >> $PATCHFILE
+	echo '1d' > $PATCHFILE
 	cat $PATCHFILE | gzip > ${PATCHFILE}.gz
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	testsuccess aptget update "$@"
 	cp -f rootdir/tmp/testsuccess.output rootdir/tmp/aptgetupdate.output
-	testsuccess grep 'have the expected hashsum' rootdir/tmp/aptgetupdate.output
+	testsuccess grep 'Hash Sum mismatch' rootdir/tmp/aptgetupdate.output
 	testnopackage oldstuff
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
@@ -201,6 +232,14 @@ SHA256-History:
 SHA256-Patches:
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 19722 2010-08-18-2013.28
  $(sha256sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE)000 $(basename $PATCHFILE)" > $PATCHINDEX
+	if $DOWNLOADHASH; then
+		echo "SHA1-Download:
+ 2365ac0ac57cde3d43c63145e8251a3bd5410213 197 2010-08-18-2013.28.gz
+ $(sha1sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)
+SHA256-Download:
+ d2a1b33187ed2d248eeae3b1223ea71791ea35f2138a713ed371332a6421f467 197 2010-08-18-2013.28.gz
+ $(sha256sum ${PATCHFILE}.gz | cut -d' ' -f 1) $(stat -c%s ${PATCHFILE}.gz)000 $(basename ${PATCHFILE}.gz)" >> $PATCHINDEX
+	fi
 	generatereleasefiles '+1hour'
 	signreleasefiles
 	#find aptarchive -name 'Packages*' -type f -delete
@@ -215,6 +254,9 @@ echo 'Debug::pkgAcquire::Diffs "true";
 Debug::Acquire::Transaction "true";
 Debug::pkgAcquire::rred "true";' > rootdir/etc/apt/apt.conf.d/rreddebug.conf
 
+testrun nohash -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
+testrun nohash -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
+
 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=1
 testrun -o Acquire::PDiffs::Merge=1 -o APT::Get::List-Cleanup=1
 testrun -o Acquire::PDiffs::Merge=0 -o APT::Get::List-Cleanup=0
-- 
cgit v1.2.3-70-g09d2


From 8d041b4f4f353079268039dcbfd8b5e575196b66 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 8 Jun 2015 15:22:01 +0200
Subject: do not request files if we expect an IMS hit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If we have a file on disk and the hashes are the same in the new Release
file and the old one we have on disk we know that if we ask the server
for the file, we will at best get an IMS hit – at worse the server
doesn't support this and sends us the (unchanged) file and we have to
run all our checks on it again for nothing. So, we can save ourselves
(and the servers) some unneeded requests if we figure this out on our
own.
---
 apt-pkg/acquire-item.cc                            | 77 ++++++++++++++--------
 test/integration/framework                         |  4 +-
 test/integration/test-apt-update-expected-size     |  4 +-
 test/integration/test-apt-update-file              | 25 +++++--
 test/integration/test-apt-update-not-modified      | 19 ++++++
 test/integration/test-apt-update-rollback          |  6 +-
 test/integration/test-apt-update-transactions      | 24 +++++--
 .../test-cve-2013-1051-InRelease-parsing           | 13 ++--
 8 files changed, 127 insertions(+), 45 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 13e971e9f..511bbbc64 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -119,6 +119,16 @@ static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser
    return false;
 }
 									/*}}}*/
+static HashStringList GetExpectedHashesFromFor(indexRecords * const Parser, std::string const MetaKey)/*{{{*/
+{
+   if (Parser == NULL)
+      return HashStringList();
+   indexRecords::checkSum * const R = Parser->Lookup(MetaKey);
+   if (R == NULL)
+      return HashStringList();
+   return R->Hashes;
+}
+									/*}}}*/
 
 // all ::HashesRequired and ::GetExpectedHashes implementations		/*{{{*/
 /* ::GetExpectedHashes is abstract and has to be implemented by all subclasses.
@@ -372,6 +382,25 @@ bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)
 }
 									/*}}}*/
 
+class APT_HIDDEN NoActionItem : public pkgAcquire::Item			/*{{{*/
+/* The sole purpose of this class is having an item which does nothing to
+   reach its done state to prevent cleanup deleting the mentioned file.
+   Handy in cases in which we know we have the file already, like IMS-Hits. */
+{
+   IndexTarget const * const Target;
+   public:
+   virtual std::string DescURI() const {return Target->URI;};
+   virtual HashStringList GetExpectedHashes()  const {return HashStringList();};
+
+   NoActionItem(pkgAcquire * const Owner, IndexTarget const * const Target) :
+      pkgAcquire::Item(Owner), Target(Target)
+   {
+      Status = StatDone;
+      DestFile = GetFinalFileNameFromURI(Target->URI);
+   }
+};
+									/*}}}*/
+
 // Acquire::Item::Item - Constructor					/*{{{*/
 APT_IGNORE_DEPRECATED_PUSH
 pkgAcquire::Item::Item(pkgAcquire * const Owner) :
@@ -644,12 +673,7 @@ pkgAcqTransactionItem::~pkgAcqTransactionItem()				/*{{{*/
 									/*}}}*/
 HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const MetaKey) const	/*{{{*/
 {
-   if (TransactionManager->MetaIndexParser == NULL)
-      return HashStringList();
-   indexRecords::checkSum * const R = TransactionManager->MetaIndexParser->Lookup(MetaKey);
-   if (R == NULL)
-      return HashStringList();
-   return R->Hashes;
+   return GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, MetaKey);
 }
 									/*}}}*/
 
@@ -939,6 +963,23 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify)			/*{{{*/
 	    return;
 	 }
 
+	 if (RealFileExists(GetFinalFileNameFromURI((*Target)->URI)))
+	 {
+	    if (TransactionManager->LastMetaIndexParser != NULL)
+	    {
+	       HashStringList const newFile = GetExpectedHashesFromFor(TransactionManager->MetaIndexParser, (*Target)->MetaKey);
+	       HashStringList const oldFile = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, (*Target)->MetaKey);
+	       if (newFile == oldFile)
+	       {
+		  // we have the file already, no point in trying to acquire it again
+		  new NoActionItem(Owner, *Target);
+		  continue;
+	       }
+	    }
+	 }
+	 else
+	    trypdiff = false; // no file to patch
+
 	 // check if we have patches available
 	 trypdiff &= TransactionManager->MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index");
       }
@@ -1085,20 +1126,6 @@ string pkgAcqMetaClearSig::Custom600Headers() const
 }
 									/*}}}*/
 // pkgAcqMetaClearSig::Done - We got a file				/*{{{*/
-class APT_HIDDEN DummyItem : public pkgAcquire::Item
-{
-   IndexTarget const * const Target;
-   public:
-   virtual std::string DescURI() const {return Target->URI;};
-   virtual HashStringList GetExpectedHashes()  const {return HashStringList();};
-
-   DummyItem(pkgAcquire * const Owner, IndexTarget const * const Target) :
-      pkgAcquire::Item(Owner), Target(Target)
-   {
-      Status = StatDone;
-      DestFile = GetFinalFileNameFromURI(Target->URI);
-   }
-};
 void pkgAcqMetaClearSig::Done(std::string const &Message,
                               HashStringList const &Hashes,
                               pkgAcquire::MethodConfig const * const Cnf)
@@ -1131,8 +1158,8 @@ void pkgAcqMetaClearSig::Done(std::string const &Message,
 	 // We got an InRelease file IMSHit, but we haven't one, which means
 	 // we had a valid Release/Release.gpg combo stepping in, which we have
 	 // to 'acquire' now to ensure list cleanup isn't removing them
-	 new DummyItem(Owner, &DetachedDataTarget);
-	 new DummyItem(Owner, &DetachedSigTarget);
+	 new NoActionItem(Owner, &DetachedDataTarget);
+	 new NoActionItem(Owner, &DetachedSigTarget);
       }
    }
 }
@@ -1578,11 +1605,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
    HashStringList LocalHashes;
    // try avoiding calculating the hash here as this is costly
    if (TransactionManager->LastMetaIndexParser != NULL)
-   {
-      indexRecords::checkSum * const R = TransactionManager->LastMetaIndexParser->Lookup(Target->MetaKey);
-      if (R != NULL)
-	 LocalHashes = R->Hashes;
-   }
+      LocalHashes = GetExpectedHashesFromFor(TransactionManager->LastMetaIndexParser, Target->MetaKey);
    if (LocalHashes.usable() == false)
    {
       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
diff --git a/test/integration/framework b/test/integration/framework
index b253deb91..56c4a1216 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -835,7 +835,9 @@ buildaptarchivefromincoming() {
 
 buildaptarchivefromfiles() {
 	msginfo "Build APT archive for ${CCMD}$(basename $0)${CINFO} based on prebuild files…"
-	find aptarchive -name 'Packages' -o -name 'Sources' -o -name 'Translation-*' | while read line; do
+	local DIR='aptarchive'
+	if [ -d "${DIR}/dists" ]; then DIR="${DIR}/dists"; fi
+	find "$DIR" -name 'Packages' -o -name 'Sources' -o -name 'Translation-*' | while read line; do
 		msgninfo "\t${line} file… "
 		compressfile "$line" "$1"
 		msgdone "info"
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 55bba8188..24ca85133 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -26,7 +26,9 @@ test_inreleasetoobig() {
 }
 
 test_packagestoobig() {
-	redatereleasefiles '+1hour'
+	insertpackage 'unstable' 'foo' 'all' '1.0'
+	buildaptarchivefromfiles '+1 hour'
+	signreleasefiles
 	# append junk at the end of the Packages.gz/Packages
 	SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
 	find aptarchive/dists -name 'Packages*' | while read pkg; do
diff --git a/test/integration/test-apt-update-file b/test/integration/test-apt-update-file
index 665f94fa5..94b604f0e 100755
--- a/test/integration/test-apt-update-file
+++ b/test/integration/test-apt-update-file
@@ -26,14 +26,29 @@ testsuccess aptget update
 # the release files aren't an IMS-hit, but the indexes are
 redatereleasefiles '+1 hour'
 
+# we don't download the index if it isn't updated
 testsuccess aptget update -o Debug::pkgAcquire::Auth=1
+# file:/ isn't shown in the log, so see if it was downloaded anyhow
 cp -a rootdir/tmp/testsuccess.output rootdir/tmp/update.output
+canary="SHA512:$(bzcat aptarchive/dists/unstable/main/binary-amd64/Packages.bz2 | sha512sum |cut -f1 -d' ')"
+testfailure grep -- "$canary" rootdir/tmp/update.output
+
+testfoo() {
+	# foo is still available
+	testsuccess aptget install -s foo
+	testsuccess aptcache showsrc foo
+	testsuccess aptget source foo --print-uris
+}
+testfoo
+
+# the release file is new again, the index still isn't, but it is somehow gone now from disk
+redatereleasefiles '+2 hour'
+find rootdir/var/lib/apt/lists -name '*_Packages*' -delete
 
-# ensure that the hash of the uncompressed file was verified even on a local ims hit
+testsuccess aptget update -o Debug::pkgAcquire::Auth=1
+# file:/ isn't shown in the log, so see if it was downloaded anyhow
+cp -a rootdir/tmp/testsuccess.output rootdir/tmp/update.output
 canary="SHA512:$(bzcat aptarchive/dists/unstable/main/binary-amd64/Packages.bz2 | sha512sum |cut -f1 -d' ')"
 testsuccess grep -- "$canary" rootdir/tmp/update.output
 
-# foo is still available
-testsuccess aptget install -s foo
-testsuccess aptcache showsrc foo
-testsuccess aptget source foo --print-uris
+testfoo
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
index a490f00de..32818658f 100755
--- a/test/integration/test-apt-update-not-modified
+++ b/test/integration/test-apt-update-not-modified
@@ -133,6 +133,25 @@ Reading package lists..." aptget update
 
 	rm -rf aptarchive/dists
 	cp -a aptarchive/dists.good aptarchive/dists
+
+	# new release file, but the indexes are the same
+	redatereleasefiles '+2 hours'
+
+	rm -rf rootdir/var/lib/apt/lists.good
+	cp -a rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
+	testsuccessequal "Get:1 $1 unstable InRelease [$(stat -c '%s' 'aptarchive/dists/unstable/InRelease') B]
+Reading package lists..." aptget update
+
+	rm -rf rootdir/var/lib/apt/lists
+	cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists
+	find rootdir/var/lib/apt/lists -name '*_Packages*' -delete
+	testsuccessequal "Get:1 $1 unstable InRelease [$(stat -c '%s' 'aptarchive/dists/unstable/InRelease') B]
+Get:2 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Get:3 $1 unstable/main i386 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-i386/Packages.gz') B]
+Reading package lists..." aptget update
+
+	rm -rf aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
 }
 
 changetowebserver
diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback
index 6fd901715..6ecf322b2 100755
--- a/test/integration/test-apt-update-rollback
+++ b/test/integration/test-apt-update-rollback
@@ -158,7 +158,10 @@ test_inrelease_to_broken_gzip() {
     msgmsg "Test InRelease to broken gzip"
     start_with_good_inrelease
 
-    redatereleasefiles '+2hours'
+    break_repository_sources_index '+1hour'
+    generatereleasefiles '+2hours'
+    signreleasefiles
+
     # append junk at the end of the compressed file
     echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz
     touch -d '+2min' $APTARCHIVE/dists/unstable/main/source/Sources.gz
@@ -166,6 +169,7 @@ test_inrelease_to_broken_gzip() {
     rm $APTARCHIVE/dists/unstable/main/source/Sources
 
     testfailure aptget update
+    testsuccess grep 'Hash Sum mismatch' rootdir/tmp/testfailure.output
     testfileequal lists.before "$(listcurrentlistsdirectory)"
 }
 
diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions
index 152e1617a..a5dac1737 100755
--- a/test/integration/test-apt-update-transactions
+++ b/test/integration/test-apt-update-transactions
@@ -29,6 +29,12 @@ restorefile() {
 }
 
 testrun() {
+	rm -rf aptarchive/dists.good
+	cp -a aptarchive/dists aptarchive/dists.good
+	insertpackage 'unstable' 'bar' 'all' '1.0'
+	insertsource 'unstable' 'bar' 'all' '1.0'
+	buildaptarchivefromfiles '+1 hour'
+
 	# produce an unsigned repository
 	find aptarchive \( -name 'Release.gpg' -o -name 'InRelease' \) -delete
 	testfailure aptget update --no-allow-insecure-repositories
@@ -37,20 +43,27 @@ testrun() {
 	# signed but broken
 	signreleasefiles
 
+	onehashbroken() {
+		testfailure aptget update
+		# each file generates two messages with this string
+		testequal '2' grep --count 'Hash Sum mismatch' rootdir/tmp/testfailure.output
+		testfileequal "$1" "$(listcurrentlistsdirectory)"
+	}
+
 	breakfile aptarchive/dists/unstable/main/binary-i386/Packages
-	testfailure aptget update
-	testfileequal "$1" "$(listcurrentlistsdirectory)"
+	onehashbroken "$1"
 	restorefile aptarchive/dists/unstable/main/binary-i386/Packages
 
 	breakfile aptarchive/dists/unstable/main/source/Sources
-	testfailure aptget update
-	testfileequal "$1" "$(listcurrentlistsdirectory)"
+	onehashbroken "$1"
 	restorefile aptarchive/dists/unstable/main/source/Sources
+
+	rm -rf aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
 }
 
 testsetup() {
 	msgmsg 'Test with no initial data over' "$1"
-	redatereleasefiles 'now'
 	rm -rf rootdir/var/lib/apt/lists
 	mkdir -p rootdir/var/lib/apt/lists/partial
 	listcurrentlistsdirectory > listsdir.lst
@@ -60,7 +73,6 @@ testsetup() {
 	rm -rf rootdir/var/lib/apt/lists
 	testsuccess aptget update -o Debug::pkgAcquire::Worker=1
 	listcurrentlistsdirectory > listsdir.lst
-	redatereleasefiles '+1hour'
 	testrun 'listsdir.lst'
 }
 
diff --git a/test/integration/test-cve-2013-1051-InRelease-parsing b/test/integration/test-cve-2013-1051-InRelease-parsing
index e38e40cc9..d99174553 100755
--- a/test/integration/test-cve-2013-1051-InRelease-parsing
+++ b/test/integration/test-cve-2013-1051-InRelease-parsing
@@ -39,10 +39,15 @@ sed -i '/^-----BEGIN PGP SIGNATURE-----/,/^-----END PGP SIGNATURE-----/ s/^$/  /
 cat aptarchive/dists/stable/Release >> aptarchive/dists/stable/InRelease
 touch -d '+1hour' aptarchive/dists/stable/InRelease
 
-# ensure the update fails
-# useful for debugging to add "-o Debug::pkgAcquire::auth=true"
-msgtest 'apt-get update for should fail with the modified' 'InRelease'
-aptget update 2>&1 | grep -E -q '(Writing more data than expected|Hash Sum mismatch)' > /dev/null && msgpass || msgfail
+# ensure the update doesn't load bad data as good data
+# Note that we will pick up the InRelease itself as we download no other
+# indexes which would trigger a hashsum mismatch, but we ignore the 'bad'
+# part of the InRelease
+listcurrentlistsdirectory | sed '/_InRelease/ d' > listsdir.lst
+msgtest 'apt-get update should ignore unsigned data in the' 'InRelease'
+testsuccessequal "Get:1 http://localhost:8080 stable InRelease [$(stat -c%s aptarchive/dists/stable/InRelease) B]
+Reading package lists..." --nomsg aptget update
+testfileequal './listsdir.lst' "$(listcurrentlistsdirectory | sed '/_InRelease/ d')"
 
 # ensure there is no package
 testfailureequal 'Reading package lists...
-- 
cgit v1.2.3-70-g09d2


From 1e0f0f28e1025f42a8172eb72f3e87984eb2b939 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 9 Jun 2015 11:59:22 +0200
Subject: configureable acquire targets to download additional files

First pass at making the acquire system capable of downloading files
based on configuration rather than hardcoded entries. It is now possible
to instruct 'deb' and 'deb-src' sources.list lines to download more than
just Packages/Translation-* and Sources files. Details on how to do that
can be found in the included documentation file.
---
 apt-pkg/contrib/configuration.cc                   |   4 +-
 apt-pkg/contrib/configuration.h                    |  11 +-
 apt-pkg/deb/debmetaindex.cc                        | 317 ++++++++++-----------
 apt-pkg/deb/debmetaindex.h                         |   8 -
 apt-pkg/init.cc                                    |  20 +-
 doc/acquire-additional-files.txt                   | 142 +++++++++
 test/integration/test-apt-acquire-additional-files |  54 ++++
 test/integration/test-apt-by-hash-update           |  12 +-
 8 files changed, 376 insertions(+), 192 deletions(-)
 create mode 100644 doc/acquire-additional-files.txt
 create mode 100755 test/integration/test-apt-acquire-additional-files

(limited to 'test')

diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index 42e35d32a..2500ab631 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -259,7 +259,7 @@ vector Configuration::FindVector(const char *Name) const
    return FindVector(Name, "");
 }
 #endif
-vector Configuration::FindVector(const char *Name, std::string const &Default) const
+vector Configuration::FindVector(const char *Name, std::string const &Default, bool const Keys) const
 {
    vector Vec;
    const Item *Top = Lookup(Name);
@@ -272,7 +272,7 @@ vector Configuration::FindVector(const char *Name, std::string const &De
    Item *I = Top->Child;
    while(I != NULL)
    {
-      Vec.push_back(I->Value);
+      Vec.push_back(Keys ? I->Tag : I->Value);
       I = I->Next;
    }
    if (Vec.empty() == true)
diff --git a/apt-pkg/contrib/configuration.h b/apt-pkg/contrib/configuration.h
index 8d7d51037..eacc26fda 100644
--- a/apt-pkg/contrib/configuration.h
+++ b/apt-pkg/contrib/configuration.h
@@ -84,15 +84,8 @@ class Configuration
     *
     * \param Name of the parent node
     * \param Default list of values separated by commas */
-#if APT_PKG_ABI >= 413
-   std::vector FindVector(const char *Name, std::string const &Default = "") const;
-   std::vector FindVector(std::string const &Name, std::string const &Default = "") const { return FindVector(Name.c_str(), Default); };
-#else
-   std::vector FindVector(const char *Name, std::string const &Default) const;
-   std::vector FindVector(std::string const &Name, std::string const &Default) const { return FindVector(Name.c_str(), Default); };
-   std::vector FindVector(const char *Name) const;
-   std::vector FindVector(std::string const &Name) const { return FindVector(Name.c_str(), ""); };
-#endif
+   std::vector FindVector(const char *Name, std::string const &Default = "", bool const Keys = false) const;
+   std::vector FindVector(std::string const &Name, std::string const &Default = "", bool const Keys = false) const { return FindVector(Name.c_str(), Default, Keys); };
 
    int FindI(const char *Name,int const &Default = 0) const;
    int FindI(std::string const &Name,int const &Default = 0) const {return FindI(Name.c_str(),Default);};
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index eb5e78e3b..8fef05ab0 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -23,25 +23,6 @@
 
 using namespace std;
 
-string debReleaseIndex::Info(const char *Type, string const &Section, string const &Arch) const
-{
-   string Info = ::URI::SiteOnly(URI) + ' ';
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Info += Dist;
-   }
-   else
-   {
-      Info += Dist + '/' + Section;
-      if (Arch.empty() != true)
-	 Info += " " + Arch;
-   }
-   Info += " ";
-   Info += Type;
-   return Info;
-}
-
 string debReleaseIndex::MetaIndexInfo(const char *Type) const
 {
    string Info = ::URI::SiteOnly(URI) + ' ';
@@ -92,81 +73,6 @@ std::string debReleaseIndex::LocalFileName() const
    return "";
 }
 
-string debReleaseIndex::IndexURISuffix(const char *Type, string const &Section, string const &Arch) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-   {
-      if (Arch == "native")
-	 Res += Section + "/binary-" + _config->Find("APT::Architecture") + '/';
-      else
-	 Res += Section + "/binary-" + Arch + '/';
-   }
-   return Res + Type;
-}
-   
-
-string debReleaseIndex::IndexURI(const char *Type, string const &Section, string const &Arch) const
-{
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      string Res;
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + '/' + IndexURISuffix(Type, Section, Arch);
- }
-
-string debReleaseIndex::SourceIndexURISuffix(const char *Type, const string &Section) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-      Res += Section + "/source/";
-   return Res + Type;
-}
-
-string debReleaseIndex::SourceIndexURI(const char *Type, const string &Section) const
-{
-   string Res;
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + "/" + SourceIndexURISuffix(Type, Section);
-}
-
-string debReleaseIndex::TranslationIndexURISuffix(const char *Type, const string &Section) const
-{
-   string Res ="";
-   if (Dist[Dist.size() - 1] != '/')
-      Res += Section + "/i18n/Translation-";
-   return Res + Type;
-}
-
-string debReleaseIndex::TranslationIndexURI(const char *Type, const string &Section) const
-{
-   string Res;
-   if (Dist[Dist.size() - 1] == '/')
-   {
-      if (Dist != "/")
-         Res = URI + Dist;
-      else 
-         Res = URI;
-      return Res + Type;
-   }
-   else
-      return URI + "dists/" + Dist + "/" + TranslationIndexURISuffix(Type, Section);
-}
-
 debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) :
 					metaIndex(URI, Dist, "deb"), Trusted(CHECK_TRUST)
 {}
@@ -184,73 +90,161 @@ debReleaseIndex::~debReleaseIndex() {
 			delete *S;
 }
 
-vector * debReleaseIndex::ComputeIndexTargets() const {
-	vector * IndexTargets = new vector ;
+vector * debReleaseIndex::ComputeIndexTargets() const
+{
+   vector * IndexTargets = new vector ;
 
-	map >::const_iterator const src = ArchEntries.find("source");
-	if (src != ArchEntries.end()) {
-		vector const SectionEntries = src->second;
-		for (vector::const_iterator I = SectionEntries.begin();
-		     I != SectionEntries.end(); ++I) {
-			char const * const ShortDesc = "Sources";
-			IndexTarget * const Target = new IndexTarget(
-			      SourceIndexURISuffix(ShortDesc, (*I)->Section),
-			      ShortDesc,
-			      Info(ShortDesc, (*I)->Section),
-			      SourceIndexURI(ShortDesc, (*I)->Section)
-			      );
-			IndexTargets->push_back (Target);
-		}
-	}
+   bool const flatArchive = (Dist[Dist.length() - 1] == '/');
+   std::string baseURI = URI;
+   if (flatArchive)
+   {
+      if (Dist != "/")
+         baseURI += Dist;
+   }
+   else
+      baseURI += "dists/" + Dist + "/";
+   std::string const Release = (Dist == "/") ? "" : Dist;
+   std::string const Site = ::URI::SiteOnly(URI);
+   std::vector lang = APT::Configuration::getLanguages(true);
+   if (lang.empty())
+      lang.push_back("none");
+   map >::const_iterator const src = ArchEntries.find("source");
+   if (src != ArchEntries.end())
+   {
+      std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true);
+      for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
+      {
+#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb-src::") + *T + "::" + (X))
+	 std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI");
+	 std::string const ShortDesc = APT_T_CONFIG("ShortDescription");
+	 std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
+	 bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true);
+#undef APT_T_CONFIG
+	 if (URI.empty())
+	    continue;
 
-	// Only source release
-	if (IndexTargets->empty() == false && ArchEntries.size() == 1)
-		return IndexTargets;
+	 vector const SectionEntries = src->second;
+	 for (vector::const_iterator I = SectionEntries.begin();
+	       I != SectionEntries.end(); ++I)
+	 {
+	    for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l)
+	    {
+	       if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos)
+		  continue;
+
+	       struct SubstVar subst[] = {
+		  { "$(SITE)", &Site },
+		  { "$(RELEASE)", &Release },
+		  { "$(COMPONENT)", &((*I)->Section) },
+		  { "$(LANGUAGE)", &(*l) },
+		  { NULL, NULL }
+	       };
+	       std::string const name = SubstVar(URI, subst);
+	       IndexTarget * Target;
+	       if (IsOptional == true)
+	       {
+		  Target = new OptionalIndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       else
+	       {
+		  Target = new IndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       IndexTargets->push_back(Target);
+
+	       if (URI.find("$(LANGUAGE)") == std::string::npos)
+		  break;
+	    }
 
-	std::set sections;
-	for (map >::const_iterator a = ArchEntries.begin();
-	     a != ArchEntries.end(); ++a) {
-		if (a->first == "source")
-			continue;
-		for (vector ::const_iterator I = a->second.begin();
-		     I != a->second.end(); ++I) {
-			char const * const ShortDesc = "Packages";
-			IndexTarget * const Target = new IndexTarget(
-			      IndexURISuffix(ShortDesc, (*I)->Section, a->first),
-			      ShortDesc,
-			      Info (ShortDesc, (*I)->Section, a->first),
-			      IndexURI(ShortDesc, (*I)->Section, a->first)
-			      );
-			IndexTargets->push_back (Target);
-			sections.insert((*I)->Section);
-		}
-	}
+	    if (URI.find("$(COMPONENT)") == std::string::npos)
+	       break;
+	 }
+      }
+   }
 
-	std::vector lang = APT::Configuration::getLanguages(true);
-	std::vector::iterator lend = std::remove(lang.begin(), lang.end(), "none");
-	if (lend != lang.end())
-		lang.erase(lend);
-
-	if (lang.empty() == true)
-		return IndexTargets;
-
-	// get the Translation-* files, later we will skip download of non-existent if we have an index
-	for (std::set::const_iterator s = sections.begin();
-	     s != sections.end(); ++s) {
-		for (std::vector::const_iterator l = lang.begin();
-		     l != lang.end(); ++l) {
-			std::string const ShortDesc = "Translation-" + *l;
-			IndexTarget * const Target = new OptionalIndexTarget(
-			      TranslationIndexURISuffix(l->c_str(), *s),
-			      ShortDesc,
-			      Info (ShortDesc.c_str(), *s),
-			      TranslationIndexURI(l->c_str(), *s)
-			      );
-			IndexTargets->push_back(Target);
-		}
-	}
+   // Only source release
+   if (IndexTargets->empty() == false && ArchEntries.size() == 1)
+      return IndexTargets;
 
-	return IndexTargets;
+   std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true);
+   for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
+   {
+#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb::") + *T + "::" + (X))
+      std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI");
+      std::string const ShortDesc = APT_T_CONFIG("ShortDescription");
+      std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
+      bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb::") + *T + "::Optional", true);
+#undef APT_T_CONFIG
+      if (URI.empty())
+	 continue;
+
+      for (map >::const_iterator a = ArchEntries.begin();
+	    a != ArchEntries.end(); ++a)
+      {
+	 if (a->first == "source")
+	    continue;
+
+	 for (vector ::const_iterator I = a->second.begin();
+	       I != a->second.end(); ++I) {
+
+	    for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l)
+	    {
+	       if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos)
+		  continue;
+
+	       struct SubstVar subst[] = {
+		  { "$(SITE)", &Site },
+		  { "$(RELEASE)", &Release },
+		  { "$(COMPONENT)", &((*I)->Section) },
+		  { "$(LANGUAGE)", &(*l) },
+		  { "$(ARCHITECTURE)", &(a->first) },
+		  { NULL, NULL }
+	       };
+	       std::string const name = SubstVar(URI, subst);
+	       IndexTarget * Target;
+	       if (IsOptional == true)
+	       {
+		  Target = new OptionalIndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       else
+	       {
+		  Target = new IndexTarget(
+			name,
+			SubstVar(ShortDesc, subst),
+			SubstVar(LongDesc, subst),
+			baseURI + name
+			);
+	       }
+	       IndexTargets->push_back(Target);
+
+	       if (URI.find("$(LANGUAGE)") == std::string::npos)
+		  break;
+	    }
+
+	    if (URI.find("$(COMPONENT)") == std::string::npos)
+	       break;
+	 }
+
+	 if (URI.find("$(ARCHITECTURE)") == std::string::npos)
+	    break;
+      }
+   }
+
+   return IndexTargets;
 }
 									/*}}}*/
 bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
@@ -361,17 +355,6 @@ void debReleaseIndex::PushSectionEntry(string const &Arch, const debSectionEntry
 	ArchEntries[Arch].push_back(Entry);
 }
 
-void debReleaseIndex::PushSectionEntry(const debSectionEntry *Entry) {
-	if (Entry->IsSrc == true)
-		PushSectionEntry("source", Entry);
-	else {
-		for (map >::iterator a = ArchEntries.begin();
-		     a != ArchEntries.end(); ++a) {
-			a->second.push_back(Entry);
-		}
-	}
-}
-
 debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section,
 		bool const &IsSrc): Section(Section), IsSrc(IsSrc)
 {}
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 94d005760..e1c1d91ef 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -47,7 +47,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual std::string ArchiveURI(std::string const &File) const {return URI + File;};
    virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const;
    std::vector * ComputeIndexTargets() const;
-   std::string Info(const char *Type, std::string const &Section, std::string const &Arch="") const;
 
    std::string MetaIndexInfo(const char *Type) const;
    std::string MetaIndexFile(const char *Types) const;
@@ -58,12 +57,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
 #endif
    std::string LocalFileName() const;
 
-   std::string IndexURI(const char *Type, std::string const &Section, std::string const &Arch="native") const;
-   std::string IndexURISuffix(const char *Type, std::string const &Section, std::string const &Arch="native") const;
-   std::string SourceIndexURI(const char *Type, const std::string &Section) const;
-   std::string SourceIndexURISuffix(const char *Type, const std::string &Section) const;
-   std::string TranslationIndexURI(const char *Type, const std::string &Section) const;
-   std::string TranslationIndexURISuffix(const char *Type, const std::string &Section) const;
    virtual std::vector  *GetIndexFiles();
 
    void SetTrusted(bool const Trusted);
@@ -71,7 +64,6 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
 
    void PushSectionEntry(std::vector const &Archs, const debSectionEntry *Entry);
    void PushSectionEntry(std::string const &Arch, const debSectionEntry *Entry);
-   void PushSectionEntry(const debSectionEntry *Entry);
 };
 
 class APT_HIDDEN debDebFileMetaIndex : public metaIndex
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index f756eab26..50ea2b49e 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -101,8 +101,26 @@ bool pkgInitConfig(Configuration &Cnf)
    // The default user we drop to in the methods
    Cnf.CndSet("APT::Sandbox::User", "_apt");
 
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::URI", "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatURI", "Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::ShortDescription", "Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::Description", "$(SITE) $(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatDescription", "$(SITE) $(RELEASE) Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::Optional", false);
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::URI", "$(COMPONENT)/i18n/Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatURI", "$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::ShortDescription", "Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::Description", "$(SITE) $(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatDescription", "$(SITE) $(RELEASE) Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::URI", "$(COMPONENT)/source/Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatURI", "Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::ShortDescription", "Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Description", "$(SITE) $(RELEASE)/$(COMPONENT) Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatDescription", "$(SITE) $(RELEASE) Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Optional", false);
+
    bool Res = true;
-   
+
    // Read an alternate config file
    const char *Cfg = getenv("APT_CONFIG");
    if (Cfg != 0 && strlen(Cfg) != 0)
diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
new file mode 100644
index 000000000..5a07c2bec
--- /dev/null
+++ b/doc/acquire-additional-files.txt
@@ -0,0 +1,142 @@
+# Acquire additional files in 'update' operations
+
+The download and verification of data from multiple sources in different
+compression formats, with partial downloads and patches is an involved
+process which is hard to implement correctly and securely.
+
+APT frontends share the code and binaries to make this happen in libapt
+with the Acquire system, supported by helpers shipped in the apt package
+itself and additional transports in individual packages like
+apt-transport-https.
+
+For its own operation libapt needs or can make use of Packages, Sources
+and Translation-* files, which it will acquire by default, but
+a repository might contain more data files (e.g.  Contents) a frontend
+might want to use and would therefore need to be downloaded as well
+(e.g. apt-file).
+
+This file describes the configuration scheme such a frontend can use to
+instruct the Acquire system to download those additional files.
+
+Note that you can't download files from other sources. It must be files
+in the same repository and below the Release file. The Release file must
+also contain hashes for the file itself as well as for the compressed
+file if wanted, otherwise a download isn't even tried!
+
+
+# The Configuration Stanza
+
+The Acquire system uses the same configuration settings to implement the
+files it downloads by default. These settings are the default, but if
+they would be written in a configuration file the configuration
+instructing the Acquire system to download the Packages files would look
+like this (see also apt.conf(5) manpage for configuration file syntax):
+
+	APT::Acquire::Targets::deb::Packages {
+		URI "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages";
+		ShortDescription "Packages";
+		Description "$(SITE) $(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages";
+
+		flatURI "Packages";
+		flatDescription "$(SITE) $(RELEASE) Packages";
+
+		Optional "false";
+	};
+
+All files which should be downloaded (nicknamed 'Targets') are mentioned
+below the APT::Acquire::Targets scope. 'deb' is here the type of the
+sources.list entry the file should be acquired for. The only other
+supported value is hence 'deb-src'. Beware: You can't specify multiple
+types here and you can't download the same URI for multiple types!
+
+After the type you can pick any valid and unique string which preferable
+refers to the file it downloads (In the example we picked 'Packages').
+This string is never shown or used.
+
+All targets have three main properties you can define:
+* URI: The identifier of the file to be downloaded as used in the
+  Release file.  It is also the relative location of the file from the
+  Release file.  You can neither download from a different server
+  entirely (absolute URI) nor should you try to access directories above
+  the Release file (e.g. "../../").
+* ShortDescription: Very short string intended to be displayed to the
+  user e.g.  while reporting progress. apt will e.g. use this string in
+  the last line to indicate progress of e.g. the download of a specific
+  item.
+* Description: A preferable human understandable and readable identifier
+  of which file is acquired exactly. Mainly used for progress reporting
+  and error messages. apt will e.g. use this string in the Get/Hit/Err
+  progress lines.
+
+Additional optional properties:
+* flat{URI,Description}: APT supports two types of repositories:
+  dists-style repositories which are the default and by far the most
+  common which are named after the fact that the files are in an
+  elaborated directory structure.  In contrast a flat-style repositories
+  lumps all files together in one directory.  Support for these flat
+  repositories exists mainly for legacy purposes only.  It is therefore
+  recommend to not set these values.
+* Optional: The default value is 'true' and should be kept at this
+  value.  If enabled the acquire system will skip the download if the
+  file isn't mentioned in the Release file. Otherwise this is treated as
+  a hard error and the update process fails.
+
+
+Note that the acquire system will automatically choose to download
+a compressed file if it is available and uncompress it for you, just as
+it will also use pdiff patching if provided by the repository and
+enabled by the user. You only have to ensure that the Release file
+contains the information about the compressed files/pdiffs to make this
+happen. NO properties have to be set to enable this.
+
+# More examples
+
+The stanzas for Translation-* files as well as for Sources files would
+look like this:
+
+APT::Acquire::Targets {
+	deb::Translations {
+		URI "$(COMPONENT)/i18n/Translation-$(LANGUAGE)";
+		ShortDescription "Translation-$(LANGUAGE)";
+		Description "$(SITE) $(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)";
+
+		flatURI "$(LANGUAGE)";
+		flatDescription "$(SITE) $(RELEASE) Translation-$(LANGUAGE)";
+	};
+
+	deb-src::Sources {
+		URI "$(COMPONENT)/source/Sources";
+		ShortDescription "Sources";
+		Description "$(SITE) $(RELEASE)/$(COMPONENT) Sources";
+
+		flatURI "Sources";
+		flatDescription "$(SITE) $(RELEASE) Sources";
+
+		Optional "false";
+	};
+};
+
+# Substitution variables
+
+As seen in the examples, properties can contain placeholders filled in
+by the acquire system. The following variables are known; note that
+unknown variables have no default value nor are they touched: They are
+printed literally.
+
+* $(SITE): An identifier of the site we access, e.g.
+  "http://example.org/".
+* $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
+  "stretch".  Note that flat-style repositories do not have a archive-
+  or codename per-se, so the value might very well be just "/" or so.
+* $(COMPONENT): as given in the sources.list, e.g. "main", "non-free" or
+  "universe".  Note that flat-style repositories again do not really
+  have a meaningful value here.
+* $(LANGUAGE): Values are all entries (expect "none") of configuration
+  option Acquire::Languages, e.g. "en", "de" or "de_AT".
+
+These values are defined both for 'deb' as well as 'deb-src' targets.
+'deb' targets additional have the variable:
+
+* $(ARCHITECTURE): Values are all entries of configuration option
+  APT::Architectures (potentially modified by sources.list options),
+  e.g. "amd64", "i386" or "armel".
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
new file mode 100755
index 000000000..150a50980
--- /dev/null
+++ b/test/integration/test-apt-acquire-additional-files
@@ -0,0 +1,54 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'amd64'
+configcompression '.' 'gz'
+
+buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
+
+setupaptarchive --no-update
+changetowebserver
+
+testsuccessequal "Get:1 http://localhost:8080 unstable InRelease [$(stat -c%s aptarchive/dists/unstable/InRelease) B]
+Get:2 http://localhost:8080 unstable/main Sources [$(stat -c%s aptarchive/dists/unstable/main/source/Sources.gz) B]
+Get:3 http://localhost:8080 unstable/main amd64 Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B]
+Get:4 http://localhost:8080 unstable/main Translation-en [$(stat -c%s aptarchive/dists/unstable/main/i18n/Translation-en.gz) B]
+Reading package lists..." aptget update
+
+testempty find rootdir/var/lib/apt/lists -name '*Contents*'
+
+cat > rootdir/etc/apt/apt.conf.d/content-target.conf < rootdir/etc/apt/apt.conf.d/content-target.conf <
Date: Wed, 10 Jun 2015 19:22:41 +0200
Subject: abstract the code to iterate over all targets a bit

We have two places in the code which need to iterate over targets and do
certain things with it. The first one is actually creating these targets
for download and the second instance pepares certain targets for
reading.

Git-Dch: Ignore
---
 apt-pkg/deb/debmetaindex.cc                        | 197 +++++++++++----------
 test/integration/framework                         |   7 +-
 test/integration/test-apt-acquire-additional-files |  33 ++++
 3 files changed, 142 insertions(+), 95 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 8fef05ab0..c35f3b0a4 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -90,10 +90,11 @@ debReleaseIndex::~debReleaseIndex() {
 			delete *S;
 }
 
-vector * debReleaseIndex::ComputeIndexTargets() const
+template
+void foreachTarget(std::string const URI, std::string const Dist,
+      std::map > const &ArchEntries,
+      CallC Call)
 {
-   vector * IndexTargets = new vector ;
-
    bool const flatArchive = (Dist[Dist.length() - 1] == '/');
    std::string baseURI = URI;
    if (flatArchive)
@@ -108,7 +109,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const
    std::vector lang = APT::Configuration::getLanguages(true);
    if (lang.empty())
       lang.push_back("none");
-   map >::const_iterator const src = ArchEntries.find("source");
+   map >::const_iterator const src = ArchEntries.find("source");
    if (src != ArchEntries.end())
    {
       std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true);
@@ -123,8 +124,8 @@ vector * debReleaseIndex::ComputeIndexTargets() const
 	 if (URI.empty())
 	    continue;
 
-	 vector const SectionEntries = src->second;
-	 for (vector::const_iterator I = SectionEntries.begin();
+	 vector const SectionEntries = src->second;
+	 for (vector::const_iterator I = SectionEntries.begin();
 	       I != SectionEntries.end(); ++I)
 	 {
 	    for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l)
@@ -139,27 +140,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const
 		  { "$(LANGUAGE)", &(*l) },
 		  { NULL, NULL }
 	       };
-	       std::string const name = SubstVar(URI, subst);
-	       IndexTarget * Target;
-	       if (IsOptional == true)
-	       {
-		  Target = new OptionalIndexTarget(
-			name,
-			SubstVar(ShortDesc, subst),
-			SubstVar(LongDesc, subst),
-			baseURI + name
-			);
-	       }
-	       else
-	       {
-		  Target = new IndexTarget(
-			name,
-			SubstVar(ShortDesc, subst),
-			SubstVar(LongDesc, subst),
-			baseURI + name
-			);
-	       }
-	       IndexTargets->push_back(Target);
+	       Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst);
 
 	       if (URI.find("$(LANGUAGE)") == std::string::npos)
 		  break;
@@ -171,10 +152,6 @@ vector * debReleaseIndex::ComputeIndexTargets() const
       }
    }
 
-   // Only source release
-   if (IndexTargets->empty() == false && ArchEntries.size() == 1)
-      return IndexTargets;
-
    std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true);
    for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
    {
@@ -187,13 +164,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const
       if (URI.empty())
 	 continue;
 
-      for (map >::const_iterator a = ArchEntries.begin();
+      for (map >::const_iterator a = ArchEntries.begin();
 	    a != ArchEntries.end(); ++a)
       {
 	 if (a->first == "source")
 	    continue;
 
-	 for (vector ::const_iterator I = a->second.begin();
+	 for (vector ::const_iterator I = a->second.begin();
 	       I != a->second.end(); ++I) {
 
 	    for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l)
@@ -209,27 +186,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const
 		  { "$(ARCHITECTURE)", &(a->first) },
 		  { NULL, NULL }
 	       };
-	       std::string const name = SubstVar(URI, subst);
-	       IndexTarget * Target;
-	       if (IsOptional == true)
-	       {
-		  Target = new OptionalIndexTarget(
-			name,
-			SubstVar(ShortDesc, subst),
-			SubstVar(LongDesc, subst),
-			baseURI + name
-			);
-	       }
-	       else
-	       {
-		  Target = new IndexTarget(
-			name,
-			SubstVar(ShortDesc, subst),
-			SubstVar(LongDesc, subst),
-			baseURI + name
-			);
-	       }
-	       IndexTargets->push_back(Target);
+	       Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst);
 
 	       if (URI.find("$(LANGUAGE)") == std::string::npos)
 		  break;
@@ -243,9 +200,51 @@ vector * debReleaseIndex::ComputeIndexTargets() const
 	    break;
       }
    }
+}
+
+
+struct ComputeIndexTargetsClass
+{
+   vector  * const IndexTargets;
+
+   void operator()(std::string const &baseURI, std::string const &/*TargetName*/,
+	 std::string const &URI, std::string const &ShortDesc, std::string const &LongDesc,
+	 bool const IsOptional, struct SubstVar const * const subst)
+   {
+      std::string const name = SubstVar(URI, subst);
+      IndexTarget * Target;
+      if (IsOptional == true)
+      {
+	 Target = new OptionalIndexTarget(
+	       name,
+	       SubstVar(ShortDesc, subst),
+	       SubstVar(LongDesc, subst),
+	       baseURI + name
+	       );
+      }
+      else
+      {
+	 Target = new IndexTarget(
+	       name,
+	       SubstVar(ShortDesc, subst),
+	       SubstVar(LongDesc, subst),
+	       baseURI + name
+	       );
+      }
+      IndexTargets->push_back(Target);
+   }
 
-   return IndexTargets;
+   ComputeIndexTargetsClass() : IndexTargets(new vector ) {}
+};
+
+vector * debReleaseIndex::ComputeIndexTargets() const
+{
+   ComputeIndexTargetsClass comp;
+   foreachTarget(URI, Dist, ArchEntries, comp);
+   return comp.IndexTargets;
 }
+
+
 									/*}}}*/
 bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 {
@@ -303,45 +302,55 @@ bool debReleaseIndex::IsTrusted() const
    return FileExists(VerifiedSigFile);
 }
 
-vector  *debReleaseIndex::GetIndexFiles() {
-	if (Indexes != NULL)
-		return Indexes;
-
-	Indexes = new vector ;
-	map >::const_iterator const src = ArchEntries.find("source");
-	if (src != ArchEntries.end()) {
-		vector const SectionEntries = src->second;
-		for (vector::const_iterator I = SectionEntries.begin();
-		     I != SectionEntries.end(); ++I)
-			Indexes->push_back(new debSourcesIndex (URI, Dist, (*I)->Section, IsTrusted()));
-	}
-
-	// Only source release
-	if (Indexes->empty() == false && ArchEntries.size() == 1)
-		return Indexes;
-
-	std::vector const lang = APT::Configuration::getLanguages(true);
-	map > sections;
-	for (map >::const_iterator a = ArchEntries.begin();
-	     a != ArchEntries.end(); ++a) {
-		if (a->first == "source")
-			continue;
-		for (vector::const_iterator I = a->second.begin();
-		     I != a->second.end(); ++I) {
-			Indexes->push_back(new debPackagesIndex (URI, Dist, (*I)->Section, IsTrusted(), a->first));
-			sections[(*I)->Section].insert(lang.begin(), lang.end());
-		}
-	}
-
-	for (map >::const_iterator s = sections.begin();
-	     s != sections.end(); ++s)
-		for (set::const_iterator l = s->second.begin();
-		     l != s->second.end(); ++l) {
-			if (*l == "none") continue;
-			Indexes->push_back(new debTranslationsIndex(URI,Dist,s->first,(*l).c_str()));
-		}
-
-	return Indexes;
+struct GetIndexFilesClass
+{
+   vector  * const Indexes;
+   std::string const URI;
+   std::string const Release;
+   bool const IsTrusted;
+
+   void operator()(std::string const &/*baseURI*/, std::string const &TargetName,
+	 std::string const &/*URI*/, std::string const &/*ShortDesc*/, std::string const &/*LongDesc*/,
+	 bool const /*IsOptional*/, struct SubstVar const * const subst)
+   {
+      if (TargetName == "Packages")
+      {
+	 Indexes->push_back(new debPackagesIndex(
+		  URI,
+		  Release,
+		  SubstVar("$(COMPONENT)", subst),
+		  IsTrusted,
+		  SubstVar("$(ARCHITECTURE)", subst)
+		  ));
+      }
+      else if (TargetName == "Sources")
+	 Indexes->push_back(new debSourcesIndex(
+		  URI,
+		  Release,
+		  SubstVar("$(COMPONENT)", subst),
+		  IsTrusted
+		  ));
+      else if (TargetName == "Translations")
+	 Indexes->push_back(new debTranslationsIndex(
+		  URI,
+		  Release,
+		  SubstVar("$(COMPONENT)", subst),
+		  SubstVar("$(LANGUAGE)", subst)
+		  ));
+   }
+
+   GetIndexFilesClass(std::string const &URI, std::string const &Release, bool const IsTrusted) :
+      Indexes(new vector ), URI(URI), Release(Release), IsTrusted(IsTrusted) {}
+};
+
+std::vector  *debReleaseIndex::GetIndexFiles()
+{
+   if (Indexes != NULL)
+      return Indexes;
+
+   GetIndexFilesClass comp(URI, Dist, IsTrusted());
+   foreachTarget(URI, Dist, ArchEntries, comp);
+   return Indexes = comp.Indexes;
 }
 
 void debReleaseIndex::PushSectionEntry(vector const &Archs, const debSectionEntry *Entry) {
diff --git a/test/integration/framework b/test/integration/framework
index 56c4a1216..110758e82 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1531,6 +1531,11 @@ aptautotest() {
 }
 
 aptautotest_aptget_update() {
+	local TESTCALL="$1"
+	while [ -n "$2" ]; do
+		if [ "$2" = '--print-uris' ]; then return; fi # simulation mode
+		shift
+	done
 	if ! test -d "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"; then return; fi
 	testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755"
 	testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755"
@@ -1538,7 +1543,7 @@ aptautotest_aptget_update() {
 	for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -type f ! -name 'lock'); do
 		testfilestats "$file" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
 	done
-	if [ "$1" = 'testsuccess' ]; then
+	if [ "$TESTCALL" = 'testsuccess' ]; then
 		# failure cases can retain partial files and such
 		testempty find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" -mindepth 1 ! \( -name 'lock' -o -name '*.FAILED' \)
 	fi
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 150a50980..0e59233e9 100755
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -6,6 +6,9 @@ TESTDIR=$(readlink -f $(dirname $0))
 
 setupenvironment
 configarchitecture 'amd64'
+
+# note that in --print-uri we talk about .bz2 because that is the default.
+# This doesn't mean it is actually attempt to download it.
 configcompression '.' 'gz'
 
 buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
@@ -13,6 +16,11 @@ buildsimplenativepackage 'foo' 'amd64' '1' 'unstable'
 setupaptarchive --no-update
 changetowebserver
 
+testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
+'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
+'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris
+
 testsuccessequal "Get:1 http://localhost:8080 unstable InRelease [$(stat -c%s aptarchive/dists/unstable/InRelease) B]
 Get:2 http://localhost:8080 unstable/main Sources [$(stat -c%s aptarchive/dists/unstable/main/source/Sources.gz) B]
 Get:3 http://localhost:8080 unstable/main amd64 Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B]
@@ -29,6 +37,12 @@ APT::Acquire::Targets::deb::Contents {
 };
 EOF
 
+testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
+'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
+'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:8080/dists/unstable/main/Contents-amd64.bz2' localhost:8080_dists_unstable_main_Contents-amd64 0 " aptget update --print-uris
+
 testsuccessequal "Hit http://localhost:8080 unstable InRelease
 Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
 Reading package lists..." aptget update
@@ -46,9 +60,28 @@ APT::Acquire::Targets::deb::Contents {
 };
 EOF
 
+# the last line is utter bogus of course, but how should apt know…
+testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
+'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
+'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 
+'http://localhost:8080/dists/unstable/main/Contents-amd64.gz.bz2' localhost:8080_dists_unstable_main_Contents-amd64.gz 0 " aptget update --print-uris
+
 testsuccessequal "Hit http://localhost:8080 unstable InRelease
 Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'
+
+rm -f rootdir/etc/apt/apt.conf.d/content-target.conf
+
+testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
+'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
+'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
+'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris
+
+testsuccessequal "Hit http://localhost:8080 unstable InRelease
+Reading package lists..." aptget update
+
+testempty find rootdir/var/lib/apt/lists -name '*Contents*'
-- 
cgit v1.2.3-70-g09d2


From d3a869e35503638e3483228fbfc95b7143568ad0 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Wed, 10 Jun 2015 21:24:47 +0200
Subject: store all targets data in IndexTarget struct

We still need an API for the targets, so slowly prepare the IndexTargets
to let them take this job.

Git-Dch: Ignore
---
 apt-pkg/acquire-item.cc                            |  15 ++-
 apt-pkg/acquire-item.h                             |  31 ++----
 apt-pkg/deb/debmetaindex.cc                        | 114 ++++++++++-----------
 apt-pkg/init.cc                                    |  12 +--
 doc/acquire-additional-files.txt                   |  29 +++---
 test/integration/test-apt-acquire-additional-files |   4 +-
 6 files changed, 97 insertions(+), 108 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index a1357fb15..e92ccc08b 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -382,6 +382,15 @@ bool pkgAcqDiffIndex::TransactionState(TransactionStates const state)
 }
 									/*}}}*/
 
+// IndexTarget - Constructor						/*{{{*/
+IndexTarget::IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
+      std::string const &LongDesc, std::string const &URI, bool const IsOptional,
+      std::map const &Options) :
+   URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey), IsOptional(IsOptional), Options(Options)
+{
+}
+									/*}}}*/
+
 class APT_HIDDEN NoActionItem : public pkgAcquire::Item			/*{{{*/
 /* The sole purpose of this class is having an item which does nothing to
    reach its done state to prevent cleanup deleting the mentioned file.
@@ -955,7 +964,7 @@ void pkgAcqMetaBase::QueueIndexes(bool const verify)			/*{{{*/
 	 if (TransactionManager->MetaIndexParser->Exists((*Target)->MetaKey) == false)
 	 {
 	    // optional targets that we do not have in the Release file are skipped
-	    if ((*Target)->IsOptional())
+	    if ((*Target)->IsOptional)
 	       continue;
 
 	    Status = StatAuthError;
@@ -2388,7 +2397,7 @@ string pkgAcqIndex::Custom600Headers() const
    if (stat(Final.c_str(),&Buf) == 0)
       msg += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 
-   if(Target->IsOptional())
+   if(Target->IsOptional)
       msg += "\nFail-Ignore: true";
 
    return msg;
@@ -2410,7 +2419,7 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const *
       }
    }
 
-   if(Target->IsOptional() && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
+   if(Target->IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
       Status = StatDone;
    else
       TransactionManager->AbortTransaction();
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 38a7a8662..a2571e1cd 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -28,6 +28,7 @@
 
 #include 
 #include 
+#include 
 
 #ifndef APT_8_CLEANER_HEADERS
 #include 
@@ -49,7 +50,7 @@ class pkgSourceList;
 class IndexTarget;
 class pkgAcqMetaBase;
 
-class APT_HIDDEN IndexTarget						/*{{{*/
+class IndexTarget							/*{{{*/
 /** \brief Information about an index file. */
 {
    public:
@@ -63,30 +64,18 @@ class APT_HIDDEN IndexTarget						/*{{{*/
    std::string const ShortDesc;
 
    /** \brief The key by which this index file should be
-    *  looked up within the meta signature file.
-    */
+       looked up within the meta index file. */
    std::string const MetaKey;
 
-   virtual bool IsOptional() const {
-      return false;
-   }
+   /** \brief Is it okay if the file isn't found in the meta index */
+   bool const IsOptional;
 
-   IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
-	 std::string const &LongDesc, std::string const &URI) :
-      URI(URI), Description(LongDesc), ShortDesc(ShortDesc), MetaKey(MetaKey) {}
-};
-									/*}}}*/
-class APT_HIDDEN OptionalIndexTarget : public IndexTarget		/*{{{*/
-/** \brief Information about an optional index file. */
-{
-   public:
-   virtual bool IsOptional() const {
-      return true;
-   }
+   /** \brief Target specific options defined by the implementation */
+   std::map const Options;
 
-   OptionalIndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
-	 std::string const &LongDesc, std::string const &URI) :
-      IndexTarget(MetaKey, ShortDesc, LongDesc, URI) {}
+   IndexTarget(std::string const &MetaKey, std::string const &ShortDesc,
+	 std::string const &LongDesc, std::string const &URI, bool const IsOptional,
+	 std::map const &Options);
 };
 									/*}}}*/
 class pkgAcquire::Item : public WeakPointable				/*{{{*/
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index c35f3b0a4..f038f12f5 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -116,12 +116,12 @@ void foreachTarget(std::string const URI, std::string const Dist,
       for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
       {
 #define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb-src::") + *T + "::" + (X))
-	 std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI");
+	 std::string const MetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey");
 	 std::string const ShortDesc = APT_T_CONFIG("ShortDescription");
 	 std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
 	 bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true);
 #undef APT_T_CONFIG
-	 if (URI.empty())
+	 if (MetaKey.empty())
 	    continue;
 
 	 vector const SectionEntries = src->second;
@@ -130,23 +130,24 @@ void foreachTarget(std::string const URI, std::string const Dist,
 	 {
 	    for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l)
 	    {
-	       if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos)
+	       if (*l == "none" && MetaKey.find("$(LANGUAGE)") != std::string::npos)
 		  continue;
 
-	       struct SubstVar subst[] = {
-		  { "$(SITE)", &Site },
-		  { "$(RELEASE)", &Release },
-		  { "$(COMPONENT)", &((*I)->Section) },
-		  { "$(LANGUAGE)", &(*l) },
-		  { NULL, NULL }
-	       };
-	       Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst);
-
-	       if (URI.find("$(LANGUAGE)") == std::string::npos)
+	       std::map Options;
+	       Options.insert(std::make_pair("SITE", Site));
+	       Options.insert(std::make_pair("RELEASE", Release));
+	       Options.insert(std::make_pair("COMPONENT", (*I)->Section));
+	       Options.insert(std::make_pair("LANGUAGE", *l));
+	       Options.insert(std::make_pair("ARCHITECTURE", "source"));
+	       Options.insert(std::make_pair("BASE_URI", baseURI));
+	       Options.insert(std::make_pair("CREATED_BY", *T));
+	       Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options);
+
+	       if (MetaKey.find("$(LANGUAGE)") == std::string::npos)
 		  break;
 	    }
 
-	    if (URI.find("$(COMPONENT)") == std::string::npos)
+	    if (MetaKey.find("$(COMPONENT)") == std::string::npos)
 	       break;
 	 }
       }
@@ -156,12 +157,12 @@ void foreachTarget(std::string const URI, std::string const Dist,
    for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
    {
 #define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb::") + *T + "::" + (X))
-      std::string const URI = APT_T_CONFIG(flatArchive ? "flatURI" : "URI");
+      std::string const MetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey");
       std::string const ShortDesc = APT_T_CONFIG("ShortDescription");
       std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description");
       bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb::") + *T + "::Optional", true);
 #undef APT_T_CONFIG
-      if (URI.empty())
+      if (MetaKey.empty())
 	 continue;
 
       for (map >::const_iterator a = ArchEntries.begin();
@@ -175,28 +176,28 @@ void foreachTarget(std::string const URI, std::string const Dist,
 
 	    for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l)
 	    {
-	       if (*l == "none" && URI.find("$(LANGUAGE)") != std::string::npos)
+	       if (*l == "none" && MetaKey.find("$(LANGUAGE)") != std::string::npos)
 		  continue;
 
-	       struct SubstVar subst[] = {
-		  { "$(SITE)", &Site },
-		  { "$(RELEASE)", &Release },
-		  { "$(COMPONENT)", &((*I)->Section) },
-		  { "$(LANGUAGE)", &(*l) },
-		  { "$(ARCHITECTURE)", &(a->first) },
-		  { NULL, NULL }
-	       };
-	       Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst);
-
-	       if (URI.find("$(LANGUAGE)") == std::string::npos)
+	       std::map Options;
+	       Options.insert(std::make_pair("SITE", Site));
+	       Options.insert(std::make_pair("RELEASE", Release));
+	       Options.insert(std::make_pair("COMPONENT", (*I)->Section));
+	       Options.insert(std::make_pair("LANGUAGE", *l));
+	       Options.insert(std::make_pair("ARCHITECTURE", a->first));
+	       Options.insert(std::make_pair("BASE_URI", baseURI));
+	       Options.insert(std::make_pair("CREATED_BY", *T));
+	       Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options);
+
+	       if (MetaKey.find("$(LANGUAGE)") == std::string::npos)
 		  break;
 	    }
 
-	    if (URI.find("$(COMPONENT)") == std::string::npos)
+	    if (MetaKey.find("$(COMPONENT)") == std::string::npos)
 	       break;
 	 }
 
-	 if (URI.find("$(ARCHITECTURE)") == std::string::npos)
+	 if (MetaKey.find("$(ARCHITECTURE)") == std::string::npos)
 	    break;
       }
    }
@@ -207,30 +208,23 @@ struct ComputeIndexTargetsClass
 {
    vector  * const IndexTargets;
 
-   void operator()(std::string const &baseURI, std::string const &/*TargetName*/,
-	 std::string const &URI, std::string const &ShortDesc, std::string const &LongDesc,
-	 bool const IsOptional, struct SubstVar const * const subst)
+   void operator()(std::string MetaKey, std::string ShortDesc, std::string LongDesc,
+	 bool const IsOptional, std::map Options)
    {
-      std::string const name = SubstVar(URI, subst);
-      IndexTarget * Target;
-      if (IsOptional == true)
-      {
-	 Target = new OptionalIndexTarget(
-	       name,
-	       SubstVar(ShortDesc, subst),
-	       SubstVar(LongDesc, subst),
-	       baseURI + name
-	       );
-      }
-      else
+      for (std::map::const_iterator O = Options.begin(); O != Options.end(); ++O)
       {
-	 Target = new IndexTarget(
-	       name,
-	       SubstVar(ShortDesc, subst),
-	       SubstVar(LongDesc, subst),
-	       baseURI + name
-	       );
+	 MetaKey = SubstVar(MetaKey, std::string("$(") + O->first + ")", O->second);
+	 ShortDesc = SubstVar(ShortDesc, std::string("$(") + O->first + ")", O->second);
+	 LongDesc = SubstVar(LongDesc, std::string("$(") + O->first + ")", O->second);
       }
+      IndexTarget * Target = new IndexTarget(
+	    MetaKey,
+	    ShortDesc,
+	    LongDesc,
+	    Options.find("BASE_URI")->second + MetaKey,
+	    IsOptional,
+	    Options
+	    );
       IndexTargets->push_back(Target);
    }
 
@@ -256,7 +250,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 
    // special case for --print-uris
    vector  const * const targets = ComputeIndexTargets();
-#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X))
+#define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map())
    pkgAcqMetaBase * const TransactionManager = new pkgAcqMetaClearSig(Owner,
 	 APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"),
 	 targets, iR);
@@ -309,33 +303,33 @@ struct GetIndexFilesClass
    std::string const Release;
    bool const IsTrusted;
 
-   void operator()(std::string const &/*baseURI*/, std::string const &TargetName,
-	 std::string const &/*URI*/, std::string const &/*ShortDesc*/, std::string const &/*LongDesc*/,
-	 bool const /*IsOptional*/, struct SubstVar const * const subst)
+   void operator()(std::string const &/*URI*/, std::string const &/*ShortDesc*/, std::string const &/*LongDesc*/,
+	 bool const /*IsOptional*/, std::map Options)
    {
+      std::string const TargetName = Options.find("CREATED_BY")->second;
       if (TargetName == "Packages")
       {
 	 Indexes->push_back(new debPackagesIndex(
 		  URI,
 		  Release,
-		  SubstVar("$(COMPONENT)", subst),
+		  Options.find("COMPONENT")->second,
 		  IsTrusted,
-		  SubstVar("$(ARCHITECTURE)", subst)
+		  Options.find("ARCHITECTURE")->second
 		  ));
       }
       else if (TargetName == "Sources")
 	 Indexes->push_back(new debSourcesIndex(
 		  URI,
 		  Release,
-		  SubstVar("$(COMPONENT)", subst),
+		  Options.find("COMPONENT")->second,
 		  IsTrusted
 		  ));
       else if (TargetName == "Translations")
 	 Indexes->push_back(new debTranslationsIndex(
 		  URI,
 		  Release,
-		  SubstVar("$(COMPONENT)", subst),
-		  SubstVar("$(LANGUAGE)", subst)
+		  Options.find("COMPONENT")->second,
+		  Options.find("LANGUAGE")->second
 		  ));
    }
 
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index 50ea2b49e..e2239a906 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -101,19 +101,19 @@ bool pkgInitConfig(Configuration &Cnf)
    // The default user we drop to in the methods
    Cnf.CndSet("APT::Sandbox::User", "_apt");
 
-   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::URI", "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages");
-   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatURI", "Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::MetaKey", "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatMetaKey", "Packages");
    Cnf.CndSet("APT::Acquire::Targets::deb::Packages::ShortDescription", "Packages");
    Cnf.CndSet("APT::Acquire::Targets::deb::Packages::Description", "$(SITE) $(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages");
    Cnf.CndSet("APT::Acquire::Targets::deb::Packages::flatDescription", "$(SITE) $(RELEASE) Packages");
    Cnf.CndSet("APT::Acquire::Targets::deb::Packages::Optional", false);
-   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::URI", "$(COMPONENT)/i18n/Translation-$(LANGUAGE)");
-   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatURI", "$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::MetaKey", "$(COMPONENT)/i18n/Translation-$(LANGUAGE)");
+   Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatMetaKey", "$(LANGUAGE)");
    Cnf.CndSet("APT::Acquire::Targets::deb::Translations::ShortDescription", "Translation-$(LANGUAGE)");
    Cnf.CndSet("APT::Acquire::Targets::deb::Translations::Description", "$(SITE) $(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)");
    Cnf.CndSet("APT::Acquire::Targets::deb::Translations::flatDescription", "$(SITE) $(RELEASE) Translation-$(LANGUAGE)");
-   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::URI", "$(COMPONENT)/source/Sources");
-   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatURI", "Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::MetaKey", "$(COMPONENT)/source/Sources");
+   Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatMetaKey", "Sources");
    Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::ShortDescription", "Sources");
    Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Description", "$(SITE) $(RELEASE)/$(COMPONENT) Sources");
    Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatDescription", "$(SITE) $(RELEASE) Sources");
diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index 5a07c2bec..ab833440b 100644
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -33,11 +33,11 @@ instructing the Acquire system to download the Packages files would look
 like this (see also apt.conf(5) manpage for configuration file syntax):
 
 	APT::Acquire::Targets::deb::Packages {
-		URI "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages";
+		MetaKey "$(COMPONENT)/binary-$(ARCHITECTURE)/Packages";
 		ShortDescription "Packages";
 		Description "$(SITE) $(RELEASE)/$(COMPONENT) $(ARCHITECTURE) Packages";
 
-		flatURI "Packages";
+		flatMetaKey "Packages";
 		flatDescription "$(SITE) $(RELEASE) Packages";
 
 		Optional "false";
@@ -47,18 +47,18 @@ All files which should be downloaded (nicknamed 'Targets') are mentioned
 below the APT::Acquire::Targets scope. 'deb' is here the type of the
 sources.list entry the file should be acquired for. The only other
 supported value is hence 'deb-src'. Beware: You can't specify multiple
-types here and you can't download the same URI for multiple types!
+types here and you can't download the same MetaKey form multiple types!
 
 After the type you can pick any valid and unique string which preferable
 refers to the file it downloads (In the example we picked 'Packages').
 This string is never shown or used.
 
 All targets have three main properties you can define:
-* URI: The identifier of the file to be downloaded as used in the
+* MetaKey: The identifier of the file to be downloaded as used in the
   Release file.  It is also the relative location of the file from the
   Release file.  You can neither download from a different server
-  entirely (absolute URI) nor should you try to access directories above
-  the Release file (e.g. "../../").
+  entirely (absolute URI) nor access directories above the Release file
+  (e.g. "../../").
 * ShortDescription: Very short string intended to be displayed to the
   user e.g.  while reporting progress. apt will e.g. use this string in
   the last line to indicate progress of e.g. the download of a specific
@@ -69,7 +69,7 @@ All targets have three main properties you can define:
   progress lines.
 
 Additional optional properties:
-* flat{URI,Description}: APT supports two types of repositories:
+* flat{MetaKey,Description}: APT supports two types of repositories:
   dists-style repositories which are the default and by far the most
   common which are named after the fact that the files are in an
   elaborated directory structure.  In contrast a flat-style repositories
@@ -96,20 +96,20 @@ look like this:
 
 APT::Acquire::Targets {
 	deb::Translations {
-		URI "$(COMPONENT)/i18n/Translation-$(LANGUAGE)";
+		MetaKey "$(COMPONENT)/i18n/Translation-$(LANGUAGE)";
 		ShortDescription "Translation-$(LANGUAGE)";
 		Description "$(SITE) $(RELEASE)/$(COMPONENT) Translation-$(LANGUAGE)";
 
-		flatURI "$(LANGUAGE)";
+		flatMetaKey "$(LANGUAGE)";
 		flatDescription "$(SITE) $(RELEASE) Translation-$(LANGUAGE)";
 	};
 
 	deb-src::Sources {
-		URI "$(COMPONENT)/source/Sources";
+		MetaKey "$(COMPONENT)/source/Sources";
 		ShortDescription "Sources";
 		Description "$(SITE) $(RELEASE)/$(COMPONENT) Sources";
 
-		flatURI "Sources";
+		flatMetaKey "Sources";
 		flatDescription "$(SITE) $(RELEASE) Sources";
 
 		Optional "false";
@@ -133,10 +133,7 @@ printed literally.
   have a meaningful value here.
 * $(LANGUAGE): Values are all entries (expect "none") of configuration
   option Acquire::Languages, e.g. "en", "de" or "de_AT".
-
-These values are defined both for 'deb' as well as 'deb-src' targets.
-'deb' targets additional have the variable:
-
 * $(ARCHITECTURE): Values are all entries of configuration option
   APT::Architectures (potentially modified by sources.list options),
-  e.g. "amd64", "i386" or "armel".
+  e.g. "amd64", "i386" or "armel" for the 'deb' type. In type 'deb-src'
+  this variable has the value "source".
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 0e59233e9..4a6845f40 100755
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -31,7 +31,7 @@ testempty find rootdir/var/lib/apt/lists -name '*Contents*'
 
 cat > rootdir/etc/apt/apt.conf.d/content-target.conf < rootdir/etc/apt/apt.conf.d/content-target.conf <
Date: Thu, 11 Jun 2015 10:56:31 +0200
Subject: show URI.Path in all acquire item descriptions

It is a rather strange sight that index items use SiteOnly which strips
the Path, while e.g. deb files are downloaded with NoUserPassword which
does not. Important to note here is that for the file transport Path is
pretty important as there is no Host which would be displayed by Site,
which always resulted in "interesting" unspecific errors for "file:".

Adding a 'middle' ground between the two which does show the Path but
potentially modifies it (it strips a pending / at the end if existing)
solves this "file:" issue, syncs the output and in the end helps to
identify which file is meant exactly in progress output and co as a
single site can have multiple repositories in different paths.
---
 apt-pkg/contrib/strutl.cc                          | 15 ++++++---
 apt-pkg/contrib/strutl.h                           |  1 +
 apt-pkg/deb/debindexfile.cc                        | 10 +++---
 apt-pkg/deb/debmetaindex.cc                        |  4 +--
 test/integration/test-apt-cache                    |  2 +-
 test/integration/test-apt-cli-show                 |  2 +-
 .../integration/test-apt-get-update-unauth-warning | 12 +++----
 .../test-apt-translation-has-no-packages           |  2 +-
 test/integration/test-apt-update-nofallback        |  6 ++--
 test/integration/test-apt-update-rollback          |  6 ++--
 .../test-bug-543966-downgrade-below-1000-pin       |  2 +-
 .../test-bug-595691-empty-and-broken-archive-files |  5 +--
 .../test-cve-2013-1051-InRelease-parsing           |  2 +-
 test/integration/test-policy-pinning               |  8 ++---
 test/integration/test-releasefile-verification     |  4 +--
 .../test-ubuntu-bug-1098738-apt-get-source-md5sum  | 28 ++++++++--------
 test/libapt/uri_test.cc                            | 39 ++++++++++++++++++++++
 17 files changed, 98 insertions(+), 50 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index 0db4c57b2..5731b5a2b 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -1637,8 +1637,6 @@ URI::operator string()
 }
 									/*}}}*/
 // URI::SiteOnly - Return the schema and site for the URI		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 string URI::SiteOnly(const string &URI)
 {
    ::URI U(URI);
@@ -1648,9 +1646,18 @@ string URI::SiteOnly(const string &URI)
    return U;
 }
 									/*}}}*/
+// URI::ArchiveOnly - Return the schema, site and cleaned path for the URI /*{{{*/
+string URI::ArchiveOnly(const string &URI)
+{
+   ::URI U(URI);
+   U.User.clear();
+   U.Password.clear();
+   if (U.Path.empty() == false && U.Path[U.Path.length() - 1] == '/')
+      U.Path.erase(U.Path.length() - 1);
+   return U;
+}
+									/*}}}*/
 // URI::NoUserPassword - Return the schema, site and path for the URI	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 string URI::NoUserPassword(const string &URI)
 {
    ::URI U(URI);
diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h
index d64270aaf..01bbfef72 100644
--- a/apt-pkg/contrib/strutl.h
+++ b/apt-pkg/contrib/strutl.h
@@ -167,6 +167,7 @@ class URI
    inline void operator =(const std::string &From) {CopyFrom(From);}
    inline bool empty() {return Access.empty();};
    static std::string SiteOnly(const std::string &URI);
+   static std::string ArchiveOnly(const std::string &URI);
    static std::string NoUserPassword(const std::string &URI);
    
    URI(std::string Path) {CopyFrom(Path);}
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index 185248619..20bf5ae50 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -57,7 +57,7 @@ string debSourcesIndex::SourceInfo(pkgSrcRecords::Parser const &Record,
 				   pkgSrcRecords::File const &File) const
 {
    string Res;
-   Res = ::URI::NoUserPassword(URI) + ' ';
+   Res = ::URI::ArchiveOnly(URI) + ' ';
    if (Dist[Dist.size() - 1] == '/')
    {
       if (Dist != "/")
@@ -116,7 +116,7 @@ string debSourcesIndex::Describe(bool Short) const
 /* */
 string debSourcesIndex::Info(const char *Type) const
 {
-   string Info = ::URI::NoUserPassword(URI) + ' ';
+   string Info = ::URI::ArchiveOnly(URI) + ' ';
    if (Dist[Dist.size() - 1] == '/')
    {
       if (Dist != "/")
@@ -210,7 +210,7 @@ debPackagesIndex::debPackagesIndex(string const &URI, string const &Dist, string
 /* This is a shorter version that is designed to be < 60 chars or so */
 string debPackagesIndex::ArchiveInfo(pkgCache::VerIterator Ver) const
 {
-   string Res = ::URI::NoUserPassword(URI) + ' ';
+   string Res = ::URI::ArchiveOnly(URI) + ' ';
    if (Dist[Dist.size() - 1] == '/')
    {
       if (Dist != "/")
@@ -248,7 +248,7 @@ string debPackagesIndex::Describe(bool Short) const
 /* */
 string debPackagesIndex::Info(const char *Type) const 
 {
-   string Info = ::URI::NoUserPassword(URI) + ' ';
+   string Info = ::URI::ArchiveOnly(URI) + ' ';
    if (Dist[Dist.size() - 1] == '/')
    {
       if (Dist != "/")
@@ -473,7 +473,7 @@ string debTranslationsIndex::Describe(bool Short) const
 /* */
 string debTranslationsIndex::Info(const char *Type) const 
 {
-   string Info = ::URI::NoUserPassword(URI) + ' ';
+   string Info = ::URI::ArchiveOnly(URI) + ' ';
    if (Dist[Dist.size() - 1] == '/')
    {
       if (Dist != "/")
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index e1f86d20f..d4e340be0 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -25,7 +25,7 @@ using namespace std;
 
 string debReleaseIndex::MetaIndexInfo(const char *Type) const
 {
-   string Info = ::URI::SiteOnly(URI) + ' ';
+   string Info = ::URI::ArchiveOnly(URI) + ' ';
    if (Dist[Dist.size() - 1] == '/')
    {
       if (Dist != "/")
@@ -105,7 +105,7 @@ void foreachTarget(std::string const URI, std::string const Dist,
    else
       baseURI += "dists/" + Dist + "/";
    std::string const Release = (Dist == "/") ? "" : Dist;
-   std::string const Site = ::URI::SiteOnly(URI);
+   std::string const Site = ::URI::ArchiveOnly(URI);
    std::vector lang = APT::Configuration::getLanguages(true);
    if (lang.empty())
       lang.push_back("none");
diff --git a/test/integration/test-apt-cache b/test/integration/test-apt-cache
index f3db8024f..97dc0f939 100755
--- a/test/integration/test-apt-cache
+++ b/test/integration/test-apt-cache
@@ -50,7 +50,7 @@ testsuccessequal 'bar' aptcache pkgnames bar
 testsuccessequal 'fancy
 foo' aptcache pkgnames f
 
-testsuccessequal "       foo |          1 | file:$(readlink -f .)/aptarchive/ unstable/main amd64 Packages" aptcache madison foo
+testsuccessequal "       foo |          1 | file:$(readlink -f .)/aptarchive unstable/main amd64 Packages" aptcache madison foo
 
 ### depends
 
diff --git a/test/integration/test-apt-cli-show b/test/integration/test-apt-cli-show
index 43072cf03..5f4ef1b48 100755
--- a/test/integration/test-apt-cli-show
+++ b/test/integration/test-apt-cli-show
@@ -32,7 +32,7 @@ Maintainer: Joe Sixpack 
 Installed-Size: 43.0 kB
 Download-Size: unknown
 APT-Manual-Installed: yes
-APT-Sources: file:$APTARCHIVE/ unstable/main i386 Packages
+APT-Sources: file:$APTARCHIVE unstable/main i386 Packages
 Description: Some description 
  That has multiple lines
 " apt show foo
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 81c01ba3e..ada7f7a26 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -19,11 +19,11 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated files leads to warning
-testfailureequal "Ign file: unstable InRelease
+testfailureequal "Ign file:$APTARCHIVE unstable InRelease
   File not found
-Err file: unstable Release
+Err file:$APTARCHIVE unstable Release
   File not found
-W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
+W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
 E: Use --allow-insecure-repositories to force the update" aptget update --no-allow-insecure-repositories
 
 # no package foo
@@ -32,12 +32,12 @@ testequal 'lock
 partial' ls rootdir/var/lib/apt/lists
 
 # allow override
-testwarningequal "Ign file: unstable InRelease
+testwarningequal "Ign file:$APTARCHIVE unstable InRelease
   File not found
-Ign file: unstable Release
+Ign file:$APTARCHIVE unstable Release
   File not found
 Reading package lists...
-W: The repository 'file: unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories
+W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories
 # ensure we can not install the package
 testfailureequal "WARNING: The following packages cannot be authenticated!
   foo
diff --git a/test/integration/test-apt-translation-has-no-packages b/test/integration/test-apt-translation-has-no-packages
index 440fd30cf..cf5b56243 100755
--- a/test/integration/test-apt-translation-has-no-packages
+++ b/test/integration/test-apt-translation-has-no-packages
@@ -38,4 +38,4 @@ testsuccessequal "foo:
   Candidate: 1.0
   Version table:
      1.0 0
-        500 file:$APTARCHIVE/ unstable/main amd64 Packages" aptcache policy foo
+        500 file:$APTARCHIVE unstable/main amd64 Packages" aptcache policy foo
diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback
index f132bcf8e..2ded73122 100755
--- a/test/integration/test-apt-update-nofallback
+++ b/test/integration/test-apt-update-nofallback
@@ -33,7 +33,7 @@ EOF
 
 assert_update_is_refused_and_last_good_state_used()
 {
-    testfailuremsg "E: The repository 'file: unstable Release' is no longer signed." aptget update
+    testfailuremsg "E: The repository 'file:${APTARCHIVE} unstable Release' is no longer signed." aptget update
 
     assert_repo_is_intact
 }
@@ -171,7 +171,7 @@ test_inrelease_to_invalid_inrelease()
     sed -i 's/^Codename:.*/Codename: evil!/' $APTARCHIVE/dists/unstable/InRelease
     inject_evil_package
 
-    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
+    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file:${APTARCHIVE} unstable InRelease: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
 W: Failed to fetch file:${APTARCHIVE}/dists/unstable/InRelease  The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
@@ -195,7 +195,7 @@ test_release_gpg_to_invalid_release_release_gpg()
     echo "Some evil data" >>  $APTARCHIVE/dists/unstable/Release
     inject_evil_package
 
-    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
+    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file:${APTARCHIVE} unstable Release: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
 W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg  The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) 
 
diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback
index 6ecf322b2..70619dd08 100755
--- a/test/integration/test-apt-update-rollback
+++ b/test/integration/test-apt-update-rollback
@@ -78,7 +78,7 @@ test_inrelease_to_valid_release() {
     rm $APTARCHIVE/dists/unstable/Release.gpg
 
     # update fails
-    testfailureequal "E: The repository 'file: unstable Release' is no longer signed." aptget update -qq
+    testfailureequal "E: The repository 'file:${APTARCHIVE} unstable Release' is no longer signed." aptget update -qq
 
     # test that security downgrade was not successful
     testfileequal lists.before "$(listcurrentlistsdirectory)"
@@ -101,7 +101,7 @@ test_inrelease_to_release_reverts_all() {
     break_repository_sources_index '+1hour'
 
     # ensure error
-    testfailureequal "E: The repository 'file: unstable Release' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1
+    testfailureequal "E: The repository 'file:${APTARCHIVE} unstable Release' is no longer signed." aptget update -qq # -o Debug::acquire::transaction=1
 
     # ensure that the Packages file is also rolled back
     testfileequal lists.before "$(listcurrentlistsdirectory)"
@@ -144,7 +144,7 @@ test_inrelease_to_unauth_inrelease() {
 
     signreleasefiles 'Marvin Paranoid'
 
-    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
+    testwarningequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file:${APTARCHIVE} unstable InRelease: The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
 
 W: Failed to fetch file:$APTARCHIVE/dists/unstable/InRelease  The following signatures couldn't be verified because the public key is not available: NO_PUBKEY E8525D47528144E2
 
diff --git a/test/integration/test-bug-543966-downgrade-below-1000-pin b/test/integration/test-bug-543966-downgrade-below-1000-pin
index e59231608..180393867 100755
--- a/test/integration/test-bug-543966-downgrade-below-1000-pin
+++ b/test/integration/test-bug-543966-downgrade-below-1000-pin
@@ -13,7 +13,7 @@ insertinstalledpackage 'base-files' 'all' '5.0.0-1'
 setupaptarchive
 
 STATUS=$(readlink -f rootdir/var/lib/dpkg/status)
-APTARCHIVE="$(readlink -f aptarchive)/"
+APTARCHIVE="$(readlink -f aptarchive)"
 
 testsuccessequal "base-files:
   Installed: 5.0.0-1
diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index 486b8ba02..b42212f5e 100755
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -49,6 +49,7 @@ createemptyfile() {
 }
 
 testoverfile() {
+	local APTARCHIVE="$(readlink -f ./aptarchive)"
 	forcecompressor "$1"
 
 	createemptyfile 'en'
@@ -63,9 +64,9 @@ testoverfile() {
 	testaptgetupdate 'Reading package lists...' "empty archive Packages.$COMPRESS over file"
 
 	createemptyfile 'Packages'
-	testaptgetupdate "Err file:  Packages
+	testaptgetupdate "Err file:$APTARCHIVE  Packages
   Empty files can't be valid archives
-W: Failed to fetch ${COMPRESSOR}:$(readlink -f aptarchive/Packages.$COMPRESS)  Empty files can't be valid archives
+W: Failed to fetch ${COMPRESSOR}:${APTARCHIVE}/Packages.$COMPRESS  Empty files can't be valid archives
 
 E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over file"
 }
diff --git a/test/integration/test-cve-2013-1051-InRelease-parsing b/test/integration/test-cve-2013-1051-InRelease-parsing
index d99174553..933cbbd92 100755
--- a/test/integration/test-cve-2013-1051-InRelease-parsing
+++ b/test/integration/test-cve-2013-1051-InRelease-parsing
@@ -12,7 +12,7 @@ insertpackage 'stable' 'good-pkg' 'all' '1.0'
 setupaptarchive
 
 changetowebserver
-ARCHIVE='http://localhost:8080/'
+ARCHIVE='http://localhost:8080'
 msgtest 'Initial apt-get update should work with' 'InRelease'
 testsuccess --nomsg aptget update
 
diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning
index 2675b51bc..9cd54264b 100755
--- a/test/integration/test-policy-pinning
+++ b/test/integration/test-policy-pinning
@@ -20,7 +20,7 @@ testequalpolicy() {
 	testsuccessequal "Package files:
  $(echo "$SP" | awk '{ printf("%3s\n",$0) }') ${STATUS}
      release a=now
- $(echo "$AP" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}/  Packages
+ $(echo "$AP" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}  Packages
      release c=
 Pinned packages:" aptcache policy $*
 }
@@ -105,11 +105,11 @@ testequalpolicycoolstuff() {
 	local BPO1ARCHIVE=""
 	local BPO2ARCHIVE=""
 	if [ ! "$7" = "2.0~bpo2" ]; then
-		BPO1ARCHIVE="        $(echo "$AB" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}/ backports/main i386 Packages"
+		BPO1ARCHIVE="        $(echo "$AB" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE} backports/main i386 Packages"
 	else
 		BPO2ARCHIVE="
      2.0~bpo2 $PB
-        $(echo "$AB" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}/ backports/main i386 Packages"
+        $(echo "$AB" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE} backports/main i386 Packages"
 		SB="$(echo "$SB" | tail -n 1)"
 		shift
 	fi
@@ -121,7 +121,7 @@ testequalpolicycoolstuff() {
  $IB 2.0~bpo1 $PB
 ${BPO1ARCHIVE}$SB
  $IS 1.0 $PB
-        $(echo "$AS" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}/ stable/main i386 Packages$SS" \
+        $(echo "$AS" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE} stable/main i386 Packages$SS" \
 		aptcache policy coolstuff -o Policy=${INSTALLED}-${CANDIDATE}-${AB}-${AS}-${PB} $*
 }
 
diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification
index 469ed34d2..e8419524c 100755
--- a/test/integration/test-releasefile-verification
+++ b/test/integration/test-releasefile-verification
@@ -41,7 +41,7 @@ The following NEW packages will be installed:
   apt
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 After this operation, 5370 kB of additional disk space will be used.
-Get:1 http://localhost:8080/  apt 0.7.25.3
+Get:1 http://localhost:8080  apt 0.7.25.3
 Download complete and in download only mode' aptget install apt -dy
 }
 
@@ -54,7 +54,7 @@ The following NEW packages will be installed:
   apt
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
 After this operation, 5808 kB of additional disk space will be used.
-Get:1 http://localhost:8080/  apt 0.8.0~pre1
+Get:1 http://localhost:8080  apt 0.8.0~pre1
 Download complete and in download only mode' aptget install apt -dy
 }
 
diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
index 555d8fcaa..6abb5d12a 100755
--- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
+++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
@@ -148,8 +148,8 @@ testok() {
 	testsuccessequal "Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
-Get:1 http://localhost:8080/  $1 1.0 (dsc) [3 B]
-Get:2 http://localhost:8080/  $1 1.0 (tar) [3 B]
+Get:1 http://localhost:8080  $1 1.0 (dsc) [3 B]
+Get:2 http://localhost:8080  $1 1.0 (tar) [3 B]
 Download complete and in download only mode" aptget source -d "$@"
 	msgtest 'Files were successfully downloaded for' "$1"
 	testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz
@@ -175,11 +175,11 @@ testmismatch() {
 	testfailureequal "Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
-Get:1 http://localhost:8080/  $1 1.0 (dsc) [3 B]
-Err http://localhost:8080/  $1 1.0 (dsc)
+Get:1 http://localhost:8080  $1 1.0 (dsc) [3 B]
+Err http://localhost:8080  $1 1.0 (dsc)
   Hash Sum mismatch
-Get:2 http://localhost:8080/  $1 1.0 (tar) [3 B]
-Err http://localhost:8080/  $1 1.0 (tar)
+Get:2 http://localhost:8080  $1 1.0 (tar) [3 B]
+Err http://localhost:8080  $1 1.0 (tar)
   Hash Sum mismatch
 E: Failed to fetch http://localhost:8080/${1}_1.0.dsc  Hash Sum mismatch
 
@@ -203,8 +203,8 @@ Download complete and in download only mode" aptget source -d "$@" -o Acquire::F
 	testsuccessequal "Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
-Get:1 http://localhost:8080/  $1 1.0 (dsc) [3 B]
-Get:2 http://localhost:8080/  $1 1.0 (tar) [3 B]
+Get:1 http://localhost:8080  $1 1.0 (dsc) [3 B]
+Get:2 http://localhost:8080  $1 1.0 (tar) [3 B]
 Download complete and in download only mode" aptget source --allow-unauthenticated -d "$@" -o Acquire::ForceHash=ROT26
 	msgtest 'Files were downloaded unauthenticated as user allowed it' "$1"
 	testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz
@@ -240,9 +240,9 @@ testok pkg-mixed-ok
 testfailureequal 'Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
-Get:1 http://localhost:8080/  pkg-mixed-sha1-bad 1.0 (tar) [3 B]
-Get:2 http://localhost:8080/  pkg-mixed-sha1-bad 1.0 (dsc) [3 B]
-Err http://localhost:8080/  pkg-mixed-sha1-bad 1.0 (dsc)
+Get:1 http://localhost:8080  pkg-mixed-sha1-bad 1.0 (tar) [3 B]
+Get:2 http://localhost:8080  pkg-mixed-sha1-bad 1.0 (dsc) [3 B]
+Err http://localhost:8080  pkg-mixed-sha1-bad 1.0 (dsc)
   Hash Sum mismatch
 E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc  Hash Sum mismatch
 
@@ -252,10 +252,10 @@ testsuccess --nomsg test ! -e pkg-mixed-sha1-bad_1.0.dsc -a -e pkg-mixed-sha1-ba
 testfailureequal 'Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
-Get:1 http://localhost:8080/  pkg-mixed-sha2-bad 1.0 (tar) [3 B]
-Err http://localhost:8080/  pkg-mixed-sha2-bad 1.0 (tar)
+Get:1 http://localhost:8080  pkg-mixed-sha2-bad 1.0 (tar) [3 B]
+Err http://localhost:8080  pkg-mixed-sha2-bad 1.0 (tar)
   Hash Sum mismatch
-Get:2 http://localhost:8080/  pkg-mixed-sha2-bad 1.0 (dsc) [3 B]
+Get:2 http://localhost:8080  pkg-mixed-sha2-bad 1.0 (dsc) [3 B]
 E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz  Hash Sum mismatch
 
 E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha2-bad
diff --git a/test/libapt/uri_test.cc b/test/libapt/uri_test.cc
index 5d5ae9679..fe6015e65 100644
--- a/test/libapt/uri_test.cc
+++ b/test/libapt/uri_test.cc
@@ -13,6 +13,9 @@ TEST(URITest, BasicHTTP)
    EXPECT_EQ("www.debian.org", U.Host);
    EXPECT_EQ("/temp/test", U.Path);
    EXPECT_EQ("http://www.debian.org:90/temp/test", (std::string)U);
+   EXPECT_EQ("http://www.debian.org:90", URI::SiteOnly(U));
+   EXPECT_EQ("http://www.debian.org:90/temp/test", URI::ArchiveOnly(U));
+   EXPECT_EQ("http://www.debian.org:90/temp/test", URI::NoUserPassword(U));
    // Login data
    U = URI("http://jgg:foo@ualberta.ca/blah");
    EXPECT_EQ("http", U.Access);
@@ -22,6 +25,9 @@ TEST(URITest, BasicHTTP)
    EXPECT_EQ("ualberta.ca", U.Host);
    EXPECT_EQ("/blah", U.Path);
    EXPECT_EQ("http://jgg:foo@ualberta.ca/blah", (std::string)U);
+   EXPECT_EQ("http://ualberta.ca", URI::SiteOnly(U));
+   EXPECT_EQ("http://ualberta.ca/blah", URI::ArchiveOnly(U));
+   EXPECT_EQ("http://ualberta.ca/blah", URI::NoUserPassword(U));
 }
 TEST(URITest, SingeSlashFile)
 {
@@ -33,6 +39,9 @@ TEST(URITest, SingeSlashFile)
    EXPECT_EQ("", U.Host);
    EXPECT_EQ("/usr/bin/foo", U.Path);
    EXPECT_EQ("file:/usr/bin/foo", (std::string)U);
+   EXPECT_EQ("file:", URI::SiteOnly(U));
+   EXPECT_EQ("file:/usr/bin/foo", URI::ArchiveOnly(U));
+   EXPECT_EQ("file:/usr/bin/foo", URI::NoUserPassword(U));
 }
 TEST(URITest, BasicCDROM)
 {
@@ -44,6 +53,9 @@ TEST(URITest, BasicCDROM)
    EXPECT_EQ("Moo Cow Rom", U.Host);
    EXPECT_EQ("/debian", U.Path);
    EXPECT_EQ("cdrom://Moo Cow Rom/debian", (std::string)U);
+   EXPECT_EQ("cdrom://Moo Cow Rom", URI::SiteOnly(U));
+   EXPECT_EQ("cdrom://Moo Cow Rom/debian", URI::ArchiveOnly(U));
+   EXPECT_EQ("cdrom://Moo Cow Rom/debian", URI::NoUserPassword(U));
 }
 TEST(URITest, RelativeGzip)
 {
@@ -55,6 +67,9 @@ TEST(URITest, RelativeGzip)
    EXPECT_EQ(".", U.Host);
    EXPECT_EQ("/bar/cow", U.Path);
    EXPECT_EQ("gzip://./bar/cow", (std::string)U);
+   EXPECT_EQ("gzip://.", URI::SiteOnly(U));
+   EXPECT_EQ("gzip://./bar/cow", URI::ArchiveOnly(U));
+   EXPECT_EQ("gzip://./bar/cow", URI::NoUserPassword(U));
 }
 TEST(URITest, NoSlashFTP)
 {
@@ -66,6 +81,9 @@ TEST(URITest, NoSlashFTP)
    EXPECT_EQ("ftp.fr.debian.org", U.Host);
    EXPECT_EQ("/debian/pool/main/x/xtel/xtel_3.2.1-15_i386.deb", U.Path);
    EXPECT_EQ("ftp://ftp.fr.debian.org/debian/pool/main/x/xtel/xtel_3.2.1-15_i386.deb", (std::string)U);
+   EXPECT_EQ("ftp://ftp.fr.debian.org", URI::SiteOnly(U));
+   EXPECT_EQ("ftp://ftp.fr.debian.org/debian/pool/main/x/xtel/xtel_3.2.1-15_i386.deb", URI::ArchiveOnly(U));
+   EXPECT_EQ("ftp://ftp.fr.debian.org/debian/pool/main/x/xtel/xtel_3.2.1-15_i386.deb", URI::NoUserPassword(U));
 }
 TEST(URITest, RFC2732)
 {
@@ -77,6 +95,9 @@ TEST(URITest, RFC2732)
    EXPECT_EQ("1080::8:800:200C:417A", U.Host);
    EXPECT_EQ("/foo", U.Path);
    EXPECT_EQ("http://[1080::8:800:200C:417A]/foo", (std::string)U);
+   EXPECT_EQ("http://[1080::8:800:200C:417A]", URI::SiteOnly(U));
+   EXPECT_EQ("http://[1080::8:800:200C:417A]/foo", URI::ArchiveOnly(U));
+   EXPECT_EQ("http://[1080::8:800:200C:417A]/foo", URI::NoUserPassword(U));
    // with port
    U = URI("http://[::FFFF:129.144.52.38]:80/index.html");
    EXPECT_EQ("http", U.Access);
@@ -86,6 +107,9 @@ TEST(URITest, RFC2732)
    EXPECT_EQ("::FFFF:129.144.52.38", U.Host);
    EXPECT_EQ("/index.html", U.Path);
    EXPECT_EQ("http://[::FFFF:129.144.52.38]:80/index.html", (std::string)U);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38]:80", URI::SiteOnly(U));
+   EXPECT_EQ("http://[::FFFF:129.144.52.38]:80/index.html", URI::ArchiveOnly(U));
+   EXPECT_EQ("http://[::FFFF:129.144.52.38]:80/index.html", URI::NoUserPassword(U));
    // extra colon
    U = URI("http://[::FFFF:129.144.52.38:]:80/index.html");
    EXPECT_EQ("http", U.Access);
@@ -95,6 +119,9 @@ TEST(URITest, RFC2732)
    EXPECT_EQ("::FFFF:129.144.52.38:", U.Host);
    EXPECT_EQ("/index.html", U.Path);
    EXPECT_EQ("http://[::FFFF:129.144.52.38:]:80/index.html", (std::string)U);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]:80", URI::SiteOnly(U));
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]:80/index.html", URI::ArchiveOnly(U));
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]:80/index.html", URI::NoUserPassword(U));
    // extra colon port
    U = URI("http://[::FFFF:129.144.52.38:]/index.html");
    EXPECT_EQ("http", U.Access);
@@ -104,6 +131,9 @@ TEST(URITest, RFC2732)
    EXPECT_EQ("::FFFF:129.144.52.38:", U.Host);
    EXPECT_EQ("/index.html", U.Path);
    EXPECT_EQ("http://[::FFFF:129.144.52.38:]/index.html", (std::string)U);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]", URI::SiteOnly(U));
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]/index.html", URI::ArchiveOnly(U));
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]/index.html", URI::NoUserPassword(U));
    // My Evil Corruption of RFC 2732 to handle CDROM names!
    // Fun for the whole family! */
    U = URI("cdrom:[The Debian 1.2 disk, 1/2 R1:6]/debian/");
@@ -114,6 +144,9 @@ TEST(URITest, RFC2732)
    EXPECT_EQ("The Debian 1.2 disk, 1/2 R1:6", U.Host);
    EXPECT_EQ("/debian/", U.Path);
    EXPECT_EQ("cdrom://[The Debian 1.2 disk, 1/2 R1:6]/debian/", (std::string)U);
+   EXPECT_EQ("cdrom://[The Debian 1.2 disk, 1/2 R1:6]", URI::SiteOnly(U));
+   EXPECT_EQ("cdrom://[The Debian 1.2 disk, 1/2 R1:6]/debian", URI::ArchiveOnly(U));
+   EXPECT_EQ("cdrom://[The Debian 1.2 disk, 1/2 R1:6]/debian/", URI::NoUserPassword(U));
    // no brackets
    U = URI("cdrom:Foo Bar Cow/debian/");
    EXPECT_EQ("cdrom", U.Access);
@@ -123,9 +156,15 @@ TEST(URITest, RFC2732)
    EXPECT_EQ("Foo Bar Cow", U.Host);
    EXPECT_EQ("/debian/", U.Path);
    EXPECT_EQ("cdrom://Foo Bar Cow/debian/", (std::string)U);
+   EXPECT_EQ("cdrom://Foo Bar Cow", URI::SiteOnly(U));
+   EXPECT_EQ("cdrom://Foo Bar Cow/debian", URI::ArchiveOnly(U));
+   EXPECT_EQ("cdrom://Foo Bar Cow/debian/", URI::NoUserPassword(U));
    // percent encoded
    U = URI("ftp://foo:b%40r@example.org");
    EXPECT_EQ("foo", U.User);
    EXPECT_EQ("b@r", U.Password);
    EXPECT_EQ("ftp://foo:b%40r@example.org/", (std::string) U);
+   EXPECT_EQ("ftp://example.org", URI::SiteOnly(U));
+   EXPECT_EQ("ftp://example.org", URI::ArchiveOnly(U));
+   EXPECT_EQ("ftp://example.org/", URI::NoUserPassword(U));
 }
-- 
cgit v1.2.3-70-g09d2


From 8881b11eacd735148d087c8c0f53827cb537b582 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Thu, 11 Jun 2015 16:40:45 +0200
Subject: implement 'apt-get files' to access index targets

Downloading additional files is only half the job. We still need a way
to allow external tools to know where the files are they requested for
download given that we don't want them to choose their own location.

'apt-get files' is our answer to this showing by default in a deb822
format information about each IndexTarget with the potential to filter
the records based on lines and an option to change the output format.

The command serves also as an example on how to get to this information
via libapt.
---
 apt-pkg/deb/debmetaindex.cc                        | 17 +++--
 apt-pkg/indexfile.cc                               | 18 ++++-
 apt-pkg/indexfile.h                                |  3 +
 apt-private/private-cmndline.cc                    |  4 +
 cmdline/apt-get.cc                                 | 87 ++++++++++++++++++++++
 doc/acquire-additional-files.txt                   | 87 +++++++++++++++++++++-
 doc/apt-get.8.xml                                  | 15 ++++
 test/integration/test-apt-acquire-additional-files |  2 +
 8 files changed, 223 insertions(+), 10 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index c05e7cdae..8e4c2be2d 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -136,11 +136,14 @@ void foreachTarget(std::string const URI, std::string const Dist,
 	       std::map Options;
 	       Options.insert(std::make_pair("SITE", Site));
 	       Options.insert(std::make_pair("RELEASE", Release));
-	       Options.insert(std::make_pair("COMPONENT", (*I)->Section));
-	       Options.insert(std::make_pair("LANGUAGE", *l));
+	       if (MetaKey.find("$(COMPONENT)") != std::string::npos)
+		  Options.insert(std::make_pair("COMPONENT", (*I)->Section));
+	       if (MetaKey.find("$(LANGUAGE)") != std::string::npos)
+		  Options.insert(std::make_pair("LANGUAGE", *l));
 	       Options.insert(std::make_pair("ARCHITECTURE", "source"));
 	       Options.insert(std::make_pair("BASE_URI", baseURI));
 	       Options.insert(std::make_pair("REPO_URI", URI));
+	       Options.insert(std::make_pair("TARGET_OF", "deb-src"));
 	       Options.insert(std::make_pair("CREATED_BY", *T));
 	       Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options);
 
@@ -183,11 +186,15 @@ void foreachTarget(std::string const URI, std::string const Dist,
 	       std::map Options;
 	       Options.insert(std::make_pair("SITE", Site));
 	       Options.insert(std::make_pair("RELEASE", Release));
-	       Options.insert(std::make_pair("COMPONENT", (*I)->Section));
-	       Options.insert(std::make_pair("LANGUAGE", *l));
-	       Options.insert(std::make_pair("ARCHITECTURE", a->first));
+	       if (MetaKey.find("$(COMPONENT)") != std::string::npos)
+		  Options.insert(std::make_pair("COMPONENT", (*I)->Section));
+	       if (MetaKey.find("$(LANGUAGE)") != std::string::npos)
+		  Options.insert(std::make_pair("LANGUAGE", *l));
+	       if (MetaKey.find("$(ARCHITECTURE)") != std::string::npos)
+		  Options.insert(std::make_pair("ARCHITECTURE", a->first));
 	       Options.insert(std::make_pair("BASE_URI", baseURI));
 	       Options.insert(std::make_pair("REPO_URI", URI));
+	       Options.insert(std::make_pair("TARGET_OF", "deb"));
 	       Options.insert(std::make_pair("CREATED_BY", *T));
 	       Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options);
 
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 72d35ddcc..33fb48e35 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -133,8 +133,10 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(ARCHITECTURE);
       APT_CASE(BASE_URI);
       APT_CASE(REPO_URI);
+      APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
 #undef APT_CASE
+      case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
    }
    std::map::const_iterator const M = Options.find(Key);
    if (M == Options.end())
@@ -142,6 +144,20 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
    return M->second;
 }
 									/*}}}*/
+std::string IndexTarget::Format(std::string format) const		/*{{{*/
+{
+   for (std::map::const_iterator O = Options.begin(); O != Options.end(); ++O)
+   {
+      format = SubstVar(format, std::string("$(") + O->first + ")", O->second);
+   }
+   format = SubstVar(format, "$(METAKEY)", MetaKey);
+   format = SubstVar(format, "$(SHORTDESC)", ShortDesc);
+   format = SubstVar(format, "$(DESCRIPTION)", Description);
+   format = SubstVar(format, "$(URI)", URI);
+   format = SubstVar(format, "$(FILENAME)", Option(IndexTarget::FILENAME));
+   return format;
+}
+									/*}}}*/
 
 pkgIndexTargetFile::pkgIndexTargetFile(IndexTarget const &Target, bool const Trusted) :/*{{{*/
    pkgIndexFile(Trusted), Target(Target)
@@ -162,7 +178,7 @@ std::string pkgIndexTargetFile::Describe(bool Short) const		/*{{{*/
 									/*}}}*/
 std::string pkgIndexTargetFile::IndexFileName() const			/*{{{*/
 {
-   std::string const s =_config->FindDir("Dir::State::lists") + URItoFileName(Target.URI);
+   std::string const s = Target.Option(IndexTarget::FILENAME);
    if (FileExists(s))
       return s;
 
diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index c38cf0bf0..220c415ac 100644
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -79,8 +79,11 @@ class IndexTarget							/*{{{*/
       BASE_URI,
       REPO_URI,
       CREATED_BY,
+      TARGET_OF,
+      FILENAME,
    };
    std::string Option(OptionKeys const Key) const;
+   std::string Format(std::string format) const;
 };
 									/*}}}*/
 
diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc
index 41aab81f6..458051bf2 100644
--- a/apt-private/private-cmndline.cc
+++ b/apt-private/private-cmndline.cc
@@ -163,6 +163,10 @@ static bool addArgumentsAPTGet(std::vector &Args, char const
       // once sbuild is fixed, this option can be removed
       addArg('f', "fix-broken", "APT::Get::Fix-Broken", 0);
    }
+   else if (CmdMatches("files"))
+   {
+      addArg(0,"format","APT::Get::Files::Format", CommandLine::HasArg);
+   }
    else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
 	    CmdMatches("markauto", "unmarkauto")) // deprecated commands
       ;
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index c1f78523c..0fff2db58 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -86,6 +86,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -1602,6 +1603,91 @@ static bool DoChangelog(CommandLine &CmdL)
    return true;
 }
 									/*}}}*/
+// DoFiles - Lists all IndexTargets					/*{{{*/
+static std::string format_key(std::string key)
+{
+   // deb822 is case-insensitive, but the human eye prefers candy
+   std::transform(key.begin(), key.end(), key.begin(), ::tolower);
+   key[0] = ::toupper(key[0]);
+   size_t found = key.find("_uri");
+   if (found != std::string::npos)
+      key.replace(found, 4, "-URI");
+   while ((found = key.find('_')) != std::string::npos)
+   {
+      key[found] = '-';
+      key[found + 1] = ::toupper(key[found + 1]);
+   }
+   return key;
+}
+static bool DoFiles(CommandLine &CmdL)
+{
+   pkgCacheFile CacheFile;
+   pkgSourceList *SrcList = CacheFile.GetSourceList();
+
+   if (SrcList == NULL)
+      return false;
+
+   std::string const Format = _config->Find("APT::Get::Files::Format");
+   bool Filtered = CmdL.FileSize() > 1;
+   for (pkgSourceList::const_iterator S = SrcList->begin(); S != SrcList->end(); ++S)
+   {
+      std::vector const targets = (*S)->GetIndexTargets();
+      std::map AddOptions;
+      AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+
+      for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
+      {
+	 std::ostringstream stanza;
+	 if (Filtered || Format.empty())
+	 {
+	    stanza << "MetaKey: " << T->MetaKey << "\n"
+	       << "ShortDesc: " << T->ShortDesc << "\n"
+	       << "Description: " << T->Description << "\n"
+	       << "URI: " << T->URI << "\n"
+	       << "Filename: " << T->Option(IndexTarget::FILENAME) << "\n"
+	       << "Optional: " << (T->IsOptional ? "yes" : "no") << "\n";
+	    for (std::map::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
+	       stanza << format_key(O->first) << ": " << O->second << "\n";
+	    for (std::map::const_iterator O = T->Options.begin(); O != T->Options.end(); ++O)
+	       stanza << format_key(O->first) << ": " << O->second << "\n";
+	    stanza << "\n";
+
+	    if (Filtered)
+	    {
+	       // that is a bit crude, but good enough for now
+	       bool found = true;
+	       std::string haystack = std::string("\n") + stanza.str() + "\n";
+	       std::transform(haystack.begin(), haystack.end(), haystack.begin(), ::tolower);
+	       size_t const filesize = CmdL.FileSize() - 1;
+	       for (size_t i = 0; i != filesize; ++i)
+	       {
+		  std::string needle = std::string("\n") + CmdL.FileList[i + 1] + "\n";
+		  std::transform(needle.begin(), needle.end(), needle.begin(), ::tolower);
+		  if (haystack.find(needle) != std::string::npos)
+		     continue;
+		  found = false;
+		  break;
+	       }
+	       if (found == false)
+		  continue;
+	    }
+	 }
+
+	 if (Format.empty())
+	    cout << stanza.str();
+	 else
+	 {
+	    std::string out = T->Format(Format);
+	    for (std::map::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
+	       out = SubstVar(out, std::string("$(") + O->first + ")", O->second);
+	    cout << out << std::endl;
+	 }
+      }
+   }
+
+   return true;
+}
+									/*}}}*/
 // ShowHelp - Show a help screen					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -1716,6 +1802,7 @@ int main(int argc,const char *argv[])					/*{{{*/
 				   {"source",&DoSource},
                                    {"download",&DoDownload},
                                    {"changelog",&DoChangelog},
+				   {"files",&DoFiles},
 				   {"moo",&DoMoo},
 				   {"help",&ShowHelp},
                                    {0,0}};
diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index ab833440b..a47dac2d4 100644
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -47,11 +47,13 @@ All files which should be downloaded (nicknamed 'Targets') are mentioned
 below the APT::Acquire::Targets scope. 'deb' is here the type of the
 sources.list entry the file should be acquired for. The only other
 supported value is hence 'deb-src'. Beware: You can't specify multiple
-types here and you can't download the same MetaKey form multiple types!
+types here and you can't download the same (evaluated) MetaKey from
+multiple types!
 
 After the type you can pick any valid and unique string which preferable
 refers to the file it downloads (In the example we picked 'Packages').
-This string is never shown or used.
+This string is used as identifier for the target class and accessible as
+'Created-By' e.g. in the "apt-get files" output as detailed below.
 
 All targets have three main properties you can define:
 * MetaKey: The identifier of the file to be downloaded as used in the
@@ -123,11 +125,12 @@ by the acquire system. The following variables are known; note that
 unknown variables have no default value nor are they touched: They are
 printed literally.
 
-* $(SITE): An identifier of the site we access, e.g.
-  "http://example.org/".
+* $(SITE): An identifier of the site we access as seen in sources.list,
+  e.g. "http://example.org/debian" or "file:/path/to/a/repository".
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
   "stretch".  Note that flat-style repositories do not have a archive-
   or codename per-se, so the value might very well be just "/" or so.
+  Again, as seen in the sources.list.
 * $(COMPONENT): as given in the sources.list, e.g. "main", "non-free" or
   "universe".  Note that flat-style repositories again do not really
   have a meaningful value here.
@@ -137,3 +140,79 @@ printed literally.
   APT::Architectures (potentially modified by sources.list options),
   e.g. "amd64", "i386" or "armel" for the 'deb' type. In type 'deb-src'
   this variable has the value "source".
+
+Note that while more variables might exist in the implementation, these
+are to be considered undefined and their usage strongly discouraged. If
+you have a need for another variable contact us instead.
+
+# Accessing files
+
+Do NOT hardcode specific file locations, names or compression types in
+your application! You will notice that the configuration options give
+you no choice over where the downloaded files will be stored. This is by
+design so multiple applications can download and use the same file
+rather than each and every one of them potentially downloads and uses
+its own copy somewhere on disk.
+
+"apt-get files" can be used to get the location as well as other
+information about all files downloaded (aka: you will see Packages,
+Sources and Translation-* files here as well). Provide a line of the
+default output format as parameter to filter out all entries which do
+not have such a line. With --format, you can further more define your
+own output style. The variables are what you see in the output, just all
+uppercase and wrapped in $(), as in the configuration file.
+
+To get all the filenames of all Translation-en files you can e.g. call:
+	apt-get files --format '$(FILENAME)' "Created-By: Translations" "Language: en"
+
+Accessing this information via libapt is done by reading the
+sources.lists (pkgSourceList), iterating over the metaIndex objects this
+creates and calling GetIndexTargets() on them. See the sourcecode of
+"apt-get files" for a complete example.
+
+Remarks on the available fields:
+* MetaKey, ShortDesc, Description, Site, Release: as defined
+  by the configuration and described further above.
+* Created-By: configuration entity responsible for this target
+* Target-Of: type of the sources.list entry
+* URI, Repo-URI: avoid using. Contains potentially username/password.
+  Prefer 'Site', especially for display.
+* Filename: The mentioned file doesn't need to exist, e.g. because the
+  file wasn't downloaded yet – or it does exist compressed. libapt users
+  are encouraged to use FileFd to open such files as it can handle
+  decompression transparently.
+* Trusted: As of the last 'apt-get update' call denoting if e.g. apt-get
+  would print the unauthenticated warning while installing packages
+  mentioned in this file (example assumes Packages file) [Not really
+  a property of the target itself, but of the metaIndex].
+* Optional: Decodes the option of the same name from the configuration.
+  Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
+* Language, Architecture, Component: as defined further above, but with
+  the catch that they might be missing if they don't effect the target
+  (aka: They weren't used while evaluating the MetaKey template).
+
+Again, additional fields might be visible in certain implementation, but
+you should avoid using them and instead talk to us about a portable
+implementation.
+
+# Multiple application requiring the same files
+
+It is highly encouraged that applications talk to each other and to us
+about which files they require. It is usually best to have a common
+package ship the configuration needed to get the files, but specific
+needs might require specific solutions. Again: talk to us.
+
+# Acquiring files not mentioned in the Release file
+
+You can't. This is by design as these files couldn't be verified to not
+be modified in transit, corrupted by the download process or simple if
+they are present at all on the server, which would require apt to probe
+for them. APT did this in the past for legacy reasons, we do not intend
+to go back to these dark times.
+
+This is also why you can't request files from a different server. It
+would have the additional problem that this server might not even be
+accessible (e.g. proxy settings) or that local sources (file:/, cdrom:/)
+start requesting online files…
+
+In other words: We would be opening Pandora's box.
diff --git a/doc/apt-get.8.xml b/doc/apt-get.8.xml
index a372a0d30..da077afa7 100644
--- a/doc/apt-get.8.xml
+++ b/doc/apt-get.8.xml
@@ -243,6 +243,21 @@
        
      
 
+     
+	Displays by default a deb822 formatted listing of
+	      information about all data files apt-get
+		 update would download. Supports a
+	       option to modify the output format as
+	      well as accepts lines of the default output to filter the records
+	      by. The command is mainly used as an interface for external tools
+	      working with APT to get information as well as filenames for
+	      downloaded files so they can use them as well instead of
+	      downloading them again on their own. Detailed documentation is
+	      omitted here and can instead be found in the source tree in
+	      doc/acquire-additional-files.txt.
+	   
+	
+     
 
 
    
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 4a6845f40..971cfac2e 100755
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -48,6 +48,7 @@ Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64'
 
 # no automatic uncompress based on the name please,
@@ -72,6 +73,7 @@ Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarch
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'
 
 rm -f rootdir/etc/apt/apt.conf.d/content-target.conf
-- 
cgit v1.2.3-70-g09d2


From b07aeb1a6e24825e534167a737043441e871de9f Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 12 Jun 2015 02:08:53 +0200
Subject: store Release files data in the Cache

We used to read the Release file for each Packages file and store the
data in the PackageFile struct even through potentially many Packages
(and Translation-*) files could use the same data. The point of the
exercise isn't the duplicated data through. Having the Release files as
first-class citizens in the Cache allows us to properly track their
state as well as allows us to use the information also for files which
aren't in the cache, but where we know to which Release file they
belong (Sources are an example for this).

This modifies the pkgCache structs, especially the PackagesFile struct
which depending on how libapt users access the data in these structs can
mean huge breakage or no visible change. As a single data point:
aptitude seems to be fine with this. Even if there is breakage it is
trivial to fix in a backportable way while avoiding breakage for
everyone would be a huge pain for us.

Note that not all PackageFile structs have a corresponding ReleaseFile.
In particular the dpkg/status file as well as *.deb files have not. As
these have only a Archive property need, the Component property takes
over this duty and the ReleaseFile remains zero. This is also the reason
why it isn't needed nor particularily recommended to change from
PackagesFile to ReleaseFile blindly. Sticking with the earlier is
usually the better option.
---
 apt-pkg/acquire-item.cc              |   6 +-
 apt-pkg/cacheiterators.h             |  46 ++++++-
 apt-pkg/clean.cc                     |   2 +-
 apt-pkg/deb/debindexfile.cc          |  42 ++-----
 apt-pkg/deb/debindexfile.h           |   1 -
 apt-pkg/deb/deblistparser.cc         |  37 ------
 apt-pkg/deb/deblistparser.h          |   6 +-
 apt-pkg/deb/debmetaindex.cc          | 124 ++++++++++++++++++-
 apt-pkg/deb/debmetaindex.h           |   6 +
 apt-pkg/deb/dpkgpm.cc                |   4 +-
 apt-pkg/depcache.cc                  |   6 +-
 apt-pkg/edsp.cc                      |   2 +-
 apt-pkg/edsp/edspindexfile.cc        |   5 +-
 apt-pkg/edsp/edsplistparser.cc       |   4 +-
 apt-pkg/edsp/edsplistparser.h        |   4 +-
 apt-pkg/metaindex.cc                 |  17 +++
 apt-pkg/metaindex.h                  |   6 +
 apt-pkg/pkgcache.cc                  |  77 ++++++++----
 apt-pkg/pkgcache.h                   |  75 ++++++++++--
 apt-pkg/pkgcachegen.cc               | 226 +++++++++++++++++++++++++++--------
 apt-pkg/pkgcachegen.h                |  10 +-
 apt-pkg/policy.cc                    |  15 ++-
 apt-pkg/versionmatch.cc              |  45 +++----
 apt-pkg/versionmatch.h               |   4 +-
 cmdline/apt-cache.cc                 |  11 +-
 test/integration/framework           |   9 +-
 test/integration/test-policy-pinning |   2 +-
 27 files changed, 573 insertions(+), 219 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index d4191f00e..d6e9ccbe0 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -2568,7 +2568,7 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour
    // Skip not source sources, they do not have file fields.
    for (; Vf.end() == false; ++Vf)
    {
-      if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
+      if (Vf.File().Flagged(pkgCache::Flag::NotSource))
 	 continue;
       break;
    }
@@ -2636,14 +2636,14 @@ bool pkgAcqArchive::QueueNext()
    {
       pkgCache::PkgFileIterator const PkgF = Vf.File();
       // Ignore not source sources
-      if ((PkgF->Flags & pkgCache::Flag::NotSource) != 0)
+      if (PkgF.Flagged(pkgCache::Flag::NotSource))
 	 continue;
 
       // Try to cross match against the source list
       pkgIndexFile *Index;
       if (Sources->FindIndex(PkgF, Index) == false)
 	    continue;
-      LocalSource = (PkgF->Flags & pkgCache::Flag::LocalSource) == pkgCache::Flag::LocalSource;
+      LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource);
 
       // only try to get a trusted package from another source if that source
       // is also trusted
diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h
index fe798799c..301da6fc4 100644
--- a/apt-pkg/cacheiterators.h
+++ b/apt-pkg/cacheiterators.h
@@ -367,27 +367,61 @@ class pkgCache::PrvIterator : public Iterator {
 	}
 };
 									/*}}}*/
-// Package file								/*{{{*/
-class pkgCache::PkgFileIterator : public Iterator {
+// Release file								/*{{{*/
+class pkgCache::RlsFileIterator : public Iterator {
 	protected:
-	inline PackageFile* OwnerPointer() const {
-		return (Owner != 0) ? Owner->PkgFileP : 0;
+	inline ReleaseFile* OwnerPointer() const {
+		return (Owner != 0) ? Owner->RlsFileP : 0;
 	}
 
 	public:
 	// Iteration
-	void operator ++(int) {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile;}
+	void operator ++(int) {if (S != Owner->RlsFileP) S = Owner->RlsFileP + S->NextFile;}
 	inline void operator ++() {operator ++(0);}
 
 	// Accessors
 	inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;}
 	inline const char *Archive() const {return S->Archive == 0?0:Owner->StrP + S->Archive;}
-	inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;}
 	inline const char *Version() const {return S->Version == 0?0:Owner->StrP + S->Version;}
 	inline const char *Origin() const {return S->Origin == 0?0:Owner->StrP + S->Origin;}
 	inline const char *Codename() const {return S->Codename ==0?0:Owner->StrP + S->Codename;}
 	inline const char *Label() const {return S->Label == 0?0:Owner->StrP + S->Label;}
 	inline const char *Site() const {return S->Site == 0?0:Owner->StrP + S->Site;}
+	inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return (S->Flags & flag) == flag; }
+
+	bool IsOk();
+	std::string RelStr();
+
+	// Constructors
+	inline RlsFileIterator() : Iterator() {}
+	inline RlsFileIterator(pkgCache &Owner) : Iterator(Owner, Owner.RlsFileP) {}
+	inline RlsFileIterator(pkgCache &Owner,ReleaseFile *Trg) : Iterator(Owner, Trg) {}
+};
+									/*}}}*/
+// Package file								/*{{{*/
+class pkgCache::PkgFileIterator : public Iterator {
+	protected:
+	inline PackageFile* OwnerPointer() const {
+		return (Owner != 0) ? Owner->PkgFileP : 0;
+	}
+
+	public:
+	// Iteration
+	void operator ++(int) {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile;}
+	inline void operator ++() {operator ++(0);}
+
+	// Accessors
+	inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;}
+	inline pkgCache::RlsFileIterator ReleaseFile() const {return RlsFileIterator(*Owner, Owner->RlsFileP + S->Release);}
+	inline const char *Archive() const {return S->Release == 0 ? Component() : ReleaseFile().Archive();}
+	inline const char *Version() const {return S->Release == 0 ? NULL : ReleaseFile().Version();}
+	inline const char *Origin() const {return S->Release == 0 ? NULL : ReleaseFile().Origin();}
+	inline const char *Codename() const {return S->Release == 0 ? NULL : ReleaseFile().Codename();}
+	inline const char *Label() const {return S->Release == 0 ? NULL : ReleaseFile().Label();}
+	inline const char *Site() const {return S->Release == 0 ? NULL : ReleaseFile().Site();}
+	inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return S->Release== 0 ? false : ReleaseFile().Flagged(flag);}
+	inline bool Flagged(pkgCache::Flag::PkgFFlags const flag) const {return (S->Flags & flag) == flag;}
+	inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;}
 	inline const char *Architecture() const {return S->Architecture == 0?0:Owner->StrP + S->Architecture;}
 	inline const char *IndexType() const {return S->IndexType == 0?0:Owner->StrP + S->IndexType;}
 
diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc
index 6edce5b6d..0fca60ba9 100644
--- a/apt-pkg/clean.cc
+++ b/apt-pkg/clean.cc
@@ -106,7 +106,7 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache)
 		 J.end() == false; ++J)
 	    {
 	       if (CleanInstalled == true &&
-		   (J.File()->Flags & pkgCache::Flag::NotSource) != 0)
+		   J.File().Flagged(pkgCache::Flag::NotSource))
 		  continue;
 	       IsFetchable = true;
 	       break;
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index 7aad65c0e..3a79cbc58 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -128,7 +128,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    if (Dist.empty())
       Dist = "/";
    ::URI Tmp(URI);
-   if (Gen.SelectFile(PackageFile,Tmp.Host,*this) == false)
+   if (Gen.SelectFile(PackageFile, *this, Target.Option(IndexTarget::COMPONENT)) == false)
       return _error->Error("Problem with SelectFile %s",PackageFile.c_str());
 
    // Store the IMS information
@@ -136,31 +136,10 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    pkgCacheGenerator::Dynamic DynFile(File);
    File->Size = Pkg.FileSize();
    File->mtime = Pkg.ModificationTime();
-   
+
    if (Gen.MergeList(Parser) == false)
       return _error->Error("Problem with MergeList %s",PackageFile.c_str());
 
-   // Check the release file
-   string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease");
-   bool releaseExists = false;
-   if (FileExists(ReleaseFile) == true)
-      releaseExists = true;
-   else
-      ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release");
-
-   if (releaseExists == true || FileExists(ReleaseFile) == true)
-   {
-      FileFd Rel;
-      // Beware: The 'Release' file might be clearsigned in case the
-      // signature for an 'InRelease' file couldn't be checked
-      if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false)
-	 return false;
-
-      if (_error->PendingError() == true)
-	 return false;
-      Parser.LoadReleaseInfo(File, Rel, Target.Option(IndexTarget::COMPONENT));
-   }
-   
    return true;
 }
 									/*}}}*/
@@ -221,17 +200,17 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
      debTranslationsParser TransParser(&Trans);
      if (_error->PendingError() == true)
        return false;
-     
+
      if (Prog != NULL)
 	Prog->SubProgress(0, Target.Description);
-     if (Gen.SelectFile(TranslationFile,string(),*this) == false)
+     if (Gen.SelectFile(TranslationFile, *this, Target.Option(IndexTarget::COMPONENT), pkgCache::Flag::NotSource) == false)
        return _error->Error("Problem with SelectFile %s",TranslationFile.c_str());
 
      // Store the IMS information
      pkgCache::PkgFileIterator TransFile = Gen.GetCurFile();
      TransFile->Size = Trans.FileSize();
      TransFile->mtime = Trans.ModificationTime();
-   
+
      if (Gen.MergeList(TransParser) == false)
        return _error->Error("Problem with MergeList %s",TranslationFile.c_str());
    }
@@ -305,18 +284,17 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 
    if (Prog != NULL)
       Prog->SubProgress(0,File);
-   if (Gen.SelectFile(File,string(),*this,pkgCache::Flag::NotSource) == false)
+   if (Gen.SelectFile(File, *this, "now", pkgCache::Flag::NotSource) == false)
       return _error->Error("Problem with SelectFile %s",File.c_str());
 
    // Store the IMS information
    pkgCache::PkgFileIterator CFile = Gen.GetCurFile();
+   pkgCacheGenerator::Dynamic DynFile(CFile);
    CFile->Size = Pkg.FileSize();
    CFile->mtime = Pkg.ModificationTime();
-   map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "now");
-   CFile->Archive = storage;
-   
+
    if (Gen.MergeList(Parser) == false)
-      return _error->Error("Problem with MergeList %s",File.c_str());   
+      return _error->Error("Problem with MergeList %s",File.c_str());
    return true;
 }
 									/*}}}*/
@@ -431,7 +409,7 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const
 
    // and give it to the list parser
    debDebFileParser Parser(DebControl, DebFile);
-   if(Gen.SelectFile(DebFile, "local", *this, pkgCache::Flag::LocalSource) == false)
+   if(Gen.SelectFile(DebFile, *this, "now", pkgCache::Flag::LocalSource) == false)
       return _error->Error("Problem with SelectFile %s", DebFile.c_str());
 
    pkgCache::PkgFileIterator File = Gen.GetCurFile();
diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h
index dd3f212b2..6b8c78e5a 100644
--- a/apt-pkg/deb/debindexfile.h
+++ b/apt-pkg/deb/debindexfile.h
@@ -45,7 +45,6 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile
    virtual bool HasPackages() const {return true;};
    virtual unsigned long Size() const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
-   bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog, unsigned long const Flag) const;
    virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const;
 
    debStatusIndex(std::string File);
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index b80b57bc4..c5e77b0ff 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -951,43 +951,6 @@ bool debListParser::Step()
    return false;
 }
 									/*}}}*/
-// ListParser::LoadReleaseInfo - Load the release information		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,
-				    FileFd &File, string component)
-{
-   // apt-secure does no longer download individual (per-section) Release
-   // file. to provide Component pinning we use the section name now
-   map_stringitem_t const storage = StoreString(pkgCacheGenerator::MIXED, component);
-   FileI->Component = storage;
-
-   pkgTagFile TagFile(&File, File.Size());
-   pkgTagSection Section;
-   if (_error->PendingError() == true || TagFile.Step(Section) == false)
-      return false;
-
-   std::string data;
-   #define APT_INRELEASE(TYPE, TAG, STORE) \
-   data = Section.FindS(TAG); \
-   if (data.empty() == false) \
-   { \
-      map_stringitem_t const storage = StoreString(pkgCacheGenerator::TYPE, data); \
-      STORE = storage; \
-   }
-   APT_INRELEASE(MIXED, "Suite", FileI->Archive)
-   APT_INRELEASE(MIXED, "Component", FileI->Component)
-   APT_INRELEASE(VERSIONNUMBER, "Version", FileI->Version)
-   APT_INRELEASE(MIXED, "Origin", FileI->Origin)
-   APT_INRELEASE(MIXED, "Codename", FileI->Codename)
-   APT_INRELEASE(MIXED, "Label", FileI->Label)
-   #undef APT_INRELEASE
-   Section.FindFlag("NotAutomatic", FileI->Flags, pkgCache::Flag::NotAutomatic);
-   Section.FindFlag("ButAutomaticUpgrades", FileI->Flags, pkgCache::Flag::ButAutomaticUpgrades);
-
-   return !_error->PendingError();
-}
-									/*}}}*/
 // ListParser::GetPrio - Convert the priority from a string		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h
index 6279d8399..420d5ff08 100644
--- a/apt-pkg/deb/deblistparser.h
+++ b/apt-pkg/deb/deblistparser.h
@@ -80,9 +80,9 @@ class APT_HIDDEN debListParser : public pkgCacheGenerator::ListParser
    virtual map_filesize_t Size() {return Section.size();};
 
    virtual bool Step();
-   
-   bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File,
-			std::string section);
+
+   bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File,
+			std::string const §ion);
 
    APT_PUBLIC static const char *ParseDepends(const char *Start,const char *Stop,
 	 std::string &Package,std::string &Ver,unsigned int &Op);
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 8e4c2be2d..994b95849 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -1,5 +1,6 @@
 #include 
 
+#include 
 #include 
 #include 
 #include 
@@ -10,10 +11,12 @@
 #include 
 #include 
 #include 
-#include 
 #include 
+#include 
+#include 
+#include 
+#include 
 
-#include 
 #include 
 #include 
 #include 
@@ -21,6 +24,11 @@
 #include 
 #include 
 
+#include 
+#include 
+#include 
+#include 
+
 using namespace std;
 
 string debReleaseIndex::MetaIndexInfo(const char *Type) const
@@ -37,6 +45,10 @@ string debReleaseIndex::MetaIndexInfo(const char *Type) const
    Info += Type;
    return Info;
 }
+std::string debReleaseIndex::Describe() const
+{
+   return MetaIndexInfo("Release");
+}
 
 string debReleaseIndex::MetaIndexFile(const char *Type) const
 {
@@ -339,6 +351,114 @@ debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section,
 		bool const &IsSrc): Section(Section), IsSrc(IsSrc)
 {}
 
+static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile)
+{
+   ReleaseFile = That->MetaIndexFile("InRelease");
+   bool releaseExists = false;
+   if (FileExists(ReleaseFile) == true)
+      releaseExists = true;
+   else
+   {
+      ReleaseFile = That->MetaIndexFile("Release");
+      if (FileExists(ReleaseFile))
+	 releaseExists = true;
+   }
+   return releaseExists;
+}
+
+bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/
+{
+   std::string ReleaseFile;
+   bool const releaseExists = ReleaseFileName(this, ReleaseFile);
+
+   ::URI Tmp(URI);
+   if (Gen.SelectReleaseFile(ReleaseFile, Tmp.Host) == false)
+      return _error->Error("Problem with SelectReleaseFile %s", ReleaseFile.c_str());
+
+   if (releaseExists == false)
+      return true;
+
+   FileFd Rel;
+   // Beware: The 'Release' file might be clearsigned in case the
+   // signature for an 'InRelease' file couldn't be checked
+   if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false)
+      return false;
+   if (_error->PendingError() == true)
+      return false;
+
+   // Store the IMS information
+   pkgCache::RlsFileIterator File = Gen.GetCurRlsFile();
+   pkgCacheGenerator::Dynamic DynFile(File);
+   // Rel can't be used as this is potentially a temporary file
+   struct stat Buf;
+   if (stat(ReleaseFile.c_str(), &Buf) != 0)
+      return _error->Errno("fstat", "Unable to stat file %s", ReleaseFile.c_str());
+   File->Size = Buf.st_size;
+   File->mtime = Buf.st_mtime;
+
+   pkgTagFile TagFile(&Rel, Rel.Size());
+   pkgTagSection Section;
+   if (_error->PendingError() == true || TagFile.Step(Section) == false)
+      return false;
+
+   std::string data;
+   #define APT_INRELEASE(TYPE, TAG, STORE) \
+   data = Section.FindS(TAG); \
+   if (data.empty() == false) \
+   { \
+      map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::TYPE, data); \
+      STORE = storage; \
+   }
+   APT_INRELEASE(MIXED, "Suite", File->Archive)
+   APT_INRELEASE(VERSIONNUMBER, "Version", File->Version)
+   APT_INRELEASE(MIXED, "Origin", File->Origin)
+   APT_INRELEASE(MIXED, "Codename", File->Codename)
+   APT_INRELEASE(MIXED, "Label", File->Label)
+   #undef APT_INRELEASE
+   Section.FindFlag("NotAutomatic", File->Flags, pkgCache::Flag::NotAutomatic);
+   Section.FindFlag("ButAutomaticUpgrades", File->Flags, pkgCache::Flag::ButAutomaticUpgrades);
+
+   return !_error->PendingError();
+}
+									/*}}}*/
+// ReleaseIndex::FindInCache - Find this index				/*{{{*/
+pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
+{
+   std::string ReleaseFile;
+   bool const releaseExists = ReleaseFileName(this, ReleaseFile);
+
+   pkgCache::RlsFileIterator File = Cache.RlsFileBegin();
+   for (; File.end() == false; ++File)
+   {
+       if (File->FileName == 0 || ReleaseFile != File.FileName())
+	 continue;
+
+       // empty means the file does not exist by "design"
+       if (releaseExists == false && File->Size == 0)
+	  return File;
+
+      struct stat St;
+      if (stat(File.FileName(),&St) != 0)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "ReleaseIndex::FindInCache - stat failed on " << File.FileName() << std::endl;
+	 return pkgCache::RlsFileIterator(Cache);
+      }
+      if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "ReleaseIndex::FindInCache - size (" << St.st_size << " <> " << File->Size
+			<< ") or mtime (" << St.st_mtime << " <> " << File->mtime
+			<< ") doesn't match for " << File.FileName() << std::endl;
+	 return pkgCache::RlsFileIterator(Cache);
+      }
+      return File;
+   }
+
+   return File;
+}
+									/*}}}*/
+
 class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type
 {
    protected:
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 0b1b08432..7e9942710 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -20,6 +20,8 @@ class pkgAcquire;
 class pkgIndexFile;
 class debDebPkgFileIndex;
 class IndexTarget;
+class pkgCacheGenerator;
+class OpProgress;
 
 class APT_HIDDEN debReleaseIndex : public metaIndex {
    public:
@@ -48,6 +50,10 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const;
    virtual std::vector GetIndexTargets() const;
 
+   virtual std::string Describe() const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
+
    std::string MetaIndexInfo(const char *Type) const;
    std::string MetaIndexFile(const char *Types) const;
    std::string MetaIndexURI(const char *Type) const;
diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index a7a66c75d..6ee939edd 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -205,8 +205,10 @@ pkgCache::VerIterator FindNowVersion(const pkgCache::PkgIterator &Pkg)
    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
       for (pkgCache::VerFileIterator Vf = Ver.FileList(); Vf.end() == false; ++Vf)
 	 for (pkgCache::PkgFileIterator F = Vf.File(); F.end() == false; ++F)
-	    if (F->Archive != 0 && strcmp(F.Archive(), "now") == 0)
+	 {
+	    if (F.Archive() != 0 && strcmp(F.Archive(), "now") == 0)
 	       return Ver;
+	 }
    return Ver;
 }
 									/*}}}*/
diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc
index b73c336db..36e1ac9ec 100644
--- a/apt-pkg/depcache.cc
+++ b/apt-pkg/depcache.cc
@@ -1685,13 +1685,13 @@ pkgCache::VerIterator pkgDepCache::Policy::GetCandidateVer(PkgIterator const &Pk
       
       for (VerFileIterator J = I.FileList(); J.end() == false; ++J)
       {
-	 if ((J.File()->Flags & Flag::NotSource) != 0)
+	 if (J.File().Flagged(Flag::NotSource))
 	    continue;
 
 	 /* Stash the highest version of a not-automatic source, we use it
 	    if there is nothing better */
-	 if ((J.File()->Flags & Flag::NotAutomatic) != 0 ||
-	     (J.File()->Flags & Flag::ButAutomaticUpgrades) != 0)
+	 if (J.File().Flagged(Flag::NotAutomatic) ||
+	     J.File().Flagged(Flag::ButAutomaticUpgrades))
 	 {
 	    if (Last.end() == true)
 	       Last = I;
diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc
index 3c6a7e30f..41cc2cdfe 100644
--- a/apt-pkg/edsp.cc
+++ b/apt-pkg/edsp.cc
@@ -129,7 +129,7 @@ void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgI
       signed short const p = Cache.GetPolicy().GetPriority(File);
       if (Pin < p)
 	 Pin = p;
-      if ((File->Flags & pkgCache::Flag::NotSource) != pkgCache::Flag::NotSource) {
+      if (File.Flagged(pkgCache::Flag::NotSource) == false) {
 	 string Release = File.RelStr();
 	 if (!Release.empty())
 	    Releases.insert(Release);
diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc
index d00536362..43dd44a79 100644
--- a/apt-pkg/edsp/edspindexfile.cc
+++ b/apt-pkg/edsp/edspindexfile.cc
@@ -49,15 +49,14 @@ bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 
    if (Prog != NULL)
       Prog->SubProgress(0,File);
-   if (Gen.SelectFile(File,std::string(),*this) == false)
+   if (Gen.SelectFile(File, *this, "edsp") == false)
       return _error->Error("Problem with SelectFile %s",File.c_str());
 
    // Store the IMS information
    pkgCache::PkgFileIterator CFile = Gen.GetCurFile();
+   pkgCacheGenerator::Dynamic DynFile(CFile);
    CFile->Size = Pkg.FileSize();
    CFile->mtime = Pkg.ModificationTime();
-   map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "edsp::scenario");
-   CFile->Archive = storage;
 
    if (Gen.MergeList(Parser) == false)
       return _error->Error("Problem with MergeList %s",File.c_str());
diff --git a/apt-pkg/edsp/edsplistparser.cc b/apt-pkg/edsp/edsplistparser.cc
index 212dc7840..d62abe709 100644
--- a/apt-pkg/edsp/edsplistparser.cc
+++ b/apt-pkg/edsp/edsplistparser.cc
@@ -86,8 +86,8 @@ bool edspListParser::ParseStatus(pkgCache::PkgIterator &Pkg,
 }
 									/*}}}*/
 // ListParser::LoadReleaseInfo - Load the release information		/*{{{*/
-APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::PkgFileIterator & /*FileI*/,
-				    FileFd & /*File*/, std::string /*component*/)
+APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::RlsFileIterator & /*FileI*/,
+				    FileFd & /*File*/, std::string const &/*component*/)
 {
    return true;
 }
diff --git a/apt-pkg/edsp/edsplistparser.h b/apt-pkg/edsp/edsplistparser.h
index 86cd77606..abe2ef139 100644
--- a/apt-pkg/edsp/edsplistparser.h
+++ b/apt-pkg/edsp/edsplistparser.h
@@ -34,8 +34,8 @@ class APT_HIDDEN edspListParser : public debListParser
    virtual MD5SumValue Description_md5();
    virtual unsigned short VersionHash();
 
-   bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File,
-			std::string section);
+   bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File,
+			std::string const §ion);
 
    edspListParser(FileFd *File, std::string const &Arch = "");
 
diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc
index 31a8ec009..35ab6c53b 100644
--- a/apt-pkg/metaindex.cc
+++ b/apt-pkg/metaindex.cc
@@ -1,4 +1,5 @@
 // Include Files                                                       /*{{{*/
+#include 
 #include 
 #include 
 
@@ -22,6 +23,22 @@ std::string metaIndex::LocalFileName() const
 }
 #endif
 
+std::string metaIndex::Describe() const
+{
+   return "Release";
+}
+
+pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache) const
+{
+   return pkgCache::RlsFileIterator(Cache);
+}
+
+bool metaIndex::Merge(pkgCacheGenerator &Gen,OpProgress *) const
+{
+   return Gen.SelectReleaseFile("", "");
+}
+
+
 metaIndex::metaIndex(std::string const &URI, std::string const &Dist,
       char const * const Type)
 : Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(false)
diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h
index 4e5de8be0..20c879a89 100644
--- a/apt-pkg/metaindex.h
+++ b/apt-pkg/metaindex.h
@@ -23,6 +23,8 @@ using std::string;
 
 class pkgAcquire;
 class IndexTarget;
+class pkgCacheGenerator;
+class OpProgress;
 
 class metaIndex
 {
@@ -51,6 +53,10 @@ class metaIndex
    virtual std::vector *GetIndexFiles() = 0;
    virtual bool IsTrusted() const = 0;
 
+   virtual std::string Describe() const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
+
    metaIndex(std::string const &URI, std::string const &Dist,
              char const * const Type);
    virtual ~metaIndex();
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index 864ae0f60..9fe382108 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -61,6 +61,7 @@ pkgCache::Header::Header()
    HeaderSz = sizeof(pkgCache::Header);
    GroupSz = sizeof(pkgCache::Group);
    PackageSz = sizeof(pkgCache::Package);
+   ReleaseFileSz = sizeof(pkgCache::ReleaseFile);
    PackageFileSz = sizeof(pkgCache::PackageFile);
    VersionSz = sizeof(pkgCache::Version);
    DescriptionSz = sizeof(pkgCache::Description);
@@ -74,6 +75,7 @@ pkgCache::Header::Header()
    VersionCount = 0;
    DescriptionCount = 0;
    DependsCount = 0;
+   ReleaseFileCount = 0;
    PackageFileCount = 0;
    VerFileCount = 0;
    DescFileCount = 0;
@@ -82,6 +84,7 @@ pkgCache::Header::Header()
    MaxDescFileSize = 0;
    
    FileList = 0;
+   RlsFileList = 0;
 #if APT_PKG_ABI < 413
    APT_IGNORE_DEPRECATED(StringList = 0;)
 #endif
@@ -102,6 +105,7 @@ bool pkgCache::Header::CheckSizes(Header &Against) const
    if (HeaderSz == Against.HeaderSz &&
        GroupSz == Against.GroupSz &&
        PackageSz == Against.PackageSz &&
+       ReleaseFileSz == Against.ReleaseFileSz &&
        PackageFileSz == Against.PackageFileSz &&
        VersionSz == Against.VersionSz &&
        DescriptionSz == Against.DescriptionSz &&
@@ -140,6 +144,7 @@ bool pkgCache::ReMap(bool const &Errorchecks)
    PkgP = (Package *)Map.Data();
    VerFileP = (VerFile *)Map.Data();
    DescFileP = (DescFile *)Map.Data();
+   RlsFileP = (ReleaseFile *)Map.Data();
    PkgFileP = (PackageFile *)Map.Data();
    VerP = (Version *)Map.Data();
    DescP = (Description *)Map.Data();
@@ -814,7 +819,7 @@ APT_PURE bool pkgCache::VerIterator::Downloadable() const
 {
    VerFileIterator Files = FileList();
    for (; Files.end() == false; ++Files)
-      if ((Files.File()->Flags & pkgCache::Flag::NotSource) != pkgCache::Flag::NotSource)
+      if (Files.File().Flagged(pkgCache::Flag::NotSource) == false)
 	 return true;
    return false;
 }
@@ -828,7 +833,7 @@ APT_PURE bool pkgCache::VerIterator::Automatic() const
    VerFileIterator Files = FileList();
    for (; Files.end() == false; ++Files)
       // Do not check ButAutomaticUpgrades here as it is kind of automatic…
-      if ((Files.File()->Flags & pkgCache::Flag::NotAutomatic) != pkgCache::Flag::NotAutomatic)
+      if (Files.File().Flagged(pkgCache::Flag::NotAutomatic) == false)
 	 return true;
    return false;
 }
@@ -861,27 +866,27 @@ string pkgCache::VerIterator::RelStr() const
    for (pkgCache::VerFileIterator I = this->FileList(); I.end() == false; ++I)
    {
       // Do not print 'not source' entries'
-      pkgCache::PkgFileIterator File = I.File();
-      if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
+      pkgCache::PkgFileIterator const File = I.File();
+      if (File.Flagged(pkgCache::Flag::NotSource))
 	 continue;
 
       // See if we have already printed this out..
       bool Seen = false;
       for (pkgCache::VerFileIterator J = this->FileList(); I != J; ++J)
       {
-	 pkgCache::PkgFileIterator File2 = J.File();
-	 if (File2->Label == 0 || File->Label == 0)
+	 pkgCache::PkgFileIterator const File2 = J.File();
+	 if (File2.Label() == 0 || File.Label() == 0)
 	    continue;
 
 	 if (strcmp(File.Label(),File2.Label()) != 0)
 	    continue;
 	 
-	 if (File2->Version == File->Version)
+	 if (File2.Version() == File.Version())
 	 {
 	    Seen = true;
 	    break;
 	 }
-	 if (File2->Version == 0 || File->Version == 0)
+	 if (File2.Version() == 0 || File.Version() == 0)
 	    break;
 	 if (strcmp(File.Version(),File2.Version()) == 0)
 	    Seen = true;
@@ -895,12 +900,12 @@ string pkgCache::VerIterator::RelStr() const
       else
 	 First = false;
       
-      if (File->Label != 0)
+      if (File.Label() != 0)
 	 Res = Res + File.Label() + ':';
 
-      if (File->Archive != 0)
+      if (File.Archive() != 0)
       {
-	 if (File->Version == 0)
+	 if (File.Version() == 0)
 	    Res += File.Archive();
 	 else
 	    Res = Res + File.Version() + '/' +  File.Archive();
@@ -908,7 +913,7 @@ string pkgCache::VerIterator::RelStr() const
       else
       {
 	 // No release file, print the host name that this came from
-	 if (File->Site == 0 || File.Site()[0] == 0)
+	 if (File.Site() == 0 || File.Site()[0] == 0)
 	    Res += "localhost";
 	 else
 	    Res += File.Site();
@@ -931,12 +936,12 @@ const char * pkgCache::VerIterator::MultiArchType() const
    return "none";
 }
 									/*}}}*/
-// PkgFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
+// RlsFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
 // ---------------------------------------------------------------------
 /* This stats the file and compares its stats with the ones that were
-   stored during generation. Date checks should probably also be 
+   stored during generation. Date checks should probably also be
    included here. */
-bool pkgCache::PkgFileIterator::IsOk()
+bool pkgCache::RlsFileIterator::IsOk()
 {
    struct stat Buf;
    if (stat(FileName(),&Buf) != 0)
@@ -948,10 +953,8 @@ bool pkgCache::PkgFileIterator::IsOk()
    return true;
 }
 									/*}}}*/
-// PkgFileIterator::RelStr - Return the release string			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-string pkgCache::PkgFileIterator::RelStr()
+// RlsFileIterator::RelStr - Return the release string			/*{{{*/
+string pkgCache::RlsFileIterator::RelStr()
 {
    string Res;
    if (Version() != 0)
@@ -964,8 +967,40 @@ string pkgCache::PkgFileIterator::RelStr()
       Res = Res + (Res.empty() == true?"n=":",n=")  + Codename();
    if (Label() != 0)
       Res = Res + (Res.empty() == true?"l=":",l=")  + Label();
-   if (Component() != 0)
-      Res = Res + (Res.empty() == true?"c=":",c=")  + Component();
+   return Res;
+}
+									/*}}}*/
+// PkgFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
+// ---------------------------------------------------------------------
+/* This stats the file and compares its stats with the ones that were
+   stored during generation. Date checks should probably also be
+   included here. */
+bool pkgCache::PkgFileIterator::IsOk()
+{
+   struct stat Buf;
+   if (stat(FileName(),&Buf) != 0)
+      return false;
+
+   if (Buf.st_size != (signed)S->Size || Buf.st_mtime != S->mtime)
+      return false;
+
+   return true;
+}
+									/*}}}*/
+string pkgCache::PkgFileIterator::RelStr()				/*{{{*/
+{
+   std::string Res;
+   if (ReleaseFile() == 0)
+   {
+      if (Component() != 0)
+	 Res = Res + (Res.empty() == true?"a=":",a=")  + Component();
+   }
+   else
+   {
+      Res = ReleaseFile().RelStr();
+      if (Component() != 0)
+	 Res = Res + (Res.empty() == true?"c=":",c=")  + Component();
+   }
    if (Architecture() != 0)
       Res = Res + (Res.empty() == true?"b=":",b=")  + Architecture();
    return Res;
diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h
index b4d56611a..696b3b94d 100644
--- a/apt-pkg/pkgcache.h
+++ b/apt-pkg/pkgcache.h
@@ -133,6 +133,7 @@ class pkgCache								/*{{{*/
    struct Header;
    struct Group;
    struct Package;
+   struct ReleaseFile;
    struct PackageFile;
    struct Version;
    struct Description;
@@ -150,6 +151,7 @@ class pkgCache								/*{{{*/
    class DescIterator;
    class DepIterator;
    class PrvIterator;
+   class RlsFileIterator;
    class PkgFileIterator;
    class VerFileIterator;
    class DescFileIterator;
@@ -192,9 +194,11 @@ class pkgCache								/*{{{*/
       enum PkgFlags {Auto=(1<<0),Essential=(1<<3),Important=(1<<4)};
       enum PkgFFlags {
 	 NotSource=(1<<0), /*!< packages can't be fetched from here, e.g. dpkg/status file */
-	 NotAutomatic=(1<<1), /*!< archive has a default pin of 1 */
-	 ButAutomaticUpgrades=(1<<2), /*!< (together with the previous) archive has a default pin of 100 */
-	 LocalSource=(1<<3), /*!< local sources can't and will not be verified by hashes */
+	 LocalSource=(1<<1), /*!< local sources can't and will not be verified by hashes */
+      };
+      enum ReleaseFileFlags {
+	 NotAutomatic=(1<<0), /*!< archive has a default pin of 1 */
+	 ButAutomaticUpgrades=(1<<1), /*!< (together with the previous) archive has a default pin of 100 */
       };
    };
    
@@ -215,6 +219,7 @@ class pkgCache								/*{{{*/
    Package *PkgP;
    VerFile *VerFileP;
    DescFile *DescFileP;
+   ReleaseFile *RlsFileP;
    PackageFile *PkgFileP;
    Version *VerP;
    Description *DescP;
@@ -247,6 +252,8 @@ class pkgCache								/*{{{*/
    inline PkgIterator PkgEnd();
    inline PkgFileIterator FileBegin();
    inline PkgFileIterator FileEnd();
+   inline RlsFileIterator RlsFileBegin();
+   inline RlsFileIterator RlsFileEnd();
 
    inline bool MultiArchCache() const { return MultiArchEnabled; }
    inline char const * NativeArch();
@@ -296,6 +303,7 @@ struct pkgCache::Header
    unsigned short HeaderSz;
    unsigned short GroupSz;
    unsigned short PackageSz;
+   unsigned short ReleaseFileSz;
    unsigned short PackageFileSz;
    unsigned short VersionSz;
    unsigned short DescriptionSz;
@@ -314,6 +322,7 @@ struct pkgCache::Header
    map_id_t VersionCount;
    map_id_t DescriptionCount;
    map_id_t DependsCount;
+   map_fileid_t ReleaseFileCount;
    map_fileid_t PackageFileCount;
    map_fileid_t VerFileCount;
    map_fileid_t DescFileCount;
@@ -324,6 +333,9 @@ struct pkgCache::Header
        The PackageFile structures are singly linked lists that represent
        all package files that have been merged into the cache. */
    map_pointer_t FileList;
+   /** \brief index of the first ReleaseFile structure */
+   map_pointer_t RlsFileList;
+
 #if APT_PKG_ABI < 413
    APT_DEPRECATED map_pointer_t StringList;
 #endif
@@ -482,15 +494,14 @@ struct pkgCache::Package
    unsigned long Flags;
 };
 									/*}}}*/
-// Package File structure						/*{{{*/
-/** \brief stores information about the files used to generate the cache
+// Release File structure						/*{{{*/
+/** \brief stores information about the release files used to generate the cache
 
-    Package files are referenced by Version structures to be able to know
-    after the generation still from which Packages file includes this Version
-    as we need this information later on e.g. for pinning. */
-struct pkgCache::PackageFile
+    PackageFiles reference ReleaseFiles as we need to keep record of which
+    version belongs to which release e.g. for pinning. */
+struct pkgCache::ReleaseFile
 {
-   /** \brief physical disk file that this PackageFile represents */
+   /** \brief physical disk file that this ReleaseFile represents */
    map_stringitem_t FileName;
    /** \brief the release information
 
@@ -498,13 +509,47 @@ struct pkgCache::PackageFile
        release information means. */
    map_stringitem_t Archive;
    map_stringitem_t Codename;
-   map_stringitem_t Component;
    map_stringitem_t Version;
    map_stringitem_t Origin;
    map_stringitem_t Label;
-   map_stringitem_t Architecture;
    /** \brief The site the index file was fetched from */
    map_stringitem_t Site;
+
+   /** \brief Size of the file
+
+       Used together with the modification time as a
+       simple check to ensure that the Packages
+       file has not been altered since Cache generation. */
+   map_filesize_t Size;
+   /** \brief Modification time for the file */
+   time_t mtime;
+
+   /** @TODO document PackageFile::Flags */
+   unsigned long Flags;
+
+   // Linked list
+   /** \brief Link to the next ReleaseFile in the Cache */
+   map_pointer_t NextFile;
+   /** \brief unique sequel ID */
+   should_be_map_fileid_t ID;
+};
+									/*}}}*/
+// Package File structure						/*{{{*/
+/** \brief stores information about the files used to generate the cache
+
+    Package files are referenced by Version structures to be able to know
+    after the generation still from which Packages file includes this Version
+    as we need this information later on e.g. for pinning. */
+struct pkgCache::PackageFile
+{
+   /** \brief physical disk file that this PackageFile represents */
+   map_stringitem_t FileName;
+   /** \brief the release information */
+   map_pointer_t Release;
+
+   map_stringitem_t Component;
+   map_stringitem_t Architecture;
+
    /** \brief indicates what sort of index file this is
 
        @TODO enumerate at least the possible indexes */
@@ -744,6 +789,11 @@ inline pkgCache::PkgFileIterator pkgCache::FileBegin()
        {return PkgFileIterator(*this,PkgFileP + HeaderP->FileList);}
 inline pkgCache::PkgFileIterator pkgCache::FileEnd()
        {return PkgFileIterator(*this,PkgFileP);}
+inline pkgCache::RlsFileIterator pkgCache::RlsFileBegin()
+       {return RlsFileIterator(*this,RlsFileP + HeaderP->RlsFileList);}
+inline pkgCache::RlsFileIterator pkgCache::RlsFileEnd()
+       {return RlsFileIterator(*this,RlsFileP);}
+
 
 // Oh I wish for Real Name Space Support
 class pkgCache::Namespace						/*{{{*/
@@ -755,6 +805,7 @@ class pkgCache::Namespace						/*{{{*/
    typedef pkgCache::DescIterator DescIterator;
    typedef pkgCache::DepIterator DepIterator;
    typedef pkgCache::PrvIterator PrvIterator;
+   typedef pkgCache::RlsFileIterator RlsFileIterator;
    typedef pkgCache::PkgFileIterator PkgFileIterator;
    typedef pkgCache::VerFileIterator VerFileIterator;
    typedef pkgCache::Version Version;
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index ba454f057..2d0e3960d 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -54,10 +54,8 @@ using std::string;
 /* We set the dirty flag and make sure that is written to the disk */
 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
 		    Map(*pMap), Cache(pMap,false), Progress(Prog),
-		    FoundFileDeps(0)
+		     CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0)
 {
-   CurrentFile = 0;
-
    if (_error->PendingError() == true)
       return;
 
@@ -145,6 +143,7 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
    Cache.ReMap(false);
 
    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
+   CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 
    for (std::vector::const_iterator i = Dynamic::toReMap.begin();
 	i != Dynamic::toReMap.end(); ++i)
@@ -167,6 +166,9 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
    for (std::vector::const_iterator i = Dynamic::toReMap.begin();
 	i != Dynamic::toReMap.end(); ++i)
       (*i)->ReMap(oldMap, newMap);
+   for (std::vector::const_iterator i = Dynamic::toReMap.begin();
+	i != Dynamic::toReMap.end(); ++i)
+      (*i)->ReMap(oldMap, newMap);
 }									/*}}}*/
 // CacheGenerator::WriteStringInMap					/*{{{*/
 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
@@ -1072,13 +1074,47 @@ bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*
    return Hash == Ver->Hash;
 }
 									/*}}}*/
+// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to	/*{{{*/
+bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
+				   unsigned long Flags)
+{
+   if (File.empty() && Site.empty())
+   {
+      CurrentRlsFile = NULL;
+      return true;
+   }
+
+   // Get some space for the structure
+   map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
+   if (unlikely(idxFile == 0))
+      return false;
+   CurrentRlsFile = Cache.RlsFileP + idxFile;
+
+   // Fill it in
+   map_stringitem_t const idxFileName = WriteStringInMap(File);
+   map_stringitem_t const idxSite = StoreString(MIXED, Site);
+   if (unlikely(idxFileName == 0 || idxSite == 0))
+      return false;
+   CurrentRlsFile->FileName = idxFileName;
+   CurrentRlsFile->Site = idxSite;
+   CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
+   CurrentRlsFile->Flags = Flags;
+   CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
+   RlsFileName = File;
+   Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
+   Cache.HeaderP->ReleaseFileCount++;
+
+   return true;
+}
+									/*}}}*/
 // CacheGenerator::SelectFile - Select the current file being parsed	/*{{{*/
 // ---------------------------------------------------------------------
 /* This is used to select which file is to be associated with all newly
    added versions. The caller is responsible for setting the IMS fields. */
-bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
-				   const pkgIndexFile &Index,
-				   unsigned long Flags)
+bool pkgCacheGenerator::SelectFile(std::string const &File,
+				   pkgIndexFile const &Index,
+				   std::string const &Component,
+				   unsigned long const Flags)
 {
    // Get some space for the structure
    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
@@ -1088,18 +1124,24 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 
    // Fill it in
    map_stringitem_t const idxFileName = WriteStringInMap(File);
-   map_stringitem_t const idxSite = StoreString(MIXED, Site);
-   if (unlikely(idxFileName == 0 || idxSite == 0))
+   if (unlikely(idxFileName == 0))
       return false;
    CurrentFile->FileName = idxFileName;
-   CurrentFile->Site = idxSite;
    CurrentFile->NextFile = Cache.HeaderP->FileList;
-   CurrentFile->Flags = Flags;
    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
    if (unlikely(idxIndexType == 0))
       return false;
    CurrentFile->IndexType = idxIndexType;
+   map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
+   if (unlikely(component == 0))
+      return false;
+   CurrentFile->Component = component;
+   CurrentFile->Flags = Flags;
+   if (CurrentRlsFile != NULL)
+      CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
+   else
+      CurrentFile->Release = 0;
    PkgFileName = File;
    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
    Cache.HeaderP->PackageFileCount++;
@@ -1174,35 +1216,59 @@ static bool CheckValidity(const string &CacheFile,
       _error->Discard();
       return false;
    }
-   
+
+   SPtrArray RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
+   memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
+   std::vector Files;
+   for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
+   {
+      if (Debug == true)
+	 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
+      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache);
+      if (RlsFile.end() == true)
+      {
+	 if (Debug == true)
+	    std::clog << "FindInCache returned end-Pointer" << std::endl;
+	 return false;
+      }
+
+      RlsVisited[RlsFile->ID] = true;
+      if (Debug == true)
+	 std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
+
+      std::vector  *Indexes = (*i)->GetIndexFiles();
+      for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
+	 if ((*j)->HasPackages())
+	    Files.push_back (*j);
+   }
+   for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
+      if (RlsVisited[I] == false)
+      {
+	 if (Debug == true)
+	    std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
+	 return false;
+      }
+
+   for (; Start != End; ++Start)
+      Files.push_back(*Start);
+
    /* Now we check every index file, see if it is in the cache,
       verify the IMS data and check that it is on the disk too.. */
    SPtrArray Visited = new bool[Cache.HeaderP->PackageFileCount];
    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
-   for (; Start != End; ++Start)
+   for (std::vector::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
    {
       if (Debug == true)
-	 std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
-      if ((*Start)->HasPackages() == false)
-      {
-         if (Debug == true)
-	    std::clog << "Has NO packages" << std::endl;
-	 continue;
-      }
-    
-      if ((*Start)->Exists() == false)
+	 std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
+      if ((*PkgFile)->Exists() == false)
       {
-#if 0 // mvo: we no longer give a message here (Default Sources spec)
-	 _error->WarningE("stat",_("Couldn't stat source package list %s"),
-			  (*Start)->Describe().c_str());
-#endif
          if (Debug == true)
 	    std::clog << "file doesn't exist" << std::endl;
 	 continue;
       }
 
       // FindInCache is also expected to do an IMS check.
-      pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
+      pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
       if (File.end() == true)
       {
 	 if (Debug == true)
@@ -1214,15 +1280,15 @@ static bool CheckValidity(const string &CacheFile,
       if (Debug == true)
 	 std::clog << "with ID " << File->ID << " is valid" << std::endl;
    }
-   
+
    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
       if (Visited[I] == false)
       {
 	 if (Debug == true)
-	    std::clog << "File with ID" << I << " wasn't visited" << std::endl;
+	    std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
 	 return false;
       }
-   
+
    if (_error->PendingError() == true)
    {
       if (Debug == true)
@@ -1243,9 +1309,20 @@ static bool CheckValidity(const string &CacheFile,
 // ---------------------------------------------------------------------
 /* Size is kind of an abstract notion that is only used for the progress
    meter */
-static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
+static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
 {
    map_filesize_t TotalSize = 0;
+   if (List !=  NULL)
+   {
+      for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
+      {
+	 std::vector  *Indexes = (*i)->GetIndexFiles();
+	 for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
+	    if ((*j)->HasPackages() == true)
+	       TotalSize += (*j)->Size();
+      }
+   }
+
    for (; Start < End; ++Start)
    {
       if ((*Start)->HasPackages() == false)
@@ -1261,11 +1338,63 @@ static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
 static bool BuildCache(pkgCacheGenerator &Gen,
 		       OpProgress *Progress,
 		       map_filesize_t &CurrentSize,map_filesize_t TotalSize,
+		       pkgSourceList const * const List,
 		       FileIterator Start, FileIterator End)
 {
+   std::vector Files;
+   bool const HasFileDeps = Gen.HasFileDeps();
+
+   if (List !=  NULL)
+   {
+      for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
+      {
+	 if ((*i)->FindInCache(Gen.GetCache()).end() == false)
+	 {
+	    _error->Warning("Duplicate sources.list entry %s",
+		  (*i)->Describe().c_str());
+	    continue;
+	 }
+
+	 if ((*i)->Merge(Gen, Progress) == false)
+	    return false;
+
+	 std::vector  *Indexes = (*i)->GetIndexFiles();
+	 for (std::vector::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
+	 {
+	    if (HasFileDeps)
+	       Files.push_back(*I);
+
+	    if ((*I)->HasPackages() == false)
+	       continue;
+
+	    if ((*I)->Exists() == false)
+	       continue;
+
+	    if ((*I)->FindInCache(Gen.GetCache()).end() == false)
+	    {
+	       _error->Warning("Duplicate sources.list entry %s",
+		     (*I)->Describe().c_str());
+	       continue;
+	    }
+
+	    map_filesize_t Size = (*I)->Size();
+	    if (Progress != NULL)
+	       Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
+	    CurrentSize += Size;
+
+	    if ((*I)->Merge(Gen,Progress) == false)
+	       return false;
+	 }
+      }
+   }
+
+   Gen.SelectReleaseFile("", "");
    FileIterator I;
    for (I = Start; I != End; ++I)
    {
+      if (HasFileDeps)
+	 Files.push_back(*I);
+
       if ((*I)->HasPackages() == false)
 	 continue;
       
@@ -1288,13 +1417,13 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 	 return false;
    }   
 
-   if (Gen.HasFileDeps() == true)
+   if (HasFileDeps == true)
    {
       if (Progress != NULL)
 	 Progress->Done();
-      TotalSize = ComputeSize(Start, End);
+      TotalSize = ComputeSize(List, Start, End);
       CurrentSize = 0;
-      for (I = Start; I != End; ++I)
+      for (std::vector::const_iterator I = Files.begin(); I != Files.end(); ++I)
       {
 	 map_filesize_t Size = (*I)->Size();
 	 if (Progress != NULL)
@@ -1339,6 +1468,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 
    std::vector Files;
+   /*
    for (std::vector::const_iterator i = List.begin();
         i != List.end();
         ++i)
@@ -1349,8 +1479,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 	   ++j)
          Files.push_back (*j);
    }
-   
-   map_filesize_t const EndOfSource = Files.size();
+*/
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
@@ -1442,8 +1571,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    // Lets try the source cache.
    map_filesize_t CurrentSize = 0;
    map_filesize_t TotalSize = 0;
-   if (CheckValidity(SrcCacheFile, List, Files.begin(),
-		     Files.begin()+EndOfSource) == true)
+   if (CheckValidity(SrcCacheFile, List, Files.end(),
+		     Files.end()) == true)
    {
       if (Debug == true)
 	 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
@@ -1455,28 +1584,28 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 				SCacheF.Size()) == false)
 	 return false;
 
-      TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
+      TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
 
       // Build the status cache
       pkgCacheGenerator Gen(Map.Get(),Progress);
       if (_error->PendingError() == true)
 	 return false;
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin()+EndOfSource,Files.end()) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
+		     Files.begin(),Files.end()) == false)
 	 return false;
    }
    else
    {
       if (Debug == true)
 	 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
-      TotalSize = ComputeSize(Files.begin(),Files.end());
+      TotalSize = ComputeSize(&List, Files.begin(),Files.end());
       
       // Build the source cache
       pkgCacheGenerator Gen(Map.Get(),Progress);
       if (_error->PendingError() == true)
 	 return false;
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin(),Files.begin()+EndOfSource) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
+		     Files.end(),Files.end()) == false)
 	 return false;
       
       // Write it back
@@ -1503,8 +1632,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
       }
       
       // Build the status cache
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin()+EndOfSource,Files.end()) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
+		     Files.begin(), Files.end()) == false)
 	 return false;
    }
    if (Debug == true)
@@ -1536,7 +1665,6 @@ APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **Ou
 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
 {
    std::vector Files;
-   map_filesize_t EndOfSource = Files.size();
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
@@ -1544,7 +1672,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    map_filesize_t CurrentSize = 0;
    map_filesize_t TotalSize = 0;
    
-   TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
+   TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
    
    // Build the status cache
    if (Progress != NULL)
@@ -1552,8 +1680,8 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    pkgCacheGenerator Gen(Map.Get(),Progress);
    if (_error->PendingError() == true)
       return false;
-   if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		  Files.begin()+EndOfSource,Files.end()) == false)
+   if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
+		  Files.begin(), Files.end()) == false)
       return false;
 
    if (_error->PendingError() == true)
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index c4ace713d..2795d09d6 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -69,7 +69,9 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    DynamicMMap ⤅
    pkgCache Cache;
    OpProgress *Progress;
-   
+
+   std::string RlsFileName;
+   pkgCache::ReleaseFile *CurrentRlsFile;
    std::string PkgFileName;
    pkgCache::PackageFile *CurrentFile;
 
@@ -100,12 +102,14 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    inline map_stringitem_t StoreString(enum StringType const type, const std::string &S) {return StoreString(type, S.c_str(),S.length());};
 
    void DropProgress() {Progress = 0;};
-   bool SelectFile(const std::string &File,const std::string &Site,pkgIndexFile const &Index,
-		   unsigned long Flags = 0);
+   bool SelectFile(const std::string &File,pkgIndexFile const &Index, std::string const &Component, unsigned long Flags = 0);
+   bool SelectReleaseFile(const std::string &File, const std::string &Site, unsigned long Flags = 0);
    bool MergeList(ListParser &List,pkgCache::VerIterator *Ver = 0);
    inline pkgCache &GetCache() {return Cache;};
    inline pkgCache::PkgFileIterator GetCurFile() 
          {return pkgCache::PkgFileIterator(Cache,CurrentFile);};
+   inline pkgCache::RlsFileIterator GetCurRlsFile() 
+         {return pkgCache::RlsFileIterator(Cache,CurrentRlsFile);};
 
    bool HasFileDeps() {return FoundFileDeps;};
    bool MergeFileProvides(ListParser &List);
diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index 00693ce54..bd40ad2d9 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -63,9 +63,9 @@ pkgPolicy::pkgPolicy(pkgCache *Owner) : Pins(0), PFPriority(0), Cache(Owner)
       pkgVersionMatch vm("", pkgVersionMatch::None);
       for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
       {
-	 if ((F->Archive != 0 && vm.ExpressionMatches(DefRel, F.Archive()) == true) ||
-	     (F->Codename != 0 && vm.ExpressionMatches(DefRel, F.Codename()) == true) ||
-	     (F->Version != 0 && vm.ExpressionMatches(DefRel, F.Version()) == true) ||
+	 if (vm.ExpressionMatches(DefRel, F.Archive()) ||
+	     vm.ExpressionMatches(DefRel, F.Codename()) ||
+	     vm.ExpressionMatches(DefRel, F.Version()) ||
 	     (DefRel.length() > 2 && DefRel[1] == '='))
 	    found = true;
       }
@@ -86,11 +86,11 @@ bool pkgPolicy::InitDefaults()
    for (pkgCache::PkgFileIterator I = Cache->FileBegin(); I != Cache->FileEnd(); ++I)
    {
       PFPriority[I->ID] = 500;
-      if ((I->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
+      if (I.Flagged(pkgCache::Flag::NotSource))
 	 PFPriority[I->ID] = 100;
-      else if ((I->Flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades)
+      else if (I.Flagged(pkgCache::Flag::ButAutomaticUpgrades))
 	 PFPriority[I->ID] = 100;
-      else if ((I->Flags & pkgCache::Flag::NotAutomatic) == pkgCache::Flag::NotAutomatic)
+      else if (I.Flagged(pkgCache::Flag::NotAutomatic))
 	 PFPriority[I->ID] = 1;
    }
 
@@ -170,8 +170,7 @@ pkgCache::VerIterator pkgPolicy::GetCandidateVer(pkgCache::PkgIterator const &Pk
 	    then it is not a candidate for installation, ever. This weeds
 	    out bogus entries that may be due to config-file states, or
 	    other. */
-	 if ((VF.File()->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource &&
-	     instVer == false)
+	 if (VF.File().Flagged(pkgCache::Flag::NotSource) && instVer == false)
 	    continue;
 
 	 signed Prio = PFPriority[VF.File()->ID];
diff --git a/apt-pkg/versionmatch.cc b/apt-pkg/versionmatch.cc
index 284098bdf..86c1b7d4a 100644
--- a/apt-pkg/versionmatch.cc
+++ b/apt-pkg/versionmatch.cc
@@ -137,7 +137,10 @@ pkgVersionMatch::pkgVersionMatch(string Data,MatchType Type) : Type(Type)
 // ---------------------------------------------------------------------
 /* */
 bool pkgVersionMatch::MatchVer(const char *A,string B,bool Prefix)
-{   
+{
+   if (A == NULL)
+      return false;
+
    const char *Ab = A;
    const char *Ae = Ab + strlen(A);
    
@@ -178,13 +181,16 @@ pkgCache::VerIterator pkgVersionMatch::Find(pkgCache::PkgIterator Pkg)
    // This will be Ended by now.
    return Ver;
 }
+									/*}}}*/
 
 #ifndef FNM_CASEFOLD
 #define FNM_CASEFOLD 0
 #endif
 
-bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string)
+bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string)/*{{{*/
 {
+   if (pattern == NULL || string == NULL)
+      return false;
    if (pattern[0] == '/') {
       size_t length = strlen(pattern);
       if (pattern[length - 1] == '/') {
@@ -230,38 +236,30 @@ bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File)
 	 return false;
 
       if (RelVerStr.empty() == false)
-	 if (File->Version == 0 ||
-	     (MatchVer(File.Version(),RelVerStr,RelVerPrefixMatch) == false &&
-	      ExpressionMatches(RelVerStr, File.Version()) == false))
+	 if (MatchVer(File.Version(),RelVerStr,RelVerPrefixMatch) == false &&
+	       ExpressionMatches(RelVerStr, File.Version()) == false)
 	    return false;
       if (RelOrigin.empty() == false)
-	 if (File->Origin == 0 || !ExpressionMatches(RelOrigin,File.Origin()))
+	 if (!ExpressionMatches(RelOrigin,File.Origin()))
 	    return false;
       if (RelArchive.empty() == false)
-	 if (File->Archive == 0 ||
-	     !ExpressionMatches(RelArchive,File.Archive()))
+	 if (!ExpressionMatches(RelArchive,File.Archive()))
             return false;
       if (RelCodename.empty() == false)
-	 if (File->Codename == 0 ||
-	     !ExpressionMatches(RelCodename,File.Codename()))
+	 if (!ExpressionMatches(RelCodename,File.Codename()))
             return false;
       if (RelRelease.empty() == false)
-	 if ((File->Archive == 0 ||
-	     !ExpressionMatches(RelRelease,File.Archive())) &&
-             (File->Codename == 0 ||
-	      !ExpressionMatches(RelRelease,File.Codename())))
+	 if (!ExpressionMatches(RelRelease,File.Archive()) &&
+             !ExpressionMatches(RelRelease,File.Codename()))
 	       return false;
       if (RelLabel.empty() == false)
-	 if (File->Label == 0 ||
-	     !ExpressionMatches(RelLabel,File.Label()))
+	 if (!ExpressionMatches(RelLabel,File.Label()))
 	    return false;
       if (RelComponent.empty() == false)
-	 if (File->Component == 0 ||
-	     !ExpressionMatches(RelComponent,File.Component()))
+	 if (!ExpressionMatches(RelComponent,File.Component()))
 	    return false;
       if (RelArchitecture.empty() == false)
-	 if (File->Architecture == 0 ||
-	     !ExpressionMatches(RelArchitecture,File.Architecture()))
+	 if (!ExpressionMatches(RelArchitecture,File.Architecture()))
 	    return false;
       return true;
    }
@@ -269,11 +267,14 @@ bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File)
    if (Type == Origin)
    {
       if (OrSite.empty() == false) {
-	 if (File->Site == 0)
+	 if (File.Site() == NULL)
 	    return false;
       } else // so we are talking about file:// or status file
-	 if (strcmp(File.Site(),"") == 0 && File->Archive != 0 && strcmp(File.Archive(),"now") == 0) // skip the status file
+      {
+	 pkgCache::RlsFileIterator const RlsFile = File.ReleaseFile();
+	 if (strcmp(File.Site(),"") == 0 && RlsFile->Archive != 0 && strcmp(RlsFile.Archive(),"now") == 0) // skip the status file
 	    return false;
+      }
       return (ExpressionMatches(OrSite, File.Site())); /* both strings match */
    }
 
diff --git a/apt-pkg/versionmatch.h b/apt-pkg/versionmatch.h
index 4c8f704c8..bb950b9c7 100644
--- a/apt-pkg/versionmatch.h
+++ b/apt-pkg/versionmatch.h
@@ -70,8 +70,8 @@ class pkgVersionMatch
    enum MatchType {None = 0,Version,Release,Origin} Type;
 
    bool MatchVer(const char *A,std::string B,bool Prefix) APT_PURE;
-   bool ExpressionMatches(const char *pattern, const char *string);
-   bool ExpressionMatches(const std::string& pattern, const char *string);
+   static bool ExpressionMatches(const char *pattern, const char *string);
+   static bool ExpressionMatches(const std::string& pattern, const char *string);
    bool FileMatch(pkgCache::PkgFileIterator File);
    pkgCache::VerIterator Find(pkgCache::PkgIterator Pkg);
 
diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index 690b03bcc..3b8ebc96d 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -413,17 +413,21 @@ static bool Stats(CommandLine &)
 	    stritems.insert(Prv->ProvideVersion);
       }
    }
-   for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
+   for (pkgCache::RlsFileIterator F = Cache->RlsFileBegin(); F != Cache->RlsFileEnd(); ++F)
    {
       stritems.insert(F->FileName);
       stritems.insert(F->Archive);
       stritems.insert(F->Codename);
-      stritems.insert(F->Component);
       stritems.insert(F->Version);
       stritems.insert(F->Origin);
       stritems.insert(F->Label);
-      stritems.insert(F->Architecture);
       stritems.insert(F->Site);
+   }
+   for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
+   {
+      stritems.insert(F->FileName);
+      stritems.insert(F->Architecture);
+      stritems.insert(F->Component);
       stritems.insert(F->IndexType);
    }
    unsigned long Size = 0;
@@ -446,6 +450,7 @@ static bool Stats(CommandLine &)
       APT_CACHESIZE(VersionCount, VersionSz) +
       APT_CACHESIZE(DescriptionCount, DescriptionSz) +
       APT_CACHESIZE(DependsCount, DependencySz) +
+      APT_CACHESIZE(ReleaseFileCount, ReleaseFileSz) +
       APT_CACHESIZE(PackageFileCount, PackageFileSz) +
       APT_CACHESIZE(VerFileCount, VerFileSz) +
       APT_CACHESIZE(DescFileCount, DescFileSz) +
diff --git a/test/integration/framework b/test/integration/framework
index 110758e82..7b03c09ef 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -137,7 +137,14 @@ dpkgcheckbuilddeps() {
 	command dpkg-checkbuilddeps --admindir=${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg "$@"
 }
 gdb() {
-	local CMD="$1"
+	local CMD
+	case "$1" in
+	aptget) CMD="apt-get";;
+	aptcache) CMD="apt-cache";;
+	aptmark) CMD="apt-mark";;
+	apthelper) CMD="apt-helper";;
+	*) CMD="$1";;
+	esac
 	shift
 	runapt command gdb --quiet -ex run "${BUILDDIRECTORY}/$CMD" --args "${BUILDDIRECTORY}/$CMD" "$@"
 }
diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning
index 9cd54264b..9f7f457ae 100755
--- a/test/integration/test-policy-pinning
+++ b/test/integration/test-policy-pinning
@@ -22,7 +22,7 @@ testequalpolicy() {
      release a=now
  $(echo "$AP" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}  Packages
      release c=
-Pinned packages:" aptcache policy $*
+Pinned packages:" aptcache policy "$@"
 }
 
 testglobalpolicy() {
-- 
cgit v1.2.3-70-g09d2


From 3fd89e62e985c89b1f9a545ab72c20987b756aff Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 12 Jun 2015 12:06:29 +0200
Subject: implement default apt-get file --release-info mode

Selecting targets based on the Release they belong to isn't to
unrealistic. In fact, it is assumed to be the most used case so it is
made the default especially as this allows to bundle another thing we
have to be careful with: Filenames and only showing targets we have
acquired.

Closes: 752702
---
 apt-pkg/deb/debmetaindex.cc                        |  4 +-
 apt-pkg/deb/debmetaindex.h                         |  2 +-
 apt-pkg/indexfile.cc                               | 12 ++++
 apt-pkg/indexfile.h                                |  1 +
 apt-pkg/metaindex.cc                               |  2 +-
 apt-pkg/metaindex.h                                |  2 +-
 apt-pkg/pkgcachegen.cc                             |  4 +-
 apt-private/private-cmndline.cc                    |  1 +
 cmdline/apt-get.cc                                 | 26 ++++++++-
 doc/acquire-additional-files.txt                   | 65 +++++++++++++---------
 test/integration/test-apt-acquire-additional-files |  6 ++
 11 files changed, 88 insertions(+), 37 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 994b95849..b328fcea8 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -422,7 +422,7 @@ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/
 }
 									/*}}}*/
 // ReleaseIndex::FindInCache - Find this index				/*{{{*/
-pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
+pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool const ModifyCheck) const
 {
    std::string ReleaseFile;
    bool const releaseExists = ReleaseFileName(this, ReleaseFile);
@@ -434,7 +434,7 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
 	 continue;
 
        // empty means the file does not exist by "design"
-       if (releaseExists == false && File->Size == 0)
+       if (ModifyCheck == false || (releaseExists == false && File->Size == 0))
 	  return File;
 
       struct stat St;
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 7e9942710..b448ecc53 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -51,7 +51,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual std::vector GetIndexTargets() const;
 
    virtual std::string Describe() const;
-   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    std::string MetaIndexInfo(const char *Type) const;
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 33fb48e35..605bbeb47 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -137,6 +137,18 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(CREATED_BY);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
+      case EXISTING_FILENAME:
+	 std::string const filename = Option(FILENAME);
+	 std::vector const types = APT::Configuration::getCompressionTypes();
+	 for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t)
+	 {
+	    if (t->empty())
+	       continue;
+	    std::string const file = (*t == "uncompressed") ? filename : (filename + "." + *t);
+	    if (FileExists(file))
+	       return file;
+	 }
+	 return "";
    }
    std::map::const_iterator const M = Options.find(Key);
    if (M == Options.end())
diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 220c415ac..042e5c2f7 100644
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -81,6 +81,7 @@ class IndexTarget							/*{{{*/
       CREATED_BY,
       TARGET_OF,
       FILENAME,
+      EXISTING_FILENAME,
    };
    std::string Option(OptionKeys const Key) const;
    std::string Format(std::string format) const;
diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc
index 35ab6c53b..3c1b696bd 100644
--- a/apt-pkg/metaindex.cc
+++ b/apt-pkg/metaindex.cc
@@ -28,7 +28,7 @@ std::string metaIndex::Describe() const
    return "Release";
 }
 
-pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache) const
+pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache, bool const) const
 {
    return pkgCache::RlsFileIterator(Cache);
 }
diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h
index 20c879a89..e1810fb27 100644
--- a/apt-pkg/metaindex.h
+++ b/apt-pkg/metaindex.h
@@ -54,7 +54,7 @@ class metaIndex
    virtual bool IsTrusted() const = 0;
 
    virtual std::string Describe() const;
-   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    metaIndex(std::string const &URI, std::string const &Dist,
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 2d0e3960d..9db4ef41f 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -1224,7 +1224,7 @@ static bool CheckValidity(const string &CacheFile,
    {
       if (Debug == true)
 	 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
-      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache);
+      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
       if (RlsFile.end() == true)
       {
 	 if (Debug == true)
@@ -1348,7 +1348,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    {
       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
       {
-	 if ((*i)->FindInCache(Gen.GetCache()).end() == false)
+	 if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
 	 {
 	    _error->Warning("Duplicate sources.list entry %s",
 		  (*i)->Describe().c_str());
diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc
index 458051bf2..11e88b1e7 100644
--- a/apt-private/private-cmndline.cc
+++ b/apt-private/private-cmndline.cc
@@ -166,6 +166,7 @@ static bool addArgumentsAPTGet(std::vector &Args, char const
    else if (CmdMatches("files"))
    {
       addArg(0,"format","APT::Get::Files::Format", CommandLine::HasArg);
+      addArg(0,"release-info","APT::Get::Files::ReleaseInfo", 0);
    }
    else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
 	    CmdMatches("markauto", "unmarkauto")) // deprecated commands
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index 0fff2db58..a4cd3c377 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1628,15 +1628,34 @@ static bool DoFiles(CommandLine &CmdL)
       return false;
 
    std::string const Format = _config->Find("APT::Get::Files::Format");
+   bool const ReleaseInfo = _config->FindB("APT::Get::Files::ReleaseInfo", true);
    bool Filtered = CmdL.FileSize() > 1;
    for (pkgSourceList::const_iterator S = SrcList->begin(); S != SrcList->end(); ++S)
    {
       std::vector const targets = (*S)->GetIndexTargets();
       std::map AddOptions;
-      AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+      if (ReleaseInfo)
+      {
+	 AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+	 pkgCache &Cache = *CacheFile.GetPkgCache();
+	 pkgCache::RlsFileIterator const RlsFile = (*S)->FindInCache(Cache, false);
+	 if (RlsFile.end())
+	    continue;
+#define APT_RELEASE(X,Y) if (RlsFile.Y() != NULL) AddOptions.insert(std::make_pair(X, RlsFile.Y()))
+	 APT_RELEASE("CODENAME", Codename);
+	 APT_RELEASE("SUITE", Archive);
+	 APT_RELEASE("VERSION", Version);
+	 APT_RELEASE("ORIGIN", Origin);
+	 APT_RELEASE("LABEL", Label);
+#undef APT_RELEASE
+      }
 
       for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
       {
+	 std::string filename = T->Option(ReleaseInfo ? IndexTarget::EXISTING_FILENAME : IndexTarget::FILENAME);
+	 if (filename.empty())
+	    continue;
+
 	 std::ostringstream stanza;
 	 if (Filtered || Format.empty())
 	 {
@@ -1644,7 +1663,7 @@ static bool DoFiles(CommandLine &CmdL)
 	       << "ShortDesc: " << T->ShortDesc << "\n"
 	       << "Description: " << T->Description << "\n"
 	       << "URI: " << T->URI << "\n"
-	       << "Filename: " << T->Option(IndexTarget::FILENAME) << "\n"
+	       << "Filename: " << filename << "\n"
 	       << "Optional: " << (T->IsOptional ? "yes" : "no") << "\n";
 	    for (std::map::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
 	       stanza << format_key(O->first) << ": " << O->second << "\n";
@@ -1677,7 +1696,8 @@ static bool DoFiles(CommandLine &CmdL)
 	    cout << stanza.str();
 	 else
 	 {
-	    std::string out = T->Format(Format);
+	    std::string out = SubstVar(Format, "$(FILENAME)", filename);
+	    out = T->Format(out);
 	    for (std::map::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
 	       out = SubstVar(out, std::string("$(") + O->first + ")", O->second);
 	    cout << out << std::endl;
diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index a47dac2d4..f9a16318d 100644
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -18,12 +18,6 @@ might want to use and would therefore need to be downloaded as well
 This file describes the configuration scheme such a frontend can use to
 instruct the Acquire system to download those additional files.
 
-Note that you can't download files from other sources. It must be files
-in the same repository and below the Release file. The Release file must
-also contain hashes for the file itself as well as for the compressed
-file if wanted, otherwise a download isn't even tried!
-
-
 # The Configuration Stanza
 
 The Acquire system uses the same configuration settings to implement the
@@ -81,15 +75,17 @@ Additional optional properties:
 * Optional: The default value is 'true' and should be kept at this
   value.  If enabled the acquire system will skip the download if the
   file isn't mentioned in the Release file. Otherwise this is treated as
-  a hard error and the update process fails.
+  a hard error and the update process fails. Note that failures while
+  downloading (e.g. 404 or hash verification errors) are failures,
+  regardless of this setting.
 
 
-Note that the acquire system will automatically choose to download
-a compressed file if it is available and uncompress it for you, just as
-it will also use pdiff patching if provided by the repository and
-enabled by the user. You only have to ensure that the Release file
-contains the information about the compressed files/pdiffs to make this
-happen. NO properties have to be set to enable this.
+The acquire system will automatically choose to download a compressed
+file if it is available and uncompress it for you, just as it will also
+use pdiff patching if provided by the repository and enabled by the
+user. You only have to ensure that the Release file contains the
+information about the compressed files/pdiffs to make this happen.
+NO properties have to be set to enable this.
 
 # More examples
 
@@ -123,10 +119,12 @@ APT::Acquire::Targets {
 As seen in the examples, properties can contain placeholders filled in
 by the acquire system. The following variables are known; note that
 unknown variables have no default value nor are they touched: They are
-printed literally.
+printed as-is.
 
 * $(SITE): An identifier of the site we access as seen in sources.list,
-  e.g. "http://example.org/debian" or "file:/path/to/a/repository".
+  e.g. "http://example.org/debian" or "file:/path/to/a/repository". You
+  can't use this field in {,flat}MetaKey, it is for description proposes
+  only.
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
   "stretch".  Note that flat-style repositories do not have a archive-
   or codename per-se, so the value might very well be just "/" or so.
@@ -143,7 +141,7 @@ printed literally.
 
 Note that while more variables might exist in the implementation, these
 are to be considered undefined and their usage strongly discouraged. If
-you have a need for another variable contact us instead.
+you have a need for other variables contact us.
 
 # Accessing files
 
@@ -170,29 +168,42 @@ sources.lists (pkgSourceList), iterating over the metaIndex objects this
 creates and calling GetIndexTargets() on them. See the sourcecode of
 "apt-get files" for a complete example.
 
-Remarks on the available fields:
+Note that by default targets are not listed if they weren't downloaded.
+If you want to see all targets, you can use the --no-release-info, which
+also removes the Codename, Suite, Version, Origin, Label and Trusted
+fields from the output as these also display data which needs to be
+downloaded first and could hence be inaccurate [on the pro-side: This
+mode is faster as it doesn't require a valid binary cache to operate].
+The most notable difference perhaps is in the Filename field through: By
+default it indicates an existing file, potentially compressed (Hint:
+libapt users can use FileFd to open compressed files transparently). In
+the --no-release-info mode the indicated file doesn't need to exist and
+it will always refer to an uncompressed file, even if the index would be
+(or is) stored compressed.
+
+Remarks on fields only available in (default) --release-info mode:
+* Trusted: Denotes with a 'yes' or 'no' if the data in this file is
+  authenticated by a trustchain rooted in a trusted gpg key. You should
+  be careful with untrusted data and warn the user if you use it.
+* Codename, Suite, Version, Origin and Label are fields from the Release
+  file, are only present if they are present in the Release file and
+  contain the same data.
+
+Remarks on other available fields:
 * MetaKey, ShortDesc, Description, Site, Release: as defined
   by the configuration and described further above.
 * Created-By: configuration entity responsible for this target
 * Target-Of: type of the sources.list entry
 * URI, Repo-URI: avoid using. Contains potentially username/password.
   Prefer 'Site', especially for display.
-* Filename: The mentioned file doesn't need to exist, e.g. because the
-  file wasn't downloaded yet – or it does exist compressed. libapt users
-  are encouraged to use FileFd to open such files as it can handle
-  decompression transparently.
-* Trusted: As of the last 'apt-get update' call denoting if e.g. apt-get
-  would print the unauthenticated warning while installing packages
-  mentioned in this file (example assumes Packages file) [Not really
-  a property of the target itself, but of the metaIndex].
 * Optional: Decodes the option of the same name from the configuration.
   Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
 * Language, Architecture, Component: as defined further above, but with
   the catch that they might be missing if they don't effect the target
   (aka: They weren't used while evaluating the MetaKey template).
 
-Again, additional fields might be visible in certain implementation, but
-you should avoid using them and instead talk to us about a portable
+Again, additional fields might be visible in certain implementations,
+but you should avoid using them and instead talk to us about a portable
 implementation.
 
 # Multiple application requiring the same files
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 971cfac2e..ffb9f3135 100755
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -37,6 +37,12 @@ APT::Acquire::Targets::deb::Contents {
 };
 EOF
 
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --no-release-info --format '$(FILENAME)' 'Created-By: Contents'
+testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'
+# lets fake the existence of a compressed Contents file
+touch ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
+
 testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
 'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
 'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
-- 
cgit v1.2.3-70-g09d2


From d56e2917f27a722b54685de13aeb1bb7592fc61b Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 13 Jun 2015 11:13:45 +0200
Subject: provide a public interface for acquiring changelogs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Provided is a specialized acquire item which given a version can figure
out the correct URI to try by itself and if not provides an error
message alongside with static methods to get just the URI it would try
to download if it should just be displayed or similar such.

The URI is constructed as follows:
Release files can provide an URI template in the "Changelogs" field,
otherwise we lookup a configuration item based on the "Label" or
"Origin" of the Release file to get a (hopefully known) default value
for now. This template should contain the string CHANGEPATH which is
replaced with the information about the version we want the changelog
for (e.g.  main/a/apt/apt_1.1). This middleway was choosen as this path
part was consistent over the three known implementations (+1 defunct),
while the rest of the URI varies widely between them.

The benefit of this construct is that it is now easy to get changelogs
for Debian packages on Ubuntu and vice versa – even at the moment where
the Changelogs field is present nowhere. Strictly better than what
apt-get had before as it would even fail to get changelogs from
security… Now it will notice that security identifies as Origin: Debian
and pick this setting (assuming again that no Changelogs field exists).
If on the other hand security would ship its changelogs in a different
location we could set it via the Label option overruling Origin.

Closes: 687147, 739854, 784027, 787190
---
 apt-pkg/acquire-item.cc                            | 211 +++++++++++++++++++++
 apt-pkg/acquire-item.h                             | 114 +++++++++++
 apt-pkg/init.cc                                    |   4 +
 cmdline/apt-get.cc                                 | 204 ++++----------------
 doc/apt-get.8.xml                                  |  16 +-
 doc/apt.conf.5.xml                                 |  27 +++
 doc/examples/configure-index                       |  11 +-
 test/integration/framework                         |  24 ++-
 test/integration/test-apt-get-changelog            | 104 +++++++---
 .../test-bug-722207-print-uris-even-if-very-quiet  |   3 +-
 test/integration/test-bug-738785-switch-protocol   |   7 +-
 vendor/ubuntu/apt.conf-01-vendor-ubuntu            |   6 -
 12 files changed, 507 insertions(+), 224 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index d6e9ccbe0..4bf4e62f8 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -31,6 +31,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -2836,6 +2837,216 @@ std::string pkgAcqArchive::ShortDesc() const				/*{{{*/
 }
 									/*}}}*/
 
+// AcqChangelog::pkgAcqChangelog - Constructors				/*{{{*/
+pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
+      std::string const &DestDir, std::string const &DestFilename) :
+   pkgAcquire::Item(Owner), d(NULL), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr())
+{
+   Desc.URI = URI(Ver);
+   Init(DestDir, DestFilename);
+}
+// some parameters are char* here as they come likely from char* interfaces – which can also return NULL
+pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile,
+      char const * const Component, char const * const SrcName, char const * const SrcVersion,
+      const string &DestDir, const string &DestFilename) :
+   pkgAcquire::Item(Owner), d(NULL), SrcName(SrcName), SrcVersion(SrcVersion)
+{
+   Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion);
+   Init(DestDir, DestFilename);
+}
+pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner,
+      std::string const &URI, char const * const SrcName, char const * const SrcVersion,
+      const string &DestDir, const string &DestFilename) :
+   pkgAcquire::Item(Owner), d(NULL), SrcName(SrcName), SrcVersion(SrcVersion)
+{
+   Desc.URI = URI;
+   Init(DestDir, DestFilename);
+}
+void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename)
+{
+   if (Desc.URI.empty())
+   {
+      Status = StatError;
+      // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
+      strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
+      // Let the error message print something sensible rather than "Failed to fetch /"
+      if (DestFilename.empty())
+	 DestFile = SrcName + ".changelog";
+      else
+	 DestFile = DestFilename;
+      Desc.URI = "changelog:/" + DestFile;
+      return;
+   }
+
+   if (DestDir.empty())
+   {
+      std::string const systemTemp = GetTempDir();
+      char tmpname[100];
+      snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str());
+      if (NULL == mkdtemp(tmpname))
+      {
+	 _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str());
+	 Status = StatError;
+	 return;
+      }
+      DestFile = TemporaryDirectory = tmpname;
+   }
+   else
+      DestFile = DestDir;
+
+   if (DestFilename.empty())
+      DestFile = flCombine(DestFile, SrcName + ".changelog");
+   else
+      DestFile = flCombine(DestFile, DestFilename);
+
+   Desc.ShortDesc = "Changelog";
+   strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str());
+   Desc.Owner = this;
+   QueueURI(Desc);
+
+   if (Status == StatDone) // this happens if we queue the same changelog two times
+   {
+      Complete = true;
+      for (pkgAcquire::UriIterator I = Owner->UriBegin(); I != Owner->UriEnd(); ++I)
+	 if (I->URI == Desc.URI)
+	    if (DestFile != I->Owner->DestFile)
+	       if (symlink(I->Owner->DestFile.c_str(), DestFile.c_str()) != 0)
+	       {
+		  ; // ignore error, there isn't anthing we could do to handle the edgecase of an edgecase
+	       }
+   }
+}
+									/*}}}*/
+std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver)	/*{{{*/
+{
+   char const * const SrcName = Ver.SourcePkgName();
+   char const * const SrcVersion = Ver.SourceVerStr();
+   pkgCache::PkgFileIterator PkgFile;
+   // find the first source for this version which promises a changelog
+   for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
+   {
+      pkgCache::PkgFileIterator const PF = VF.File();
+      if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
+	 continue;
+      PkgFile = PF;
+      pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
+      std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion);
+      if (uri.empty())
+	 continue;
+      return uri;
+   }
+   return "";
+}
+std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls)
+{
+   if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0))
+      return "";
+   std::string const serverConfig = "Acquire::Changelogs::URI";
+   std::string server;
+#define APT_EMPTY_SERVER \
+   if (server.empty() == false) \
+   { \
+      if (server != "no") \
+	 return server; \
+      return ""; \
+   }
+#define APT_CHECK_SERVER(X, Y) \
+   if (Rls->X != 0) \
+   { \
+      std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \
+      server = _config->Find(specialServerConfig); \
+      APT_EMPTY_SERVER \
+   }
+   // this way e.g. Debian-Security can fallback to Debian
+   APT_CHECK_SERVER(Label, "Override::")
+   APT_CHECK_SERVER(Origin, "Override::")
+
+   if (RealFileExists(Rls.FileName()))
+   {
+      _error->PushToStack();
+      FileFd rf;
+      /* This can be costly. A caller wanting to get millions of URIs might
+	 want to do this on its own once and use Override settings.
+	 We don't do this here as Origin/Label are not as unique as they
+	 should be so this could produce request order-dependent anomalies */
+      if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true)
+      {
+	 pkgTagFile TagFile(&rf, rf.Size());
+	 pkgTagSection Section;
+	 if (TagFile.Step(Section) == true)
+	    server = Section.FindS("Changelogs");
+      }
+      _error->RevertToStack();
+      APT_EMPTY_SERVER
+   }
+
+   APT_CHECK_SERVER(Label, "")
+   APT_CHECK_SERVER(Origin, "")
+#undef APT_CHECK_SERVER
+#undef APT_EMPTY_SERVER
+   return "";
+}
+std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls,
+	 char const * const Component, char const * const SrcName,
+	 char const * const SrcVersion)
+{
+   return URI(URITemplate(Rls), Component, SrcName, SrcVersion);
+}
+std::string pkgAcqChangelog::URI(std::string const &Template,
+	 char const * const Component, char const * const SrcName,
+	 char const * const SrcVersion)
+{
+   if (Template.find("CHANGEPATH") == std::string::npos)
+      return "";
+
+   // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/1.1 or contrib/liba/libapt/2.0
+   std::string Src = SrcName;
+   std::string path = APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1);
+   path.append("/").append(Src).append("/");
+   path.append(Src).append("_").append(StripEpoch(SrcVersion));
+   // we omit component for releases without one (= flat-style repositories)
+   if (Component != NULL && strlen(Component) != 0)
+      path = std::string(Component) + "/" + path;
+
+   return SubstVar(Template, "CHANGEPATH", path);
+}
+									/*}}}*/
+// AcqChangelog::Failed - Failure handler				/*{{{*/
+void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
+{
+   Item::Failed(Message,Cnf);
+
+   std::string errText;
+   // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
+   strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
+
+   // Error is probably something techy like 404 Not Found
+   if (ErrorText.empty())
+      ErrorText = errText;
+   else
+      ErrorText = errText + " (" + ErrorText + ")";
+   return;
+}
+									/*}}}*/
+// AcqChangelog::Done - Item downloaded OK				/*{{{*/
+void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes,
+		      pkgAcquire::MethodConfig const * const Cnf)
+{
+   Item::Done(Message,CalcHashes,Cnf);
+
+   Complete = true;
+}
+									/*}}}*/
+pkgAcqChangelog::~pkgAcqChangelog()					/*{{{*/
+{
+   if (TemporaryDirectory.empty() == false)
+   {
+      unlink(DestFile.c_str());
+      rmdir(TemporaryDirectory.c_str());
+   }
+}
+									/*}}}*/
+
 // AcqFile::pkgAcqFile - Constructor					/*{{{*/
 pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes,
 		       unsigned long long const Size,string const &Dsc,string const &ShortDesc,
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index b6d569737..606fd4173 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -1031,6 +1031,120 @@ class pkgAcqArchive : public pkgAcquire::Item
 		 std::string &StoreFilename);
 };
 									/*}}}*/
+/** \brief Retrieve the changelog for the given version			{{{
+ *
+ *  Downloads the changelog to a temporary file it will also remove again
+ *  while it is deconstructed or downloads it to a named location.
+ */
+class pkgAcqChangelog : public pkgAcquire::Item
+{
+   void *d;
+   std::string TemporaryDirectory;
+   std::string const SrcName;
+   std::string const SrcVersion;
+
+   public:
+   // we will never have hashes for changelogs.
+   // If you need verified ones, download the deb and extract the changelog.
+   virtual HashStringList GetExpectedHashes() const { return HashStringList(); }
+   virtual bool HashesRequired() const { return false; }
+
+   // Specialized action members
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &CalcHashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string DescURI() const {return Desc.URI;};
+
+   /** returns the URI to the changelog of this version
+    *
+    * @param Ver is the version to get the changelog for
+    * @return the URI which will be used to acquire the changelog
+    */
+   static std::string URI(pkgCache::VerIterator const &Ver);
+
+   /** returns the URI to the changelog of this version
+    *
+    *  \param Rls is the Release file the package comes from
+    *  \param Component in which the package resides, can be empty
+    *  \param SrcName is the source package name
+    *  \param SrcVersion is the source package version
+    * @return the URI which will be used to acquire the changelog
+    */
+   static std::string URI(pkgCache::RlsFileIterator const &Rls,
+	 char const * const Component, char const * const SrcName,
+	 char const * const SrcVersion);
+
+   /** returns the URI to the changelog of this version
+    *
+    *  \param Template URI where CHANGEPATH has to be filled in
+    *  \param Component in which the package resides, can be empty
+    *  \param SrcName is the source package name
+    *  \param SrcVersion is the source package version
+    * @return the URI which will be used to acquire the changelog
+    */
+   static std::string URI(std::string const &Template,
+	 char const * const Component, char const * const SrcName,
+	 char const * const SrcVersion);
+
+   /** returns the URI template for this release file
+    *
+    *  \param Rls is a Release file
+    * @return the URI template to use for this release file
+    */
+   static std::string URITemplate(pkgCache::RlsFileIterator const &Rls);
+
+   /** \brief Create a new pkgAcqChangelog object.
+    *
+    *  \param Owner The pkgAcquire object with which this object is
+    *  associated.
+    *  \param Ver is the version to get the changelog for
+    *  \param DestDir The directory the file should be downloaded into.
+    *  Will be an autocreated (and cleaned up) temporary directory if not set.
+    *  \param DestFilename The filename the file should have in #DestDir
+    *  Defaults to sourcepackagename.changelog if not set.
+    */
+   pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
+	 std::string const &DestDir="", std::string const &DestFilename="");
+
+   /** \brief Create a new pkgAcqChangelog object.
+    *
+    *  \param Owner The pkgAcquire object with which this object is
+    *  associated.
+    *  \param Rls is the Release file the package comes from
+    *  \param Component in which the package resides, can be empty
+    *  \param SrcName is the source package name
+    *  \param SrcVersion is the source package version
+    *  \param DestDir The directory the file should be downloaded into.
+    *  Will be an autocreated (and cleaned up) temporary directory if not set.
+    *  \param DestFilename The filename the file should have in #DestDir
+    *  Defaults to sourcepackagename.changelog if not set.
+    */
+   pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &Rls,
+	 char const * const Component, char const * const SrcName, char const * const SrcVersion,
+	 std::string const &DestDir="", std::string const &DestFilename="");
+
+   /** \brief Create a new pkgAcqChangelog object.
+    *
+    *  \param Owner The pkgAcquire object with which this object is
+    *  associated.
+    *  \param URI is to be used to get the changelog
+    *  \param SrcName is the source package name
+    *  \param SrcVersion is the source package version
+    *  \param DestDir The directory the file should be downloaded into.
+    *  Will be an autocreated (and cleaned up) temporary directory if not set.
+    *  \param DestFilename The filename the file should have in #DestDir
+    *  Defaults to sourcepackagename.changelog if not set.
+    */
+   pkgAcqChangelog(pkgAcquire * const Owner, std::string const &URI,
+	 char const * const SrcName, char const * const SrcVersion,
+	 std::string const &DestDir="", std::string const &DestFilename="");
+
+   virtual ~pkgAcqChangelog();
+
+private:
+   APT_HIDDEN void Init(std::string const &DestDir, std::string const &DestFilename);
+};
+									/*}}}*/
 /** \brief Retrieve an arbitrary file to the current directory.		{{{
  *
  *  The file is retrieved even if it is accessed via a URL type that
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index e2239a906..96966b249 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -119,6 +119,10 @@ bool pkgInitConfig(Configuration &Cnf)
    Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatDescription", "$(SITE) $(RELEASE) Sources");
    Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Optional", false);
 
+   Cnf.CndSet("Acquire::Changelogs::URI::Origin::Debian", "http://metadata.ftp-master.debian.org/changelogs/CHANGEPATH_changelog");
+   Cnf.CndSet("Acquire::Changelogs::URI::Origin::Ubuntu", "http://changelogs.ubuntu.com/changelogs/pool/CHANGEPATH/changelog");
+   Cnf.CndSet("Acquire::Changelogs::URI::Origin::Ultimedia", "http://packages.ultimediaos.com/changelogs/pool/CHANGEPATH/changelog.txt");
+
    bool Res = true;
 
    // Read an alternate config file
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index a4cd3c377..184b51d23 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1410,196 +1410,70 @@ static bool DoBuildDep(CommandLine &CmdL)
    return true;
 }
 									/*}}}*/
-// GetChangelogPath - return a path pointing to a changelog file or dir /*{{{*/
-// ---------------------------------------------------------------------
-/* This returns a "path" string for the changelog url construction.
- * Please note that its not complete, it either needs a "/changelog"
- * appended (for the packages.debian.org/changelogs site) or a
- * ".changelog" (for third party sites that store the changelog in the
- * pool/ next to the deb itself)
- * Example return: "pool/main/a/apt/apt_0.8.8ubuntu3" 
- */
-static string GetChangelogPath(CacheFile &Cache,
-                        pkgCache::VerIterator Ver)
-{
-   pkgRecords Recs(Cache);
-   pkgRecords::Parser &rec=Recs.Lookup(Ver.FileList());
-   string path = flNotFile(rec.FileName());
-#if APT_PKG_ABI >= 413
-   path.append(Ver.SourcePkgName());
-   path.append("_");
-   path.append(StripEpoch(Ver.SourceVerStr()));
-#else
-   string srcpkg = rec.SourcePkg().empty() ? Ver.ParentPkg().Name() : rec.SourcePkg();
-   string ver = Ver.VerStr();
-   // if there is a source version it always wins
-   if (rec.SourceVer() != "")
-      ver = rec.SourceVer();
-   path += srcpkg + "_" + StripEpoch(ver);
-#endif
-   return path;
-}
-									/*}}}*/
-// GuessThirdPartyChangelogUri - return url 			        /*{{{*/
-// ---------------------------------------------------------------------
-/* Contruct a changelog file path for third party sites that do not use
- * packages.debian.org/changelogs
- * This simply uses the ArchiveURI() of the source pkg and looks for
- * a .changelog file there, Example for "mediabuntu":
- * apt-get changelog mplayer-doc:
- *  http://packages.medibuntu.org/pool/non-free/m/mplayer/mplayer_1.0~rc4~try1.dsfg1-1ubuntu1+medibuntu1.changelog
- */
-static bool GuessThirdPartyChangelogUri(CacheFile &Cache, 
-                                 pkgCache::VerIterator Ver,
-                                 string &out_uri)
-{
-   // get the binary deb server path
-   pkgCache::VerFileIterator Vf = Ver.FileList();
-   if (Vf.end() == true)
-      return false;
-   pkgCache::PkgFileIterator F = Vf.File();
-   pkgIndexFile *index;
-   pkgSourceList *SrcList = Cache.GetSourceList();
-   if(SrcList->FindIndex(F, index) == false)
-      return false;
-
-   // get archive uri for the binary deb
-   string path_without_dot_changelog = GetChangelogPath(Cache, Ver);
-   out_uri = index->ArchiveURI(path_without_dot_changelog + ".changelog");
-
-   // now strip away the filename and add srcpkg_srcver.changelog
-   return true;
-}
-									/*}}}*/
-// DownloadChangelog - Download the changelog 			        /*{{{*/
-// ---------------------------------------------------------------------
-static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, 
-                       pkgCache::VerIterator Ver, string targetfile)
-/* Download a changelog file for the given package version to
- * targetfile. This will first try the server from Apt::Changelogs::Server
- * (http://packages.debian.org/changelogs by default) and if that gives
- * a 404 tries to get it from the archive directly (see 
- * GuessThirdPartyChangelogUri for details how)
- */
-{
-   // make the server root configurable
-   string const server = _config->Find("Apt::Changelogs::Server",
-                          "http://packages.debian.org/changelogs");
-   string const path = GetChangelogPath(CacheFile, Ver);
-   string changelog_uri;
-   if (APT::String::Endswith(server, "/") == true)
-      strprintf(changelog_uri, "%s%s/changelog", server.c_str(), path.c_str());
-   else
-      strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str());
-   if (_config->FindB("APT::Get::Print-URIs", false) == true)
-   {
-      std::cout << '\'' << changelog_uri << '\'' << std::endl;
-      return true;
-   }
-   pkgCache::PkgIterator const Pkg = Ver.ParentPkg();
-
-   string descr;
-   strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), changelog_uri.c_str());
-   // queue it
-   pkgAcquire::Item const * itm = new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile);
-
-   // Disable drop-privs if "_apt" can not write to the target dir
-   CheckDropPrivsMustBeDisabled(Fetcher);
-
-   // try downloading it, if that fails, try third-party-changelogs location
-   // FIXME: Fetcher.Run() is "Continue" even if I get a 404?!?
-   Fetcher.Run();
-   if (itm->Status != pkgAcquire::Item::StatDone)
-   {
-      string third_party_uri;
-      if (GuessThirdPartyChangelogUri(CacheFile, Ver, third_party_uri))
-      {
-         strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), third_party_uri.c_str());
-         itm = new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile);
-         Fetcher.Run();
-      }
-   }
-
-   if (itm->Status == pkgAcquire::Item::StatDone)
-      return true;
-
-   // error
-   return _error->Error("changelog download failed");
-}
-									/*}}}*/
 // DoChangelog - Get changelog from the command line			/*{{{*/
-// ---------------------------------------------------------------------
 static bool DoChangelog(CommandLine &CmdL)
 {
    CacheFile Cache;
    if (Cache.ReadOnlyOpen() == false)
       return false;
-   
+
    APT::CacheSetHelper helper;
    APT::VersionList verset = APT::VersionList::FromCommandLine(Cache,
 		CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper);
    if (verset.empty() == true)
       return false;
    pkgAcquire Fetcher;
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
+   Fetcher.SetLog(&Stat);
 
-   if (_config->FindB("APT::Get::Print-URIs", false) == true)
+   bool const downOnly = _config->FindB("APT::Get::Download-Only", false);
+   bool const printOnly = _config->FindB("APT::Get::Print-URIs", false);
+
+   for (APT::VersionList::const_iterator Ver = verset.begin();
+        Ver != verset.end();
+        ++Ver)
    {
-      bool Success = true;
-      for (APT::VersionList::const_iterator Ver = verset.begin();
-	   Ver != verset.end(); ++Ver)
-	 Success &= DownloadChangelog(Cache, Fetcher, Ver, "");
-      return Success;
+      if (printOnly)
+	 new pkgAcqChangelog(&Fetcher, Ver, "/dev/null");
+      else if (downOnly)
+	 new pkgAcqChangelog(&Fetcher, Ver, ".");
+      else
+	 new pkgAcqChangelog(&Fetcher, Ver);
    }
 
-   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
-   Fetcher.SetLog(&Stat);
+   if (printOnly == false)
+   {
+      // Disable drop-privs if "_apt" can not write to the target dir
+      CheckDropPrivsMustBeDisabled(Fetcher);
+      if (_error->PendingError() == true)
+	 return false;
 
-   bool const downOnly = _config->FindB("APT::Get::Download-Only", false);
+      bool Failed = false;
+      if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true)
+	 return false;
+   }
 
-   char tmpname[100];
-   const char* tmpdir = NULL;
-   if (downOnly == false)
+   if (downOnly == false || printOnly == true)
    {
-      std::string systemTemp = GetTempDir();
-      snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", 
-               systemTemp.c_str());
-      tmpdir = mkdtemp(tmpname);
-      if (tmpdir == NULL)
-	 return _error->Errno("mkdtemp", "mkdtemp failed");
-
-      std::string const SandboxUser = _config->Find("APT::Sandbox::User");
-      if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
+      bool Failed = false;
+      for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin(); I != Fetcher.ItemsEnd(); ++I)
       {
-	 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
-	 struct group const * const gr = getgrnam("root");
-	 if (pw != NULL && gr != NULL)
+	 if (printOnly)
 	 {
-	    // chown the tmp dir directory we use to the sandbox user
-	    if(chown(tmpdir, pw->pw_uid, gr->gr_gid) != 0)
-	       _error->WarningE("DoChangelog", "chown to %s:%s of directory %s failed", SandboxUser.c_str(), "root", tmpdir);
+	    if ((*I)->ErrorText.empty() == false)
+	    {
+	       Failed = true;
+	       _error->Error("%s", (*I)->ErrorText.c_str());
+	    }
+	    else
+	       cout << '\'' << (*I)->DescURI() << "' " << flNotDir((*I)->DestFile)  << std::endl;
 	 }
+	 else
+	    DisplayFileInPager((*I)->DestFile);
       }
+      return Failed == false;
    }
 
-   for (APT::VersionList::const_iterator Ver = verset.begin(); 
-        Ver != verset.end(); 
-        ++Ver) 
-   {
-      string changelogfile;
-      if (downOnly == false)
-	 changelogfile.append(tmpname).append("/changelog");
-      else
-	 changelogfile.append(Ver.ParentPkg().Name()).append(".changelog");
-      if (DownloadChangelog(Cache, Fetcher, Ver, changelogfile) && downOnly == false)
-      {
-         DisplayFileInPager(changelogfile);
-         // cleanup temp file
-	 unlink(changelogfile.c_str());
-      }
-   }
-   // clenaup tmp dir
-   if (tmpdir != NULL)
-      rmdir(tmpdir);
    return true;
 }
 									/*}}}*/
diff --git a/doc/apt-get.8.xml b/doc/apt-get.8.xml
index da077afa7..5b6788ed4 100644
--- a/doc/apt-get.8.xml
+++ b/doc/apt-get.8.xml
@@ -230,16 +230,12 @@
      
 
      
-       changelog downloads a package changelog and displays
-           it through sensible-pager. The server name and base
-           directory is defined in the APT::Changelogs::Server
-	   variable (e.g. packages.debian.org/changelogs for
-	   Debian or changelogs.ubuntu.com/changelogs for
-           Ubuntu).
-           By default it displays the changelog for the version that is
-           installed.  However, you can specify the same options as for
-           the  command.
-         
+	changelog tries to download the
+	      changelog of a package and displays it through
+	      sensible-pager.  By default it
+	      displays the changelog for the version that is installed.
+	      However, you can specify the same options as for the
+	       command.
        
      
 
diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml
index efe986ea8..7d5f7e9b3 100644
--- a/doc/apt.conf.5.xml
+++ b/doc/apt.conf.5.xml
@@ -618,6 +618,33 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
          
      
 
+      scope
+	
+	  Acquiring changelogs can only be done if an URI is known from where to get them.
+	  Preferable the Release file indicates this in a 'Changelogs' field. If this isn't
+	  available the Label/Origin field of the Release file is used to check if a
+	  Acquire::Changelogs::URI::Label::LABEL or
+	  Acquire::Changelogs::URI::Origin::ORIGIN option
+	  exists and if so this value is taken. The value in the Release file can be overridden
+	  with Acquire::Changelogs::URI::Override::Label::LABEL
+	  or Acquire::Changelogs::URI::Override::Origin::ORIGIN.
+
+	  The value should be a normal URI to a text file, expect that package specific data is
+	  replaced with the placeholder CHANGEPATH. The
+	  value for it is: 1. if the package is from a component (e.g. main)
+	  this is the first part otherwise it is omitted, 2. the first letter of source package name,
+	  expect if the source package name starts with 'lib' in which case it will
+	  be the first four letters. 3. The complete source package name. 4. the complete name again and
+	  5. the source version.
+	  The first (if present), second, third and fourth part are separated by a slash ('/')
+	  and between the fourth and fifth part is an underscore ('_').
+
+	  The special value 'no' is available for this option indicating that
+	  this source can't be used to acquire changelog files from. Another source will be tried
+	  if available in this case.
+	
+    
+
    
  
 
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index ef1ae056d..1339335fa 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -117,14 +117,6 @@ APT
   // does a ExecFork)
   Keep-Fds {};
 
-  Changelogs
-  {
-    // server the provides the changelogs, the code will assume
-    // the changlogs are in the pool/ under a srcpkg_ver directory 
-    // with the name "changelog"
-    Server "http://packages.debian.org/changelogs";
-  }:
-
   // control parameters for cron jobs by /etc/cron.daily/apt
   Periodic
   {
@@ -305,6 +297,9 @@ Acquire
      "none";
      "fr";
   };
+
+  // Location of the changelogs with the placeholder CHANGEPATH (e.g. "main/a/apt/apt_1.1")
+  Changelogs::URI::Origin::Debian "http://metadata.ftp-master.debian.org/changelogs/CHANGEPATH_changelog";
 };
 
 // Directory layout
diff --git a/test/integration/framework b/test/integration/framework
index 7b03c09ef..1b99929e2 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -520,6 +520,12 @@ Package: $NAME" > debian/control
 
 buildsimplenativepackage() {
 	local NAME="$1"
+	local NM
+	if [ "$(echo "$NAME" | cut -c 1-3)" = 'lib' ]; then
+		NM="$(echo "$NAME" | cut -c 1-4)"
+	else
+		NM="$(echo "$NAME" | cut -c 1)"
+	fi
 	local ARCH="$2"
 	local VERSION="$3"
 	local RELEASE="${4:-unstable}"
@@ -600,15 +606,15 @@ Package: $NAME" >> ${BUILDDIR}/debian/control
 		(cd ${BUILDDIR}; dpkg-gencontrol -DArchitecture=$arch)
 		(cd ${BUILDDIR}/debian/tmp; md5sum $(find usr/ -type f) > DEBIAN/md5sums)
 		local LOG="${BUILDDIR}/../${NAME}_${VERSION}_${arch}.dpkg-deb.log"
-		# ensure the right permissions as dpkg-deb ensists
+		# ensure the right permissions as dpkg-deb insists
 		chmod 755 ${BUILDDIR}/debian/tmp/DEBIAN
 		testsuccess --nomsg dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/..
 		echo "pool/${NAME}_${VERSION}_${arch}.deb" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.pkglist
 	done
 
-	mkdir -p ${BUILDDIR}/../${NAME}_${VERSION}
-	cp ${BUILDDIR}/debian/changelog ${BUILDDIR}/../${NAME}_${VERSION}/
-	cp ${BUILDDIR}/debian/changelog ${BUILDDIR}/../${NAME}_${VERSION}.changelog
+	local CHANGEPATH="${BUILDDIR}/../${DISTSECTION}/${NM}/${NAME}/${NAME}_${VERSION}"
+	mkdir -p $CHANGEPATH
+	cp ${BUILDDIR}/debian/changelog $CHANGEPATH
 	rm -rf "${BUILDDIR}"
 	msgdone "info"
 }
@@ -876,6 +882,7 @@ getcodenamefromsuite() {
 }
 getreleaseversionfromsuite() { true; }
 getlabelfromsuite() { true; }
+getoriginfromsuite() { true; }
 
 generatereleasefiles() {
 	# $1 is the Date header and $2 is the ValidUntil header to be set
@@ -887,16 +894,21 @@ generatereleasefiles() {
 			local CODENAME="$(getcodenamefromsuite $SUITE)"
 			local VERSION="$(getreleaseversionfromsuite $SUITE)"
 			local LABEL="$(getlabelfromsuite $SUITE)"
+			local ORIGIN="$(getoriginfromsuite $SUITE)"
 			if [ -n "$VERSION" ]; then
 				VERSION="-o APT::FTPArchive::Release::Version=${VERSION}"
 			fi
 			if [ -n "$LABEL" ]; then
 				LABEL="-o APT::FTPArchive::Release::Label=${LABEL}"
 			fi
+			if [ -n "$ORIGIN" ]; then
+				ORIGIN="-o APT::FTPArchive::Release::Origin=${ORIGIN}"
+			fi
 			aptftparchive -qq release $dir \
 				-o APT::FTPArchive::Release::Suite="${SUITE}" \
 				-o APT::FTPArchive::Release::Codename="${CODENAME}" \
 				${LABEL} \
+				${ORIGIN} \
 				${VERSION} \
 					| sed -e '/0 Release$/ d' > $dir/Release # remove the self reference
 			if [ "$SUITE" = "experimental" -o "$SUITE" = "experimental2" ]; then
@@ -1450,9 +1462,9 @@ testfilestats() {
 		msgpass
 	else
 		echo >&2
-		ls -ld >&2 "$1"
+		ls -ld >&2 "$1" || true
 		echo -n >&2 "stat(1) reports for $2: "
-		stat --format "$2" "$1"
+		stat --format "$2" "$1" || true
 		msgfail
 	fi
 }
diff --git a/test/integration/test-apt-get-changelog b/test/integration/test-apt-get-changelog
index 7e81c71b6..5fa8543b9 100755
--- a/test/integration/test-apt-get-changelog
+++ b/test/integration/test-apt-get-changelog
@@ -5,44 +5,98 @@ TESTDIR=$(readlink -f $(dirname $0))
 . $TESTDIR/framework
 
 setupenvironment
-configarchitecture "i386"
+configarchitecture 'native'
 
-buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
+buildsimplenativepackage 'foo' 'all' '1.0' 'stable'
+buildsimplenativepackage 'libbar' 'all' '1.0' 'stable'
+
+getlabelfromsuite() { echo 'Testcases'; }
+getoriginfromsuite() { echo 'Debian'; }
 
 setupaptarchive --no-update
 changetowebserver
 testsuccess aptget update
 
-# simulate normal user with non-existent root-owned directories
-rm -rf rootdir/var/cache/apt/archives/
-mkdir rootdir/var/cache/apt/archives/
-addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/cache/apt/archives || true;"
-chmod -R -w rootdir/var/cache/apt/archives
+testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/f/foo/foo_1.0_changelog' foo.changelog
+'http://metadata.ftp-master.debian.org/changelogs/main/libb/libbar/libbar_1.0_changelog' libbar.changelog" aptget changelog foo libbar --print-uris
+
+releasechanger() {
+	# modifying the Release files in lists… bad stuff. Good that this is only a test…
+	sed -i "s#^${1}: .*#${1}: ${2}#" $(find rootdir/var/lib/apt/lists -name '*Release')
+	rm -f rootdir/var/cache/apt/*.bin
+}
+releasechanger 'Origin' 'Ubuntu'
+testsuccessequal "'http://changelogs.ubuntu.com/changelogs/pool/main/f/foo/foo_1.0/changelog' foo.changelog
+'http://changelogs.ubuntu.com/changelogs/pool/main/libb/libbar/libbar_1.0/changelog' libbar.changelog" aptget changelog foo libbar --print-uris
+
+releasechanger 'Label' 'Debian'
+testsuccessequal "'http://changelogs.ubuntu.com/changelogs/pool/main/f/foo/foo_1.0/changelog' foo.changelog
+'http://changelogs.ubuntu.com/changelogs/pool/main/libb/libbar/libbar_1.0/changelog' libbar.changelog" aptget changelog foo libbar --print-uris
+
+testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0.changelog' foo.changelog
+'http://localhost:8080/main/libb/libbar/libbar_1.0.changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Label::Debian='http://localhost:8080/CHANGEPATH.changelog'
+
+sed -i '/^Origin: / a\
+Changelogs: http://example.org/CHANGEPATH-changelog' $(find rootdir/var/lib/apt/lists -name '*Release')
+rm -f rootdir/var/cache/apt/*.bin
 
-echo 'Apt::Changelogs::Server "http://localhost:8080/";' > rootdir/etc/apt/apt.conf.d/changelog.conf
+testsuccessequal "'http://example.org/main/f/foo/foo_1.0-changelog' foo.changelog
+'http://example.org/main/libb/libbar/libbar_1.0-changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Label::Debian='http://localhost:8080/CHANGEPATH.changelog'
 
-testsuccessequal "'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
+testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0.changelog' foo.changelog
+'http://localhost:8080/main/libb/libbar/libbar_1.0.changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Override::Label::Debian='http://localhost:8080/CHANGEPATH.changelog'
 
-testsuccessequal "'http://localhost:8080/pool/apt_1.0/changelog'
-'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt apt --print-uris
+releasechanger 'Changelogs' 'no'
+testequal 'E: Failed to fetch changelog:/foo.changelog  Changelog unavailable for foo=1.0
+' aptget changelog foo -qq -d
+
+sed -i '/^Changelogs: / d' $(find rootdir/var/lib/apt/lists -name '*Release')
+releasechanger 'Label' 'Testcases'
+
+echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/CHANGEPATH/change.txt";' > rootdir/etc/apt/apt.conf.d/changelog.conf
+testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0/change.txt' foo.changelog
+'http://localhost:8080/main/libb/libbar/libbar_1.0/change.txt' libbar.changelog" aptget changelog foo libbar --print-uris
+
+echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/pool/CHANGEPATH/changelog";' > rootdir/etc/apt/apt.conf.d/changelog.conf
+testsuccessequal "'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog" aptget changelog foo --print-uris
 
 cd downloaded
 
-testsuccess aptget changelog apt -qq
-testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/apt_1.0/changelog)"
+testsuccess aptget changelog foo -qq
+testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+
+testsuccess aptget changelog foo libbar -qq
+testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)
+$(cat ../aptarchive/pool/main/libb/libbar/libbar_1.0/changelog)"
+
+testsuccess aptget changelog foo -d
+testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
+testfileequal 'foo.changelog' "$(cat  ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+rm -f foo.changelog
 
-testsuccess aptget changelog apt -d
-testfileequal 'apt.changelog' "$(cat  ../aptarchive/pool/apt_1.0/changelog)"
-testfilestats 'apt.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
-rm -f apt.changelog ../aptarchive/pool/apt_1.0/changelog
+testsuccess aptget changelog libbar foo -d
+testfilestats 'libbar.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
+testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
+testfileequal 'libbar.changelog' "$(cat  ../aptarchive/pool/main/libb/libbar/libbar_1.0/changelog)"
+testfileequal 'foo.changelog' "$(cat  ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+rm -f libbar.changelog foo.changelog
 
-testequal "$(cat ../aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \
-	-qq -o APT::Changelogs::Server='http://not-on-the-main-server:8080/'
+# as such bogus, but can happen with multiple binaries from the same source
+testsuccessequal "'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog
+'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog" aptget changelog foo foo --print-uris
+testsuccess aptget changelog foo foo -qq
+testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)
+$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+testsuccess aptget changelog foo foo -d
+testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
+testfileequal 'foo.changelog' "$(cat  ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+rm -f foo.changelog
 
-testsuccess aptget changelog apt -d
-testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0.changelog)"
-testfilestats 'apt.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
-rm -f apt.changelog ../aptarchive/pool/apt_1.0.changelog
+# no CHANGEPATH in the URI
+testequal 'E: Failed to fetch changelog:/foo.changelog  Changelog unavailable for foo=1.0
+' aptget changelog foo -qq -d -o Acquire::Changelogs::URI::Label::Testcases='http://localhost:8080/change.txt'
+testfailure test -e foo.changelog
 
-testequal 'E: changelog download failed' aptget changelog apt -qq -d -o APT::Changelogs::Server='http://not-on-the-main-server:8080/'
-testfailure test -e apt.changelog
+testequal 'E: Failed to fetch http://localhost:8080/does/not/exist/main/f/foo/foo_1.0/change.txt  Changelog unavailable for foo=1.0 (404  Not Found)
+' aptget changelog foo -qq -d -o Acquire::Changelogs::URI::Label::Testcases='http://localhost:8080/does/not/exist/CHANGEPATH/change.txt'
+testfailure test -e foo.changelog
diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
index 2cad929cc..e51d72ccd 100755
--- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet
+++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
@@ -12,6 +12,7 @@ insertpackage 'unstable' 'apt' 'all' '2'
 insertsource 'unstable' 'apt' 'all' '2'
 insertsource 'unstable' 'apt2' 'all' '1'
 
+getoriginfromsuite() { echo 'Debian'; }
 setupaptarchive
 
 APTARCHIVE=$(readlink -f ./aptarchive)
@@ -22,7 +23,7 @@ testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.d
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris
-testsuccessequal "'http://packages.debian.org/changelogs/pool/main/apt/apt_2/changelog'" aptget changelog apt -qq --print-uris
+testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris
 
 testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index f6336ffe3..539080200 100755
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -10,6 +10,7 @@ configarchitecture "i386"
 buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
 
 # setup http redirecting to https
+getlabelfromsuite() { echo 'Testcases'; }
 setupaptarchive --no-update
 changetowebserver -o 'aptwebserver::redirect::replace::/redirectme/=https://localhost:4433/' \
 	-o 'aptwebserver::redirect::replace::/downgrademe/=http://localhost:8080/' \
@@ -20,10 +21,10 @@ sed -i -e 's#:4433/#:8080/redirectme#' -e 's# https:# http:#' rootdir/etc/apt/so
 testsuccess aptget update -o Debug::Acquire::http=1 -o Debug::Acquire::https=1 -o Debug::pkgAcquire::Worker=1
 
 msgtest 'Test that the webserver does not answer' 'http requests'
-downloadfile 'http://localhost:8080/pool/apt_1.0/changelog' changelog >/dev/null 2>&1 && msgfail || msgpass
+downloadfile 'http://localhost:8080/pool/main/a/apt/apt_1.0/changelog' changelog >/dev/null 2>&1 && msgfail || msgpass
 
-echo 'Apt::Changelogs::Server "http://localhost:8080/redirectme";' > rootdir/etc/apt/apt.conf.d/changelog.conf
-testsuccessequal "'http://localhost:8080/redirectme/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
+echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/redirectme/pool/CHANGEPATH/changelog";' > rootdir/etc/apt/apt.conf.d/changelog.conf
+testsuccessequal "'http://localhost:8080/redirectme/pool/main/a/apt/apt_1.0/changelog' apt.changelog" aptget changelog apt --print-uris
 
 cd downloaded
 testsuccess aptget changelog apt -d
diff --git a/vendor/ubuntu/apt.conf-01-vendor-ubuntu b/vendor/ubuntu/apt.conf-01-vendor-ubuntu
index c4092ff44..e69de29bb 100644
--- a/vendor/ubuntu/apt.conf-01-vendor-ubuntu
+++ b/vendor/ubuntu/apt.conf-01-vendor-ubuntu
@@ -1,6 +0,0 @@
-// Server information for apt-changelog
-APT {
-  Changelogs {
-    Server "http://changelogs.ubuntu.com/changelogs";
-  };
-};
-- 
cgit v1.2.3-70-g09d2


From 08ea7806458de0995414eaae852e0a5985875642 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 15 Jun 2015 13:16:43 +0200
Subject: deal better with acquiring the same URI multiple times

This is an unlikely event for indexes and co, but it can happen quiet
easily e.g. for changelogs where you want to get the changelogs for
multiple binary package(version)s which happen to all be built from a
single source.

The interesting part is that the Acquire system actually detected this
already and set the item requesting the URI again to StatDone - expect
that this is hardly sufficient: an Item must be Complete=true as well
to be considered truely done and that is only the tip of the ::Done
handling iceberg. So instead of this StatDone hack we allow QItems to be
owned by multiple items and notify all owners about everything now,
so that for the point of each item they got it downloaded just for them.
---
 apt-pkg/acquire-item.cc                            |  25 +-
 apt-pkg/acquire-item.h                             |   1 +
 apt-pkg/acquire-worker.cc                          | 280 +++++++++++++--------
 apt-pkg/acquire-worker.h                           |   3 +
 apt-pkg/acquire.cc                                 | 126 ++++++++--
 apt-pkg/acquire.h                                  |  44 +++-
 .../test-acquire-same-file-multiple-times          |  80 ++++++
 test/integration/test-apt-get-download             |   2 +-
 8 files changed, 409 insertions(+), 152 deletions(-)
 create mode 100755 test/integration/test-acquire-same-file-multiple-times

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 4bf4e62f8..dc4f61b56 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -438,6 +438,11 @@ APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const		/*{{{*/
    return Owner;
 }
 									/*}}}*/
+pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc()			/*{{{*/
+{
+   return Desc;
+}
+									/*}}}*/
 APT_CONST bool pkgAcquire::Item::IsTrusted() const			/*{{{*/
 {
    return false;
@@ -840,7 +845,7 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st
       return false;
    }
 
-   if (FileName != I->DestFile)
+   if (FileName != I->DestFile && RealFileExists(I->DestFile) == false)
    {
       I->Local = true;
       I->Desc.URI = "copy:" + FileName;
@@ -2482,7 +2487,7 @@ void pkgAcqIndex::StageDownloadDone(string const &Message, HashStringList const
 
    // Methods like e.g. "file:" will give us a (compressed) FileName that is
    // not the "DestFile" we set, in this case we uncompress from the local file
-   if (FileName != DestFile)
+   if (FileName != DestFile && RealFileExists(DestFile) == false)
       Local = true;
    else
       EraseFileName = FileName;
@@ -2760,7 +2765,7 @@ void pkgAcqArchive::Done(string const &Message, HashStringList const &Hashes,
    }
 
    // Reference filename
-   if (FileName != DestFile)
+   if (DestFile !=  FileName && RealFileExists(DestFile) == false)
    {
       StoreFilename = DestFile = FileName;
       Local = true;
@@ -2903,18 +2908,6 @@ void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFi
    strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str());
    Desc.Owner = this;
    QueueURI(Desc);
-
-   if (Status == StatDone) // this happens if we queue the same changelog two times
-   {
-      Complete = true;
-      for (pkgAcquire::UriIterator I = Owner->UriBegin(); I != Owner->UriEnd(); ++I)
-	 if (I->URI == Desc.URI)
-	    if (DestFile != I->Owner->DestFile)
-	       if (symlink(I->Owner->DestFile.c_str(), DestFile.c_str()) != 0)
-	       {
-		  ; // ignore error, there isn't anthing we could do to handle the edgecase of an edgecase
-	       }
-   }
 }
 									/*}}}*/
 std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver)	/*{{{*/
@@ -3107,7 +3100,7 @@ void pkgAcqFile::Done(string const &Message,HashStringList const &CalcHashes,
       return;
 
    // We have to copy it into place
-   if (FileName != DestFile)
+   if (RealFileExists(DestFile.c_str()) == false)
    {
       Local = true;
       if (_config->FindB("Acquire::Source-Symlinks",true) == false ||
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 606fd4173..9dbacc1ea 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -249,6 +249,7 @@ class pkgAcquire::Item : public WeakPointable				/*{{{*/
 
    /** \return the acquire process with which this item is associated. */
    pkgAcquire *GetOwner() const;
+   pkgAcquire::ItemDesc &GetItemDesc();
 
    /** \return \b true if this object is being fetched from a trusted source. */
    virtual bool IsTrusted() const;
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 099a1f87d..d6318a21b 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -258,18 +258,27 @@ bool pkgAcquire::Worker::RunMessages()
 
 	    ItemDone();
 
-	    pkgAcquire::Item *Owner = Itm->Owner;
-	    pkgAcquire::ItemDesc Desc = *Itm;
-
 	    // Change the status so that it can be dequeued
-	    Owner->Status = pkgAcquire::Item::StatIdle;
+	    for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
+	    {
+	       pkgAcquire::Item *Owner = *O;
+	       Owner->Status = pkgAcquire::Item::StatIdle;
+	    }
 	    // Mark the item as done (taking care of all queues)
 	    // and then put it in the main queue again
+	    std::vector const ItmOwners = Itm->Owners;
 	    OwnerQ->ItemDone(Itm);
-	    OwnerQ->Owner->Enqueue(Desc);
+	    Itm = NULL;
+	    for (pkgAcquire::Queue::QItem::owner_iterator O = ItmOwners.begin(); O != ItmOwners.end(); ++O)
+	    {
+	       pkgAcquire::Item *Owner = *O;
+	       pkgAcquire::ItemDesc desc = Owner->GetItemDesc();
+	       desc.URI = NewURI;
+	       OwnerQ->Owner->Enqueue(desc);
 
-	    if (Log != 0)
-	       Log->Done(Desc);
+	       if (Log != 0)
+		  Log->Done(Owner->GetItemDesc());
+	    }
             break;
          }
 
@@ -286,14 +295,18 @@ bool pkgAcquire::Worker::RunMessages()
 	    CurrentSize = 0;
 	    TotalSize = strtoull(LookupTag(Message,"Size","0").c_str(), NULL, 10);
 	    ResumePoint = strtoull(LookupTag(Message,"Resume-Point","0").c_str(), NULL, 10);
-	    Itm->Owner->Start(Message, TotalSize);
-
-	    // Display update before completion
-	    if (Log != 0 && Log->MorePulses == true)
-	       Log->Pulse(Itm->Owner->GetOwner());
+	    for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
+	    {
+	       (*O)->Start(Message, TotalSize);
 
-	    if (Log != 0)
-	       Log->Fetch(*Itm);
+	       // Display update before completion
+	       if (Log != 0)
+	       {
+		  if (Log->MorePulses == true)
+		     Log->Pulse((*O)->GetOwner());
+		  Log->Fetch((*O)->GetItemDesc());
+	       }
+	    }
 
 	    break;
 	 }
@@ -307,105 +320,110 @@ bool pkgAcquire::Worker::RunMessages()
 	       break;
 	    }
 
-	    pkgAcquire::Item *Owner = Itm->Owner;
-	    pkgAcquire::ItemDesc Desc = *Itm;
-
-	    if (RealFileExists(Owner->DestFile))
-	       ChangeOwnerAndPermissionOfFile("201::URIDone", Owner->DestFile.c_str(), "root", "root", 0644);
+	    PrepareFiles("201::URIDone", Itm);
 
 	    // Display update before completion
 	    if (Log != 0 && Log->MorePulses == true)
-	       Log->Pulse(Owner->GetOwner());
+	       for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
+		  Log->Pulse((*O)->GetOwner());
 
-	    OwnerQ->ItemDone(Itm);
-
-	    HashStringList const ExpectedHashes = Owner->GetExpectedHashes();
-	    // see if we got hashes to verify
+	    std::string const filename = LookupTag(Message, "Filename", Itm->Owner->DestFile.c_str());
 	    HashStringList ReceivedHashes;
-	    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 	    {
-	       std::string const tagname = std::string(*type) + "-Hash";
-	       std::string const hashsum = LookupTag(Message, tagname.c_str());
-	       if (hashsum.empty() == false)
-		  ReceivedHashes.push_back(HashString(*type, hashsum));
-	    }
-	    // not all methods always sent Hashes our way
-	    if (ExpectedHashes.usable() == true && ReceivedHashes.usable() == false)
-	    {
-	       std::string const filename = LookupTag(Message, "Filename", Owner->DestFile.c_str());
-	       if (filename.empty() == false && RealFileExists(filename))
+	       // see if we got hashes to verify
+	       for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 	       {
-		  Hashes calc(ExpectedHashes);
-		  FileFd file(filename, FileFd::ReadOnly, FileFd::None);
-		  calc.AddFD(file);
-		  ReceivedHashes = calc.GetHashStringList();
+		  std::string const tagname = std::string(*type) + "-Hash";
+		  std::string const hashsum = LookupTag(Message, tagname.c_str());
+		  if (hashsum.empty() == false)
+		     ReceivedHashes.push_back(HashString(*type, hashsum));
+	       }
+	       // not all methods always sent Hashes our way
+	       if (ReceivedHashes.usable() == false)
+	       {
+		  HashStringList const ExpectedHashes = Itm->GetExpectedHashes();
+		  if (ExpectedHashes.usable() == true && RealFileExists(filename))
+		  {
+		     Hashes calc(ExpectedHashes);
+		     FileFd file(filename, FileFd::ReadOnly, FileFd::None);
+		     calc.AddFD(file);
+		     ReceivedHashes = calc.GetHashStringList();
+		  }
 	       }
 	    }
 
-	    if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
-	    {
-	       std::clog << "201 URI Done: " << Owner->DescURI() << endl
-		  << "ReceivedHash:" << endl;
-	       for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs)
-		  std::clog <<  "\t- " << hs->toStr() << std::endl;
-	       std::clog << "ExpectedHash:" << endl;
-	       for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
-		  std::clog <<  "\t- " << hs->toStr() << std::endl;
-	       std::clog << endl;
-	    }
+	    // only local files can refer other filenames and counting them as fetched would be unfair
+	    if (Log !=  NULL && filename != Itm->Owner->DestFile)
+	       Log->Fetched(ReceivedHashes.FileSize(),atoi(LookupTag(Message,"Resume-Point","0").c_str()));
+
+	    std::vector const ItmOwners = Itm->Owners;
+	    OwnerQ->ItemDone(Itm);
+	    Itm = NULL;
 
-	    // decide if what we got is what we expected
-	    bool consideredOkay = false;
 	    bool const isIMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) ||
 	       StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false);
-	    if (ExpectedHashes.usable())
+
+	    for (pkgAcquire::Queue::QItem::owner_iterator O = ItmOwners.begin(); O != ItmOwners.end(); ++O)
 	    {
-	       if (ReceivedHashes.usable() == false)
+	       pkgAcquire::Item * const Owner = *O;
+	       HashStringList const ExpectedHashes = Owner->GetExpectedHashes();
+	       if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
 	       {
-		  /* IMS-Hits can't be checked here as we will have uncompressed file,
-		     but the hashes for the compressed file. What we have was good through
-		     so all we have to ensure later is that we are not stalled. */
-		  consideredOkay = isIMSHit;
+		  std::clog << "201 URI Done: " << Owner->DescURI() << endl
+		     << "ReceivedHash:" << endl;
+		  for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs)
+		     std::clog <<  "\t- " << hs->toStr() << std::endl;
+		  std::clog << "ExpectedHash:" << endl;
+		  for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
+		     std::clog <<  "\t- " << hs->toStr() << std::endl;
+		  std::clog << endl;
 	       }
-	       else if (ReceivedHashes == ExpectedHashes)
-		  consideredOkay = true;
-	       else
-		  consideredOkay = false;
 
-	    }
-	    else if (Owner->HashesRequired() == true)
-	       consideredOkay = false;
-	    else
-	       consideredOkay = true;
+	       // decide if what we got is what we expected
+	       bool consideredOkay = false;
+	       if (ExpectedHashes.usable())
+	       {
+		  if (ReceivedHashes.usable() == false)
+		  {
+		     /* IMS-Hits can't be checked here as we will have uncompressed file,
+			but the hashes for the compressed file. What we have was good through
+			so all we have to ensure later is that we are not stalled. */
+		     consideredOkay = isIMSHit;
+		  }
+		  else if (ReceivedHashes == ExpectedHashes)
+		     consideredOkay = true;
+		  else
+		     consideredOkay = false;
 
-	    if (consideredOkay == true)
-	    {
-	       Owner->Done(Message, ReceivedHashes, Config);
-	       ItemDone();
+	       }
+	       else if (Owner->HashesRequired() == true)
+		  consideredOkay = false;
+	       else
+		  consideredOkay = true;
 
-	       // Log that we are done
-	       if (Log != 0)
+	       if (consideredOkay == true)
 	       {
-		  if (isIMSHit)
+		  Owner->Done(Message, ReceivedHashes, Config);
+
+		  // Log that we are done
+		  if (Log != 0)
 		  {
-		     /* Hide 'hits' for local only sources - we also manage to
-			hide gets */
-		     if (Config->LocalOnly == false)
-			Log->IMSHit(Desc);
+		     if (isIMSHit)
+			Log->IMSHit(Owner->GetItemDesc());
+		     else
+			Log->Done(Owner->GetItemDesc());
 		  }
-		  else
-		     Log->Done(Desc);
 	       }
-	    }
-	    else
-	    {
-	       Owner->Status = pkgAcquire::Item::StatAuthError;
-	       Owner->Failed(Message,Config);
-	       ItemDone();
+	       else
+	       {
+		  Owner->Status = pkgAcquire::Item::StatAuthError;
+		  Owner->Failed(Message,Config);
 
-	       if (Log != 0)
-		  Log->Fail(Desc);
+		  if (Log != 0)
+		     Log->Fail(Owner->GetItemDesc());
+	       }
 	    }
+	    ItemDone();
 	    break;
 	 }
 
@@ -419,30 +437,32 @@ bool pkgAcquire::Worker::RunMessages()
 	       break;
 	    }
 
+	    PrepareFiles("400::URIFailure", Itm);
+
 	    // Display update before completion
 	    if (Log != 0 && Log->MorePulses == true)
-	       Log->Pulse(Itm->Owner->GetOwner());
-
-	    pkgAcquire::Item *Owner = Itm->Owner;
-	    pkgAcquire::ItemDesc Desc = *Itm;
-
-	    if (RealFileExists(Owner->DestFile))
-	       ChangeOwnerAndPermissionOfFile("400::URIFailure", Owner->DestFile.c_str(), "root", "root", 0644);
+	       for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
+		  Log->Pulse((*O)->GetOwner());
 
+	    std::vector const ItmOwners = Itm->Owners;
 	    OwnerQ->ItemDone(Itm);
+	    Itm = NULL;
 
-	    // set some status
-	    if(LookupTag(Message,"FailReason") == "Timeout" || 
-	       LookupTag(Message,"FailReason") == "TmpResolveFailure" ||
-	       LookupTag(Message,"FailReason") == "ResolveFailure" ||
-	       LookupTag(Message,"FailReason") == "ConnectionRefused") 
-	       Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
+	    for (pkgAcquire::Queue::QItem::owner_iterator O = ItmOwners.begin(); O != ItmOwners.end(); ++O)
+	    {
+	       // set some status
+	       if(LookupTag(Message,"FailReason") == "Timeout" ||
+		     LookupTag(Message,"FailReason") == "TmpResolveFailure" ||
+		     LookupTag(Message,"FailReason") == "ResolveFailure" ||
+		     LookupTag(Message,"FailReason") == "ConnectionRefused")
+		  (*O)->Status = pkgAcquire::Item::StatTransientNetworkError;
 
-	    Owner->Failed(Message,Config);
-	    ItemDone();
+	       (*O)->Failed(Message,Config);
 
-	    if (Log != 0)
-	       Log->Fail(Desc);
+	       if (Log != 0)
+		  Log->Fail((*O)->GetItemDesc());
+	    }
+	    ItemDone();
 
 	    break;
 	 }
@@ -578,16 +598,24 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
    Message.reserve(300);
    Message += "URI: " + Item->URI;
    Message += "\nFilename: " + Item->Owner->DestFile;
-   HashStringList const hsl = Item->Owner->GetExpectedHashes();
+
+   HashStringList const hsl = Item->GetExpectedHashes();
    for (HashStringList::const_iterator hs = hsl.begin(); hs != hsl.end(); ++hs)
       Message += "\nExpected-" + hs->HashType() + ": " + hs->HashValue();
-   if(Item->Owner->FileSize > 0)
+
+   if (hsl.FileSize() == 0)
    {
-      string MaximumSize;
-      strprintf(MaximumSize, "%llu", Item->Owner->FileSize);
-      Message += "\nMaximum-Size: " + MaximumSize;
+      unsigned long long FileSize = Item->GetMaximumSize();
+      if(FileSize > 0)
+      {
+	 string MaximumSize;
+	 strprintf(MaximumSize, "%llu", FileSize);
+	 Message += "\nMaximum-Size: " + MaximumSize;
+      }
    }
-   Message += Item->Owner->Custom600Headers();
+
+   Item->SyncDestinationFiles();
+   Message += Item->Custom600Headers();
    Message += "\n\n";
 
    if (RealFileExists(Item->Owner->DestFile))
@@ -686,3 +714,33 @@ void pkgAcquire::Worker::ItemDone()
    Status = string();
 }
 									/*}}}*/
+void pkgAcquire::Worker::PrepareFiles(char const * const caller, pkgAcquire::Queue::QItem const * const Itm)/*{{{*/
+{
+   if (RealFileExists(Itm->Owner->DestFile))
+   {
+      ChangeOwnerAndPermissionOfFile(caller, Itm->Owner->DestFile.c_str(), "root", "root", 0644);
+      std::string const filename = Itm->Owner->DestFile;
+      for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
+      {
+	 pkgAcquire::Item const * const Owner = *O;
+	 if (Owner->DestFile == filename)
+	    continue;
+	 unlink(Owner->DestFile.c_str());
+	 if (link(filename.c_str(), Owner->DestFile.c_str()) != 0)
+	 {
+	    // diferent mounts can't happen for us as we download to lists/ by default,
+	    // but if the system is reused by others the locations can potentially be on
+	    // different disks, so use symlink as poor-men replacement.
+	    // FIXME: Real copying as last fallback, but that is costly, so offload to a method preferable
+	    if (symlink(filename.c_str(), Owner->DestFile.c_str()) != 0)
+	       _error->Error("Can't create (sym)link of file %s to %s", filename.c_str(), Owner->DestFile.c_str());
+	 }
+      }
+   }
+   else
+   {
+      for (pkgAcquire::Queue::QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
+	 unlink((*O)->DestFile.c_str());
+   }
+}
+									/*}}}*/
diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h
index db8889c8e..3a3ef706d 100644
--- a/apt-pkg/acquire-worker.h
+++ b/apt-pkg/acquire-worker.h
@@ -326,6 +326,9 @@ class pkgAcquire::Worker : public WeakPointable
     *  \b false, also rudely interrupts the worker with a SIGINT.
     */
    virtual ~Worker();
+
+private:
+   APT_HIDDEN void PrepareFiles(char const * const caller, pkgAcquire::Queue::QItem const * const Itm);
 };
 
 /** @} */
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 0c815c005..14c8863dc 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -23,6 +23,7 @@
 #include 
 #include 
 
+#include 
 #include 
 #include 
 #include 
@@ -682,7 +683,8 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
    for (; *I != 0; I = &(*I)->Next)
       if (Item.URI == (*I)->URI) 
       {
-	 Item.Owner->Status = Item::StatDone;
+	 (*I)->Owners.push_back(Item.Owner);
+	 Item.Owner->Status = (*I)->Owner->Status;
 	 return false;
       }
 
@@ -705,13 +707,13 @@ bool pkgAcquire::Queue::Dequeue(Item *Owner)
 {
    if (Owner->Status == pkgAcquire::Item::StatFetching)
       return _error->Error("Tried to dequeue a fetching object");
-       
+
    bool Res = false;
-   
+
    QItem **I = &Items;
    for (; *I != 0;)
    {
-      if ((*I)->Owner == Owner)
+      if (Owner == (*I)->Owner)
       {
 	 QItem *Jnk= *I;
 	 *I = (*I)->Next;
@@ -722,7 +724,7 @@ bool pkgAcquire::Queue::Dequeue(Item *Owner)
       else
 	 I = &(*I)->Next;
    }
-   
+
    return Res;
 }
 									/*}}}*/
@@ -799,9 +801,12 @@ pkgAcquire::Queue::QItem *pkgAcquire::Queue::FindItem(string URI,pkgAcquire::Wor
 bool pkgAcquire::Queue::ItemDone(QItem *Itm)
 {
    PipeDepth--;
-   if (Itm->Owner->Status == pkgAcquire::Item::StatFetching)
-      Itm->Owner->Status = pkgAcquire::Item::StatDone;
-   
+   for (QItem::owner_iterator O = Itm->Owners.begin(); O != Itm->Owners.end(); ++O)
+   {
+      if ((*O)->Status == pkgAcquire::Item::StatFetching)
+	 (*O)->Status = pkgAcquire::Item::StatDone;
+   }
+
    if (Itm->Owner->QueueCounter <= 1)
       Owner->Dequeue(Itm->Owner);
    else
@@ -809,7 +814,7 @@ bool pkgAcquire::Queue::ItemDone(QItem *Itm)
       Dequeue(Itm->Owner);
       Owner->Bump();
    }
-   
+
    return Cycle();
 }
 									/*}}}*/
@@ -824,7 +829,7 @@ bool pkgAcquire::Queue::Cycle()
 
    if (PipeDepth < 0)
       return _error->Error("Pipedepth failure");
-			   
+
    // Look for a queable item
    QItem *I = Items;
    while (PipeDepth < (signed)MaxPipeDepth)
@@ -832,18 +837,19 @@ bool pkgAcquire::Queue::Cycle()
       for (; I != 0; I = I->Next)
 	 if (I->Owner->Status == pkgAcquire::Item::StatIdle)
 	    break;
-      
+
       // Nothing to do, queue is idle.
       if (I == 0)
 	 return true;
-      
+
       I->Worker = Workers;
-      I->Owner->Status = pkgAcquire::Item::StatFetching;
+      for (QItem::owner_iterator O = I->Owners.begin(); O != I->Owners.end(); ++O)
+	 (*O)->Status = pkgAcquire::Item::StatFetching;
       PipeDepth++;
       if (Workers->QueueItem(I) == false)
 	 return false;
    }
-   
+
    return true;
 }
 									/*}}}*/
@@ -855,6 +861,94 @@ void pkgAcquire::Queue::Bump()
    Cycle();
 }
 									/*}}}*/
+HashStringList pkgAcquire::Queue::QItem::GetExpectedHashes() const	/*{{{*/
+{
+   /* each Item can have multiple owners and each owner might have different
+      hashes, even if that is unlikely in practice and if so at least some
+      owners will later fail. There is one situation through which is not a
+      failure and still needs this handling: Two owners who expect the same
+      file, but one owner only knows the SHA1 while the other only knows SHA256. */
+   HashStringList superhsl;
+   for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
+   {
+      HashStringList const hsl = (*O)->GetExpectedHashes();
+      if (hsl.usable() == false)
+	 continue;
+      if (superhsl.usable() == false)
+	 superhsl = hsl;
+      else
+      {
+	 // we merge both lists - if we find disagreement send no hashes
+	 HashStringList::const_iterator hs = hsl.begin();
+	 for (; hs != hsl.end(); ++hs)
+	    if (superhsl.push_back(*hs) == false)
+	       break;
+	 if (hs != hsl.end())
+	 {
+	    superhsl.clear();
+	    break;
+	 }
+      }
+   }
+   return superhsl;
+}
+									/*}}}*/
+APT_PURE unsigned long long pkgAcquire::Queue::QItem::GetMaximumSize() const	/*{{{*/
+{
+   unsigned long long Maximum = std::numeric_limits::max();
+   for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
+   {
+      if ((*O)->FileSize == 0)
+	 continue;
+      Maximum = std::min(Maximum, (*O)->FileSize);
+   }
+   if (Maximum == std::numeric_limits::max())
+      return 0;
+   return Maximum;
+}
+									/*}}}*/
+void pkgAcquire::Queue::QItem::SyncDestinationFiles() const		/*{{{*/
+{
+   /* ensure that the first owner has the best partial file of all and
+      the rest have (potentially dangling) symlinks to it so that
+      everything (like progress reporting) finds it easily */
+   std::string superfile = Owner->DestFile;
+   off_t supersize = 0;
+   for (pkgAcquire::Queue::QItem::owner_iterator O = Owners.begin(); O != Owners.end(); ++O)
+   {
+      if ((*O)->DestFile == superfile)
+	 continue;
+      struct stat file;
+      if (lstat((*O)->DestFile.c_str(),&file) == 0)
+      {
+	 if ((file.st_mode & S_IFREG) == 0)
+	    unlink((*O)->DestFile.c_str());
+	 else if (supersize < file.st_size)
+	 {
+	    supersize = file.st_size;
+	    unlink(superfile.c_str());
+	    rename((*O)->DestFile.c_str(), superfile.c_str());
+	 }
+	 else
+	    unlink((*O)->DestFile.c_str());
+	 if (symlink(superfile.c_str(), (*O)->DestFile.c_str()) != 0)
+	 {
+	    ; // not a problem per-se and no real alternative
+	 }
+      }
+   }
+}
+									/*}}}*/
+std::string pkgAcquire::Queue::QItem::Custom600Headers() const		/*{{{*/
+{
+   /* The others are relatively easy to merge, but this one?
+      Lets not merge and see how far we can run with it…
+      Likely, nobody will ever notice as all the items will
+      be of the same class and hence generate the same headers. */
+   return Owner->Custom600Headers();
+}
+									/*}}}*/
+
 // AcquireStatus::pkgAcquireStatus - Constructor			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -914,9 +1008,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       {
 	 CurrentBytes += I->CurrentSize;
 	 ResumeSize += I->ResumePoint;
-	 
+
 	 // Files with unknown size always have 100% completion
-	 if (I->CurrentItem->Owner->FileSize == 0 && 
+	 if (I->CurrentItem->Owner->FileSize == 0 &&
 	     I->CurrentItem->Owner->Complete == false)
 	    TotalBytes += I->CurrentSize;
       }
diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index fc90624e1..02031dafd 100644
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -68,9 +68,10 @@
 
 #include 
 #include 
+#include 
 
-#include 
 #include 
+#include 
 
 #include 
 #include 
@@ -390,13 +391,13 @@ class pkgAcquire
  */
 struct pkgAcquire::ItemDesc : public WeakPointable
 {
-   /** \brief The URI from which to download this item. */
+   /** \brief URI from which to download this item. */
    std::string URI;
-   /** brief A description of this item. */
+   /** \brief description of this item. */
    std::string Description;
-   /** brief A shorter description of this item. */
+   /** \brief shorter description of this item. */
    std::string ShortDesc;
-   /** brief The underlying item which is to be downloaded. */
+   /** \brief underlying item which is to be downloaded. */
    Item *Owner;
 };
 									/*}}}*/
@@ -419,13 +420,26 @@ class pkgAcquire::Queue
    protected:
 
    /** \brief A single item placed in this queue. */
-   struct QItem : pkgAcquire::ItemDesc
+   struct QItem : public WeakPointable
    {
       /** \brief The next item in the queue. */
       QItem *Next;
       /** \brief The worker associated with this item, if any. */
       pkgAcquire::Worker *Worker;
 
+      /** \brief The URI from which to download this item. */
+      std::string URI;
+      /** \brief A description of this item. */
+      std::string Description;
+      /** \brief A shorter description of this item. */
+      std::string ShortDesc;
+      /** \brief The underlying items interested in the download */
+      std::vector Owners;
+      // both, backward compatibility and easy access as syncing is interal
+      Item * Owner;
+
+      typedef std::vector::const_iterator owner_iterator;
+
       /** \brief Assign the ItemDesc portion of this QItem from
        *  another ItemDesc
        */
@@ -434,10 +448,24 @@ class pkgAcquire::Queue
 	 URI = I.URI;
 	 Description = I.Description;
 	 ShortDesc = I.ShortDesc;
+	 Owners.clear();
+	 Owners.push_back(I.Owner);
 	 Owner = I.Owner;
       };
+
+      /** @return the sum of all expected hashes by all owners */
+      HashStringList GetExpectedHashes() const;
+
+      /** @return smallest maximum size of all owners */
+      unsigned long long GetMaximumSize() const;
+
+      /** \brief get partial files in order */
+      void SyncDestinationFiles() const;
+
+      /** @return the custom headers to use for this item */
+      std::string Custom600Headers() const;
    };
-   
+
    /** \brief The name of this queue. */
    std::string Name;
 
@@ -590,7 +618,7 @@ class pkgAcquire::UriIterator
       }
    };
    
-   inline pkgAcquire::ItemDesc const *operator ->() const {return CurItem;};
+   inline pkgAcquire::Queue::QItem const *operator ->() const {return CurItem;};
    inline bool operator !=(UriIterator const &rhs) const {return rhs.CurQ != CurQ || rhs.CurItem != CurItem;};
    inline bool operator ==(UriIterator const &rhs) const {return rhs.CurQ == CurQ && rhs.CurItem == CurItem;};
    
diff --git a/test/integration/test-acquire-same-file-multiple-times b/test/integration/test-acquire-same-file-multiple-times
new file mode 100755
index 000000000..d329a39cb
--- /dev/null
+++ b/test/integration/test-acquire-same-file-multiple-times
@@ -0,0 +1,80 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+TESTFILE="$TESTDIR/framework"
+cp $TESTFILE aptarchive/foo
+APTARCHIVE="$(readlink -f ./aptarchive)"
+
+filedown() {
+	msgtest 'Downloading the same URI twice over file' "$1"
+	testsuccess --nomsg apthelper download-file file:///$APTARCHIVE/foo ./downloaded/foo1 '' file:///$APTARCHIVE/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	#cat download.log
+	testsuccess cmp $TESTFILE ./downloaded/foo1
+	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
+	#testequal '1' grep -c '200%20URI%20Start' ./download.log
+	testequal '1' grep -c '201%20URI%20Done' ./download.log
+	rm -f ./downloaded/foo1 ./downloaded/foo2
+}
+
+testrun() {
+	$1 'no partial'
+	cp $TESTFILE ./downloaded/foo1
+	$1 'complete partial 1'
+	cp $TESTFILE ./downloaded/foo2
+	$1 'complete partial 2'
+	cp $TESTFILE ./downloaded/foo1
+	cp $TESTFILE ./downloaded/foo2
+	$1 'complete partial 1+2'
+	dd if=$TESTFILE of=./downloaded/foo1 bs=500 count=1 2>/dev/null
+	$1 'partial partial 1'
+	dd if=$TESTFILE of=./downloaded/foo2 bs=500 count=1 2>/dev/null
+	$1 'partial partial 2'
+	dd if=$TESTFILE of=./downloaded/foo1 bs=500 count=1 2>/dev/null
+	dd if=$TESTFILE of=./downloaded/foo2 bs=500 count=1 2>/dev/null
+	$1 'partial partial 1+2'
+}
+testrun 'filedown'
+
+changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo
+
+httpdown() {
+	msgtest 'Downloading the same URI to different files' 'twice over http'
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
+	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
+	testequal '1' grep -c '200%20URI%20Start' ./download.log
+	testequal '1' grep -c '201%20URI%20Done' ./download.log
+	rm -f ./downloaded/foo1 ./downloaded/foo2
+}
+testrun 'httpdown'
+
+httpredirectdown() {
+	msgtest 'Redirect leads' 'first URI to the second URI'
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
+	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
+	testequal '1' grep -c '200%20URI%20Start' ./download.log
+	testequal '1' grep -c '103%20Redirect' ./download.log
+	testequal '1' grep -c '201%20URI%20Done' ./download.log
+	rm -f ./downloaded/foo1 ./downloaded/foo2
+}
+testrun 'httpredirectdown'
+
+httpsamedown() {
+	msgtest 'Downloading two files via the same URI to' 'the same file'
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo1 '' -o Debug::pkgAcquire::Worker=1
+	cp rootdir/tmp/testsuccess.output download.log
+	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
+	testequal '1' grep -c '200%20URI%20Start' ./download.log
+	testequal '1' grep -c '201%20URI%20Done' ./download.log
+	rm -f ./downloaded/foo1
+}
+testrun 'httpsamedown'
diff --git a/test/integration/test-apt-get-download b/test/integration/test-apt-get-download
index 6503bbd1c..fa0b65672 100755
--- a/test/integration/test-apt-get-download
+++ b/test/integration/test-apt-get-download
@@ -38,7 +38,7 @@ testdownload() {
 		APT="${APT}/${3}"
 	fi
 	msgtest "Test download of package file $1 with" "$APT"
-	testsuccess --nomsg aptget download ${APT}
+	testsuccess --nomsg aptget download ${APT} -o Debug::pkgAcquire::Worker=1 -o Debug::pkgAcquire::Auth=1
 	testsuccess test -f "$1"
 	rm -f "$1"
 }
-- 
cgit v1.2.3-70-g09d2


From ff86d7df6a53ff6283de4b9a858c1dad98ed887f Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 15 Jun 2015 13:36:11 +0200
Subject: call URIStart in cdrom and file method
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All other methods call it, so they should follow along even if the work
they do afterwards is hardly breathtaking and usually results in a
URIDone pretty soon, but the acquire system tells the individual item
about this via a virtual method call, so even through none of our
existing items contains any critical code in these, maybe one day they
might. Consistency at least once…

Which is also why this has a good sideeffect: file: and cdrom: requests
appear now in the 'apt-get update' output. Finally - it never made sense
to hide them for me. Okay, I guess it made before the new hit behavior,
but now that you can actually see the difference in an update it makes
sense to see if a file: repository changed or not as well.
---
 apt-pkg/acquire-item.cc                              | 13 ++++---------
 apt-pkg/acquire-worker.cc                            |  1 -
 methods/cdrom.cc                                     |  3 ++-
 methods/file.cc                                      | 18 ++++++++++--------
 test/integration/test-apt-cdrom                      |  6 ++++--
 test/integration/test-apt-get-update-unauth-warning  |  7 +++++++
 test/integration/test-apt-progress-fd                |  6 ++++--
 .../test-bug-595691-empty-and-broken-archive-files   | 20 ++++----------------
 .../test-bug-596498-trusted-unsigned-repo            |  8 +++++++-
 9 files changed, 42 insertions(+), 40 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index dc4f61b56..50936b627 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -519,19 +519,14 @@ void pkgAcquire::Item::Done(string const &Message, HashStringList const &Hashes,
 			    pkgAcquire::MethodConfig const * const /*Cnf*/)
 {
    // We just downloaded something..
-   string FileName = LookupTag(Message,"Filename");
    UsedMirror = LookupTag(Message,"UsedMirror");
-   unsigned long long const downloadedSize = Hashes.FileSize();
-   if (downloadedSize != 0)
+   if (FileSize == 0)
    {
-      if (Complete == false && !Local && FileName == DestFile)
+      unsigned long long const downloadedSize = Hashes.FileSize();
+      if (downloadedSize != 0)
       {
-	 if (Owner->Log != 0)
-	    Owner->Log->Fetched(Hashes.FileSize(),atoi(LookupTag(Message,"Resume-Point","0").c_str()));
+	 FileSize = downloadedSize;
       }
-
-      if (FileSize == 0)
-	 FileSize= downloadedSize;
    }
    Status = StatDone;
    ErrorText = string();
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index d6318a21b..ef195d44b 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -362,7 +362,6 @@ bool pkgAcquire::Worker::RunMessages()
 
 	    bool const isIMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) ||
 	       StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false);
-
 	    for (pkgAcquire::Queue::QItem::owner_iterator O = ItmOwners.begin(); O != ItmOwners.end(); ++O)
 	    {
 	       pkgAcquire::Item * const Owner = *O;
diff --git a/methods/cdrom.cc b/methods/cdrom.cc
index 10cb29f66..67265cfa3 100644
--- a/methods/cdrom.cc
+++ b/methods/cdrom.cc
@@ -260,7 +260,8 @@ bool CDROMMethod::Fetch(FetchItem *Itm)
    struct stat Buf;
    if (stat(Res.Filename.c_str(),&Buf) != 0)
       return _error->Error(_("File not found"));
-   
+
+   URIStart(Res);
    if (NewID.empty() == false)
       CurrentID = NewID;
    Res.LastModified = Buf.st_mtime;
diff --git a/methods/file.cc b/methods/file.cc
index 5d5fffa67..5c76ec122 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -58,7 +58,10 @@ bool FileMethod::Fetch(FetchItem *Itm)
 	 {
 	    HashStringList const hsl = Itm->ExpectedHashes;
 	    if (Itm->ExpectedHashes.VerifyFile(File))
+	    {
+	       Res.Filename = Itm->DestFile;
 	       Res.IMSHit = true;
+	    }
 	 }
       }
    }
@@ -78,7 +81,14 @@ bool FileMethod::Fetch(FetchItem *Itm)
 	 if (filesize != 0 && filesize == Res.Size)
 	    Res.IMSHit = true;
       }
+
+      Hashes Hash(Itm->ExpectedHashes);
+      FileFd Fd(File, FileFd::ReadOnly);
+      Hash.AddFD(Fd);
+      Res.TakeHashes(Hash);
    }
+   if (Res.IMSHit == false)
+      URIStart(Res);
 
    // See if the uncompressed file exists and reuse it
    FetchResult AltRes;
@@ -103,14 +113,6 @@ bool FileMethod::Fetch(FetchItem *Itm)
       }
    }
 
-   if (Res.Filename.empty() == false)
-   {
-      Hashes Hash(Itm->ExpectedHashes);
-      FileFd Fd(Res.Filename, FileFd::ReadOnly);
-      Hash.AddFD(Fd);
-      Res.TakeHashes(Hash);
-   }
-
    if (AltRes.Filename.empty() == false)
       URIDone(Res,&AltRes);
    else if (Res.Filename.empty() == false)
diff --git a/test/integration/test-apt-cdrom b/test/integration/test-apt-cdrom
index 34b35f745..6a218ffb8 100755
--- a/test/integration/test-apt-cdrom
+++ b/test/integration/test-apt-cdrom
@@ -136,13 +136,15 @@ aptcache show testing -o Acquire::Languages=en | grep -q '^Description-en: ' &&
 mv rootdir/media/cdrom-unmounted rootdir/media/cdrom-ejected
 msgmsg "ensure an update doesn't mess with cdrom sources"
 testsuccess aptget update
-testfileequal rootdir/tmp/testsuccess.output 'Reading package lists...'
+testfileequal rootdir/tmp/testsuccess.output 'Hit cdrom://Debian APT Testdisk 0.8.15 stable InRelease
+Reading package lists...'
 mv rootdir/media/cdrom-ejected rootdir/media/cdrom-unmounted
 testcdromusage
 
 msgmsg 'and again to check that it withstands the temptation even if it could mount'
 testsuccess aptget update
-testfileequal rootdir/tmp/testsuccess.output 'Reading package lists...'
+testfileequal rootdir/tmp/testsuccess.output 'Hit cdrom://Debian APT Testdisk 0.8.15 stable InRelease
+Reading package lists...'
 testcdromusage
 
 msgmsg 'Check that nothing touched our' 'CD-ROM'
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index ada7f7a26..1f4a14e23 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -9,6 +9,7 @@ TESTDIR=$(readlink -f $(dirname $0))
 
 setupenvironment
 configarchitecture "i386"
+configcompression '.' 'gz'
 
 # a "normal" package with source and binary
 buildsimplenativepackage 'foo' 'all' '2.0'
@@ -31,11 +32,17 @@ testsuccessequal 'Listing...' apt list foo
 testequal 'lock
 partial' ls rootdir/var/lib/apt/lists
 
+filesize() {
+	stat -c%s "$(aptget files --no-release-info --format '$(URI)' "Created-By: $1" | cut -d'/' -f 3- ).gz"
+}
 # allow override
 testwarningequal "Ign file:$APTARCHIVE unstable InRelease
   File not found
 Ign file:$APTARCHIVE unstable Release
   File not found
+Get:1 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
+Get:2 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B]
+Get:3 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
 Reading package lists...
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories
 # ensure we can not install the package
diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd
index 0c11aba7e..99b4ea050 100755
--- a/test/integration/test-apt-progress-fd
+++ b/test/integration/test-apt-progress-fd
@@ -16,6 +16,7 @@ setupaptarchive
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
+dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -32,6 +33,7 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
+dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -48,6 +50,7 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
+dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -73,9 +76,8 @@ pmstatus:dpkg-exec:75:Running dpkg'
 # install non-native and ensure we get proper progress info
 exec 3> apt-progress.log
 testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3
-
-# and compare
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
+dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)
diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index b42212f5e..3042d116d 100755
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -27,9 +27,6 @@ testaptgetupdate() {
 
 createemptyarchive() {
 	find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete
-	if [ "en" = "$1" ]; then
-		echo -n "" | $COMPRESSOR_CMD > aptarchive/Packages.$COMPRESS
-	fi
 	touch aptarchive/Packages
 	echo -n "" | $COMPRESSOR_CMD > aptarchive/${1}.$COMPRESS
 	generatereleasefiles
@@ -39,9 +36,6 @@ createemptyarchive() {
 
 createemptyfile() {
 	find aptarchive/ \( -name "Packages*" -o -name "en*" \) -type f -delete
-	if [ "en" = "$1" ]; then
-		echo -n "" | $COMPRESSOR_CMD > aptarchive/Packages.$COMPRESS
-	fi
 	touch aptarchive/Packages aptarchive/${1}.$COMPRESS
 	generatereleasefiles
 	signreleasefiles
@@ -52,19 +46,13 @@ testoverfile() {
 	local APTARCHIVE="$(readlink -f ./aptarchive)"
 	forcecompressor "$1"
 
-	createemptyfile 'en'
-	testaptgetupdate 'Reading package lists...' "empty file en.$COMPRESS over file"
-
-	createemptyarchive 'en'
-	testaptgetupdate 'Reading package lists...' "empty archive en.$COMPRESS over file"
-
 	createemptyarchive 'Packages'
-	# FIXME: Why omits the file transport the Packages Get line?
-	#Get:3 file:  Packages []
-	testaptgetupdate 'Reading package lists...' "empty archive Packages.$COMPRESS over file"
+	testaptgetupdate "Get: file:$APTARCHIVE  Packages []
+Reading package lists..." "empty archive Packages.$COMPRESS over file"
 
 	createemptyfile 'Packages'
-	testaptgetupdate "Err file:$APTARCHIVE  Packages
+	testaptgetupdate "Get: file:$APTARCHIVE  Packages
+Err file:$APTARCHIVE  Packages
   Empty files can't be valid archives
 W: Failed to fetch ${COMPRESSOR}:${APTARCHIVE}/Packages.$COMPRESS  Empty files can't be valid archives
 
diff --git a/test/integration/test-bug-596498-trusted-unsigned-repo b/test/integration/test-bug-596498-trusted-unsigned-repo
index 4eb77b9a4..1ff0f1d8d 100755
--- a/test/integration/test-bug-596498-trusted-unsigned-repo
+++ b/test/integration/test-bug-596498-trusted-unsigned-repo
@@ -15,13 +15,17 @@ aptgetupdate() {
 	${1:-testwarning} aptget update --allow-insecure-repositories
 }
 
-PKGTEXT="$(aptget install cool --assume-no -d | head -n 7)"
+PKGTEXT="$(aptget install cool --assume-no -d | head -n 8)"
+DOWNLOG="$(echo "$PKGTEXT" | tail -n 1)"
+PKGTEXT="$(echo "$PKGTEXT" | head -n 7)"
 DEBFILE='rootdir/etc/apt/sources.list.d/apt-test-unstable-deb.list'
 
 testsuccessequal "$PKGTEXT
+$DOWNLOG
 Download complete and in download only mode" aptget install cool --assume-no -d
 
 testsuccessequal "$PKGTEXT
+$DOWNLOG
 Download complete and in download only mode" aptget install cool --assume-no -d --allow-unauthenticated
 
 sed -i -e 's#deb#deb [trusted=no]#' $DEBFILE
@@ -47,10 +51,12 @@ testsuccessequal "$PKGTEXT
 WARNING: The following packages cannot be authenticated!
   cool
 Authentication warning overridden.
+$DOWNLOG
 Download complete and in download only mode" aptget install cool --assume-no -d --allow-unauthenticated
 
 sed -i -e 's#deb#deb [trusted=yes]#' $DEBFILE
 aptgetupdate
 
 testsuccessequal "$PKGTEXT
+$DOWNLOG
 Download complete and in download only mode" aptget install cool --assume-no -d
-- 
cgit v1.2.3-70-g09d2


From 1eb1836f4b5397497bd34f0cf516e6e4e73117bf Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 15 Jun 2015 16:41:43 +0200
Subject: show item ID in Hit, Ign and Err lines as well

Again, consistency is the main sellingpoint here, but this way it is now
also easier to explain that some files move through different stages and
lines are printed for them hence multiple times: That is a bit hard to
believe if the number is changing all the time, but now that it keeps
consistent.
---
 apt-pkg/acquire-item.cc                            |  2 +-
 apt-private/acqprogress.cc                         | 38 +++++++++++++++-----
 apt-private/acqprogress.h                          |  3 +-
 po/apt-all.pot                                     | 12 ++++---
 po/ar.po                                           | 18 ++++++----
 po/ast.po                                          | 20 ++++++-----
 po/bg.po                                           | 20 ++++++-----
 po/bs.po                                           |  8 ++---
 po/ca.po                                           | 20 ++++++-----
 po/cs.po                                           | 20 ++++++-----
 po/cy.po                                           | 20 ++++++-----
 po/da.po                                           | 20 ++++++-----
 po/de.po                                           | 20 ++++++-----
 po/dz.po                                           | 20 ++++++-----
 po/el.po                                           | 20 ++++++-----
 po/es.po                                           | 20 ++++++-----
 po/eu.po                                           | 20 ++++++-----
 po/fi.po                                           | 20 ++++++-----
 po/fr.po                                           | 20 ++++++-----
 po/gl.po                                           | 20 ++++++-----
 po/he.po                                           | 12 ++++---
 po/hu.po                                           | 20 ++++++-----
 po/it.po                                           | 21 +++++++-----
 po/ja.po                                           | 20 ++++++-----
 po/km.po                                           | 20 ++++++-----
 po/ko.po                                           | 20 ++++++-----
 po/ku.po                                           | 16 +++++----
 po/lt.po                                           | 20 ++++++-----
 po/mr.po                                           | 20 ++++++-----
 po/nb.po                                           | 20 ++++++-----
 po/ne.po                                           | 20 ++++++-----
 po/nl.po                                           | 20 ++++++-----
 po/nn.po                                           | 20 ++++++-----
 po/pl.po                                           | 20 ++++++-----
 po/pt.po                                           | 20 ++++++-----
 po/pt_BR.po                                        | 20 ++++++-----
 po/ro.po                                           | 20 ++++++-----
 po/ru.po                                           | 20 ++++++-----
 po/sk.po                                           | 20 ++++++-----
 po/sl.po                                           | 20 ++++++-----
 po/sv.po                                           | 20 ++++++-----
 po/th.po                                           | 20 ++++++-----
 po/tl.po                                           | 20 ++++++-----
 po/tr.po                                           | 20 ++++++-----
 po/uk.po                                           | 20 ++++++-----
 po/vi.po                                           | 20 ++++++-----
 po/zh_CN.po                                        | 20 ++++++-----
 po/zh_TW.po                                        | 20 ++++++-----
 test/integration/test-apt-acquire-additional-files | 10 +++---
 test/integration/test-apt-cdrom                    |  4 +--
 .../integration/test-apt-get-update-unauth-warning | 14 ++++----
 test/integration/test-apt-update-ims               | 36 +++++++++----------
 test/integration/test-apt-update-not-modified      | 40 +++++++++++-----------
 .../test-bug-595691-empty-and-broken-archive-files | 14 ++++----
 test/integration/test-bug-602412-dequote-redirect  |  2 +-
 test/integration/test-pdiff-usage                  |  2 +-
 .../test-ubuntu-bug-1098738-apt-get-source-md5sum  |  8 ++---
 57 files changed, 618 insertions(+), 422 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 50936b627..5460280ec 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -405,7 +405,7 @@ class APT_HIDDEN NoActionItem : public pkgAcquire::Item			/*{{{*/
 // Acquire::Item::Item - Constructor					/*{{{*/
 APT_IGNORE_DEPRECATED_PUSH
 pkgAcquire::Item::Item(pkgAcquire * const Owner) :
-   FileSize(0), PartialSize(0), Mode(0), Complete(false), Local(false),
+   FileSize(0), PartialSize(0), Mode(0), ID(0), Complete(false), Local(false),
     QueueCounter(0), ExpectedAdditionalItems(0), Owner(Owner)
 {
    Owner->Add(this);
diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index 0c606e48e..dc92e3b2a 100644
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -49,6 +49,16 @@ void AcqTextStatus::Start()
    ID = 1;
 }
 									/*}}}*/
+void AcqTextStatus::AssignItemID(pkgAcquire::ItemDesc &Itm)		/*{{{*/
+{
+   /* In theory calling it from Fetch() would be enough, but to be
+      safe we call it from IMSHit and Fail as well.
+      Also, an Item can pass through multiple stages, so ensure
+      that it keeps the same number */
+   if (Itm.Owner->ID == 0)
+      Itm.Owner->ID = ID++;
+}
+									/*}}}*/
 // AcqTextStatus::IMSHit - Called when an item got a HIT response	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -57,9 +67,11 @@ void AcqTextStatus::IMSHit(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
+   AssignItemID(Itm);
    clearLastLine();
 
-   out << _("Hit ") << Itm.Description;
+   // TRANSLATOR: Very short word to be displayed before unchanged files in 'apt-get update'
+   ioprintf(out, _("Hit:%lu %s"), Itm.Owner->ID, Itm.Description.c_str());
    out << std::endl;
    Update = true;
 }
@@ -72,15 +84,16 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
    Update = true;
    if (Itm.Owner->Complete == true)
       return;
-
-   Itm.Owner->ID = ID++;
+   AssignItemID(Itm);
 
    if (Quiet > 1)
       return;
 
    clearLastLine();
 
-   out << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
+   // TRANSLATOR: Very short word to be displayed for files processed in 'apt-get update'
+   // Potentially replaced later by "Hit:", "Ign:" or "Err:" if something (bad) happens
+   ioprintf(out, _("Get:%lu %s"), Itm.Owner->ID, Itm.Description.c_str());
    if (Itm.Owner->FileSize != 0)
       out << " [" << SizeToStr(Itm.Owner->FileSize) << "B]";
    out << std::endl;
@@ -89,9 +102,10 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
 // AcqTextStatus::Done - Completed a download				/*{{{*/
 // ---------------------------------------------------------------------
 /* We don't display anything... */
-void AcqTextStatus::Done(pkgAcquire::ItemDesc &/*Itm*/)
+void AcqTextStatus::Done(pkgAcquire::ItemDesc &Itm)
 {
    Update = true;
+   AssignItemID(Itm);
 }
 									/*}}}*/
 // AcqTextStatus::Fail - Called when an item fails to download		/*{{{*/
@@ -106,19 +120,25 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
    if (Itm.Owner->Status == pkgAcquire::Item::StatIdle)
       return;
 
+   AssignItemID(Itm);
    clearLastLine();
 
    if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
    {
-      out << _("Ign ") << Itm.Description << std::endl;
+      // TRANSLATOR: Very short word to be displayed for files in 'apt-get update'
+      // which failed to download, but the error is ignored (compare "Err:")
+      ioprintf(out, _("Ign:%lu %s"), Itm.Owner->ID, Itm.Description.c_str());
       if (Itm.Owner->ErrorText.empty() == false &&
 	    _config->FindB("Acquire::Progress::Ignore::ShowErrorText", false) == true)
-	 out << "  " << Itm.Owner->ErrorText << std::endl;
+	 out << std::endl << "  " << Itm.Owner->ErrorText;
+      out << std::endl;
    }
    else
    {
-      out << _("Err ") << Itm.Description << std::endl;
-      out << "  " << Itm.Owner->ErrorText << std::endl;
+      // TRANSLATOR: Very short word to be displayed for files in 'apt-get update'
+      // which failed to download and the error is critical (compare "Ign:")
+      ioprintf(out, _("Err:%lu %s"), Itm.Owner->ID, Itm.Description.c_str());
+      out << std::endl << "  " << Itm.Owner->ErrorText << std::endl;
    }
 
    Update = true;
diff --git a/apt-private/acqprogress.h b/apt-private/acqprogress.h
index 7cf990c65..cbb06fbec 100644
--- a/apt-private/acqprogress.h
+++ b/apt-private/acqprogress.h
@@ -23,7 +23,8 @@ class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
    unsigned long ID;
    unsigned long Quiet;
 
-   void clearLastLine();
+   APT_HIDDEN void clearLastLine();
+   APT_HIDDEN void AssignItemID(pkgAcquire::ItemDesc &Itm);
 
    public:
 
diff --git a/po/apt-all.pot b/po/apt-all.pot
index b68d801ea..73b033876 100644
--- a/po/apt-all.pot
+++ b/po/apt-all.pot
@@ -1529,19 +1529,23 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
+#, c-format
+msgid "Hit:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
+#, c-format
+msgid "Get:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
+#, c-format
+msgid "Ign:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
+#, c-format
+msgid "Err:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:150
diff --git a/po/ar.po b/po/ar.po
index e2b179d0e..3e71fef80 100644
--- a/po/ar.po
+++ b/po/ar.po
@@ -1555,20 +1555,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
+#, c-format
+msgid "Hit:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "جلب:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "جلب:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "تجاهل"
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "تجاهل:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "خطأ"
+#, c-format
+msgid "Err:%lu %s"
+msgstr "خطأ:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/ast.po b/po/ast.po
index 1efbbc1d2..1995c12f6 100644
--- a/po/ast.po
+++ b/po/ast.po
@@ -1684,20 +1684,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Oxe "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Oxe:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Des:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Des:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/bg.po b/po/bg.po
index bbe6df3ae..7dff005e6 100644
--- a/po/bg.po
+++ b/po/bg.po
@@ -1720,20 +1720,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Поп "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Поп:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Изт:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Изт:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Игн "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Игн:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Грш "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Грш:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/bs.po b/po/bs.po
index 9cd1adc85..abb9a570a 100644
--- a/po/bs.po
+++ b/po/bs.po
@@ -1555,19 +1555,19 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
+msgid "Hit:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
+msgid "Get:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
+msgid "Ign:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
+msgid "Err:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:150
diff --git a/po/ca.po b/po/ca.po
index cdb3ceeb8..065ad0ea7 100644
--- a/po/ca.po
+++ b/po/ca.po
@@ -1708,20 +1708,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Obj "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Obj:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Bai:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Bai:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/cs.po b/po/cs.po
index 76fc37b6b..3f6bcb4a7 100644
--- a/po/cs.po
+++ b/po/cs.po
@@ -1734,20 +1734,24 @@ msgid "Full Text Search"
 msgstr "Fulltextové hledání"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Cíl "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Cíl:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Mám:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Mám:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/cy.po b/po/cy.po
index e29221cd1..b05bc7c59 100644
--- a/po/cy.po
+++ b/po/cy.po
@@ -1706,20 +1706,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Presennol "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Presennol:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Cyrchu:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Cyrchu:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Anwybyddu "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Anwybyddu:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Gwall "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Gwall:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/da.po b/po/da.po
index 53d3de059..d6b8e07a1 100644
--- a/po/da.po
+++ b/po/da.po
@@ -1749,20 +1749,24 @@ msgid "Full Text Search"
 msgstr "Fuldtekst-søgning"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Havde "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Havde:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Henter:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Henter:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ignorerer "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ignorerer:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Fejl "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Fejl:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/de.po b/po/de.po
index 85b0cb7ac..d4ca0d5a0 100644
--- a/po/de.po
+++ b/po/de.po
@@ -1810,20 +1810,24 @@ msgid "Full Text Search"
 msgstr "Volltextsuche"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "OK   "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "OK:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Holen: "
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Holen:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ign:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Fehl "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Fehl:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/dz.po b/po/dz.po
index 8d1e57b35..a4c50d245 100644
--- a/po/dz.po
+++ b/po/dz.po
@@ -1674,20 +1674,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "ཨེབ།"
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "ཨེབ།:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "ལེན:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "ལེན:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "ཨེལ་ཇི་ཨེན:"
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "ཨེལ་ཇི་ཨེན:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "ཨི་ཨར་ཨར།"
+#, c-format
+msgid "Err:%lu %s"
+msgstr "ཨི་ཨར་ཨར།:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/el.po b/po/el.po
index e0750b61e..98e1f31c9 100644
--- a/po/el.po
+++ b/po/el.po
@@ -1695,20 +1695,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Hit "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Hit:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Φέρε:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Φέρε:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Αγνόησε "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Αγνόησε:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Σφάλμα "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Σφάλμα:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/es.po b/po/es.po
index e094ccafa..a2a9a71e9 100644
--- a/po/es.po
+++ b/po/es.po
@@ -1820,20 +1820,24 @@ msgid "Full Text Search"
 msgstr "Buscar en todo el texto"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Obj "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Obj:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Des:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Des:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/eu.po b/po/eu.po
index e268f6059..f1e90411a 100644
--- a/po/eu.po
+++ b/po/eu.po
@@ -1678,20 +1678,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Atzituta "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Atzituta:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Hartu:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Hartu:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ez ikusi "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ez ikusi:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/fi.po b/po/fi.po
index d58fb9b04..c40ed8e9e 100644
--- a/po/fi.po
+++ b/po/fi.po
@@ -1669,20 +1669,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Löytyi "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Löytyi:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Nouda:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Nouda:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Siv "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Siv:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Vrhe "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Vrhe:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/fr.po b/po/fr.po
index cb9d82856..ecb752f5e 100644
--- a/po/fr.po
+++ b/po/fr.po
@@ -1774,20 +1774,24 @@ msgid "Full Text Search"
 msgstr "Recherche en texte intégral"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Atteint "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Atteint:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Réception de : "
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Réception de:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ign:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Err:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/gl.po b/po/gl.po
index d6bb1e92e..2adaa302a 100644
--- a/po/gl.po
+++ b/po/gl.po
@@ -1706,20 +1706,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Teño "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Teño:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Rcb:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Rcb:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/he.po b/po/he.po
index b3ef8f831..ab7b50745 100644
--- a/po/he.po
+++ b/po/he.po
@@ -1188,19 +1188,23 @@ msgid ""
 msgstr ""
 
 #: cmdline/acqprogress.cc:55
-msgid "Hit "
+#, c-format
+msgid "Hit:%lu %s"
 msgstr ""
 
 #: cmdline/acqprogress.cc:79
-msgid "Get:"
+#, c-format
+msgid "Get:%lu %s"
 msgstr ""
 
 #: cmdline/acqprogress.cc:110
-msgid "Ign "
+#, c-format
+msgid "Ign:%lu %s"
 msgstr ""
 
 #: cmdline/acqprogress.cc:114
-msgid "Err "
+#, c-format
+msgid "Err:%lu %s"
 msgstr ""
 
 #: cmdline/acqprogress.cc:135
diff --git a/po/hu.po b/po/hu.po
index 313f3a492..635184c75 100644
--- a/po/hu.po
+++ b/po/hu.po
@@ -1714,20 +1714,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Találat "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Találat:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Letöltés:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Letöltés:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Mellőz "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Mellőz:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Hiba "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Hiba:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/it.po b/po/it.po
index 36173e92c..178754c52 100644
--- a/po/it.po
+++ b/po/it.po
@@ -1766,21 +1766,24 @@ msgid "Full Text Search"
 msgstr "Ricerca sul testo"
 
 #: apt-private/acqprogress.cc:62
-msgid "Hit "
-msgstr "Trovato "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Trovato:%lu %s"
 
 #: apt-private/acqprogress.cc:83
-msgid "Get:"
-msgstr "Scaricamento di:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Scaricamento di:%lu %s"
 
-# (ndt) questa non so cosa voglia dire
 #: apt-private/acqprogress.cc:113
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:120
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:143
 #, c-format
diff --git a/po/ja.po b/po/ja.po
index 6c49f2608..f8f55f8aa 100644
--- a/po/ja.po
+++ b/po/ja.po
@@ -1750,20 +1750,24 @@ msgid "Full Text Search"
 msgstr "全文検索"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "ヒット "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "ヒット:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "取得:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "取得:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "無視 "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "無視:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "エラー "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "エラー:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/km.po b/po/km.po
index 26bfe4361..8c72865f8 100644
--- a/po/km.po
+++ b/po/km.po
@@ -1652,20 +1652,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "វាយ​"
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "វាយ​:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "យក​ ៖"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "យក​ ៖:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/ko.po b/po/ko.po
index 5479ab790..51cc27d61 100644
--- a/po/ko.po
+++ b/po/ko.po
@@ -1666,20 +1666,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "기존 "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "기존:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "받기:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "받기:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "무시"
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "무시:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "오류 "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "오류:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/ku.po b/po/ku.po
index c14ca94bd..540307937 100644
--- a/po/ku.po
+++ b/po/ku.po
@@ -1558,20 +1558,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
+#, c-format
+msgid "Hit:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Anîn:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Anîn:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
+#, c-format
+msgid "Ign:%lu %s"
 msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Çewt"
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Çewt:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, fuzzy, c-format
diff --git a/po/lt.po b/po/lt.po
index b85a0fbe1..1f485b28d 100644
--- a/po/lt.po
+++ b/po/lt.po
@@ -1577,20 +1577,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Imamas "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Imamas:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Gauti:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Gauti:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ignoruotas "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ignoruotas:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Klaida "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Klaida:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/mr.po b/po/mr.po
index 286c5d859..24e470cef 100644
--- a/po/mr.po
+++ b/po/mr.po
@@ -1656,20 +1656,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "दाबा"
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "दाबा:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "मिळवा:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "मिळवा:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "आय.जी.एन."
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "आय.जी.एन.:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "दोष इ.आर.आर."
+#, c-format
+msgid "Err:%lu %s"
+msgstr "दोष इ.आर.आर.:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/nb.po b/po/nb.po
index 07247be4f..433e33909 100644
--- a/po/nb.po
+++ b/po/nb.po
@@ -1684,20 +1684,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Funnet "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Funnet:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Hent:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Hent:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ign:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Feil "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Feil:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/ne.po b/po/ne.po
index 54dd771b4..4c6f2d098 100644
--- a/po/ne.po
+++ b/po/ne.po
@@ -1652,20 +1652,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "हान्नुहोस्"
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "हान्नुहोस्:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "प्राप्त गर्नुहोस्:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "प्राप्त गर्नुहोस्:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/nl.po b/po/nl.po
index 8d226dfef..5c774fe8d 100644
--- a/po/nl.po
+++ b/po/nl.po
@@ -1776,20 +1776,24 @@ msgid "Full Text Search"
 msgstr "Volledige tekst doorzoeken"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Geraakt "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Geraakt:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Ophalen:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Ophalen:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Genegeerd "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Genegeerd:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Fout "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Fout:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/nn.po b/po/nn.po
index b3622b6cd..0886c266a 100644
--- a/po/nn.po
+++ b/po/nn.po
@@ -1668,20 +1668,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Treff "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Treff:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Hent:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Hent:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ign:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Feil "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Feil:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/pl.po b/po/pl.po
index 0dda22a47..1851fa805 100644
--- a/po/pl.po
+++ b/po/pl.po
@@ -1749,22 +1749,26 @@ msgstr ""
 
 # Ujednolicono z aptitude
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Stary "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Stary:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Pobieranie:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Pobieranie:%lu %s"
 
 # Wyrównane do Hit i Err.
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign.  "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 # Wyrównane do Hit i Ign.
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Błąd  "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Błąd:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/pt.po b/po/pt.po
index a4b1105f8..4f05ada82 100644
--- a/po/pt.po
+++ b/po/pt.po
@@ -1718,20 +1718,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Hit "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Obter:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Obter:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/pt_BR.po b/po/pt_BR.po
index 0c4d92d02..5e2abd988 100644
--- a/po/pt_BR.po
+++ b/po/pt_BR.po
@@ -1687,20 +1687,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Atingido "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Atingido:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Obter:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Obter:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/ro.po b/po/ro.po
index 63df026ab..9b0a1494a 100644
--- a/po/ro.po
+++ b/po/ro.po
@@ -1695,20 +1695,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Atins "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Atins:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Luat:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Luat:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ignorat "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ignorat:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Eroare"
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Eroare:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/ru.po b/po/ru.po
index 260bdd513..2028e5420 100644
--- a/po/ru.po
+++ b/po/ru.po
@@ -1742,20 +1742,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "В кэше "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "В кэше:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Получено:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Получено:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Игн "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Игн:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Ош  "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Ош:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/sk.po b/po/sk.po
index 912d1fcf7..631ddd826 100644
--- a/po/sk.po
+++ b/po/sk.po
@@ -1716,20 +1716,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Už existuje "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Už existuje:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Získava sa:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Získava sa:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr ""
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Chyba "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Chyba:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/sl.po b/po/sl.po
index 3210fa504..845fe3c5f 100644
--- a/po/sl.po
+++ b/po/sl.po
@@ -1717,20 +1717,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Zadetek "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Zadetek:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Dobi:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Dobi:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Prezr "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Prezr:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Nap "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Nap:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/sv.po b/po/sv.po
index 70f4b2d2d..52f65b598 100644
--- a/po/sv.po
+++ b/po/sv.po
@@ -1698,23 +1698,27 @@ msgstr ""
 # Måste vara tre bokstäver(?)
 # "Hit" = aktuell version är fortfarande giltig
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Bra "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Bra:%lu %s"
 
 # "Get:" = hämtar ny version
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Läs:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Läs:%lu %s"
 
 # "Ign" = hoppar över
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ign "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ign:%lu %s"
 
 # "Err" = fel vid hämtning
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Fel "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Fel:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/th.po b/po/th.po
index 147c3b8b8..60a468ea4 100644
--- a/po/th.po
+++ b/po/th.po
@@ -1696,20 +1696,24 @@ msgid "Full Text Search"
 msgstr "ค้นทั่วทั้งเนื้อความ"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "เจอ "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "เจอ:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "ดึง:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "ดึง:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "ข้าม "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "ข้าม:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "ปัญหา "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "ปัญหา:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/tl.po b/po/tl.po
index e3da79c8a..970b0b387 100644
--- a/po/tl.po
+++ b/po/tl.po
@@ -1677,20 +1677,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Tumama "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Tumama:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Kunin: "
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Kunin:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "DiPansin "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "DiPansin:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Err "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Err:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/tr.po b/po/tr.po
index 5d9c3cee3..1e6f38903 100644
--- a/po/tr.po
+++ b/po/tr.po
@@ -1753,20 +1753,24 @@ msgid "Full Text Search"
 msgstr "Tam Metin Arama"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Bağlandı  "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Bağlandı:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Alınıyor: "
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Alınıyor:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Yoksay    "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Yoksay:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Hata      "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Hata:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/uk.po b/po/uk.po
index faeedf941..47976d39c 100644
--- a/po/uk.po
+++ b/po/uk.po
@@ -1736,20 +1736,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "В кеші "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "В кеші:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Отр:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Отр:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Ігн "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Ігн:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Пом "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Пом:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/vi.po b/po/vi.po
index 089d8461a..2edb52f22 100644
--- a/po/vi.po
+++ b/po/vi.po
@@ -1752,20 +1752,24 @@ msgid "Full Text Search"
 msgstr "Tìm kiếm toàn văn"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "Tìm thấy "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "Tìm thấy:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "Lấy:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "Lấy:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "Bỏq "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "Bỏq:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "Lỗi "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "Lỗi:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/zh_CN.po b/po/zh_CN.po
index 2ff5bcf01..6e9610ac9 100644
--- a/po/zh_CN.po
+++ b/po/zh_CN.po
@@ -1693,20 +1693,24 @@ msgid "Full Text Search"
 msgstr "全文搜索"
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "命中 "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "命中:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "获取："
+#, c-format
+msgid "Get:%lu %s"
+msgstr "获取:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "忽略 "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "忽略:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "错误 "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "错误:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/po/zh_TW.po b/po/zh_TW.po
index e35910df2..201d9d675 100644
--- a/po/zh_TW.po
+++ b/po/zh_TW.po
@@ -1646,20 +1646,24 @@ msgid "Full Text Search"
 msgstr ""
 
 #: apt-private/acqprogress.cc:66
-msgid "Hit "
-msgstr "已有 "
+#, c-format
+msgid "Hit:%lu %s"
+msgstr "已有:%lu %s"
 
 #: apt-private/acqprogress.cc:88
-msgid "Get:"
-msgstr "下載:"
+#, c-format
+msgid "Get:%lu %s"
+msgstr "下載:%lu %s"
 
 #: apt-private/acqprogress.cc:119
-msgid "Ign "
-msgstr "略過 "
+#, c-format
+msgid "Ign:%lu %s"
+msgstr "略過:%lu %s"
 
 #: apt-private/acqprogress.cc:126
-msgid "Err "
-msgstr "錯誤 "
+#, c-format
+msgid "Err:%lu %s"
+msgstr "錯誤:%lu %s"
 
 #: apt-private/acqprogress.cc:150
 #, c-format
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index ffb9f3135..3465c0a16 100755
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -49,8 +49,8 @@ testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists
 'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 
 'http://localhost:8080/dists/unstable/main/Contents-amd64.bz2' localhost:8080_dists_unstable_main_Contents-amd64 0 " aptget update --print-uris
 
-testsuccessequal "Hit http://localhost:8080 unstable InRelease
-Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
+Get:2 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' find rootdir/var/lib/apt/lists -name '*Contents*'
@@ -74,8 +74,8 @@ testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists
 'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 
 'http://localhost:8080/dists/unstable/main/Contents-amd64.gz.bz2' localhost:8080_dists_unstable_main_Contents-amd64.gz 0 " aptget update --print-uris
 
-testsuccessequal "Hit http://localhost:8080 unstable InRelease
-Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
+testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
+Get:2 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B]
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
@@ -89,7 +89,7 @@ testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists
 'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
 'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris
 
-testsuccessequal "Hit http://localhost:8080 unstable InRelease
+testsuccessequal "Hit:1 http://localhost:8080 unstable InRelease
 Reading package lists..." aptget update
 
 testempty find rootdir/var/lib/apt/lists -name '*Contents*'
diff --git a/test/integration/test-apt-cdrom b/test/integration/test-apt-cdrom
index 6a218ffb8..108805daa 100755
--- a/test/integration/test-apt-cdrom
+++ b/test/integration/test-apt-cdrom
@@ -136,14 +136,14 @@ aptcache show testing -o Acquire::Languages=en | grep -q '^Description-en: ' &&
 mv rootdir/media/cdrom-unmounted rootdir/media/cdrom-ejected
 msgmsg "ensure an update doesn't mess with cdrom sources"
 testsuccess aptget update
-testfileequal rootdir/tmp/testsuccess.output 'Hit cdrom://Debian APT Testdisk 0.8.15 stable InRelease
+testfileequal rootdir/tmp/testsuccess.output 'Hit:1 cdrom://Debian APT Testdisk 0.8.15 stable InRelease
 Reading package lists...'
 mv rootdir/media/cdrom-ejected rootdir/media/cdrom-unmounted
 testcdromusage
 
 msgmsg 'and again to check that it withstands the temptation even if it could mount'
 testsuccess aptget update
-testfileequal rootdir/tmp/testsuccess.output 'Hit cdrom://Debian APT Testdisk 0.8.15 stable InRelease
+testfileequal rootdir/tmp/testsuccess.output 'Hit:1 cdrom://Debian APT Testdisk 0.8.15 stable InRelease
 Reading package lists...'
 testcdromusage
 
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 1f4a14e23..5b81d56d2 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -20,9 +20,9 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated files leads to warning
-testfailureequal "Ign file:$APTARCHIVE unstable InRelease
+testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease
   File not found
-Err file:$APTARCHIVE unstable Release
+Err:2 file:$APTARCHIVE unstable Release
   File not found
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
 E: Use --allow-insecure-repositories to force the update" aptget update --no-allow-insecure-repositories
@@ -36,13 +36,13 @@ filesize() {
 	stat -c%s "$(aptget files --no-release-info --format '$(URI)' "Created-By: $1" | cut -d'/' -f 3- ).gz"
 }
 # allow override
-testwarningequal "Ign file:$APTARCHIVE unstable InRelease
+testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease
   File not found
-Ign file:$APTARCHIVE unstable Release
+Ign:2 file:$APTARCHIVE unstable Release
   File not found
-Get:1 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
-Get:2 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B]
-Get:3 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
+Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
+Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B]
+Get:5 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
 Reading package lists...
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository." aptget update --allow-insecure-repositories
 # ensure we can not install the package
diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 7385e701a..2b662171c 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -43,7 +43,7 @@ runtest() {
     testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
 
     # ensure that we still do a hash check for other files on ims hit of Release
-    if grep -q '^Hit .* InRelease$' expected.output || ! grep -q '^Ign .* Release\(\.gpg\)\?$' expected.output; then
+    if grep -q '^Hit:[0-9]\+ .* InRelease$' expected.output || ! grep -q '^Ign:[0-9]\+ .* Release\(\.gpg\)\?$' expected.output; then
 	    $TEST aptget update -o Debug::Acquire::gpgv=1
 	    cp rootdir/tmp/${TEST}.output goodsign.output
 	    testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
@@ -55,7 +55,7 @@ runtest() {
 }
 
 msgmsg 'InRelease'
-EXPECT='Hit http://localhost:8080 unstable InRelease
+EXPECT='Hit:1 http://localhost:8080 unstable InRelease
 Reading package lists...'
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
@@ -63,9 +63,9 @@ echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
 msgmsg 'Release/Release.gpg'
-EXPECT='Ign http://localhost:8080 unstable InRelease
+EXPECT='Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
-Hit http://localhost:8080 unstable Release
+Hit:2 http://localhost:8080 unstable Release
 Reading package lists...'
 find aptarchive -name 'InRelease' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
@@ -74,10 +74,10 @@ echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest
 
 msgmsg 'Release only'
-EXPECT="Ign http://localhost:8080 unstable InRelease
+EXPECT="Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
-Hit http://localhost:8080 unstable Release
-Ign http://localhost:8080 unstable Release.gpg
+Hit:2 http://localhost:8080 unstable Release
+Ign:3 http://localhost:8080 unstable Release.gpg
   404  Not Found
 Reading package lists...
 W: The data from 'http://localhost:8080 unstable Release' is not signed. Packages from that repository can not be authenticated."
@@ -96,7 +96,7 @@ Valid-Until: $(date -d '-1 weeks' '+%a, %d %b %Y %H:%M:%S %Z')" '{}' \;
 signreleasefiles
 
 msgmsg 'expired InRelease'
-EXPECT='Hit http://localhost:8080 unstable InRelease
+EXPECT='Hit:1 http://localhost:8080 unstable InRelease
 E: Release file for http://localhost:8080/dists/unstable/InRelease is expired (invalid since). Updates for this repository will not be applied.'
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'failure'
@@ -104,9 +104,9 @@ echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'failure'
 
 msgmsg 'expired Release/Release.gpg'
-EXPECT='Ign http://localhost:8080 unstable InRelease
+EXPECT='Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
-Hit http://localhost:8080 unstable Release
+Hit:2 http://localhost:8080 unstable Release
 E: Release file for http://localhost:8080/dists/unstable/Release is expired (invalid since). Updates for this repository will not be applied.'
 find aptarchive -name 'InRelease' -delete
 echo 'Acquire::GzipIndexes "0";' > rootdir/etc/apt/apt.conf.d/02compressindex
@@ -115,10 +115,10 @@ echo 'Acquire::GzipIndexes "1";' > rootdir/etc/apt/apt.conf.d/02compressindex
 runtest 'failure'
 
 msgmsg 'expired Release only'
-EXPECT="Ign http://localhost:8080 unstable InRelease
+EXPECT="Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
-Hit http://localhost:8080 unstable Release
-Ign http://localhost:8080 unstable Release.gpg
+Hit:2 http://localhost:8080 unstable Release
+Ign:3 http://localhost:8080 unstable Release.gpg
   404  Not Found
 W: The data from 'http://localhost:8080 unstable Release' is not signed. Packages from that repository can not be authenticated.
 E: Release file for http://localhost:8080/dists/unstable/Release is expired (invalid since). Updates for this repository will not be applied."
@@ -130,13 +130,13 @@ runtest 'failure' 'warning'
 
 
 msgmsg 'no Release at all'
-EXPECT="Ign http://localhost:8080 unstable InRelease
+EXPECT="Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
-Ign http://localhost:8080 unstable Release
+Ign:2 http://localhost:8080 unstable Release
   404  Not Found
-Hit http://localhost:8080 unstable/main Sources
-Hit http://localhost:8080 unstable/main amd64 Packages
-Hit http://localhost:8080 unstable/main Translation-en
+Hit:3 http://localhost:8080 unstable/main Sources
+Hit:4 http://localhost:8080 unstable/main amd64 Packages
+Hit:5 http://localhost:8080 unstable/main Translation-en
 Reading package lists...
 W: The repository 'http://localhost:8080 unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository."
 find aptarchive -name '*Release*' -delete
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
index 32818658f..6d176a655 100755
--- a/test/integration/test-apt-update-not-modified
+++ b/test/integration/test-apt-update-not-modified
@@ -20,14 +20,14 @@ methodtest() {
 	listcurrentlistsdirectory > listsdir.lst
 
 	# hit again with a good cache
-	testsuccessequal "Hit $1 unstable InRelease
+	testsuccessequal "Hit:1 $1 unstable InRelease
 Reading package lists..." aptget update
 	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
 
 	# drop an architecture, which means the file should be gone now
 	configarchitecture 'i386'
 	sed '/_binary-amd64_Packages/ d' listsdir.lst > listsdir-without-amd64.lst
-	testsuccessequal "Hit $1 unstable InRelease
+	testsuccessequal "Hit:1 $1 unstable InRelease
 Reading package lists..." aptget update
 	testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)"
 
@@ -42,9 +42,9 @@ Architecture: amd64
 Version: 1
 EOF
 	compressfile aptarchive/dists/unstable/main/binary-amd64/Packages
-	testfailureequal "Hit $1 unstable InRelease
-Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
-Err $1 unstable/main amd64 Packages
+	testfailureequal "Hit:1 $1 unstable InRelease
+Get:2 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Err:2 $1 unstable/main amd64 Packages
   Hash Sum mismatch
 W: Failed to fetch $1/dists/unstable/main/binary-amd64/Packages.gz  Hash Sum mismatch
 
@@ -54,8 +54,8 @@ E: Some index files failed to download. They have been ignored, or old ones used
 	cp -a aptarchive/dists.good aptarchive/dists
 
 	# … now everything is fine again
-	testsuccessequal "Hit $1 unstable InRelease
-Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+	testsuccessequal "Hit:1 $1 unstable InRelease
+Get:2 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
 Reading package lists..." aptget update
 	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
 
@@ -74,18 +74,18 @@ Reading package lists..." aptget update
 	listcurrentlistsdirectory > listsdir.lst
 
 	# hit again with a good cache
-	testsuccessequal "Ign $1 unstable InRelease
+	testsuccessequal "Ign:1 $1 unstable InRelease
   404  Not Found
-Hit $1 unstable Release
+Hit:2 $1 unstable Release
 Reading package lists..." aptget update
 	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
 
 	# drop an architecture, which means the file should be gone now
 	configarchitecture 'i386'
 	sed '/_binary-amd64_Packages/ d' listsdir.lst > listsdir-without-amd64.lst
-	testsuccessequal "Ign $1 unstable InRelease
+	testsuccessequal "Ign:1 $1 unstable InRelease
   404  Not Found
-Hit $1 unstable Release
+Hit:2 $1 unstable Release
 Reading package lists..." aptget update
 	testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)"
 
@@ -100,11 +100,11 @@ Architecture: amd64
 Version: 1
 EOF
 	compressfile aptarchive/dists/unstable/main/binary-amd64/Packages
-	testfailureequal "Ign $1 unstable InRelease
+	testfailureequal "Ign:1 $1 unstable InRelease
   404  Not Found
-Hit $1 unstable Release
-Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
-Err $1 unstable/main amd64 Packages
+Hit:2 $1 unstable Release
+Get:4 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Err:4 $1 unstable/main amd64 Packages
   Hash Sum mismatch
 W: Failed to fetch $1/dists/unstable/main/binary-amd64/Packages.gz  Hash Sum mismatch
 
@@ -115,18 +115,18 @@ E: Some index files failed to download. They have been ignored, or old ones used
 	find aptarchive/dists -name 'InRelease' -delete
 
 	# … now everything is fine again
-	testsuccessequal "Ign $1 unstable InRelease
+	testsuccessequal "Ign:1 $1 unstable InRelease
   404  Not Found
-Hit $1 unstable Release
-Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Hit:2 $1 unstable Release
+Get:4 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
 Reading package lists..." aptget update
 	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
 
 	webserverconfig 'aptwebserver::support::modified-since' 'false'
 	webserverconfig 'aptwebserver::support::last-modified' 'false'
-	testsuccessequal "Ign $1 unstable InRelease
+	testsuccessequal "Ign:1 $1 unstable InRelease
   404  Not Found
-Get:1 $1 unstable Release [$(stat -c '%s' 'aptarchive/dists/unstable/Release') B]
+Get:2 $1 unstable Release [$(stat -c '%s' 'aptarchive/dists/unstable/Release') B]
 Reading package lists..." aptget update
 	webserverconfig 'aptwebserver::support::modified-since' 'true'
 	webserverconfig 'aptwebserver::support::last-modified' 'true'
diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index 3042d116d..47dd62712 100755
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -13,7 +13,7 @@ setupflataptarchive
 testaptgetupdate() {
 	rm -rf rootdir/var/lib/apt
 	aptget update >testaptgetupdate.diff 2>&1 || true
-	sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#Get:[0-9]\+ #Get: #' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
+	sed -i -e '/Ign /,+1d' -e '/Release/ d' -e 's#\[[0-9]* [kMGTPY]*B\]#\[\]#' testaptgetupdate.diff
 	GIVEN="$1"
 	shift
 	msgtest "Test for correctness of" "apt-get update with $*"
@@ -47,12 +47,12 @@ testoverfile() {
 	forcecompressor "$1"
 
 	createemptyarchive 'Packages'
-	testaptgetupdate "Get: file:$APTARCHIVE  Packages []
+	testaptgetupdate "Get:2 file:$APTARCHIVE  Packages []
 Reading package lists..." "empty archive Packages.$COMPRESS over file"
 
 	createemptyfile 'Packages'
-	testaptgetupdate "Get: file:$APTARCHIVE  Packages
-Err file:$APTARCHIVE  Packages
+	testaptgetupdate "Get:2 file:$APTARCHIVE  Packages
+Err:2 file:$APTARCHIVE  Packages
   Empty files can't be valid archives
 W: Failed to fetch ${COMPRESSOR}:${APTARCHIVE}/Packages.$COMPRESS  Empty files can't be valid archives
 
@@ -63,13 +63,13 @@ testoverhttp() {
 	forcecompressor "$1"
 
 	createemptyarchive 'Packages'
-	testaptgetupdate "Get: http://localhost:8080  Packages []
+	testaptgetupdate "Get:2 http://localhost:8080  Packages []
 Reading package lists..." "empty archive Packages.$COMPRESS over http"
 
 	createemptyfile 'Packages'
 	#FIXME: we should response with a good error message instead
-	testaptgetupdate "Get: http://localhost:8080  Packages
-Err http://localhost:8080  Packages
+	testaptgetupdate "Get:2 http://localhost:8080  Packages
+Err:2 http://localhost:8080  Packages
   Empty files can't be valid archives
 W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages.${COMPRESS})  Empty files can't be valid archives
 
diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect
index ca2378c19..b9d232f90 100755
--- a/test/integration/test-bug-602412-dequote-redirect
+++ b/test/integration/test-bug-602412-dequote-redirect
@@ -21,7 +21,7 @@ testrun() {
 	testsuccess --nomsg aptget update
 
 	# check that I-M-S header is kept in redirections
-	testsuccessequal "Hit $1 unstable InRelease
+	testsuccessequal "Hit:1 $1 unstable InRelease
 Reading package lists..." aptget update
 
 	msgtest 'Test redirection works in' 'package download'
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 3295d5497..e5fe21e0f 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -98,7 +98,7 @@ SHA256-Download:
 	msgmsg "Testcase: index is already up-to-date: $*"
 	find rootdir/var/lib/apt/lists -name '*diff_Index' -type f -delete
 	testsuccess aptget update "$@"
-	testequal 'Hit http://localhost:8080  InRelease
+	testequal 'Hit:1 http://localhost:8080  InRelease
 Reading package lists...' aptget update "$@" -o Debug::Acquire::Transaction=0 -o Debug::pkgAcquire::Diffs=0
 	testsuccessequal "$(cat ${PKGFILE}-new)
 " aptcache show apt newstuff
diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
index 6abb5d12a..48a7f0562 100755
--- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
+++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
@@ -176,10 +176,10 @@ testmismatch() {
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080  $1 1.0 (dsc) [3 B]
-Err http://localhost:8080  $1 1.0 (dsc)
+Err:1 http://localhost:8080  $1 1.0 (dsc)
   Hash Sum mismatch
 Get:2 http://localhost:8080  $1 1.0 (tar) [3 B]
-Err http://localhost:8080  $1 1.0 (tar)
+Err:2 http://localhost:8080  $1 1.0 (tar)
   Hash Sum mismatch
 E: Failed to fetch http://localhost:8080/${1}_1.0.dsc  Hash Sum mismatch
 
@@ -242,7 +242,7 @@ Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080  pkg-mixed-sha1-bad 1.0 (tar) [3 B]
 Get:2 http://localhost:8080  pkg-mixed-sha1-bad 1.0 (dsc) [3 B]
-Err http://localhost:8080  pkg-mixed-sha1-bad 1.0 (dsc)
+Err:2 http://localhost:8080  pkg-mixed-sha1-bad 1.0 (dsc)
   Hash Sum mismatch
 E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc  Hash Sum mismatch
 
@@ -253,7 +253,7 @@ testfailureequal 'Reading package lists...
 Building dependency tree...
 Need to get 6 B of source archives.
 Get:1 http://localhost:8080  pkg-mixed-sha2-bad 1.0 (tar) [3 B]
-Err http://localhost:8080  pkg-mixed-sha2-bad 1.0 (tar)
+Err:1 http://localhost:8080  pkg-mixed-sha2-bad 1.0 (tar)
   Hash Sum mismatch
 Get:2 http://localhost:8080  pkg-mixed-sha2-bad 1.0 (dsc) [3 B]
 E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz  Hash Sum mismatch
-- 
cgit v1.2.3-70-g09d2


From 9d2a8a7388cf3b0bbbe92f6b0b30a533e1167f40 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 15 Jun 2015 23:06:56 +0200
Subject: condense parallel requests with the same hashes to one

It shouldn't be too common, but sometimes people have multiple mirrors
in the sources or otherwise repositories with the same content. Now that
we gracefully can handle multiple requests to the same URI, we can also
fold multiple requests with the same expected hashes into one. Note that
this isn't trying to find oppertunities for merging, but just merges if
it happens to encounter the oppertunity for it.

This is most obvious in the new testcase actually as it needs to delay
the action to give the acquire system enough time to figure out that
they can be merged.
---
 apt-pkg/acquire.cc                                 |  5 +-
 apt-private/acqprogress.cc                         |  6 +-
 test/integration/framework                         |  8 ++-
 test/integration/skip-aptwebserver                 | 25 -------
 .../test-acquire-same-file-multiple-times          |  4 +-
 .../test-acquire-same-repository-multiple-times    | 81 ++++++++++++++++++++++
 test/integration/test-apt-get-source               | 42 ++++++-----
 test/integration/test-apt-get-source-arch          | 23 +++---
 test/integration/test-apt-get-source-multisources  | 12 ++--
 .../integration/test-apt-get-update-unauth-warning | 37 +++++++++-
 test/integration/test-apt-progress-fd              |  4 --
 test/integration/test-apt-update-ims               | 18 +++++
 .../test-bug-722207-print-uris-even-if-very-quiet  | 12 ++--
 test/integration/test-bug-738785-switch-protocol   | 22 +++---
 14 files changed, 198 insertions(+), 101 deletions(-)
 delete mode 100755 test/integration/skip-aptwebserver
 create mode 100755 test/integration/test-acquire-same-repository-multiple-times

(limited to 'test')

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 14c8863dc..34afab181 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -680,9 +680,12 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
 {
    QItem **I = &Items;
    // move to the end of the queue and check for duplicates here
+   HashStringList const hsl = Item.Owner->GetExpectedHashes();
    for (; *I != 0; I = &(*I)->Next)
-      if (Item.URI == (*I)->URI) 
+      if (Item.URI == (*I)->URI || hsl == (*I)->Owner->GetExpectedHashes())
       {
+	 if (_config->FindB("Debug::pkgAcquire::Worker",false) == true)
+	    std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl;
 	 (*I)->Owners.push_back(Item.Owner);
 	 Item.Owner->Status = (*I)->Owner->Status;
 	 return false;
diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index dc92e3b2a..f6c3d1204 100644
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -116,14 +116,10 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
-   // Ignore certain kinds of transient failures (bad code)
-   if (Itm.Owner->Status == pkgAcquire::Item::StatIdle)
-      return;
-
    AssignItemID(Itm);
    clearLastLine();
 
-   if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
+   if (Itm.Owner->Status == pkgAcquire::Item::StatDone || Itm.Owner->Status == pkgAcquire::Item::StatIdle)
    {
       // TRANSLATOR: Very short word to be displayed for files in 'apt-get update'
       // which failed to download, but the error is ignored (compare "Err:")
diff --git a/test/integration/framework b/test/integration/framework
index 1b99929e2..d8f7567d9 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -245,7 +245,7 @@ setupenvironment() {
 	echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf
 	echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf
 	echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf
-	echo "Dir::Bin::Methods \"${METHODSDIR}\";" >> aptconfig.conf
+	echo "Dir::Bin::Methods \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/methods\";" >> aptconfig.conf
 	# store apt-key were we can access it, even if we run it as a different user
 	# destroys coverage reporting though, so just do it for root for now
 	if [ "$(id -u)" = '0' ]; then
@@ -786,6 +786,8 @@ insertsource() {
 	local SPATH="aptarchive/dists/${RELEASE}/main/source"
 	mkdir -p $SPATH
 	local FILE="${SPATH}/Sources"
+	local DSCFILE="${NAME}_${VERSION}.dsc"
+	local TARFILE="${NAME}_${VERSION}.tar.gz"
 	echo "Package: $NAME
 Binary: $NAME
 Version: $VERSION
@@ -793,8 +795,8 @@ Maintainer: Joe Sixpack 
 Architecture: $ARCH" >> $FILE
 	test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE
 	echo "Files:
- d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.dsc
- d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.tar.gz
+ $(echo -n "$DSCFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$DSCFILE" | wc -c) $DSCFILE
+ $(echo -n "$TARFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$TARFILE" | wc -c) $TARFILE
 " >> $FILE
 }
 
diff --git a/test/integration/skip-aptwebserver b/test/integration/skip-aptwebserver
deleted file mode 100755
index 0622941ce..000000000
--- a/test/integration/skip-aptwebserver
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/sh
-set -e
-
-TESTDIR=$(readlink -f $(dirname $0))
-. $TESTDIR/framework
-
-setupenvironment
-configarchitecture 'amd64'
-
-buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
-
-setupaptarchive
-changetowebserver
-
-rm -rf rootdir/var/lib/apt/lists
-aptget update -qq
-testequal 'Hit http://localhost stable InRelease
-Hit http://localhost stable/main Sources
-Hit http://localhost stable/main amd64 Packages
-Hit http://localhost stable/main Translation-en
-Reading package lists...' aptget update
-
-mv rootdir/var/lib/apt/lists/localhost* rootdir/var/lib/apt/lists/partial
-aptget update
-
diff --git a/test/integration/test-acquire-same-file-multiple-times b/test/integration/test-acquire-same-file-multiple-times
index d329a39cb..526765521 100755
--- a/test/integration/test-acquire-same-file-multiple-times
+++ b/test/integration/test-acquire-same-file-multiple-times
@@ -45,7 +45,7 @@ changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo
 
 httpdown() {
 	msgtest 'Downloading the same URI to different files' 'twice over http'
-	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
 	cp rootdir/tmp/testsuccess.output download.log
 	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
 	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
@@ -57,7 +57,7 @@ testrun 'httpdown'
 
 httpredirectdown() {
 	msgtest 'Redirect leads' 'first URI to the second URI'
-	testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1
+	testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1
 	cp rootdir/tmp/testsuccess.output download.log
 	testsuccess cmp $TESTDIR/framework ./downloaded/foo1
 	testsuccess cmp ./downloaded/foo1 ./downloaded/foo2
diff --git a/test/integration/test-acquire-same-repository-multiple-times b/test/integration/test-acquire-same-repository-multiple-times
new file mode 100755
index 000000000..bfeaf88db
--- /dev/null
+++ b/test/integration/test-acquire-same-repository-multiple-times
@@ -0,0 +1,81 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+TESTFILE="$TESTDIR/framework"
+cp $TESTFILE aptarchive/foo
+APTARCHIVE="$(readlink -f ./aptarchive)"
+
+getcodenamefromsuite() { echo "jessie"; }
+buildsimplenativepackage 'foo' 'all' '1.0' 'stable'
+setupaptarchive --no-update
+ln -s "${APTARCHIVE}/dists/stable" "${APTARCHIVE}/dists/jessie"
+for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do
+	sed 's#stable#jessie#g' $FILE > $(echo "$FILE" | sed 's#stable#jessie#g')
+done
+
+# install a slowed down file: otherwise its to fast to reproduce combining
+NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
+OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
+rm $NEWMETHODS
+mkdir $NEWMETHODS
+for METH in $(find $OLDMETHODS ! -type d); do
+	ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS
+done
+rm $NEWMETHODS/file
+cat >$NEWMETHODS/file < file.lst
+	testequal "$(find $(readlink -f ./rootdir/var/lib/apt/lists) -name '*_dists_*' \( ! -name '*InRelease' \) -type f | sort)" cat file.lst
+	testsuccess aptcache policy
+	testequal "foo:
+  Installed: (none)
+  Candidate: 1.0
+  Version table:
+     1.0 0
+        500 $1:$2 jessie/main amd64 Packages
+        500 $1:$2 stable/main amd64 Packages" aptcache policy foo
+	testfailure aptcache show foo/unstable
+	testsuccess aptcache show foo/stable
+	testsuccess aptcache show foo/jessie
+}
+
+tworepos 'file' "$APTARCHIVE" 'no partial'
+testequal '12' grep -c '200%20URI%20Start' ./download.log
+testequal '12' grep -c '201%20URI%20Done' ./download.log
+testequal '6' grep -c '^ @ Queue: Action combined' ./download.log
+tworepos 'file' "$APTARCHIVE" 'hit'
+testequal '6' grep -c '200%20URI%20Start' ./download.log
+testequal '6' grep -c '201%20URI%20Done' ./download.log
+testequal '0' grep -c '^ @ Queue: Action combined' ./download.log
+rm -rf rootdir/var/lib/apt/lists
+
+changetowebserver
+
+tworepos 'http' '//localhost:8080' 'no partial'
+testequal '10' grep -c '200%20URI%20Start' ./download.log
+testequal '10' grep -c '201%20URI%20Done' ./download.log
+testequal '6' grep -c '^ @ Queue: Action combined' ./download.log
+tworepos 'http' '//localhost:8080' 'hit'
+testequal '2' grep -c '200%20URI%20Start' ./download.log
+testequal '4' grep -c '201%20URI%20Done' ./download.log
+testequal '0' grep -c '^ @ Queue: Action combined' ./download.log
+rm -rf rootdir/var/lib/apt/lists
diff --git a/test/integration/test-apt-get-source b/test/integration/test-apt-get-source
index 9db24370f..22f01b997 100755
--- a/test/integration/test-apt-get-source
+++ b/test/integration/test-apt-get-source
@@ -34,43 +34,40 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 # normal operation gets highest version number
 HEADER="Reading package lists...
 Building dependency tree..."
+DOWNLOAD1="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef
+'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184"
+DOWNLOAD2="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55
+'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f"
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo
+$DOWNLOAD2" aptget source -q --print-uris foo
 
 # select by release: suite
 testsuccessequal "$HEADER
 Selected version '1.0' (stable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/stable
+$DOWNLOAD1" aptget source -q --print-uris foo/stable
 testsuccessequal "$HEADER
 Selected version '2.0' (unstable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/unstable
+$DOWNLOAD2" aptget source -q --print-uris foo/unstable
 
 # select by release: codename
 testsuccessequal "$HEADER
 Selected version '2.0' (sid) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/sid
+$DOWNLOAD2" aptget source -q --print-uris foo/sid
 
 # select by version
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo=1.0
+$DOWNLOAD1" aptget source -q --print-uris foo=1.0
 
 # select by release with no binary package (Bug#731102) but ensure to get
 # highest version
+DOWNLOAD01="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af
+'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61"
 testsuccessequal "$HEADER
 Selected version '0.1' (wheezy) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/wheezy
+$DOWNLOAD01" aptget source -q --print-uris foo/wheezy
 
 # unavailable one
 testfailureequal "$HEADER
@@ -78,11 +75,12 @@ E: Can not find version '9.9-not-there' of package 'foo'
 E: Unable to find a source package for foo" aptget source -q --print-uris foo=9.9-not-there
 
 # version and release
+DOWNLOAD001="Need to get 0 B/29 B of source archives.
+'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 13 MD5Sum:6c819ebf0a21b1a480e1dbf6b8edfebd
+'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 16 MD5Sum:a3c7e1ac2159fc0faf522e110d6906fd"
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris -t unstable foo=0.0.1
+$DOWNLOAD001" aptget source -q --print-uris -t unstable foo=0.0.1
 
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
+Need to get 0 B/25 B of source archives.
 Fetch source foo" aptget source -q -s foo
diff --git a/test/integration/test-apt-get-source-arch b/test/integration/test-apt-get-source-arch
index c75798209..f54bb6012 100755
--- a/test/integration/test-apt-get-source-arch
+++ b/test/integration/test-apt-get-source-arch
@@ -28,31 +28,30 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 
 HEADER="Reading package lists...
 Building dependency tree..."
+DOWNLOAD10="Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef
+'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184"
 
 # pick :amd64
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:amd64
+$DOWNLOAD10" aptget source -q --print-uris foo:amd64
 
 # pick :i386
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386
+Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55
+'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f" aptget source -q --print-uris foo:i386
 
 # pick :i386 by release
 testsuccessequal "$HEADER
 Selected version '0.1' (oldstable) for foo
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386/oldstable
+Need to get 0 B/25 B of source archives.
+'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af
+'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61" aptget source -q --print-uris foo:i386/oldstable
 
 # pick :i386 by version
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386=1.0
+$DOWNLOAD10" aptget source -q --print-uris foo:i386=1.0
 
 # error on unknown arch
 testfailureequal "$HEADER
diff --git a/test/integration/test-apt-get-source-multisources b/test/integration/test-apt-get-source-multisources
index 03d0400a0..887a30685 100755
--- a/test/integration/test-apt-get-source-multisources
+++ b/test/integration/test-apt-get-source-multisources
@@ -20,11 +20,11 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 HEADER="Reading package lists...
 Building dependency tree..."
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11
+Need to get 0 B/43 B of source archives.
+'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b
+'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11
 
 testsuccessequal "$HEADER
-Need to get 0 B of source archives.
-'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e"  aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113
+Need to get 0 B/43 B of source archives.
+'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df
+'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b" aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 5b81d56d2..fcabf244a 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -20,8 +20,10 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 rm -f $APTARCHIVE/dists/unstable/*Release*
 
 # update without authenticated files leads to warning
-testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease
+testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease
+Ign:1 file:$APTARCHIVE unstable InRelease
   File not found
+Get:2 file:$APTARCHIVE unstable Release
 Err:2 file:$APTARCHIVE unstable Release
   File not found
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository.
@@ -36,10 +38,41 @@ filesize() {
 	stat -c%s "$(aptget files --no-release-info --format '$(URI)' "Created-By: $1" | cut -d'/' -f 3- ).gz"
 }
 # allow override
-testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease
+#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1
+#exit
+testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease
+Ign:1 file:$APTARCHIVE unstable InRelease
   File not found
+Get:2 file:$APTARCHIVE unstable Release
 Ign:2 file:$APTARCHIVE unstable Release
   File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
+Get:3 file:$APTARCHIVE unstable/main Sources
+Ign:3 file:$APTARCHIVE unstable/main Sources
+  File not found
+Get:4 file:$APTARCHIVE unstable/main i386 Packages
+Ign:4 file:$APTARCHIVE unstable/main i386 Packages
+  File not found
+Get:5 file:$APTARCHIVE unstable/main Translation-en
+Ign:5 file:$APTARCHIVE unstable/main Translation-en
+  File not found
 Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
 Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B]
 Get:5 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd
index 99b4ea050..e30d503cb 100755
--- a/test/integration/test-apt-progress-fd
+++ b/test/integration/test-apt-progress-fd
@@ -16,7 +16,6 @@ setupaptarchive
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -33,7 +32,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -50,7 +48,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing:0:Installing testing (amd64)
@@ -77,7 +74,6 @@ pmstatus:dpkg-exec:75:Running dpkg'
 exec 3> apt-progress.log
 testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1
-dlstatus:1:0:Retrieving file 1 of 1
 dlstatus:1:20:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0:Running dpkg
 pmstatus:testing2:0:Installing testing2 (i386)
diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index 2b662171c..33b4ed1b9 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -134,6 +134,24 @@ EXPECT="Ign:1 http://localhost:8080 unstable InRelease
   404  Not Found
 Ign:2 http://localhost:8080 unstable Release
   404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
+Ign:3 http://localhost:8080 unstable/main Sources
+  404  Not Found
+Ign:4 http://localhost:8080 unstable/main amd64 Packages
+  404  Not Found
+Ign:5 http://localhost:8080 unstable/main Translation-en
+  404  Not Found
 Hit:3 http://localhost:8080 unstable/main Sources
 Hit:4 http://localhost:8080 unstable/main amd64 Packages
 Hit:5 http://localhost:8080 unstable/main Translation-en
diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
index e51d72ccd..1fa94de7d 100755
--- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet
+++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
@@ -21,11 +21,11 @@ testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.d
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris
-testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a
+'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373" aptget source apt -qq --print-uris
 testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris
 
-testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
-'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt apt2 -qq --print-uris
+testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a
+'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373
+'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 10 MD5Sum:4c572ce45f1e2bedbb30da7f5e1c241c
+'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 13 MD5Sum:2a96fec139f8722d93312a1ff8281232" aptget source apt apt2 -qq --print-uris
diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index 539080200..e86f28824 100755
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -39,18 +39,15 @@ cd - >/dev/null
 testsuccess aptget install apt -y
 testdpkginstalled 'apt'
 
-# create a copy of all methods, expect https
-eval `aptconfig shell METHODS Dir::Bin::Methods/d`
-COPYMETHODS='usr/lib/apt/methods'
-mv rootdir/${COPYMETHODS} rootdir/${COPYMETHODS}.bak
-mkdir -p rootdir/$COPYMETHODS
-cd rootdir/$COPYMETHODS
-find $METHODS \! -type d | while read meth; do
-	ln -s $meth
+# install a slowed down file: otherwise its to fast to reproduce combining
+NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods"
+OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)"
+rm $NEWMETHODS
+mkdir $NEWMETHODS
+for METH in $(find $OLDMETHODS ! -type d); do
+	ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS
 done
-rm https
-cd - >/dev/null
-echo "Dir::Bin::Methods \"${COPYMETHODS}\";" >> aptconfig.conf
+rm $NEWMETHODS/https
 
 cd downloaded
 testfailureequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.
@@ -59,8 +56,7 @@ testfailure test -e apt_1.0_all.deb
 cd - >/dev/null
 
 # revert to all methods
-rm -rf rootdir/$COPYMETHODS
-mv rootdir/${COPYMETHODS}.bak rootdir/${COPYMETHODS}
+ln -s $OLDMETHODS/https $NEWMETHODS
 
 # check that downgrades from https to http are not allowed
 webserverconfig 'aptwebserver::support::http' 'true'
-- 
cgit v1.2.3-70-g09d2


From 533fe3d13927798c17bdef84eba60ed9441d9608 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 15 Jun 2015 23:59:14 +0200
Subject: allow ratelimiting progress reporting for testcases

Progress reports once in a while which is a bit to unpredictable for
testcases, so we enforce a steady progress for them in the hope that
this makes the tests (mostly test-apt-progress-fd) a bit more stable.

Git-Dch: Ignore
---
 apt-pkg/acquire.cc                                 | 14 ++++++++-----
 test/integration/framework                         | 23 ++++++++++++----------
 .../test-acquire-same-repository-multiple-times    |  2 +-
 3 files changed, 23 insertions(+), 16 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 34afab181..5e5bec95c 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -955,7 +955,7 @@ std::string pkgAcquire::Queue::QItem::Custom600Headers() const		/*{{{*/
 // AcquireStatus::pkgAcquireStatus - Constructor			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(0), Update(true), MorePulses(false)
+pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false)
 {
    Start();
 }
@@ -1054,13 +1054,17 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
       Time = NewTime;
    }
 
+   double const OldPercent = Percent;
    // calculate the percentage, if we have too little data assume 1%
    if (TotalBytes > 0 && UnfetchedReleaseFiles)
       Percent = 0;
-   else 
+   else
       // use both files and bytes because bytes can be unreliable
-      Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + 
+      Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) +
                  0.2 * (CurrentItems/float(TotalItems)*100.0));
+   double const DiffPercent = Percent - OldPercent;
+   if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true)
+      return true;
 
    int fd = _config->FindI("APT::Status-Fd",-1);
    if(fd > 0) 
@@ -1078,11 +1082,11 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner)
 	 snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str());
       else
 	 snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems);
-	 
+
       // build the status str
       status << "dlstatus:" << i
              << ":"  << std::setprecision(3) << Percent
-             << ":" << msg 
+             << ":" << msg
              << endl;
 
       std::string const dlstatus = status.str();
diff --git a/test/integration/framework b/test/integration/framework
index d8f7567d9..5d949009f 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -273,16 +273,19 @@ EOF
 	chmod +x "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg"
 	echo "Dir::Bin::dpkg \"${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg\";" > rootdir/etc/apt/apt.conf.d/99dpkg
 
-	if ! command dpkg --assert-multi-arch >/dev/null 2>&1; then
-		echo "DPKG::options:: \"--force-architecture\";" >> aptconfig.conf # Added to test multiarch before dpkg is ready for it…
-	fi
-	echo 'quiet::NoUpdate "true";' >> aptconfig.conf
-	echo 'quiet::NoStatistic "true";' >> aptconfig.conf
-	# too distracting for users, but helpful to detect changes
-	echo 'Acquire::Progress::Ignore::ShowErrorText "true";' >> aptconfig.conf
-	# in testcases, it can appear as if localhost has a rotation setup,
-	# hide this as we can't really deal with it properly
-	echo 'Acquire::Failure::ShowIP "false";' >> aptconfig.conf
+	{
+		if ! command dpkg --assert-multi-arch >/dev/null 2>&1; then
+			echo "DPKG::options:: \"--force-architecture\";" # Added to test multiarch before dpkg is ready for it…
+		fi
+		echo 'quiet::NoUpdate "true";'
+		echo 'quiet::NoStatistic "true";'
+		# too distracting for users, but helpful to detect changes
+		echo 'Acquire::Progress::Ignore::ShowErrorText "true";'
+		echo 'Acquire::Progress::Diffpercent "true";'
+		# in testcases, it can appear as if localhost has a rotation setup,
+		# hide this as we can't really deal with it properly
+		echo 'Acquire::Failure::ShowIP "false";'
+	} >> aptconfig.conf
 
 	cp "${TESTDIRECTORY}/apt.pem" "${TMPWORKINGDIRECTORY}/rootdir/etc/webserver.pem"
 	if [ "$(id -u)" = '0' ]; then
diff --git a/test/integration/test-acquire-same-repository-multiple-times b/test/integration/test-acquire-same-repository-multiple-times
index bfeaf88db..a46e0d73c 100755
--- a/test/integration/test-acquire-same-repository-multiple-times
+++ b/test/integration/test-acquire-same-repository-multiple-times
@@ -32,7 +32,7 @@ cat >$NEWMETHODS/file <
Date: Tue, 16 Jun 2015 16:22:46 +0200
Subject: add d-pointer, virtual destructors and de-inline de/constructors

To have a chance to keep the ABI for a while we need all three to team
up. One of them missing and we might loose, so ensuring that they are
available is a very tedious but needed task once in a while.

Git-Dch: Ignore
---
 apt-pkg/acquire-item.cc         |  11 +++-
 apt-pkg/acquire-item.h          |  16 ++++--
 apt-pkg/acquire-method.cc       |   6 +-
 apt-pkg/acquire-method.h        |   9 ++-
 apt-pkg/acquire-worker.h        |   1 -
 apt-pkg/acquire.cc              |   9 +++
 apt-pkg/acquire.h               |  10 +---
 apt-pkg/algorithms.h            |   6 +-
 apt-pkg/aptconfiguration.h      |  21 ++++---
 apt-pkg/cachefile.h             |   1 -
 apt-pkg/cacheset.cc             |  15 ++++-
 apt-pkg/cacheset.h              |  24 +++++---
 apt-pkg/cdrom.cc                |   6 ++
 apt-pkg/cdrom.h                 |  11 +++-
 apt-pkg/clean.cc                |   1 +
 apt-pkg/clean.h                 |   2 +
 apt-pkg/deb/debindexfile.cc     |   1 +
 apt-pkg/deb/debindexfile.h      |   8 ++-
 apt-pkg/deb/debmetaindex.cc     |   2 +
 apt-pkg/deb/debmetaindex.h      |   3 +-
 apt-pkg/deb/debrecords.cc       |   7 +++
 apt-pkg/deb/debrecords.h        |  15 +++--
 apt-pkg/deb/debsrcrecords.cc    |   4 ++
 apt-pkg/deb/debsrcrecords.h     |   4 +-
 apt-pkg/deb/debversion.h        |   1 -
 apt-pkg/depcache.h              |   6 +-
 apt-pkg/edsp.cc                 | 123 ++++++++++++++++++++--------------------
 apt-pkg/edsp.h                  |  42 +++++---------
 apt-pkg/edsp/edspindexfile.cc   |   2 +
 apt-pkg/edsp/edspindexfile.h    |   1 +
 apt-pkg/edsp/edsplistparser.cc  |   2 +
 apt-pkg/edsp/edsplistparser.h   |   2 +
 apt-pkg/edsp/edspsystem.h       |   2 +-
 apt-pkg/indexcopy.cc            |  10 ++++
 apt-pkg/indexcopy.h             |  23 ++++++--
 apt-pkg/indexfile.cc            |   3 +
 apt-pkg/indexfile.h             |   6 +-
 apt-pkg/indexrecords.h          |   3 -
 apt-pkg/install-progress.cc     |   6 ++
 apt-pkg/install-progress.h      |  11 +++-
 apt-pkg/metaindex.h             |   1 +
 apt-pkg/orderlist.h             |   4 +-
 apt-pkg/packagemanager.h        |   2 +-
 apt-pkg/pkgcache.cc             |   2 +
 apt-pkg/pkgcache.h              |   5 +-
 apt-pkg/pkgcachegen.cc          |   3 +
 apt-pkg/pkgcachegen.h           |  17 +++---
 apt-pkg/pkgrecords.cc           |   3 +
 apt-pkg/pkgrecords.h            |   8 ++-
 apt-pkg/pkgsystem.cc            |   2 +
 apt-pkg/pkgsystem.h             |   5 +-
 apt-pkg/policy.cc               |   2 +
 apt-pkg/policy.h                |   5 +-
 apt-pkg/sourcelist.h            |   4 +-
 apt-pkg/srcrecords.h            |   1 -
 apt-pkg/tagfile.h               |   2 -
 apt-pkg/version.cc              |   3 +
 apt-pkg/version.h               |   2 +-
 test/libapt/acqprogress_test.cc |  46 ++++++++-------
 59 files changed, 348 insertions(+), 205 deletions(-)

(limited to 'test')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 5460280ec..3313aaabc 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -438,7 +438,7 @@ APT_PURE pkgAcquire * pkgAcquire::Item::GetOwner() const		/*{{{*/
    return Owner;
 }
 									/*}}}*/
-pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc()			/*{{{*/
+APT_CONST pkgAcquire::ItemDesc &pkgAcquire::Item::GetItemDesc()		/*{{{*/
 {
    return Desc;
 }
@@ -1098,6 +1098,7 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message)		/*{{{*/
    return true;
 }
 									/*}}}*/
+pkgAcqMetaBase::~pkgAcqMetaBase() {}
 
 pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner,	/*{{{*/
       IndexTarget const &ClearsignedTarget,
@@ -1319,6 +1320,7 @@ std::string pkgAcqMetaIndex::DescURI() const				/*{{{*/
    return Target.URI;
 }
 									/*}}}*/
+pkgAcqMetaIndex::~pkgAcqMetaIndex() {}
 
 // AcqMetaSig::AcqMetaSig - Constructor					/*{{{*/
 pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner,
@@ -1491,6 +1493,7 @@ pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner,
 {
 }
 									/*}}}*/
+pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
 
 // AcqDiffIndex::AcqDiffIndex - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
@@ -1893,6 +1896,7 @@ void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes,	/
    return;
 }
 									/*}}}*/
+pkgAcqDiffIndex::~pkgAcqDiffIndex() {}
 
 // AcqIndexDiffs::AcqIndexDiffs - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
@@ -2119,6 +2123,7 @@ std::string pkgAcqIndexDiffs::Custom600Headers() const			/*{{{*/
    return patchhashes.str();
 }
 									/*}}}*/
+pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {}
 
 // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor			/*{{{*/
 pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner,
@@ -2263,6 +2268,7 @@ std::string pkgAcqIndexMergeDiffs::Custom600Headers() const		/*{{{*/
    return patchhashes.str();
 }
 									/*}}}*/
+pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
 
 // AcqIndex::AcqIndex - Constructor					/*{{{*/
 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
@@ -2539,6 +2545,7 @@ void pkgAcqIndex::StageDecompressDone(string const &,
    return;
 }
 									/*}}}*/
+pkgAcqIndex::~pkgAcqIndex() {}
 
 
 // AcqArchive::AcqArchive - Constructor					/*{{{*/
@@ -2836,6 +2843,7 @@ std::string pkgAcqArchive::ShortDesc() const				/*{{{*/
    return Desc.ShortDesc;
 }
 									/*}}}*/
+pkgAcqArchive::~pkgAcqArchive() {}
 
 // AcqChangelog::pkgAcqChangelog - Constructors				/*{{{*/
 pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
@@ -3156,3 +3164,4 @@ string pkgAcqFile::Custom600Headers() const				/*{{{*/
    return "";
 }
 									/*}}}*/
+pkgAcqFile::~pkgAcqFile() {}
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 9dbacc1ea..df1380b5e 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-item.h,v 1.26.2.3 2004/01/02 18:51:00 mdz Exp $
 /* ######################################################################
 
    Acquire Item - Item to acquire
@@ -345,6 +344,7 @@ class pkgAcquire::Item : public WeakPointable				/*{{{*/
 class APT_HIDDEN pkgAcqTransactionItem: public pkgAcquire::Item		/*{{{*/
 /** \brief baseclass for the indexes files to manage them all together */
 {
+   void *d;
    protected:
    IndexTarget const Target;
    HashStringList GetExpectedHashesFor(std::string const MetaKey) const;
@@ -380,7 +380,6 @@ class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem		/*{{{*/
 /** \brief the manager of a transaction */
 {
    void *d;
-
  protected:
    std::vector Transaction;
 
@@ -478,6 +477,7 @@ class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem		/*{{{*/
 		  std::vector const IndexTargets,
 		  IndexTarget const &DataTarget,
 		  indexRecords* const MetaIndexParser);
+   virtual ~pkgAcqMetaBase();
 };
 									/*}}}*/
 /** \brief An item that is responsible for downloading the meta-index	{{{
@@ -493,7 +493,6 @@ class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem		/*{{{*/
 class APT_HIDDEN pkgAcqMetaIndex : public pkgAcqMetaBase
 {
    void *d;
-
    protected:
    IndexTarget const DetachedSigTarget;
 
@@ -513,6 +512,7 @@ class APT_HIDDEN pkgAcqMetaIndex : public pkgAcqMetaBase
    pkgAcqMetaIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
 		   IndexTarget const &DataTarget, IndexTarget const &DetachedSigTarget,
 		   std::vector const IndexTargets, indexRecords * const MetaIndexParser);
+   virtual ~pkgAcqMetaIndex();
 
    friend class pkgAcqMetaSig;
 };
@@ -589,6 +589,7 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem
 
    pkgAcqBaseIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
                    IndexTarget const Target);
+   virtual ~pkgAcqBaseIndex();
 };
 									/*}}}*/
 /** \brief An item that is responsible for fetching an index file of	{{{
@@ -652,6 +653,7 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
     */
    pkgAcqDiffIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
                    IndexTarget const Target);
+   virtual ~pkgAcqDiffIndex();
  private:
    APT_HIDDEN void QueueOnIMSHit() const;
 };
@@ -751,6 +753,7 @@ class APT_HIDDEN pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex
    pkgAcqIndexMergeDiffs(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
                          IndexTarget const Target, DiffInfo const &patch,
                          std::vector const * const allPatches);
+   virtual ~pkgAcqIndexMergeDiffs();
 };
 									/*}}}*/
 /** \brief An item that is responsible for fetching server-merge patches {{{
@@ -864,6 +867,7 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex
    pkgAcqIndexDiffs(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
                     IndexTarget const Target,
 		    std::vector const &diffs=std::vector());
+   virtual ~pkgAcqIndexDiffs();
 };
 									/*}}}*/
 /** \brief An acquire item that is responsible for fetching an index	{{{
@@ -940,8 +944,10 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
 
    pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager,
                IndexTarget const Target);
+   virtual ~pkgAcqIndex();
 
-   void Init(std::string const &URI, std::string const &URIDesc,
+   private:
+   APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc,
              std::string const &ShortDesc);
 };
 									/*}}}*/
@@ -1030,6 +1036,7 @@ class pkgAcqArchive : public pkgAcquire::Item
    pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sources,
 		 pkgRecords * const Recs,pkgCache::VerIterator const &Version,
 		 std::string &StoreFilename);
+   virtual ~pkgAcqArchive();
 };
 									/*}}}*/
 /** \brief Retrieve the changelog for the given version			{{{
@@ -1211,6 +1218,7 @@ class pkgAcqFile : public pkgAcquire::Item
 	      std::string const &Desc, std::string const &ShortDesc,
 	      std::string const &DestDir="", std::string const &DestFilename="",
 	      bool const IsIndexFile=false);
+   virtual ~pkgAcqFile();
 };
 									/*}}}*/
 /** @} */
diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc
index a8fc75f8e..d3aff4d5e 100644
--- a/apt-pkg/acquire-method.cc
+++ b/apt-pkg/acquire-method.cc
@@ -478,5 +478,9 @@ void pkgAcqMethod::Dequeue() {						/*{{{*/
    delete Tmp;
 }
 									/*}}}*/
-
 pkgAcqMethod::~pkgAcqMethod() {}
+
+pkgAcqMethod::FetchItem::FetchItem() {}
+pkgAcqMethod::FetchItem::~FetchItem() {}
+
+pkgAcqMethod::FetchResult::~FetchResult() {}
diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h
index 6480eb4b5..f6659ef1f 100644
--- a/apt-pkg/acquire-method.h
+++ b/apt-pkg/acquire-method.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-method.h,v 1.15.2.1 2003/12/24 23:09:17 mdz Exp $
 /* ######################################################################
 
    Acquire Method - Method helper class + functions
@@ -53,6 +52,11 @@ class pkgAcqMethod
       // for when we know it or a arbitrary limit when we don't know the
       // filesize (like a InRelease file)
       unsigned long long MaximumSize;
+
+      FetchItem();
+      virtual ~FetchItem();
+      private:
+      void *d;
    };
    
    struct FetchResult
@@ -67,6 +71,9 @@ class pkgAcqMethod
       
       void TakeHashes(class Hashes &Hash);
       FetchResult();
+      virtual ~FetchResult();
+      private:
+      void *d;
    };
 
    // State
diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h
index 3a3ef706d..b8e8fefed 100644
--- a/apt-pkg/acquire-worker.h
+++ b/apt-pkg/acquire-worker.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-worker.h,v 1.12 2001/02/20 07:03:17 jgg Exp $
 /* ######################################################################
 
    Acquire Worker - Worker process manager
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 5e5bec95c..75df858a8 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -1143,6 +1143,15 @@ void pkgAcquireStatus::Fetched(unsigned long long Size,unsigned long long Resume
 }
 									/*}}}*/
 
+pkgAcquire::UriIterator::UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
+{
+   while (CurItem == 0 && CurQ != 0)
+   {
+      CurItem = CurQ->Items;
+      CurQ = CurQ->Next;
+   }
+}
+
 APT_CONST pkgAcquire::UriIterator::~UriIterator() {}
 APT_CONST pkgAcquire::MethodConfig::~MethodConfig() {}
 APT_CONST pkgAcquireStatus::~pkgAcquireStatus() {}
diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index 02031dafd..b7e6c68f1 100644
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire.h,v 1.29.2.1 2003/12/24 23:09:17 mdz Exp $
 /* ######################################################################
 
    Acquire - File Acquiration
@@ -626,14 +625,7 @@ class pkgAcquire::UriIterator
     *
     *  \param Q The queue over which this UriIterator should iterate.
     */
-   UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0)
-   {
-      while (CurItem == 0 && CurQ != 0)
-      {
-	 CurItem = CurQ->Items;
-	 CurQ = CurQ->Next;
-      }
-   }   
+   UriIterator(pkgAcquire::Queue *Q);
    virtual ~UriIterator();
 };
 									/*}}}*/
diff --git a/apt-pkg/algorithms.h b/apt-pkg/algorithms.h
index 2ac28c0d7..dab844220 100644
--- a/apt-pkg/algorithms.h
+++ b/apt-pkg/algorithms.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: algorithms.h,v 1.10 2001/05/22 04:17:41 jgg Exp $
 /* ######################################################################
 
    Algorithms - A set of misc algorithms
@@ -55,6 +54,7 @@ using std::ostream;
 
 class pkgSimulate : public pkgPackageManager				/*{{{*/
 {
+   void *d;
    protected:
 
    class Policy : public pkgDepCache::Policy
@@ -88,7 +88,7 @@ private:
    public:
 
    pkgSimulate(pkgDepCache *Cache);
-   ~pkgSimulate();
+   virtual ~pkgSimulate();
 };
 									/*}}}*/
 class pkgProblemResolver						/*{{{*/
@@ -156,7 +156,7 @@ class pkgProblemResolver						/*{{{*/
    APT_DEPRECATED void InstallProtect();
 
    pkgProblemResolver(pkgDepCache *Cache);
-   ~pkgProblemResolver();
+   virtual ~pkgProblemResolver();
 };
 									/*}}}*/
 bool pkgApplyStatus(pkgDepCache &Cache);
diff --git a/apt-pkg/aptconfiguration.h b/apt-pkg/aptconfiguration.h
index c7b8d2d73..353843c3e 100644
--- a/apt-pkg/aptconfiguration.h
+++ b/apt-pkg/aptconfiguration.h
@@ -16,8 +16,7 @@
 #include 
 									/*}}}*/
 namespace APT {
-class Configuration {							/*{{{*/
-public:									/*{{{*/
+namespace Configuration {							/*{{{*/
 	/** \brief Returns a vector of usable Compression Types
 	 *
 	 *  Files can be compressed in various ways to decrease the size of the
@@ -39,7 +38,7 @@ public:									/*{{{*/
 	 *
 	 *  \return a vector of the compression types in the preferred usage order
 	 */
-	std::vector static const getCompressionTypes(bool const &Cached = true);
+	std::vector const getCompressionTypes(bool const &Cached = true);
 
 	/** \brief Returns a vector of Language Codes
 	 *
@@ -64,7 +63,7 @@ public:									/*{{{*/
 	 *
 	 *  \return a vector of (all) Language Codes in the preferred usage order
 	 */
-	std::vector static const getLanguages(bool const &All = false,
+	std::vector const getLanguages(bool const &All = false,
 			bool const &Cached = true, char const ** const Locale = 0);
 
 	/** \brief Are we interested in the given Language?
@@ -73,7 +72,7 @@ public:									/*{{{*/
 	 *  \param All defines if we check against all codes or only against used codes
 	 *  \return true if we are interested, false otherwise
 	 */
-	bool static checkLanguage(std::string Lang, bool const All = false);
+	bool checkLanguage(std::string Lang, bool const All = false);
 
 	/** \brief Returns a vector of Architectures we support
 	 *
@@ -82,14 +81,14 @@ public:									/*{{{*/
 	 *
 	 *  \return a vector of Architectures in preferred order
 	 */
-	std::vector static const getArchitectures(bool const &Cached = true);
+	std::vector const getArchitectures(bool const &Cached = true);
 
 	/** \brief Are we interested in the given Architecture?
 	 *
 	 *  \param Arch we want to check
 	 *  \return true if we are interested, false otherwise
 	 */
-	bool static checkArchitecture(std::string const &Arch);
+	bool checkArchitecture(std::string const &Arch);
 
 	/** \brief Representation of supported compressors */
 	struct Compressor {
@@ -113,15 +112,15 @@ public:									/*{{{*/
 	 *
 	 *  \return a vector of Compressors
 	 */
-	std::vector static const getCompressors(bool const Cached = true);
+	std::vector const getCompressors(bool const Cached = true);
 
 	/** \brief Return a vector of extensions supported for data.tar's */
-	std::vector static const getCompressorExtensions();
+	std::vector const getCompressorExtensions();
 
 	/** \return Return a vector of enabled build profile specifications */
-	std::vector static const getBuildProfiles();
+	std::vector const getBuildProfiles();
 	/** \return Return a comma-separated list of enabled build profile specifications */
-	std::string static const getBuildProfilesString();
+	std::string const getBuildProfilesString();
 									/*}}}*/
 };
 									/*}}}*/
diff --git a/apt-pkg/cachefile.h b/apt-pkg/cachefile.h
index 36b20893a..74a092593 100644
--- a/apt-pkg/cachefile.h
+++ b/apt-pkg/cachefile.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: cachefile.h,v 1.5 2002/04/27 04:28:04 jgg Exp $
 /* ######################################################################
    
    CacheFile - Simple wrapper class for opening, generating and whatnot
diff --git a/apt-pkg/cacheset.cc b/apt-pkg/cacheset.cc
index 0ad99713a..c42f76112 100644
--- a/apt-pkg/cacheset.cc
+++ b/apt-pkg/cacheset.cc
@@ -37,7 +37,6 @@
 #include 
 									/*}}}*/
 namespace APT {
-
 // PackageFrom - selecting the appropriate method for package selection	/*{{{*/
 bool CacheSetHelper::PackageFrom(enum PkgSelector const select, PackageContainerInterface * const pci,
       pkgCacheFile &Cache, std::string const &pattern) {
@@ -812,4 +811,18 @@ APT_CONST void CacheSetHelper::showSelectedVersion(pkgCache::PkgIterator const &
 					 bool const /*verIsRel*/) {
 }
 									/*}}}*/
+
+CacheSetHelper::CacheSetHelper(bool const ShowError, GlobalError::MsgType ErrorType) :
+   ShowError(ShowError), ErrorType(ErrorType) {}
+CacheSetHelper::~CacheSetHelper() {}
+
+PackageContainerInterface::PackageContainerInterface() : ConstructedBy(CacheSetHelper::UNKNOWN) {}
+PackageContainerInterface::PackageContainerInterface(CacheSetHelper::PkgSelector const by) : ConstructedBy(by) {}
+PackageContainerInterface::~PackageContainerInterface() {}
+
+PackageUniverse::PackageUniverse(pkgCache * const Owner) : _cont(Owner) { }
+PackageUniverse::~PackageUniverse() {}
+
+VersionContainerInterface::VersionContainerInterface() {}
+VersionContainerInterface::~VersionContainerInterface() {}
 }
diff --git a/apt-pkg/cacheset.h b/apt-pkg/cacheset.h
index 97aee8c2d..1a6feb5f7 100644
--- a/apt-pkg/cacheset.h
+++ b/apt-pkg/cacheset.h
@@ -50,9 +50,8 @@ class CacheSetHelper {							/*{{{*/
 */
 public:									/*{{{*/
 	CacheSetHelper(bool const ShowError = true,
-		GlobalError::MsgType ErrorType = GlobalError::ERROR) :
-			ShowError(ShowError), ErrorType(ErrorType) {}
-	virtual ~CacheSetHelper() {}
+		GlobalError::MsgType ErrorType = GlobalError::ERROR);
+	virtual ~CacheSetHelper();
 
 	enum PkgSelector { UNKNOWN, REGEX, TASK, FNMATCH, PACKAGENAME, STRING };
 
@@ -203,6 +202,8 @@ protected:
 	bool PackageFromFnmatch(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern);
 	bool PackageFromPackageName(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern);
 	bool PackageFromString(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern);
+private:
+	void *d;
 };									/*}}}*/
 
 class PackageContainerInterface {					/*{{{*/
@@ -263,8 +264,9 @@ APT_IGNORE_DEPRECATED_POP
 
 	void setConstructor(CacheSetHelper::PkgSelector const by) { ConstructedBy = by; }
 	CacheSetHelper::PkgSelector getConstructor() const { return ConstructedBy; }
-	PackageContainerInterface() : ConstructedBy(CacheSetHelper::UNKNOWN) {}
-	PackageContainerInterface(CacheSetHelper::PkgSelector const by) : ConstructedBy(by) {}
+	PackageContainerInterface();
+	PackageContainerInterface(CacheSetHelper::PkgSelector const by);
+	virtual ~PackageContainerInterface();
 
 	APT_DEPRECATED static bool FromTask(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern, CacheSetHelper &helper) {
 	   return helper.PackageFrom(CacheSetHelper::TASK, pci, Cache, pattern); }
@@ -292,6 +294,7 @@ APT_IGNORE_DEPRECATED_POP
 
 private:
 	CacheSetHelper::PkgSelector ConstructedBy;
+	void *d;
 };
 									/*}}}*/
 template class PackageContainer : public PackageContainerInterface {/*{{{*/
@@ -355,7 +358,7 @@ public:									/*{{{*/
 	iterator end() { return iterator(_cont.end()); }
 	const_iterator find(pkgCache::PkgIterator const &P) const { return const_iterator(_cont.find(P)); }
 
-	PackageContainer() : PackageContainerInterface() {}
+	PackageContainer() : PackageContainerInterface(CacheSetHelper::UNKNOWN) {}
 	PackageContainer(CacheSetHelper::PkgSelector const &by) : PackageContainerInterface(by) {}
 APT_IGNORE_DEPRECATED_PUSH
 	APT_DEPRECATED PackageContainer(Constructor const &by) : PackageContainerInterface((CacheSetHelper::PkgSelector)by) {}
@@ -552,6 +555,7 @@ template<> template inline bool PackageContainerPkgBegin(); }
 	APT_PUBLIC iterator end() { return _cont->PkgEnd(); }
 
-	APT_PUBLIC PackageUniverse(pkgCache * const Owner) : _cont(Owner) { }
+	APT_PUBLIC PackageUniverse(pkgCache * const Owner);
+	APT_PUBLIC virtual ~PackageUniverse();
 
 private:
 	bool insert(pkgCache::PkgIterator const &) { return true; }
@@ -701,6 +706,11 @@ APT_IGNORE_DEPRECATED_PUSH
 	}
 APT_IGNORE_DEPRECATED_POP
 
+	VersionContainerInterface();
+	virtual ~VersionContainerInterface();
+private:
+	void *d;
+
 protected:								/*{{{*/
 
 	/** \brief returns the candidate version of the package
diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc
index 8cec4b78e..de5cd0657 100644
--- a/apt-pkg/cdrom.cc
+++ b/apt-pkg/cdrom.cc
@@ -1022,3 +1022,9 @@ pkgUdevCdromDevices::~pkgUdevCdromDevices()                             /*{{{*/
       dlclose(libudev_handle);
 }
 									/*}}}*/
+
+pkgCdromStatus::pkgCdromStatus() : totalSteps(0) {}
+pkgCdromStatus::~pkgCdromStatus() {}
+
+pkgCdrom::pkgCdrom() {}
+pkgCdrom::~pkgCdrom() {}
diff --git a/apt-pkg/cdrom.h b/apt-pkg/cdrom.h
index bd0902176..5626b5059 100644
--- a/apt-pkg/cdrom.h
+++ b/apt-pkg/cdrom.h
@@ -18,12 +18,13 @@ class OpProgress;
 
 class pkgCdromStatus							/*{{{*/
 {
+   void *d;
  protected:
    int totalSteps;
 
  public:
-   pkgCdromStatus() : totalSteps(0) {};
-   virtual ~pkgCdromStatus() {};
+   pkgCdromStatus();
+   virtual ~pkgCdromStatus();
 
    // total steps
    virtual void SetTotal(int total) { totalSteps = total; };
@@ -74,7 +75,12 @@ class pkgCdrom								/*{{{*/
    bool Ident(std::string &ident, pkgCdromStatus *log);
    bool Add(pkgCdromStatus *log);
 
+   pkgCdrom();
+   virtual ~pkgCdrom();
+
  private:
+   void *d;
+
    APT_HIDDEN bool MountAndIdentCDROM(Configuration &Database, std::string &CDROM,
 	 std::string &ident, pkgCdromStatus * const log, bool const interactive);
    APT_HIDDEN bool UnmountCDROM(std::string const &CDROM, pkgCdromStatus * const log);
@@ -92,6 +98,7 @@ struct CdromDevice							/*{{{*/
 									/*}}}*/
 class pkgUdevCdromDevices						/*{{{*/
 {
+   void *d;
  protected:
    // libudev dlopen structure
    void *libudev_handle;
diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc
index 0fca60ba9..d05ae83b9 100644
--- a/apt-pkg/clean.cc
+++ b/apt-pkg/clean.cc
@@ -132,4 +132,5 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache)
 }
 									/*}}}*/
 
+pkgArchiveCleaner::pkgArchiveCleaner() {}
 APT_CONST pkgArchiveCleaner::~pkgArchiveCleaner() {}
diff --git a/apt-pkg/clean.h b/apt-pkg/clean.h
index 466cb67a9..a1495702b 100644
--- a/apt-pkg/clean.h
+++ b/apt-pkg/clean.h
@@ -30,6 +30,8 @@ class pkgArchiveCleaner
    public:
 
    bool Go(std::string Dir,pkgCache &Cache);
+
+   pkgArchiveCleaner();
    virtual ~pkgArchiveCleaner();
 };
 
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index 944cbe0bf..0fffa52b0 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -576,3 +576,4 @@ debTranslationsIndex::~debTranslationsIndex() {}
 debSourcesIndex::~debSourcesIndex() {}
 
 debDebPkgFileIndex::~debDebPkgFileIndex() {}
+debDscFileIndex::~debDscFileIndex() {}
diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h
index 6b8c78e5a..6285a9e5c 100644
--- a/apt-pkg/deb/debindexfile.h
+++ b/apt-pkg/deb/debindexfile.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: debindexfile.h,v 1.3.2.1 2003/12/24 23:09:17 mdz Exp $
 /* ######################################################################
 
    Debian Index Files
@@ -30,6 +29,7 @@ class pkgCacheGenerator;
 
 class APT_HIDDEN debStatusIndex : public pkgIndexFile
 {
+   void *d;
    protected:
    std::string File;
 
@@ -53,6 +53,7 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile
 
 class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile
 {
+   void *d;
    public:
 
    virtual const Type *GetType() const APT_CONST;
@@ -71,6 +72,7 @@ class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile
 
 class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile
 {
+   void *d;
    public:
 
    virtual const Type *GetType() const APT_CONST;
@@ -86,6 +88,7 @@ class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile
 
 class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile
 {
+   void *d;
    public:
 
    virtual const Type *GetType() const APT_CONST;
@@ -145,6 +148,7 @@ class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile
 class APT_HIDDEN debDscFileIndex : public pkgIndexFile
 {
  private:
+   void *d;
    std::string DscFile;
  public:
    virtual const Type *GetType() const APT_CONST;
@@ -157,7 +161,7 @@ class APT_HIDDEN debDscFileIndex : public pkgIndexFile
    };
 
    debDscFileIndex(std::string &DscFile);
-   virtual ~debDscFileIndex() {};
+   virtual ~debDscFileIndex();
 };
 
 class APT_HIDDEN debDebianSourceDirIndex : public debDscFileIndex
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index b328fcea8..34fc98838 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -459,6 +459,8 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool con
 }
 									/*}}}*/
 
+debDebFileMetaIndex::~debDebFileMetaIndex() {}
+
 class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type
 {
    protected:
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index b448ecc53..f2706e08a 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -75,6 +75,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
 class APT_HIDDEN debDebFileMetaIndex : public metaIndex
 {
  private:
+    void *d;
    std::string DebFile;
    debDebPkgFileIndex *DebIndex;
  public:
@@ -94,7 +95,7 @@ class APT_HIDDEN debDebFileMetaIndex : public metaIndex
       return true;
    }
    debDebFileMetaIndex(std::string const &DebFile);
-   virtual ~debDebFileMetaIndex() {};
+   virtual ~debDebFileMetaIndex();
 
 };
 
diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc
index 335bcfda0..f527042e4 100644
--- a/apt-pkg/deb/debrecords.cc
+++ b/apt-pkg/deb/debrecords.cc
@@ -51,6 +51,7 @@ bool debRecordParser::Jump(pkgCache::DescFileIterator const &Desc)
 									/*}}}*/
 debRecordParser::~debRecordParser() {}
 
+debRecordParserBase::debRecordParserBase() : Parser() {}
 // RecordParserBase::FileName - Return the archive filename on the site	/*{{{*/
 string debRecordParserBase::FileName()
 {
@@ -207,3 +208,9 @@ bool debDebFileRecordParser::LoadContent()
       return _error->Error(_("Unable to parse package file %s (%d)"), debFileName.c_str(), 3);
    return true;
 }
+bool debDebFileRecordParser::Jump(pkgCache::VerFileIterator const &) { return LoadContent(); }
+bool debDebFileRecordParser::Jump(pkgCache::DescFileIterator const &) { return LoadContent(); }
+std::string debDebFileRecordParser::FileName() { return debFileName; }
+
+debDebFileRecordParser::debDebFileRecordParser(std::string FileName) : debRecordParserBase(), debFileName(FileName) {}
+debDebFileRecordParser::~debDebFileRecordParser() {}
diff --git a/apt-pkg/deb/debrecords.h b/apt-pkg/deb/debrecords.h
index 38e071940..8efcec8cd 100644
--- a/apt-pkg/deb/debrecords.h
+++ b/apt-pkg/deb/debrecords.h
@@ -27,6 +27,7 @@
 
 class APT_HIDDEN debRecordParserBase : public pkgRecords::Parser
 {
+   void *d;
  protected:
    pkgTagSection Section;
 
@@ -50,12 +51,13 @@ class APT_HIDDEN debRecordParserBase : public pkgRecords::Parser
 
    virtual void GetRec(const char *&Start,const char *&Stop);
 
-   debRecordParserBase() : Parser() {}
+   debRecordParserBase();
    virtual ~debRecordParserBase();
 };
 
 class APT_HIDDEN debRecordParser : public debRecordParserBase
 {
+   void *d;
  protected:
    FileFd File;
    pkgTagFile Tags;
@@ -71,20 +73,21 @@ class APT_HIDDEN debRecordParser : public debRecordParserBase
 // custom record parser that reads deb files directly
 class APT_HIDDEN debDebFileRecordParser : public debRecordParserBase
 {
+   void *d;
    std::string debFileName;
    std::string controlContent;
 
    APT_HIDDEN bool LoadContent();
  protected:
    // single file files, so no jumping whatsoever
-   bool Jump(pkgCache::VerFileIterator const &) { return LoadContent(); }
-   bool Jump(pkgCache::DescFileIterator const &) { return LoadContent(); }
+   bool Jump(pkgCache::VerFileIterator const &);
+   bool Jump(pkgCache::DescFileIterator const &);
 
  public:
-   virtual std::string FileName() { return debFileName; }
+   virtual std::string FileName();
 
-   debDebFileRecordParser(std::string FileName)
-      : debRecordParserBase(), debFileName(FileName) {};
+   debDebFileRecordParser(std::string FileName);
+   virtual ~debDebFileRecordParser();
 };
 
 #endif
diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc
index ca6d09896..21a4ff8ea 100644
--- a/apt-pkg/deb/debsrcrecords.cc
+++ b/apt-pkg/deb/debsrcrecords.cc
@@ -32,6 +32,10 @@
 using std::max;
 using std::string;
 
+debSrcRecordParser::debSrcRecordParser(std::string const &File,pkgIndexFile const *Index)
+   : Parser(Index), Fd(File,FileFd::ReadOnly, FileFd::Extension), Tags(&Fd,102400),
+   iOffset(0), Buffer(NULL) {}
+
 // SrcRecordParser::Binaries - Return the binaries field		/*{{{*/
 // ---------------------------------------------------------------------
 /* This member parses the binaries field into a pair of class arrays and
diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h
index cd246d624..7aeb2db88 100644
--- a/apt-pkg/deb/debsrcrecords.h
+++ b/apt-pkg/deb/debsrcrecords.h
@@ -56,9 +56,7 @@ class APT_HIDDEN debSrcRecordParser : public pkgSrcRecords::Parser
    virtual bool Files(std::vector &F);
    bool Files2(std::vector &F);
 
-   debSrcRecordParser(std::string const &File,pkgIndexFile const *Index) 
-      : Parser(Index), Fd(File,FileFd::ReadOnly, FileFd::Extension), Tags(&Fd,102400),
-        iOffset(0), Buffer(NULL) {}
+   debSrcRecordParser(std::string const &File,pkgIndexFile const *Index);
    virtual ~debSrcRecordParser();
 };
 
diff --git a/apt-pkg/deb/debversion.h b/apt-pkg/deb/debversion.h
index 434ff4a2e..7befe6372 100644
--- a/apt-pkg/deb/debversion.h
+++ b/apt-pkg/deb/debversion.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: debversion.h,v 1.3 2001/05/03 05:25:04 jgg Exp $
 /* ######################################################################
 
    Debian Version - Versioning system for Debian
diff --git a/apt-pkg/depcache.h b/apt-pkg/depcache.h
index 20d263c67..94c1088f2 100644
--- a/apt-pkg/depcache.h
+++ b/apt-pkg/depcache.h
@@ -1,6 +1,5 @@
 // -*- mode: c++; mode: fold -*-
 // Description								/*{{{*/
-// $Id: depcache.h,v 1.14 2001/02/20 07:03:17 jgg Exp $
 /* ######################################################################
 
    DepCache - Dependency Extension data for the cache
@@ -164,6 +163,7 @@ class pkgDepCache : protected pkgCache::Namespace
     */
    class ActionGroup
    {
+       void *d;
        pkgDepCache &cache;
 
        bool released;
@@ -192,7 +192,7 @@ class pkgDepCache : protected pkgCache::Namespace
 	*  If this is the last action group, the automatic cache
 	*  cleanup operations will be undertaken.
 	*/
-       ~ActionGroup();
+       virtual ~ActionGroup();
    };
 
    /** \brief Returns \b true for packages matching a regular
@@ -503,6 +503,8 @@ class pkgDepCache : protected pkgCache::Namespace
 	 bool const rPurge, unsigned long const Depth, bool const FromUser);
 
    private:
+   void *d;
+
    APT_HIDDEN bool IsModeChangeOk(ModeList const mode, PkgIterator const &Pkg,
 			unsigned long const Depth, bool const FromUser);
 };
diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc
index 41cc2cdfe..25d53747c 100644
--- a/apt-pkg/edsp.cc
+++ b/apt-pkg/edsp.cc
@@ -39,60 +39,15 @@
 using std::string;
 
 // we could use pkgCache::DepType and ::Priority, but these would be localized strings…
-const char * const EDSP::PrioMap[] = {0, "important", "required", "standard",
+const char * const PrioMap[] = {0, "important", "required", "standard",
 				      "optional", "extra"};
-const char * const EDSP::DepMap[] = {"", "Depends", "Pre-Depends", "Suggests",
+const char * const DepMap[] = {"", "Depends", "Pre-Depends", "Suggests",
 				     "Recommends" , "Conflicts", "Replaces",
 				     "Obsoletes", "Breaks", "Enhances"};
 
-// EDSP::WriteScenario - to the given file descriptor			/*{{{*/
-bool EDSP::WriteScenario(pkgDepCache &Cache, FILE* output, OpProgress *Progress)
-{
-   if (Progress != NULL)
-      Progress->SubProgress(Cache.Head().VersionCount, _("Send scenario to solver"));
-   unsigned long p = 0;
-   std::vector archs = APT::Configuration::getArchitectures();
-   for (pkgCache::PkgIterator Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg)
-   {
-      std::string const arch = Pkg.Arch();
-      if (std::find(archs.begin(), archs.end(), arch) == archs.end())
-	 continue;
-      for (pkgCache::VerIterator Ver = Pkg.VersionList(); Ver.end() == false; ++Ver, ++p)
-      {
-	 WriteScenarioVersion(Cache, output, Pkg, Ver);
-	 WriteScenarioDependency(output, Ver);
-	 fprintf(output, "\n");
-	 if (Progress != NULL && p % 100 == 0)
-	    Progress->Progress(p);
-      }
-   }
-   return true;
-}
-									/*}}}*/
-// EDSP::WriteLimitedScenario - to the given file descriptor		/*{{{*/
-bool EDSP::WriteLimitedScenario(pkgDepCache &Cache, FILE* output,
-				APT::PackageSet const &pkgset,
-				OpProgress *Progress)
-{
-   if (Progress != NULL)
-      Progress->SubProgress(Cache.Head().VersionCount, _("Send scenario to solver"));
-   unsigned long p  = 0;
-   for (APT::PackageSet::const_iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg, ++p)
-      for (pkgCache::VerIterator Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
-      {
-	 WriteScenarioVersion(Cache, output, Pkg, Ver);
-	 WriteScenarioLimitedDependency(output, Ver, pkgset);
-	 fprintf(output, "\n");
-	 if (Progress != NULL && p % 100 == 0)
-	    Progress->Progress(p);
-      }
-   if (Progress != NULL)
-      Progress->Done();
-   return true;
-}
-									/*}}}*/
-// EDSP::WriteScenarioVersion						/*{{{*/
-void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgIterator const &Pkg,
+
+// WriteScenarioVersion							/*{{{*/
+static void WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgIterator const &Pkg,
 				pkgCache::VerIterator const &Ver)
 {
    fprintf(output, "Package: %s\n", Pkg.Name());
@@ -147,8 +102,8 @@ void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgI
       fprintf(output, "APT-Automatic: yes\n");
 }
 									/*}}}*/
-// EDSP::WriteScenarioDependency					/*{{{*/
-void EDSP::WriteScenarioDependency( FILE* output, pkgCache::VerIterator const &Ver)
+// WriteScenarioDependency						/*{{{*/
+static void WriteScenarioDependency( FILE* output, pkgCache::VerIterator const &Ver)
 {
    std::string dependencies[pkgCache::Dep::Enhances + 1];
    bool orGroup = false;
@@ -183,8 +138,8 @@ void EDSP::WriteScenarioDependency( FILE* output, pkgCache::VerIterator const &V
       fprintf(output, "Provides: %s\n", provides.c_str()+2);
 }
 									/*}}}*/
-// EDSP::WriteScenarioLimitedDependency					/*{{{*/
-void EDSP::WriteScenarioLimitedDependency(FILE* output,
+// WriteScenarioLimitedDependency					/*{{{*/
+static void WriteScenarioLimitedDependency(FILE* output,
 					  pkgCache::VerIterator const &Ver,
 					  APT::PackageSet const &pkgset)
 {
@@ -235,6 +190,52 @@ void EDSP::WriteScenarioLimitedDependency(FILE* output,
       fprintf(output, "Provides: %s\n", provides.c_str()+2);
 }
 									/*}}}*/
+// EDSP::WriteScenario - to the given file descriptor			/*{{{*/
+bool EDSP::WriteScenario(pkgDepCache &Cache, FILE* output, OpProgress *Progress)
+{
+   if (Progress != NULL)
+      Progress->SubProgress(Cache.Head().VersionCount, _("Send scenario to solver"));
+   unsigned long p = 0;
+   std::vector archs = APT::Configuration::getArchitectures();
+   for (pkgCache::PkgIterator Pkg = Cache.PkgBegin(); Pkg.end() == false; ++Pkg)
+   {
+      std::string const arch = Pkg.Arch();
+      if (std::find(archs.begin(), archs.end(), arch) == archs.end())
+	 continue;
+      for (pkgCache::VerIterator Ver = Pkg.VersionList(); Ver.end() == false; ++Ver, ++p)
+      {
+	 WriteScenarioVersion(Cache, output, Pkg, Ver);
+	 WriteScenarioDependency(output, Ver);
+	 fprintf(output, "\n");
+	 if (Progress != NULL && p % 100 == 0)
+	    Progress->Progress(p);
+      }
+   }
+   return true;
+}
+									/*}}}*/
+// EDSP::WriteLimitedScenario - to the given file descriptor		/*{{{*/
+bool EDSP::WriteLimitedScenario(pkgDepCache &Cache, FILE* output,
+				APT::PackageSet const &pkgset,
+				OpProgress *Progress)
+{
+   if (Progress != NULL)
+      Progress->SubProgress(Cache.Head().VersionCount, _("Send scenario to solver"));
+   unsigned long p  = 0;
+   for (APT::PackageSet::const_iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg, ++p)
+      for (pkgCache::VerIterator Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
+      {
+	 WriteScenarioVersion(Cache, output, Pkg, Ver);
+	 WriteScenarioLimitedDependency(output, Ver, pkgset);
+	 fprintf(output, "\n");
+	 if (Progress != NULL && p % 100 == 0)
+	    Progress->Progress(p);
+      }
+   if (Progress != NULL)
+      Progress->Done();
+   return true;
+}
+									/*}}}*/
 // EDSP::WriteRequest - to the given file descriptor			/*{{{*/
 bool EDSP::WriteRequest(pkgDepCache &Cache, FILE* output, bool const Upgrade,
 			bool const DistUpgrade, bool const AutoRemove,
@@ -365,13 +366,13 @@ bool EDSP::ReadResponse(int const input, pkgDepCache &Cache, OpProgress *Progres
 	return true;
 }
 									/*}}}*/
-// EDSP::ReadLine - first line from the given file descriptor		/*{{{*/
+// ReadLine - first line from the given file descriptor			/*{{{*/
 // ---------------------------------------------------------------------
 /* Little helper method to read a complete line into a string. Similar to
    fgets but we need to use the low-level read() here as otherwise the
    listparser will be confused later on as mixing of fgets and read isn't
    a supported action according to the manpages and results are undefined */
-bool EDSP::ReadLine(int const input, std::string &line) {
+static bool ReadLine(int const input, std::string &line) {
 	char one;
 	ssize_t data = 0;
 	line.erase();
@@ -390,11 +391,11 @@ bool EDSP::ReadLine(int const input, std::string &line) {
 	return false;
 }
 									/*}}}*/
-// EDSP::StringToBool - convert yes/no to bool				/*{{{*/
+// StringToBool - convert yes/no to bool				/*{{{*/
 // ---------------------------------------------------------------------
 /* we are not as lazy as we are in the global StringToBool as we really
    only accept yes/no here - but we will ignore leading spaces */
-bool EDSP::StringToBool(char const *answer, bool const defValue) {
+static bool StringToBool(char const *answer, bool const defValue) {
    for (; isspace(*answer) != 0; ++answer);
    if (strncasecmp(answer, "yes", 3) == 0)
       return true;
@@ -443,11 +444,11 @@ bool EDSP::ReadRequest(int const input, std::list &install,
 	    request = &remove;
 	 }
 	 else if (line.compare(0, 8, "Upgrade:") == 0)
-	    upgrade = EDSP::StringToBool(line.c_str() + 9, false);
+	    upgrade = StringToBool(line.c_str() + 9, false);
 	 else if (line.compare(0, 13, "Dist-Upgrade:") == 0)
-	    distUpgrade = EDSP::StringToBool(line.c_str() + 14, false);
+	    distUpgrade = StringToBool(line.c_str() + 14, false);
 	 else if (line.compare(0, 11, "Autoremove:") == 0)
-	    autoRemove = EDSP::StringToBool(line.c_str() + 12, false);
+	    autoRemove = StringToBool(line.c_str() + 12, false);
 	 else if (line.compare(0, 13, "Architecture:") == 0)
 	    _config->Set("APT::Architecture", line.c_str() + 14);
 	 else if (line.compare(0, 14, "Architectures:") == 0)
diff --git a/apt-pkg/edsp.h b/apt-pkg/edsp.h
index 9e833556a..72b886a31 100644
--- a/apt-pkg/edsp.h
+++ b/apt-pkg/edsp.h
@@ -27,24 +27,8 @@
 class pkgDepCache;
 class OpProgress;
 
-class EDSP								/*{{{*/
+namespace EDSP								/*{{{*/
 {
-	// we could use pkgCache::DepType and ::Priority, but these would be localized strings…
-	static const char * const PrioMap[];
-	static const char * const DepMap[];
-
-	APT_HIDDEN bool static ReadLine(int const input, std::string &line);
-	APT_HIDDEN bool static StringToBool(char const *answer, bool const defValue);
-
-	APT_HIDDEN void static WriteScenarioVersion(pkgDepCache &Cache, FILE* output,
-					 pkgCache::PkgIterator const &Pkg,
-					 pkgCache::VerIterator const &Ver);
-	APT_HIDDEN void static WriteScenarioDependency(FILE* output,
-					    pkgCache::VerIterator const &Ver);
-	APT_HIDDEN void static WriteScenarioLimitedDependency(FILE* output,
-						   pkgCache::VerIterator const &Ver,
-						   APT::PackageSet const &pkgset);
-public:
 	/** \brief creates the EDSP request stanza
 	 *
 	 *  In the EDSP protocol the first thing send to the resolver is a stanza
@@ -61,7 +45,7 @@ public:
 	 *
 	 *  \return true if request was composed successfully, otherwise false
 	 */
-	bool static WriteRequest(pkgDepCache &Cache, FILE* output,
+	bool WriteRequest(pkgDepCache &Cache, FILE* output,
 				 bool const upgrade = false,
 				 bool const distUpgrade = false,
 				 bool const autoRemove = false,
@@ -84,7 +68,7 @@ public:
 	 *
 	 *  \return true if universe was composed successfully, otherwise false
 	 */
-	bool static WriteScenario(pkgDepCache &Cache, FILE* output, OpProgress *Progress = NULL);
+	bool WriteScenario(pkgDepCache &Cache, FILE* output, OpProgress *Progress = NULL);
 
 	/** \brief creates a limited scenario representing the package universe
 	 *
@@ -101,7 +85,7 @@ public:
 	 *
 	 *  \return true if universe was composed successfully, otherwise false
 	 */
-	bool static WriteLimitedScenario(pkgDepCache &Cache, FILE* output,
+	bool WriteLimitedScenario(pkgDepCache &Cache, FILE* output,
 					 APT::PackageSet const &pkgset,
 					 OpProgress *Progress = NULL);
 
@@ -118,7 +102,7 @@ public:
 	 *
 	 *  \return true if a solution is found and applied correctly, otherwise false
 	 */
-	bool static ReadResponse(int const input, pkgDepCache &Cache, OpProgress *Progress = NULL);
+	bool ReadResponse(int const input, pkgDepCache &Cache, OpProgress *Progress = NULL);
 
 	/** \brief search and read the request stanza for action later
 	 *
@@ -136,7 +120,7 @@ public:
 	 *
 	 *  \return true if the request could be found and worked on, otherwise false
 	 */
-	bool static ReadRequest(int const input, std::list &install,
+	bool ReadRequest(int const input, std::list &install,
 			std::list &remove, bool &upgrade,
 			bool &distUpgrade, bool &autoRemove);
 
@@ -152,7 +136,7 @@ public:
 	 *
 	 *  \return false if the request couldn't be applied, true otherwise
 	 */
-	bool static ApplyRequest(std::list const &install,
+	bool ApplyRequest(std::list const &install,
 				 std::list const &remove,
 				 pkgDepCache &Cache);
 
@@ -168,7 +152,7 @@ public:
 	 *
 	 *  \return true if solution could be written, otherwise false
 	 */
-	bool static WriteSolution(pkgDepCache &Cache, FILE* output);
+	bool WriteSolution(pkgDepCache &Cache, FILE* output);
 
 	/** \brief sends a progress report
 	 *
@@ -176,7 +160,7 @@ public:
 	 *  \param message the solver wants the user to see
 	 *  \param output the front-end listens for progress report
 	 */
-	bool static WriteProgress(unsigned short const percent, const char* const message, FILE* output);
+	bool WriteProgress(unsigned short const percent, const char* const message, FILE* output);
 
 	/** \brief sends an error report
 	 *
@@ -193,7 +177,7 @@ public:
 	 *  \param message is free form text to describe the error
 	 *  \param output the front-end listens for error messages
 	 */
-	bool static WriteError(char const * const uuid, std::string const &message, FILE* output);
+	bool WriteError(char const * const uuid, std::string const &message, FILE* output);
 
 
 	/** \brief executes the given solver and returns the pipe ends
@@ -207,8 +191,8 @@ public:
 	 *
 	 *  \return PID of the started solver or 0 if failure occurred
 	 */
-	pid_t static ExecuteSolver(const char* const solver, int * const solver_in, int * const solver_out, bool /*overload*/);
-	APT_DEPRECATED bool static ExecuteSolver(const char* const solver, int *solver_in, int *solver_out);
+	pid_t ExecuteSolver(const char* const solver, int * const solver_in, int * const solver_out, bool /*overload*/);
+	APT_DEPRECATED bool ExecuteSolver(const char* const solver, int *solver_in, int *solver_out);
 
 	/** \brief call an external resolver to handle the request
 	 *
@@ -224,7 +208,7 @@ public:
 	 *  \return true if the solver has successfully solved the problem,
 	 *  otherwise false
 	 */
-	bool static ResolveExternal(const char* const solver, pkgDepCache &Cache,
+	bool ResolveExternal(const char* const solver, pkgDepCache &Cache,
 				    bool const upgrade, bool const distUpgrade,
 				    bool const autoRemove, OpProgress *Progress = NULL);
 };
diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc
index a2ec0a19b..5d9383e94 100644
--- a/apt-pkg/edsp/edspindexfile.cc
+++ b/apt-pkg/edsp/edspindexfile.cc
@@ -81,3 +81,5 @@ const pkgIndexFile::Type *edspIndex::GetType() const
    return &_apt_Universe;
 }
 									/*}}}*/
+
+edspIndex::~edspIndex() {}
diff --git a/apt-pkg/edsp/edspindexfile.h b/apt-pkg/edsp/edspindexfile.h
index 8c18d8cbd..0f63b7b2a 100644
--- a/apt-pkg/edsp/edspindexfile.h
+++ b/apt-pkg/edsp/edspindexfile.h
@@ -30,6 +30,7 @@ class APT_HIDDEN edspIndex : public debStatusIndex
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    edspIndex(std::string File);
+   virtual ~edspIndex();
 };
 
 #endif
diff --git a/apt-pkg/edsp/edsplistparser.cc b/apt-pkg/edsp/edsplistparser.cc
index d62abe709..d1c0cf7e8 100644
--- a/apt-pkg/edsp/edsplistparser.cc
+++ b/apt-pkg/edsp/edsplistparser.cc
@@ -92,3 +92,5 @@ APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::RlsFileIterator & /*Fil
    return true;
 }
 									/*}}}*/
+
+edspListParser::~edspListParser() {}
diff --git a/apt-pkg/edsp/edsplistparser.h b/apt-pkg/edsp/edsplistparser.h
index abe2ef139..ef5179e68 100644
--- a/apt-pkg/edsp/edsplistparser.h
+++ b/apt-pkg/edsp/edsplistparser.h
@@ -27,6 +27,7 @@ class FileFd;
 
 class APT_HIDDEN edspListParser : public debListParser
 {
+   void *d;
    public:
    virtual bool NewVersion(pkgCache::VerIterator &Ver);
    virtual std::string Description();
@@ -38,6 +39,7 @@ class APT_HIDDEN edspListParser : public debListParser
 			std::string const §ion);
 
    edspListParser(FileFd *File, std::string const &Arch = "");
+   virtual ~edspListParser();
 
    protected:
    virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver);
diff --git a/apt-pkg/edsp/edspsystem.h b/apt-pkg/edsp/edspsystem.h
index 06a63f40c..1e27d2cb0 100644
--- a/apt-pkg/edsp/edspsystem.h
+++ b/apt-pkg/edsp/edspsystem.h
@@ -42,7 +42,7 @@ class APT_HIDDEN edspSystem : public pkgSystem
 			  pkgIndexFile *&Found) const;
 
    edspSystem();
-   ~edspSystem();
+   virtual ~edspSystem();
 };
 
 #endif
diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc
index 461aa4217..120d061ad 100644
--- a/apt-pkg/indexcopy.cc
+++ b/apt-pkg/indexcopy.cc
@@ -770,4 +770,14 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
 }
 									/*}}}*/
 
+IndexCopy::IndexCopy() {}
 APT_CONST IndexCopy::~IndexCopy() {}
+
+PackageCopy::PackageCopy() : IndexCopy() {}
+APT_CONST PackageCopy::~PackageCopy() {}
+SourceCopy::SourceCopy() : IndexCopy() {}
+APT_CONST SourceCopy::~SourceCopy() {}
+TranslationsCopy::TranslationsCopy() {}
+APT_CONST TranslationsCopy::~TranslationsCopy() {}
+SigVerify::SigVerify() {}
+APT_CONST SigVerify::~SigVerify() {}
diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h
index 729b0c8cb..7ee162542 100644
--- a/apt-pkg/indexcopy.h
+++ b/apt-pkg/indexcopy.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: indexcopy.h,v 1.3 2001/05/27 04:46:54 jgg Exp $
 /* ######################################################################
 
    Index Copying - Aid for copying and verifying the index files
@@ -54,39 +53,52 @@ class IndexCopy								/*{{{*/
 
    bool CopyPackages(std::string CDROM,std::string Name,std::vector &List,
 		     pkgCdromStatus *log);
+   IndexCopy();
    virtual ~IndexCopy();
 };
 									/*}}}*/
 class PackageCopy : public IndexCopy					/*{{{*/
 {
+   void *d;
    protected:
-   
+
    virtual bool GetFile(std::string &Filename,unsigned long long &Size);
    virtual bool RewriteEntry(FileFd &Target, std::string const &File);
    virtual const char *GetFileName() {return "Packages";};
    virtual const char *Type() {return "Package";};
-   
+
+   public:
+   PackageCopy();
+   virtual ~PackageCopy();
 };
 									/*}}}*/
 class SourceCopy : public IndexCopy					/*{{{*/
 {
+   void *d;
    protected:
    
    virtual bool GetFile(std::string &Filename,unsigned long long &Size);
    virtual bool RewriteEntry(FileFd &Target, std::string const &File);
    virtual const char *GetFileName() {return "Sources";};
    virtual const char *Type() {return "Source";};
-   
+
+   public:
+   SourceCopy();
+   virtual ~SourceCopy();
 };
 									/*}}}*/
 class TranslationsCopy							/*{{{*/
 {
+   void *d;
    protected:
    pkgTagSection *Section;
 
    public:
    bool CopyTranslations(std::string CDROM,std::string Name,std::vector &List,
 			 pkgCdromStatus *log);
+
+   TranslationsCopy();
+   virtual ~TranslationsCopy();
 };
 									/*}}}*/
 class SigVerify								/*{{{*/
@@ -106,6 +118,9 @@ class SigVerify								/*{{{*/
 		       int const &statusfd, int fd[2]);
    APT_DEPRECATED static bool RunGPGV(std::string const &File, std::string const &FileOut,
 			      int const &statusfd = -1);
+
+   SigVerify();
+   virtual ~SigVerify();
 };
 									/*}}}*/
 
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 605bbeb47..b3c5cf229 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -227,3 +227,6 @@ bool pkgIndexTargetFile::Exists() const					/*{{{*/
    return FileExists(IndexFileName());
 }
 									/*}}}*/
+
+APT_CONST pkgIndexFile::~pkgIndexFile() {}
+APT_CONST pkgIndexTargetFile::~pkgIndexTargetFile() {}
diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 042e5c2f7..c51879bb8 100644
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: indexfile.h,v 1.6.2.1 2003/12/24 23:09:17 mdz Exp $
 /* ######################################################################
 
    Index File - Abstraction for an index of archive/source file.
@@ -90,6 +89,7 @@ class IndexTarget							/*{{{*/
 
 class pkgIndexFile
 {
+   void *d;
    protected:
    bool Trusted;
 
@@ -145,11 +145,12 @@ class pkgIndexFile
    bool IsTrusted() const { return Trusted; };
 
    pkgIndexFile(bool Trusted);
-   virtual ~pkgIndexFile() {};
+   virtual ~pkgIndexFile();
 };
 
 class pkgIndexTargetFile : public pkgIndexFile
 {
+   void *d;
 protected:
    IndexTarget const Target;
 
@@ -162,6 +163,7 @@ public:
    virtual unsigned long Size() const;
 
    pkgIndexTargetFile(IndexTarget const &Target, bool const Trusted);
+   virtual ~pkgIndexTargetFile();
 };
 
 #endif
diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h
index 6ed5f0c2b..f7dfa3235 100644
--- a/apt-pkg/indexrecords.h
+++ b/apt-pkg/indexrecords.h
@@ -1,7 +1,4 @@
 // -*- mode: cpp; mode: fold -*-
-// Description								/*{{{*/
-// $Id: indexrecords.h,v 1.1.2.1 2003/12/24 23:09:17 mdz Exp $
-									/*}}}*/
 #ifndef PKGLIB_INDEXRECORDS_H
 #define PKGLIB_INDEXRECORDS_H
 
diff --git a/apt-pkg/install-progress.cc b/apt-pkg/install-progress.cc
index 5ea8bf4d0..ee2ef683f 100644
--- a/apt-pkg/install-progress.cc
+++ b/apt-pkg/install-progress.cc
@@ -69,6 +69,7 @@ PackageManagerProgressFd::PackageManagerProgressFd(int progress_fd)
 {
    OutStatusFd = progress_fd;
 }
+PackageManagerProgressFd::~PackageManagerProgressFd() {}
 
 void PackageManagerProgressFd::WriteToStatusFd(std::string s)
 {
@@ -157,6 +158,7 @@ PackageManagerProgressDeb822Fd::PackageManagerProgressDeb822Fd(int progress_fd)
 {
    OutStatusFd = progress_fd;
 }
+PackageManagerProgressDeb822Fd::~PackageManagerProgressDeb822Fd() {}
 
 void PackageManagerProgressDeb822Fd::WriteToStatusFd(std::string s)
 {
@@ -433,6 +435,10 @@ bool PackageManagerText::StatusChanged(std::string PackageName,
    return true;
 }
 
+PackageManagerText::PackageManagerText() : PackageManager() {}
+PackageManagerText::~PackageManagerText() {}
+
+
 
 
 } // namespace progress
diff --git a/apt-pkg/install-progress.h b/apt-pkg/install-progress.h
index d8b4a5c82..a4c5daf7f 100644
--- a/apt-pkg/install-progress.h
+++ b/apt-pkg/install-progress.h
@@ -61,6 +61,7 @@ namespace Progress {
 
  class PackageManagerProgressFd : public PackageManager
  {
+    void *d;
  protected:
     int OutStatusFd;
     int StepsDone;
@@ -69,6 +70,7 @@ namespace Progress {
 
  public:
     PackageManagerProgressFd(int progress_fd);
+    virtual ~PackageManagerProgressFd();
 
     virtual void StartDpkg();
     virtual void Stop();
@@ -90,6 +92,7 @@ namespace Progress {
 
  class PackageManagerProgressDeb822Fd : public PackageManager
  {
+    void *d;
  protected:
     int OutStatusFd;
     int StepsDone;
@@ -98,6 +101,7 @@ namespace Progress {
 
  public:
     PackageManagerProgressDeb822Fd(int progress_fd);
+    virtual ~PackageManagerProgressDeb822Fd();
 
     virtual void StartDpkg();
     virtual void Stop();
@@ -118,6 +122,7 @@ namespace Progress {
 
  class PackageManagerFancy : public PackageManager
  {
+    void *d;
  private:
     APT_HIDDEN static void staticSIGWINCH(int);
     static std::vector instances;
@@ -138,7 +143,7 @@ namespace Progress {
 
  public:
     PackageManagerFancy();
-    ~PackageManagerFancy();
+    virtual ~PackageManagerFancy();
     virtual void Start(int child_pty=-1);
     virtual void Stop();
     virtual bool StatusChanged(std::string PackageName,
@@ -153,11 +158,15 @@ namespace Progress {
 
  class PackageManagerText : public PackageManager
  {
+    void *d;
  public:
     virtual bool StatusChanged(std::string PackageName,
                                unsigned int StepsDone,
                                unsigned int TotalSteps,
                                std::string HumanReadableAction);
+
+    PackageManagerText();
+    virtual ~PackageManagerText();
  };
 
 
diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h
index e1810fb27..760c7dd15 100644
--- a/apt-pkg/metaindex.h
+++ b/apt-pkg/metaindex.h
@@ -28,6 +28,7 @@ class OpProgress;
 
 class metaIndex
 {
+   void *d;
    protected:
    std::vector  *Indexes;
    const char *Type;
diff --git a/apt-pkg/orderlist.h b/apt-pkg/orderlist.h
index b8bad81b3..29ef79b84 100644
--- a/apt-pkg/orderlist.h
+++ b/apt-pkg/orderlist.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: orderlist.h,v 1.9 2001/02/20 07:03:17 jgg Exp $
 /* ######################################################################
 
    Order List - Represents and Manipulates an ordered list of packages.
@@ -25,6 +24,7 @@
 class pkgDepCache;
 class pkgOrderList : protected pkgCache::Namespace
 {
+   void *d;
    protected:
 
    pkgDepCache &Cache;   
@@ -123,7 +123,7 @@ class pkgOrderList : protected pkgCache::Namespace
    int Score(PkgIterator Pkg);
 
    pkgOrderList(pkgDepCache *Cache);
-   ~pkgOrderList();
+   virtual ~pkgOrderList();
 };
 
 #endif
diff --git a/apt-pkg/packagemanager.h b/apt-pkg/packagemanager.h
index fce0ad301..60414ae1c 100644
--- a/apt-pkg/packagemanager.h
+++ b/apt-pkg/packagemanager.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: packagemanager.h,v 1.14 2001/05/07 04:24:08 jgg Exp $
 /* ######################################################################
 
    Package Manager - Abstacts the package manager
@@ -146,6 +145,7 @@ class pkgPackageManager : protected pkgCache::Namespace
    virtual ~pkgPackageManager();
 
    private:
+   void *d;
    enum APT_HIDDEN SmartAction { UNPACK_IMMEDIATE, UNPACK, CONFIGURE };
    APT_HIDDEN bool NonLoopingSmart(SmartAction const action, pkgCache::PkgIterator &Pkg,
       pkgCache::PkgIterator DepPkg, int const Depth, bool const PkgLoop,
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index 9fe382108..dc7698edd 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -1059,3 +1059,5 @@ bool pkgCache::PrvIterator::IsMultiArchImplicit() const
    return false;
 }
 									/*}}}*/
+
+pkgCache::~pkgCache() {}
diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h
index 3cc85f1e8..b7bf26c2a 100644
--- a/apt-pkg/pkgcache.h
+++ b/apt-pkg/pkgcache.h
@@ -266,11 +266,12 @@ class pkgCache								/*{{{*/
    static const char *CompTypeDeb(unsigned char Comp) APT_CONST;
    static const char *CompType(unsigned char Comp) APT_CONST;
    static const char *DepType(unsigned char Dep);
-   
+
    pkgCache(MMap *Map,bool DoMap = true);
-   virtual ~pkgCache() {}
+   virtual ~pkgCache();
 
 private:
+   void *d;
    bool MultiArchEnabled;
    APT_HIDDEN PkgIterator SingleArchFindPkg(const std::string &Name);
 };
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index ea0205944..54e2ef19c 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -1720,3 +1720,6 @@ bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/)
    return true;
 }
 									/*}}}*/
+
+pkgCacheGenerator::ListParser::ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false) {}
+pkgCacheGenerator::ListParser::~ListParser() {}
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index ade93795b..3c1a40972 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: pkgcachegen.h,v 1.19 2002/07/08 03:13:30 jgg Exp $
 /* ######################################################################
    
    Package Cache Generator - Generator for the cache structure.
@@ -123,9 +122,10 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    void ReMap(void const * const oldMap, void const * const newMap);
 
    pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress);
-   ~pkgCacheGenerator();
+   virtual ~pkgCacheGenerator();
 
    private:
+   void *d;
    APT_HIDDEN bool MergeListGroup(ListParser &List, std::string const &GrpName);
    APT_HIDDEN bool MergeListPackage(ListParser &List, pkgCache::PkgIterator &Pkg);
    APT_HIDDEN bool MergeListVersion(ListParser &List, pkgCache::PkgIterator &Pkg,
@@ -151,7 +151,9 @@ class APT_HIDDEN pkgCacheGenerator::ListParser
 
    // Flag file dependencies
    bool FoundFileDeps;
-      
+
+   void *d;
+
    protected:
 
    inline map_stringitem_t StoreString(pkgCacheGenerator::StringType const type, std::string const &S) {return Owner->StoreString(type, S);};
@@ -182,10 +184,7 @@ class APT_HIDDEN pkgCacheGenerator::ListParser
     * \param Hash of the currently parsed version
     * \param Ver to compare with
     */
-#if APT_PKG_ABI >= 413
-   virtual
-#endif
-      APT_PURE bool SameVersion(unsigned short const Hash, pkgCache::VerIterator const &Ver);
+   virtual bool SameVersion(unsigned short const Hash, pkgCache::VerIterator const &Ver);
    virtual bool UsePackage(pkgCache::PkgIterator &Pkg,
 			   pkgCache::VerIterator &Ver) = 0;
    virtual map_filesize_t Offset() = 0;
@@ -197,8 +196,8 @@ class APT_HIDDEN pkgCacheGenerator::ListParser
    virtual bool CollectFileProvides(pkgCache &/*Cache*/,
 				    pkgCache::VerIterator &/*Ver*/) {return true;};
 
-   ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false) {};
-   virtual ~ListParser() {};
+   ListParser();
+   virtual ~ListParser();
 };
 									/*}}}*/
 
diff --git a/apt-pkg/pkgrecords.cc b/apt-pkg/pkgrecords.cc
index 859af3a09..87c965f87 100644
--- a/apt-pkg/pkgrecords.cc
+++ b/apt-pkg/pkgrecords.cc
@@ -76,3 +76,6 @@ pkgRecords::Parser &pkgRecords::Lookup(pkgCache::DescFileIterator const &Desc)
    return *Files[Desc.File()->ID];
 }
 									/*}}}*/
+
+pkgRecords::Parser::Parser() {}
+pkgRecords::Parser::~Parser() {}
diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h
index bcc05baba..66eb17857 100644
--- a/apt-pkg/pkgrecords.h
+++ b/apt-pkg/pkgrecords.h
@@ -43,7 +43,7 @@ class pkgRecords							/*{{{*/
 
    // Construct destruct
    pkgRecords(pkgCache &Cache);
-   ~pkgRecords();
+   virtual ~pkgRecords();
 };
 									/*}}}*/
 class pkgRecords::Parser						/*{{{*/
@@ -106,10 +106,12 @@ class pkgRecords::Parser						/*{{{*/
 
    // The record in binary form
    virtual void GetRec(const char *&Start,const char *&Stop) {Start = Stop = 0;};
-   
-   virtual ~Parser() {};
+
+   Parser();
+   virtual ~Parser();
 
    private:
+   void *d;
    APT_HIDDEN std::string GetHashFromHashes(char const * const type) const
    {
       HashStringList const hashes = Hashes();
diff --git a/apt-pkg/pkgsystem.cc b/apt-pkg/pkgsystem.cc
index 14d090c7a..98daeb2b9 100644
--- a/apt-pkg/pkgsystem.cc
+++ b/apt-pkg/pkgsystem.cc
@@ -45,3 +45,5 @@ APT_PURE pkgSystem *pkgSystem::GetSystem(const char *Label)
    return 0;   
 }
 									/*}}}*/
+
+pkgSystem::~pkgSystem() {}
diff --git a/apt-pkg/pkgsystem.h b/apt-pkg/pkgsystem.h
index f88ffa7c8..3a447da8b 100644
--- a/apt-pkg/pkgsystem.h
+++ b/apt-pkg/pkgsystem.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: pkgsystem.h,v 1.6 2002/11/11 06:55:50 doogie Exp $
 /* ######################################################################
 
    System - Abstraction for running on different systems.
@@ -93,7 +92,9 @@ class pkgSystem
    };
    
    pkgSystem();
-   virtual ~pkgSystem() {};
+   virtual ~pkgSystem();
+   private:
+   void *d;
 };
 
 // The environment we are operating in.
diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index bd40ad2d9..6da6ed606 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -439,3 +439,5 @@ bool ReadPinFile(pkgPolicy &Plcy,string File)
    return true;
 }
 									/*}}}*/
+
+pkgPolicy::~pkgPolicy() {delete [] PFPriority; delete [] Pins;}
diff --git a/apt-pkg/policy.h b/apt-pkg/policy.h
index f15d8c0a0..c4b1cbadd 100644
--- a/apt-pkg/policy.h
+++ b/apt-pkg/policy.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: policy.h,v 1.4 2001/05/07 04:24:08 jgg Exp $
 /* ######################################################################
 
    Package Version Policy implementation
@@ -85,7 +84,9 @@ class pkgPolicy : public pkgDepCache::Policy
    bool InitDefaults();
    
    pkgPolicy(pkgCache *Owner);
-   virtual ~pkgPolicy() {delete [] PFPriority; delete [] Pins;};
+   virtual ~pkgPolicy();
+   private:
+   void *d;
 };
 
 bool ReadPinFile(pkgPolicy &Plcy, std::string File = "");
diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h
index 998357509..c92643829 100644
--- a/apt-pkg/sourcelist.h
+++ b/apt-pkg/sourcelist.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: sourcelist.h,v 1.12.2.1 2003/12/24 23:09:17 mdz Exp $
 /* ######################################################################
 
    SourceList - Manage a list of sources
@@ -54,6 +53,7 @@ class metaIndex;
 
 class pkgSourceList
 {
+   void *d;
    public:
 
    // List of supported source list types
@@ -118,7 +118,7 @@ class pkgSourceList
 
    pkgSourceList();
    pkgSourceList(std::string File);
-   ~pkgSourceList();      
+   virtual ~pkgSourceList();
 };
 
 #endif
diff --git a/apt-pkg/srcrecords.h b/apt-pkg/srcrecords.h
index c931e17b7..dda66ce48 100644
--- a/apt-pkg/srcrecords.h
+++ b/apt-pkg/srcrecords.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: srcrecords.h,v 1.8.2.1 2003/12/26 16:27:34 mdz Exp $
 /* ######################################################################
    
    Source Package Records - Allows access to source package records
diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h
index 118954541..24eda02f7 100644
--- a/apt-pkg/tagfile.h
+++ b/apt-pkg/tagfile.h
@@ -1,6 +1,5 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: tagfile.h,v 1.20 2003/05/19 17:13:57 doogie Exp $
 /* ######################################################################
 
    Fast scanner for RFC-822 type header information
@@ -47,7 +46,6 @@ class pkgTagSection
    APT_DEPRECATED unsigned int TagCount;
 #endif
 
-   // dpointer placeholder (for later in case we need it)
    pkgTagSectionPrivate *d;
 
    protected:
diff --git a/apt-pkg/version.cc b/apt-pkg/version.cc
index 29bee46da..f32d39a69 100644
--- a/apt-pkg/version.cc
+++ b/apt-pkg/version.cc
@@ -40,3 +40,6 @@ pkgVersioningSystem *pkgVersioningSystem::GetVS(const char *Label)
    return 0;
 }
 									/*}}}*/
+
+
+pkgVersioningSystem::~pkgVersioningSystem() {}
diff --git a/apt-pkg/version.h b/apt-pkg/version.h
index d98809f7e..5110ecaa7 100644
--- a/apt-pkg/version.h
+++ b/apt-pkg/version.h
@@ -55,7 +55,7 @@ class pkgVersioningSystem
    APT_MKSTRCMP(CmpReleaseVer,DoCmpReleaseVer);
    
    pkgVersioningSystem();
-   virtual ~pkgVersioningSystem() {};
+   virtual ~pkgVersioningSystem();
 };
 
 #endif
diff --git a/test/libapt/acqprogress_test.cc b/test/libapt/acqprogress_test.cc
index c634733d4..dc31423fc 100644
--- a/test/libapt/acqprogress_test.cc
+++ b/test/libapt/acqprogress_test.cc
@@ -25,21 +25,23 @@ TEST(AcqProgress, IMSHit)
    AcqTextStatus Stat(out, width, 0);
    Stat.Start();
 
+   pkgAcquire Acq(&Stat);
    pkgAcquire::ItemDesc hit;
    hit.URI = "http://example.org/file";
    hit.Description = "Example File from example.org";
    hit.ShortDesc = "Example File";
-   hit.Owner = NULL;
+   TestItem hitO(&Acq);
+   hit.Owner = &hitO;
 
    EXPECT_EQ("", out.str());
    Stat.IMSHit(hit);
-   EXPECT_EQ("Hit Example File from example.org\n", out.str());
+   EXPECT_EQ("Hit:1 Example File from example.org\n", out.str());
    Stat.IMSHit(hit);
-   EXPECT_EQ("Hit Example File from example.org\n"
-	     "Hit Example File from example.org\n", out.str());
+   EXPECT_EQ("Hit:1 Example File from example.org\n"
+	     "Hit:1 Example File from example.org\n", out.str());
    Stat.Stop();
-   EXPECT_EQ("Hit Example File from example.org\n"
-	     "Hit Example File from example.org\n", out.str());
+   EXPECT_EQ("Hit:1 Example File from example.org\n"
+	     "Hit:1 Example File from example.org\n", out.str());
 }
 TEST(AcqProgress, FetchNoFileSize)
 {
@@ -61,10 +63,10 @@ TEST(AcqProgress, FetchNoFileSize)
    EXPECT_EQ("Get:1 Example File from example.org\n", out.str());
    Stat.Fetch(fetch);
    EXPECT_EQ("Get:1 Example File from example.org\n"
-	     "Get:2 Example File from example.org\n", out.str());
+	     "Get:1 Example File from example.org\n", out.str());
    Stat.Stop();
    EXPECT_EQ("Get:1 Example File from example.org\n"
-	     "Get:2 Example File from example.org\n", out.str());
+	     "Get:1 Example File from example.org\n", out.str());
 }
 TEST(AcqProgress, FetchFileSize)
 {
@@ -88,10 +90,10 @@ TEST(AcqProgress, FetchFileSize)
    fetchO.FileSize = 42;
    Stat.Fetch(fetch);
    EXPECT_EQ("Get:1 Example File from example.org [100 B]\n"
-	     "Get:2 Example File from example.org [42 B]\n", out.str());
+	     "Get:1 Example File from example.org [42 B]\n", out.str());
    Stat.Stop();
    EXPECT_EQ("Get:1 Example File from example.org [100 B]\n"
-	     "Get:2 Example File from example.org [42 B]\n", out.str());
+	     "Get:1 Example File from example.org [42 B]\n", out.str());
 }
 TEST(AcqProgress, Fail)
 {
@@ -112,30 +114,34 @@ TEST(AcqProgress, Fail)
 
    EXPECT_EQ("", out.str());
    Stat.Fail(fetch);
-   EXPECT_EQ("", out.str());
+   EXPECT_EQ("Ign:1 Example File from example.org\n", out.str());
    fetchO.Status = pkgAcquire::Item::StatDone;
    Stat.Fail(fetch);
-   EXPECT_EQ("Ign Example File from example.org\n", out.str());
+   EXPECT_EQ("Ign:1 Example File from example.org\n"
+	     "Ign:1 Example File from example.org\n", out.str());
    fetchO.Status = pkgAcquire::Item::StatError;
    fetchO.ErrorText = "An error test!";
    Stat.Fail(fetch);
-   EXPECT_EQ("Ign Example File from example.org\n"
-	     "Err Example File from example.org\n"
+   EXPECT_EQ("Ign:1 Example File from example.org\n"
+	     "Ign:1 Example File from example.org\n"
+	     "Err:1 Example File from example.org\n"
 	     "  An error test!\n", out.str());
    _config->Set("Acquire::Progress::Ignore::ShowErrorText", true);
    fetchO.Status = pkgAcquire::Item::StatDone;
    Stat.Fail(fetch);
-   EXPECT_EQ("Ign Example File from example.org\n"
-	     "Err Example File from example.org\n"
+   EXPECT_EQ("Ign:1 Example File from example.org\n"
+	     "Ign:1 Example File from example.org\n"
+	     "Err:1 Example File from example.org\n"
 	     "  An error test!\n"
-	     "Ign Example File from example.org\n"
+	     "Ign:1 Example File from example.org\n"
 	     "  An error test!\n", out.str());
    _config->Set("Acquire::Progress::Ignore::ShowErrorText", true);
    Stat.Stop();
-   EXPECT_EQ("Ign Example File from example.org\n"
-	     "Err Example File from example.org\n"
+   EXPECT_EQ("Ign:1 Example File from example.org\n"
+	     "Ign:1 Example File from example.org\n"
+	     "Err:1 Example File from example.org\n"
 	     "  An error test!\n"
-	     "Ign Example File from example.org\n"
+	     "Ign:1 Example File from example.org\n"
 	     "  An error test!\n", out.str());
 }
 TEST(AcqProgress, Pulse)
-- 
cgit v1.2.3-70-g09d2