Merge remote-tracking branch 'upstream/debian/experimental' into feature/no-more-acquire-guessing

Conflicts:
	apt-pkg/acquire-item.cc

41 files changed, 868 insertions, 341 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 0e406f368..605ca7ae4 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -44,9 +44,6 @@
 #include <sstream>
 #include <stdio.h>
 #include <ctime>
-#include <sys/types.h>
-#include <pwd.h>
-#include <grp.h>
 
 #include <apti18n.h>
 									/*}}}*/
@@ -65,20 +62,6 @@ static void printHashSumComparision(std::string const &URI, HashStringList const
       std::cerr <<  "\t- " << hs->toStr() << std::endl;
 }
 									/*}}}*/
-static void ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) /*{{{*/
-{
-   // ensure the file is owned by root and has good permissions
-   struct passwd const * const pw = getpwnam(user);
-   struct group const * const gr = getgrnam(group);
-   if (getuid() == 0) // if we aren't root, we can't chown, so don't try it
-   {
-      if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0)
-	 _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
-   }
-   if (chmod(file, mode) != 0)
-      _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
-}
-									/*}}}*/
 static std::string GetPartialFileName(std::string const &file)		/*{{{*/
 {
    std::string DestFile = _config->FindDir("Dir::State::lists") + "partial/";
@@ -155,7 +138,7 @@ pkgAcquire::Item::~Item()
    fetch this object */
 void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
-   if(ErrorText == "")
+   if(ErrorText.empty())
       ErrorText = LookupTag(Message,"Message");
    UsedMirror =  LookupTag(Message,"UsedMirror");
    if (QueueCounter <= 1)
@@ -179,9 +162,9 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       Status = StatIdle;
 
    // check fail reason
-   string FailReason = LookupTag(Message, "FailReason");
+   string const FailReason = LookupTag(Message, "FailReason");
    if(FailReason == "MaximumSizeExceeded")
-      Rename(DestFile, DestFile+".FAILED");
+      RenameOnError(MaximumSizeExceeded);
 
    // report mirror failure back to LP if we actually use a mirror
    if(FailReason.size() != 0)
@@ -197,6 +180,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size)
 {
    Status = StatFetching;
+   ErrorText.clear();
    if (FileSize == 0 && Complete == false)
       FileSize = Size;
 }
@@ -242,12 +226,6 @@ bool pkgAcquire::Item::Rename(string From,string To)
 									/*}}}*/
 void pkgAcquire::Item::QueueURI(ItemDesc &Item)				/*{{{*/
 {
-   if (RealFileExists(DestFile))
-   {
-      std::string SandboxUser = _config->Find("APT::Sandbox::User");
-      ChangeOwnerAndPermissionOfFile("Item::QueueURI", DestFile.c_str(),
-                                     SandboxUser.c_str(), "root", 0600);
-   }
    Owner->Enqueue(Item);
 }
 									/*}}}*/
@@ -258,7 +236,7 @@ void pkgAcquire::Item::Dequeue()					/*{{{*/
 									/*}}}*/
 bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/
 {
-   if(FileExists(DestFile))
+   if (RealFileExists(DestFile))
       Rename(DestFile, DestFile + ".FAILED");
 
    switch (error)
@@ -283,7 +261,11 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const
 	 Status = StatError;
 	 break;
       case NotClearsigned:
-         ErrorText = _("Does not start with a cleartext signature");
+	 ErrorText = _("Does not start with a cleartext signature");
+	 Status = StatError;
+	 break;
+      case MaximumSizeExceeded:
+	 // the method is expected to report a good error for this
 	 Status = StatError;
 	 break;
    }
@@ -702,14 +684,14 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
 									/*}}}*/
 void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
 {
+   Item::Failed(Message,Cnf);
+   Status = StatDone;
+
    if(Debug)
       std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl
 		<< "Falling back to normal index file acquire" << std::endl;
 
    new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser);
-
-   Item::Failed(Message,Cnf);
-   Status = StatDone;
 }
 									/*}}}*/
 void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes,	/*{{{*/
@@ -792,8 +774,11 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner,
    }
 }
 									/*}}}*/
-void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/
+void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/
 {
+   Item::Failed(Message,Cnf);
+   Status = StatDone;
+
    if(Debug)
       std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl
 		<< "Falling back to normal index file acquire" << std::endl;
@@ -1284,11 +1269,14 @@ string pkgAcqIndex::Custom600Headers() const
 // pkgAcqIndex::Failed - getting the indexfile failed			/*{{{*/
 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
+   Item::Failed(Message,Cnf);
+
    size_t const nextExt = CompressionExtensions.find(' ');
    if (nextExt != std::string::npos)
    {
       CompressionExtensions = CompressionExtensions.substr(nextExt+1);
       Init(RealURI, Desc.Description, Desc.ShortDesc);
+      Status = StatIdle;
       return;
    }
 
@@ -1515,19 +1503,10 @@ void pkgAcqMetaBase::AbortTransaction()
          std::clog << "  Cancel: " << (*I)->DestFile << std::endl;
       // the transaction will abort, so stop anything that is idle
       if ((*I)->Status == pkgAcquire::Item::StatIdle)
-         (*I)->Status = pkgAcquire::Item::StatDone;
-
-      // kill failed files in partial
-      if ((*I)->Status == pkgAcquire::Item::StatError)
       {
-         std::string const PartialFile = GetPartialFileName(flNotDir((*I)->DestFile));
-         if(FileExists(PartialFile))
-            Rename(PartialFile, PartialFile + ".FAILED");
+         (*I)->Status = pkgAcquire::Item::StatDone;
+         (*I)->Dequeue();
       }
-      // fix permissions for existing files which were part of a reverify
-      // like InRelease files or files in partial we might work with next time
-      else if (FileExists((*I)->DestFile))
-	 ChangeOwnerAndPermissionOfFile("AbortTransaction", (*I)->DestFile.c_str(), "root", "root", 0644);
    }
    Transaction.clear();
 }
@@ -1562,8 +1541,6 @@ void pkgAcqMetaBase::CommitTransaction()
 	       << (*I)->DescURI() << std::endl;
 
 	 Rename((*I)->PartialFile, (*I)->DestFile);
-	 ChangeOwnerAndPermissionOfFile("CommitTransaction", (*I)->DestFile.c_str(), "root", "root", 0644);
-
       } else {
          if(_config->FindB("Debug::Acquire::Transaction", false) == true)
             std::clog << "rm "
@@ -1712,16 +1689,17 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size,
 									/*}}}*/
 void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
 {
-   string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
+   Item::Failed(Message,Cnf);
 
    // check if we need to fail at this point 
    if (AuthPass == true && CheckStopAuthentication(RealURI, Message))
          return;
 
    // FIXME: meh, this is not really elegant
-   string InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12,
+   string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI);
+   string const InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12,
                                          "InRelease");
-   string FinalInRelease = _config->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI);
+   string const FinalInRelease = _config->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI);
 
    if (RealFileExists(Final) || RealFileExists(FinalInRelease))
    {
@@ -1736,7 +1714,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
          _error->Warning(_("This is normally not allowed, but the option "
                            "Acquire::AllowDowngradeToInsecureRepositories was "
                            "given to override it."));
-         
+         Status = StatDone;
       } else {
          _error->Error("%s", downgrade_msg.c_str());
          Rename(MetaIndexFile, MetaIndexFile+".FAILED");
@@ -1764,8 +1742,6 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
       QueueIndexes(true);
    }
 
-   Item::Failed(Message,Cnf);
-
    // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor
    if (Cnf->LocalOnly == true ||
        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
@@ -2152,10 +2128,12 @@ string pkgAcqMetaClearSig::Custom600Headers() const
 									/*}}}*/
 // pkgAcqMetaClearSig::Done - We got a file                     	/*{{{*/
 // ---------------------------------------------------------------------
-void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long /*Size*/,
-                              HashStringList const &/*Hashes*/,
+void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size,
+                              HashStringList const &Hashes,
                               pkgAcquire::MethodConfig *Cnf)
 {
+   Item::Done(Message, Size, Hashes, Cnf);
+
    // if we expect a ClearTextSignature (InRelase), ensure that
    // this is what we get and if not fail to queue a 
    // Release/Release.gpg, see #346386
@@ -2420,11 +2398,7 @@ bool pkgAcqArchive::QueueNext()
 	 if ((unsigned long long)Buf.st_size > Version->Size)
 	    unlink(DestFile.c_str());
 	 else
-	 {
 	    PartialSize = Buf.st_size;
-            std::string SandboxUser = _config->Find("APT::Sandbox::User");
-	    ChangeOwnerAndPermissionOfFile("pkgAcqArchive::QueueNext",DestFile.c_str(), SandboxUser.c_str(), "root", 0600);
-	 }
       }
 
       // Disables download of archives - useful if no real installation follows,
@@ -2493,7 +2467,6 @@ void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList
    string FinalFile = _config->FindDir("Dir::Cache::Archives");
    FinalFile += flNotDir(StoreFilename);
    Rename(DestFile,FinalFile);
-   ChangeOwnerAndPermissionOfFile("pkgAcqArchive::Done", FinalFile.c_str(), "root", "root", 0644);
    StoreFilename = DestFile = FinalFile;
    Complete = true;
 }
@@ -2503,8 +2476,8 @@ void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList
 /* Here we try other sources */
 void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
-   ErrorText = LookupTag(Message,"Message");
-   
+   Item::Failed(Message,Cnf);
+
    /* We don't really want to retry on failed media swaps, this prevents 
       that. An interesting observation is that permanent failures are not
       recorded. */
@@ -2514,10 +2487,10 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       // Vf = Version.FileList();
       while (Vf.end() == false) ++Vf;
       StoreFilename = string();
-      Item::Failed(Message,Cnf);
       return;
    }
-   
+
+   Status = StatIdle;
    if (QueueNext() == false)
    {
       // This is the retry counter
@@ -2530,9 +2503,9 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 	 if (QueueNext() == true)
 	    return;
       }
-      
+
       StoreFilename = string();
-      Item::Failed(Message,Cnf);
+      Status = StatError;
    }
 }
 									/*}}}*/
@@ -2589,11 +2562,7 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashe
       if ((Size > 0) && (unsigned long long)Buf.st_size > Size)
 	 unlink(DestFile.c_str());
       else
-      {
 	 PartialSize = Buf.st_size;
-         std::string SandboxUser = _config->Find("APT::Sandbox::User");
-	 ChangeOwnerAndPermissionOfFile("pkgAcqFile", DestFile.c_str(), SandboxUser.c_str(), "root", 0600);
-      }
    }
 
    QueueURI(Desc);
@@ -2652,7 +2621,12 @@ void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList cons
       // Symlink the file
       if (symlink(FileName.c_str(),DestFile.c_str()) != 0)
       {
-	 ErrorText = "Link to " + DestFile + " failure ";
+	 _error->PushToStack();
+	 _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str());
+	 std::stringstream msg;
+	 _error->DumpErrors(msg);
+	 _error->RevertToStack();
+	 ErrorText = msg.str();
 	 Status = StatError;
 	 Complete = false;
       }      
@@ -2664,19 +2638,19 @@ void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList cons
 /* Here we try other sources */
 void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
-   ErrorText = LookupTag(Message,"Message");
-   
+   Item::Failed(Message,Cnf);
+
    // This is the retry counter
    if (Retries != 0 &&
        Cnf->LocalOnly == false &&
        StringToBool(LookupTag(Message,"Transient-Failure"),false) == true)
    {
-      Retries--;
+      --Retries;
       QueueURI(Desc);
+      Status = StatIdle;
       return;
    }
-   
-   Item::Failed(Message,Cnf);
+
 }
 									/*}}}*/
 // AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 7adc1976b..10d7e6a09 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -63,6 +63,8 @@ class pkgAcqMetaBase;
  */
 class pkgAcquire::Item : public WeakPointable
 {  
+   friend class pkgAcqMetaBase;
+
    void *d;
 
    protected:
@@ -323,6 +325,7 @@ class pkgAcquire::Item : public WeakPointable
       InvalidFormat,
       SignatureError,
       NotClearsigned,
+      MaximumSizeExceeded
    };
 
    /** \brief Rename failed file and set error
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 64df3c80f..724bdfd49 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -34,12 +34,29 @@
 #include <signal.h>
 #include <stdio.h>
 #include <errno.h>
+#include <sys/types.h>
+#include <pwd.h>
+#include <grp.h>
 
 #include <apti18n.h>
 									/*}}}*/
 
 using namespace std;
 
+static void ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) /*{{{*/
+{
+   if (getuid() == 0 && strlen(user) != 0 && strlen(group) != 0) // if we aren't root, we can't chown, so don't try it
+   {
+      // ensure the file is owned by root and has good permissions
+      struct passwd const * const pw = getpwnam(user);
+      struct group const * const gr = getgrnam(group);
+      if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0)
+	 _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file);
+   }
+   if (chmod(file, mode) != 0)
+      _error->WarningE(requester, "chmod 0%o of file %s failed", mode, file);
+}
+									/*}}}*/
 // Worker::Worker - Constructor for Queue startup			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -306,7 +323,10 @@ bool pkgAcquire::Worker::RunMessages()
 	    
 	    pkgAcquire::Item *Owner = Itm->Owner;
 	    pkgAcquire::ItemDesc Desc = *Itm;
-	    
+
+	    if (RealFileExists(Owner->DestFile))
+	       ChangeOwnerAndPermissionOfFile("201::URIDone", Owner->DestFile.c_str(), "root", "root", 0644);
+
 	    // Display update before completion
 	    if (Log != 0 && Log->MorePulses == true)
 	       Log->Pulse(Owner->GetOwner());
@@ -379,9 +399,13 @@ bool pkgAcquire::Worker::RunMessages()
 	    // Display update before completion
 	    if (Log != 0 && Log->MorePulses == true)
 	       Log->Pulse(Itm->Owner->GetOwner());
-	    
+
 	    pkgAcquire::Item *Owner = Itm->Owner;
 	    pkgAcquire::ItemDesc Desc = *Itm;
+
+	    if (RealFileExists(Owner->DestFile))
+	       ChangeOwnerAndPermissionOfFile("400::URIFailure", Owner->DestFile.c_str(), "root", "root", 0644);
+
 	    OwnerQ->ItemDone(Itm);
 
 	    // set some status
@@ -542,7 +566,14 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
    }
    Message += Item->Owner->Custom600Headers();
    Message += "\n\n";
-   
+
+   if (RealFileExists(Item->Owner->DestFile))
+   {
+      std::string SandboxUser = _config->Find("APT::Sandbox::User");
+      ChangeOwnerAndPermissionOfFile("Item::QueueURI", Item->Owner->DestFile.c_str(),
+                                     SandboxUser.c_str(), "root", 0600);
+   }
+
    if (Debug == true)
       clog << " -> " << Access << ':' << QuoteString(Message,"\n") << endl;
    OutQueue += Message;
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 1e20b74be..2c89c2dea 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -54,23 +54,38 @@ pkgAcquire::pkgAcquire() : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NU
 			   Debug(_config->FindB("Debug::pkgAcquire",false)),
 			   Running(false)
 {
-   string const Mode = _config->Find("Acquire::Queue-Mode","host");
-   if (strcasecmp(Mode.c_str(),"host") == 0)
-      QueueMode = QueueHost;
-   if (strcasecmp(Mode.c_str(),"access") == 0)
-      QueueMode = QueueAccess;
+   Initialize();
 }
 pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0),
 			   Configs(0), Log(NULL), ToFetch(0),
 			   Debug(_config->FindB("Debug::pkgAcquire",false)),
 			   Running(false)
 {
+   Initialize();
+   SetLog(Progress);
+}
+void pkgAcquire::Initialize()
+{
    string const Mode = _config->Find("Acquire::Queue-Mode","host");
    if (strcasecmp(Mode.c_str(),"host") == 0)
       QueueMode = QueueHost;
    if (strcasecmp(Mode.c_str(),"access") == 0)
       QueueMode = QueueAccess;
-   SetLog(Progress);
+
+   // chown the auth.conf file as it will be accessed by our methods
+   std::string const SandboxUser = _config->Find("APT::Sandbox::User");
+   if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
+   {
+      struct passwd const * const pw = getpwnam(SandboxUser.c_str());
+      struct group const * const gr = getgrnam("root");
+      if (pw != NULL && gr != NULL)
+      {
+	 std::string const AuthConf = _config->FindFile("Dir::Etc::netrc");
+	 if(AuthConf.empty() == false && RealFileExists(AuthConf) &&
+	       chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0)
+	    _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str());
+      }
+   }
 }
 									/*}}}*/
 // Acquire::GetLock - lock directory and prepare for action		/*{{{*/
@@ -81,21 +96,16 @@ static bool SetupAPTPartialDirectory(std::string const &grand, std::string const
 	 CreateAPTDirectoryIfNeeded(parent, partial) == false)
       return false;
 
-   if (getuid() == 0) // if we aren't root, we can't chown, so don't try it
+   std::string const SandboxUser = _config->Find("APT::Sandbox::User");
+   if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
    {
-      std::string SandboxUser = _config->Find("APT::Sandbox::User");
-      struct passwd *pw = getpwnam(SandboxUser.c_str());
-      struct group *gr = getgrnam("root");
+      struct passwd const * const pw = getpwnam(SandboxUser.c_str());
+      struct group const * const gr = getgrnam("root");
       if (pw != NULL && gr != NULL)
       {
          // chown the partial dir
          if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0)
             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
-         // chown the auth.conf file
-         std::string const AuthConf = _config->FindFile("Dir::Etc::netrc");
-	 if(AuthConf.empty() == false && RealFileExists(AuthConf) &&
-	       chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0)
-	    _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str());
       }
    }
    if (chmod(partial.c_str(), 0700) != 0)
diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index a1a192d5f..f33362922 100644
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -378,6 +378,8 @@ class pkgAcquire
     */
    virtual ~pkgAcquire();
 
+   private:
+   APT_HIDDEN void Initialize();
 };
 
 /** \brief Represents a single download source from which an item
diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index 417982343..55180c642 100644
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -29,7 +29,7 @@
 
 const char * HashString::_SupportedHashes[] =
 {
-   "SHA512", "SHA256", "SHA1", "MD5Sum", NULL
+   "SHA512", "SHA256", "SHA1", "MD5Sum", "Checksum-FileSize", NULL
 };
 
 HashString::HashString()
@@ -111,6 +111,8 @@ std::string HashString::GetHashForFile(std::string filename) const      /*{{{*/
       SHA512.AddFD(Fd);
       fileHash = (std::string)SHA512.Result();
    }
+   else if (strcasecmp(Type.c_str(), "Checksum-FileSize") == 0)
+      strprintf(fileHash, "%llu", Fd.FileSize());
    Fd.Close();
 
    return fileHash;
@@ -147,7 +149,13 @@ bool HashStringList::usable() const					/*{{{*/
       return false;
    std::string const forcedType = _config->Find("Acquire::ForceHash", "");
    if (forcedType.empty() == true)
-      return true;
+   {
+      // FileSize alone isn't usable
+      for (std::vector<HashString>::const_iterator hs = list.begin(); hs != list.end(); ++hs)
+	 if (hs->HashType() != "Checksum-FileSize")
+	    return true;
+      return false;
+   }
    return find(forcedType) != NULL;
 }
 									/*}}}*/
@@ -201,6 +209,9 @@ bool HashStringList::VerifyFile(std::string filename) const		/*{{{*/
    HashString const * const hs = find(NULL);
    if (hs == NULL || hs->VerifyFile(filename) == false)
       return false;
+   HashString const * const hsf = find("Checksum-FileSize");
+   if (hsf != NULL && hsf->VerifyFile(filename) == false)
+      return false;
    return true;
 }
 									/*}}}*/
@@ -235,6 +246,14 @@ bool HashStringList::operator!=(HashStringList const &other) const
 }
 									/*}}}*/
 
+// PrivateHashes							/*{{{*/
+class PrivateHashes {
+public:
+   unsigned long long FileSize;
+
+   PrivateHashes() : FileSize(0) {}
+};
+									/*}}}*/
 // Hashes::Add* - Add the contents of data or FD			/*{{{*/
 bool Hashes::Add(const unsigned char * const Data,unsigned long long const Size, unsigned int const Hashes)
 {
@@ -254,6 +273,7 @@ bool Hashes::Add(const unsigned char * const Data,unsigned long long const Size,
 #if __GNUC__ >= 4
 	#pragma GCC diagnostic pop
 #endif
+   d->FileSize += Size;
    return Res;
 }
 bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes)
@@ -314,15 +334,17 @@ HashStringList Hashes::GetHashStringList()
 #if __GNUC__ >= 4
 	#pragma GCC diagnostic pop
 #endif
+   std::string SizeStr;
+   strprintf(SizeStr, "%llu", d->FileSize);
+   hashes.push_back(HashString("Checksum-FileSize", SizeStr));
    return hashes;
 }
 #if __GNUC__ >= 4
 	#pragma GCC diagnostic push
 	#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-	#pragma GCC diagnostic ignored "-Wsuggest-attribute=const"
 #endif
-Hashes::Hashes() {}
-Hashes::~Hashes() {}
+Hashes::Hashes() { d = new PrivateHashes(); }
+Hashes::~Hashes() { delete d; }
 #if __GNUC__ >= 4
 	#pragma GCC diagnostic pop
 #endif
diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h
index caeba006d..e2e213855 100644
--- a/apt-pkg/contrib/hashes.h
+++ b/apt-pkg/contrib/hashes.h
@@ -161,10 +161,10 @@ class HashStringList
    std::vector<HashString> list;
 };
 
+class PrivateHashes;
 class Hashes
 {
-   /** \brief dpointer placeholder */
-   void *d;
+   PrivateHashes *d;
 
    public:
    /* those will disappear in the future as it is hard to add new ones this way.
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index aad358a55..ebf9c9ea6 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -779,86 +779,94 @@ string TimeRFC1123(time_t Date)
 
    In particular: this reads blocks from the input until it believes
    that it's run out of input text.  Each block is terminated by a
-   double newline ('\n' followed by '\n').  As noted below, there is a
-   bug in this code: it assumes that all the blocks have been read if
-   it doesn't see additional text in the buffer after the last one is
-   parsed, which will cause it to lose blocks if the last block
-   coincides with the end of the buffer.
+   double newline ('\n' followed by '\n').
  */
 bool ReadMessages(int Fd, vector<string> &List)
 {
    char Buffer[64000];
-   char *End = Buffer;
    // Represents any left-over from the previous iteration of the
    // parse loop.  (i.e., if a message is split across the end
    // of the buffer, it goes here)
    string PartialMessage;
-   
-   while (1)
-   {
-      int Res = read(Fd,End,sizeof(Buffer) - (End-Buffer));
+
+   do {
+      int const Res = read(Fd, Buffer, sizeof(Buffer));
       if (Res < 0 && errno == EINTR)
 	 continue;
-      
-      // Process is dead, this is kind of bad..
+
+      // process we read from has died
       if (Res == 0)
 	 return false;
-      
+
       // No data
-      if (Res < 0 && errno == EAGAIN)
+      if (Res < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
 	 return true;
       if (Res < 0)
 	 return false;
-			      
-      End += Res;
-      
-      // Look for the end of the message
-      for (char *I = Buffer; I + 1 < End; I++)
+
+      // extract the message(s) from the buffer
+      char const *Start = Buffer;
+      char const * const End = Buffer + Res;
+
+      char const * NL = (char const *) memchr(Start, '\n', End - Start);
+      if (NL == NULL)
       {
-	 if (I[1] != '\n' ||
-	       (I[0] != '\n' && strncmp(I, "\r\n\r\n", 4) != 0))
-	    continue;
-	 
-	 // Pull the message out
-	 string Message(Buffer,I-Buffer);
-	 PartialMessage += Message;
-
-	 // Fix up the buffer
-	 for (; I < End && (*I == '\n' || *I == '\r'); ++I);
-	 End -= I-Buffer;	 
-	 memmove(Buffer,I,End-Buffer);
-	 I = Buffer;
-	 
-	 List.push_back(PartialMessage);
-	 PartialMessage.clear();
+	 // end of buffer: store what we have so far and read new data in
+	 PartialMessage.append(Start, End - Start);
+	 Start = End;
       }
-      if (End != Buffer)
-	{
-	  // If there's text left in the buffer, store it
-	  // in PartialMessage and throw the rest of the buffer
-	  // away.  This allows us to handle messages that
-	  // are longer than the static buffer size.
-	  PartialMessage += string(Buffer, End);
-	  End = Buffer;
-	}
       else
-	{
-	  // BUG ALERT: if a message block happens to end at a
-	  // multiple of 64000 characters, this will cause it to
-	  // terminate early, leading to a badly formed block and
-	  // probably crashing the method.  However, this is the only
-	  // way we have to find the end of the message block.  I have
-	  // an idea of how to fix this, but it will require changes
-	  // to the protocol (essentially to mark the beginning and
-	  // end of the block).
-	  //
-	  //  -- dburrows 2008-04-02
-	  return true;
-	}
+	 ++NL;
+
+      if (PartialMessage.empty() == false && Start < End)
+      {
+	 // if we start with a new line, see if the partial message we have ended with one
+	 // so that we properly detect records ending between two read() runs
+	 // cases are: \n|\n  ,  \r\n|\r\n  and  \r\n\r|\n
+	 // the case \r|\n\r\n is handled by the usual double-newline handling
+	 if ((NL - Start) == 1 || ((NL - Start) == 2 && *Start == '\r'))
+	 {
+	    if (APT::String::Endswith(PartialMessage, "\n") || APT::String::Endswith(PartialMessage, "\r\n\r"))
+	    {
+	       PartialMessage.erase(PartialMessage.find_last_not_of("\r\n") + 1);
+	       List.push_back(PartialMessage);
+	       PartialMessage.clear();
+	       while (NL < End && (*NL == '\n' || *NL == '\r')) ++NL;
+	       Start = NL;
+	    }
+	 }
+      }
+
+      while (Start < End) {
+	 char const * NL2 = (char const *) memchr(NL, '\n', End - NL);
+	 if (NL2 == NULL)
+	 {
+	    // end of buffer: store what we have so far and read new data in
+	    PartialMessage.append(Start, End - Start);
+	    break;
+	 }
+	 ++NL2;
+
+	 // did we find a double newline?
+	 if ((NL2 - NL) == 1 || ((NL2 - NL) == 2 && *NL == '\r'))
+	 {
+	    PartialMessage.append(Start, NL2 - Start);
+	    PartialMessage.erase(PartialMessage.find_last_not_of("\r\n") + 1);
+	    List.push_back(PartialMessage);
+	    PartialMessage.clear();
+	    while (NL2 < End && (*NL2 == '\n' || *NL2 == '\r')) ++NL2;
+	    Start = NL2;
+	 }
+	 NL = NL2;
+      }
+
+      // we have read at least one complete message and nothing left
+      if (PartialMessage.empty() == true)
+	 return true;
 
       if (WaitFd(Fd) == false)
 	 return false;
-   }   
+   } while (true);
 }
 									/*}}}*/
 // MonthConv - Converts a month string into a number			/*{{{*/
diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc
index bf1901e11..e1e9ba657 100644
--- a/apt-pkg/indexrecords.cc
+++ b/apt-pkg/indexrecords.cc
@@ -116,6 +116,9 @@ bool indexRecords::Load(const string Filename)				/*{{{*/
             indexRecords::checkSum *Sum = new indexRecords::checkSum;
             Sum->MetaKeyFilename = Name;
             Sum->Size = Size;
+	    std::string SizeStr;
+	    strprintf(SizeStr, "%llu", Size);
+	    Sum->Hashes.push_back(HashString("Checksum-FileSize", SizeStr));
 #if __GNUC__ >= 4
        #pragma GCC diagnostic push
        #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc
index 8cabf14b5..37fae18e9 100644
--- a/apt-private/private-download.cc
+++ b/apt-private/private-download.cc
@@ -49,16 +49,23 @@ bool CheckDropPrivsMustBeDisabled(pkgAcquire &Fetcher)			/*{{{*/
    for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin();
 	I != Fetcher.ItemsEnd() && res == true; ++I)
    {
-      int fd = open((*I)->DestFile.c_str(), O_CREAT | O_RDWR, 0600);
+      if ((*I)->DestFile.empty())
+	 continue;
+      // we assume that an existing (partial) file means that we have sufficient rights
+      if (RealFileExists((*I)->DestFile))
+	 continue;
+      int fd = open((*I)->DestFile.c_str(), O_CREAT | O_EXCL | O_RDWR, 0600);
       if (fd < 0)
       {
 	 res = false;
 	 std::string msg;
 	 strprintf(msg, _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."),
 	       (*I)->DestFile.c_str(), SandboxUser.c_str());
-	 c0out << msg << std::endl;
+	 std::cerr << "W: " << msg << std::endl;
 	 _config->Set("APT::Sandbox::User", "");
+	 break;
       }
+      unlink((*I)->DestFile.c_str());
       close(fd);
    }
 
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index e176a3350..b6786faf8 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -80,6 +80,9 @@
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <unistd.h>
+#include <pwd.h>
+#include <grp.h>
+
 #include <algorithm>
 #include <fstream>
 #include <iostream>
@@ -659,6 +662,8 @@ static bool DoDownload(CommandLine &CmdL)
 	 Ver != verset.end(); ++Ver, ++i)
    {
       pkgAcquire::Item *I = new pkgAcqArchive(&Fetcher, SrcList, &Recs, *Ver, storefile[i]);
+      if (storefile[i].empty())
+	 continue;
       std::string const filename = cwd + flNotDir(storefile[i]);
       storefile[i].assign(filename);
       I->DestFile.assign(filename);
@@ -1481,7 +1486,10 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher,
                           "http://packages.debian.org/changelogs");
    string const path = GetChangelogPath(CacheFile, Ver);
    string changelog_uri;
-   strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str());
+   if (APT::String::Endswith(server, "/") == true)
+      strprintf(changelog_uri, "%s%s/changelog", server.c_str(), path.c_str());
+   else
+      strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str());
    if (_config->FindB("APT::Get::Print-URIs", false) == true)
    {
       std::cout << '\'' << changelog_uri << '\'' << std::endl;
@@ -1492,7 +1500,7 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher,
    string descr;
    strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), changelog_uri.c_str());
    // queue it
-   new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile);
+   pkgAcquire::Item const * itm = new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile);
 
    // Disable drop-privs if "_apt" can not write to the target dir
    CheckDropPrivsMustBeDisabled(Fetcher);
@@ -1500,18 +1508,18 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher,
    // try downloading it, if that fails, try third-party-changelogs location
    // FIXME: Fetcher.Run() is "Continue" even if I get a 404?!?
    Fetcher.Run();
-   if (!FileExists(targetfile))
+   if (itm->Status != pkgAcquire::Item::StatDone)
    {
       string third_party_uri;
       if (GuessThirdPartyChangelogUri(CacheFile, Ver, third_party_uri))
       {
          strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), third_party_uri.c_str());
-         new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile);
+         itm = new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile);
          Fetcher.Run();
       }
    }
 
-   if (FileExists(targetfile))
+   if (itm->Status == pkgAcquire::Item::StatDone)
       return true;
 
    // error
@@ -1557,6 +1565,19 @@ static bool DoChangelog(CommandLine &CmdL)
       tmpdir = mkdtemp(tmpname);
       if (tmpdir == NULL)
 	 return _error->Errno("mkdtemp", "mkdtemp failed");
+
+      std::string const SandboxUser = _config->Find("APT::Sandbox::User");
+      if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
+      {
+	 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
+	 struct group const * const gr = getgrnam("root");
+	 if (pw != NULL && gr != NULL)
+	 {
+	    // chown the tmp dir directory we use to the sandbox user
+	    if(chown(tmpdir, pw->pw_uid, gr->gr_gid) != 0)
+	       _error->WarningE("DoChangelog", "chown to %s:%s of directory %s failed", SandboxUser.c_str(), "root", tmpdir);
+	 }
+      }
    }
 
    for (APT::VersionList::const_iterator Ver = verset.begin(); 
@@ -1572,7 +1593,7 @@ static bool DoChangelog(CommandLine &CmdL)
       {
          DisplayFileInPager(changelogfile);
          // cleanup temp file
-         unlink(changelogfile.c_str());
+	 unlink(changelogfile.c_str());
       }
    }
    // clenaup tmp dir
diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc
index da45eb8d2..df5eb1451 100644
--- a/ftparchive/cachedb.cc
+++ b/ftparchive/cachedb.cc
@@ -473,6 +473,10 @@ bool CacheDB::GetHashes(bool const GenOnly, unsigned int const DoHashes)
 	    hex2bytes(CurStat.MD5, hs->HashValue().data(), sizeof(CurStat.MD5));
 	    CurStat.Flags |= FlMD5;
 	 }
+	 else if (strcasecmp(hs->HashType().c_str(), "Checksum-FileSize") == 0)
+	 {
+	    // we store it in a different field already
+	 }
 	 else
 	    return _error->Error("Got unknown unrequested hashtype %s", hs->HashType().c_str());
       }
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index db617e92a..ded8715f8 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -471,6 +471,8 @@ bool PackagesWriter::DoPackage(string FileName)
    {
       if (hs->HashType() == "MD5Sum")
 	 Changes.push_back(SetTFRewriteData("MD5sum", hs->HashValue().c_str()));
+      else if (hs->HashType() == "Checksum-FileSize")
+	 continue;
       else
 	 Changes.push_back(SetTFRewriteData(hs->HashType().c_str(), hs->HashValue().c_str()));
    }
@@ -785,7 +787,7 @@ bool SourcesWriter::DoPackage(string FileName)
 
          for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
 	 {
-	    if (hs->HashType() == "MD5Sum")
+	    if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
 	       continue;
 	    char const * fieldname;
 	    std::ostream * out;
diff --git a/test/integration/framework b/test/integration/framework
index d576712e5..153c5bb25 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -32,7 +32,7 @@ msgprintf() {
 		printf "$START " "$1"
 		shift
 		while [ -n "$1" ]; do
-			printf "$MIDDLE " "$(echo "$1" | sed -e 's#^apt\([cgfs]\)#apt-\1#')"
+			printf "$MIDDLE " "$(echo "$1" | sed -e 's#^apt\([cfghs]\)#apt-\1#')"
 			shift
 		done
 	fi
@@ -57,9 +57,9 @@ msgskip() {
 msgfail() {
 	if [ $# -gt 0 ]; then printf "${CFAIL}FAIL: $*${CNORMAL}\n" >&2;
 	else printf "${CFAIL}FAIL${CNORMAL}\n" >&2; fi
-        if [ -n "$APT_DEBUG_TESTS" ]; then
-            bash
-        fi
+	if [ -n "$APT_DEBUG_TESTS" ]; then
+		$SHELL
+	fi
 	EXIT_CODE=$((EXIT_CODE+1));
 }
 
@@ -77,12 +77,6 @@ if [ $MSGLEVEL -le 2 ]; then
 	msgnmsg() { true; }
 	msgtest() { true; }
 	msgpass() { printf " ${CPASS}P${CNORMAL}"; }
-	msgskip() { printf " ${CWARNING}S${CNORMAL}" >&2; }
-	if [ -n "$CFAIL" ]; then
-		msgfail() { printf " ${CFAIL}FAIL${CNORMAL}" >&2; EXIT_CODE=$((EXIT_CODE+1)); }
-	else
-		msgfail() { printf " ###FAILED###" >&2; EXIT_CODE=$((EXIT_CODE+1)); }
-	fi
 fi
 if [ $MSGLEVEL -le 3 ]; then
 	msginfo() { true; }
@@ -184,10 +178,12 @@ setupenvironment() {
 	addtrap "cd /; rm -rf $TMPWORKINGDIRECTORY;"
 	msgninfo "Preparing environment for ${CCMD}$(basename $0)${CINFO} in ${TMPWORKINGDIRECTORY}… "
 
+	mkdir -m 700 "${TMPWORKINGDIRECTORY}/downloaded"
 	if [ "$(id -u)" = '0' ]; then
 		# relax permissions so that running as root with user switching works
 		umask 022
-		chmod o+rx "$TMPWORKINGDIRECTORY"
+		chmod 711 "$TMPWORKINGDIRECTORY"
+		chown _apt:root "${TMPWORKINGDIRECTORY}/downloaded"
 	fi
 
 	TESTDIRECTORY=$(readlink -f $(dirname $0))
@@ -662,40 +658,52 @@ buildaptftparchivedirectorystructure() {
 }
 
 insertpackage() {
-	local RELEASE="$1"
+	local RELEASES="$1"
 	local NAME="$2"
 	local ARCH="$3"
 	local VERSION="$4"
 	local DEPENDENCIES="$5"
 	local PRIORITY="${6:-optional}"
-	local DESCRIPTION="${7:-"an autogenerated dummy ${NAME}=${VERSION}/${RELEASE}
+	local DESCRIPTION="${7:-"an autogenerated dummy ${NAME}=${VERSION}/${RELEASES}
  If you find such a package installed on your system,
  something went horribly wrong! They are autogenerated
  und used only by testcases and surf no other propose…"}"
 	local ARCHS=""
-	for arch in $(getarchitecturesfromcommalist "$ARCH"); do
-		if [ "$arch" = 'all' -o "$arch" = 'none' ]; then
-			ARCHS="$(getarchitectures)"
-		else
-			ARCHS="$arch"
+	for RELEASE in $(printf '%s' "$RELEASES" | tr ',' '\n'); do
+		if [ "$RELEASE" = 'installed' ]; then
+			insertinstalledpackage "$2" "$3" "$4" "$5" "$6" "$7"
+			continue
 		fi
-		for BUILDARCH in $ARCHS; do
-			local PPATH="aptarchive/dists/${RELEASE}/main/binary-${BUILDARCH}"
-			mkdir -p $PPATH aptarchive/dists/${RELEASE}/main/source
-			touch aptarchive/dists/${RELEASE}/main/source/Sources
-			local FILE="${PPATH}/Packages"
-			echo "Package: $NAME
+		for arch in $(getarchitecturesfromcommalist "$ARCH"); do
+			if [ "$arch" = 'all' -o "$arch" = 'none' ]; then
+				ARCHS="$(getarchitectures)"
+			else
+				ARCHS="$arch"
+			fi
+			for BUILDARCH in $ARCHS; do
+				local PPATH="aptarchive/dists/${RELEASE}/main/binary-${BUILDARCH}"
+				mkdir -p $PPATH
+				local FILE="${PPATH}/Packages"
+				echo "Package: $NAME
 Priority: $PRIORITY
 Section: other
 Installed-Size: 42
 Maintainer: Joe Sixpack <joe@example.org>" >> $FILE
-			test "$arch" = 'none' || echo "Architecture: $arch" >> $FILE
-			echo "Version: $VERSION
+				test "$arch" = 'none' || echo "Architecture: $arch" >> $FILE
+				echo "Version: $VERSION
 Filename: pool/main/${NAME}/${NAME}_${VERSION}_${arch}.deb" >> $FILE
-			test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE
-			echo "Description: $DESCRIPTION" >> $FILE
-			echo >> $FILE
+				test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE
+				echo "Description: $(printf '%s' "$DESCRIPTION" | head -n 1)" >> $FILE
+				echo "Description-md5: $(printf '%s' "$DESCRIPTION" | md5sum | cut -d' ' -f 1)" >> $FILE
+				echo >> $FILE
+			done
 		done
+		mkdir -p aptarchive/dists/${RELEASE}/main/source aptarchive/dists/${RELEASE}/main/i18n
+		touch aptarchive/dists/${RELEASE}/main/source/Sources
+		echo "Package: $NAME
+Description-md5: $(printf '%s' "$DESCRIPTION" | md5sum | cut -d' ' -f 1)
+Description-en: $DESCRIPTION
+" >> aptarchive/dists/${RELEASE}/main/i18n/Translation-en
 	done
 }
 
@@ -771,7 +779,7 @@ buildaptarchivefromincoming() {
 
 buildaptarchivefromfiles() {
 	msginfo "Build APT archive for ${CCMD}$(basename $0)${CINFO} based on prebuild files…"
-	find aptarchive -name 'Packages' -o -name 'Sources' | while read line; do
+	find aptarchive -name 'Packages' -o -name 'Sources' -o -name 'Translation-*' | while read line; do
 		msgninfo "\t${line} file… "
 		compressfile "$line" "$1"
 		msgdone "info"
@@ -942,23 +950,35 @@ signreleasefiles() {
 }
 
 webserverconfig() {
-	msgtest "Set webserver config option '${1}' to" "$2"
+	local NOCHECK=false
+	if [ "$1" = '--no-check' ]; then
+		NOCHECK=true
+		shift
+	fi
 	local DOWNLOG='rootdir/tmp/download-testfile.log'
-	local STATUS='rootdir/tmp/webserverconfig.status'
+	local STATUS='downloaded/webserverconfig.status'
 	rm -f "$STATUS" "$DOWNLOG"
-	if downloadfile "http://localhost:8080/_config/set/${1}/${2}" "$STATUS" > "$DOWNLOG"; then
+	local URI
+	if [ -n "$2" ]; then
+		msgtest "Set webserver config option '${1}' to" "$2"
+		URI="http://localhost:8080/_config/set/${1}/${2}"
+	else
+		msgtest 'Clear webserver config option' "${1}"
+		URI="http://localhost:8080/_config/clear/${1}"
+	fi
+	if downloadfile "$URI" "$STATUS" > "$DOWNLOG"; then
 		msgpass
 	else
-		cat "$DOWNLOG" "$STATUS"
+		cat "$DOWNLOG" "$STATUS" || true
 		msgfail
 	fi
-	testwebserverlaststatuscode '200'
+	$NOCHECK || testwebserverlaststatuscode '200'
 }
 
 rewritesourceslist() {
 	local APTARCHIVE="file://$(readlink -f "${TMPWORKINGDIRECTORY}/aptarchive")"
 	for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
-		sed -i $LIST -e "s#$APTARCHIVE#${1}#" -e "s#http://localhost:8080/#${1}#" -e "s#http://localhost:4433/#${1}#"
+		sed -i $LIST -e "s#$APTARCHIVE#${1}#" -e "s#http://localhost:8080/#${1}#" -e "s#https://localhost:4433/#${1}#"
 	done
 }
 
@@ -1045,7 +1065,7 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 	mv "${CD}" "${CD}-unmounted"
 	# we don't want the disk to be modifiable
 	addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/media/cdrom/dists/ $PWD/rootdir/media/cdrom-unmounted/dists/ || true;"
-	chmod -R -w rootdir/media/cdrom-unmounted/dists
+	chmod -R 555 rootdir/media/cdrom-unmounted/dists
 }
 
 downloadfile() {
@@ -1053,7 +1073,7 @@ downloadfile() {
 	apthelper -o Debug::Acquire::${PROTO}=1 \
 		download-file "$1" "$2" 2>&1 || true
 	# only if the file exists the download was successful
-	if [ -e "$2" ]; then
+	if [ -r "$2" ]; then
 		return 0
 	else
 		return 1
@@ -1061,7 +1081,7 @@ downloadfile() {
 }
 
 checkdiff() {
-	local DIFFTEXT="$(command diff -u "$@" | sed -e '/^---/ d' -e '/^+++/ d' -e '/^@@/ d')"
+	local DIFFTEXT="$(command diff -u "$@" 2>&1 | sed -e '/^---/ d' -e '/^+++/ d' -e '/^@@/ d')"
 	if [ -n "$DIFFTEXT" ]; then
 		echo >&2
 		echo >&2 "$DIFFTEXT"
@@ -1202,6 +1222,36 @@ testmarkedauto() {
 	aptmark showauto 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
 }
 
+msgfailoutput() {
+	local MSG="$1"
+	local OUTPUT="$2"
+	shift 2
+	echo >&2
+	if [ "$1" = 'grep' ]; then
+		while [ -n "$2" ]; do shift; done
+		echo "#### Complete file: $1 ####"
+		cat >&2 "$1" || true
+		echo '#### grep output ####'
+	elif [ "$1" = 'test' ]; then
+		# doesn't support ! or non-file flags
+		msgfailoutputstatfile() {
+			local FILEFLAGS='^-[bcdefgGhkLOprsStuwx]$'
+			if expr match "$1" "$FILEFLAGS" >/dev/null; then
+				echo "#### stat(2) of file: $2 ####"
+				stat "$2" || true
+			fi
+		}
+		msgfailoutputstatfile "$2" "$3"
+		while [ -n "$5" ] && [ "$4" = '-o' -o "$4" = '-a' ]; do
+			shift 3
+			msgfailoutputstatfile "$2" "$3"
+		done
+		echo '#### test output ####'
+	fi
+	cat >&2 $OUTPUT
+	msgfail "$MSG"
+}
+
 testsuccess() {
 	if [ "$1" = '--nomsg' ]; then
 		shift
@@ -1211,10 +1261,10 @@ testsuccess() {
 	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
 	if "$@" >${OUTPUT} 2>&1; then
 		if expr match "$1" '^apt.*' >/dev/null; then
-			if grep -q -E '^[WE]: ' "$OUTPUT"; then
-				echo >&2
-				cat >&2 $OUTPUT
-				msgfail 'successful run, but output contains warnings/errors'
+			if grep -q -E ' runtime error: ' "$OUTPUT"; then
+				msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@"
+			elif grep -q -E '^[WE]: ' "$OUTPUT"; then
+				msgfailoutput 'successful run, but output contains warnings/errors' "$OUTPUT" "$@"
 			else
 				msgpass
 			fi
@@ -1223,9 +1273,7 @@ testsuccess() {
 		fi
 	else
 		local EXITCODE=$?
-		echo >&2
-		cat >&2 $OUTPUT
-		msgfail "exitcode $EXITCODE"
+		msgfailoutput "exitcode $EXITCODE" "$OUTPUT" "$@"
 	fi
 	aptautotest 'testsuccess' "$@"
 }
@@ -1238,14 +1286,12 @@ testwarning() {
 	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
 	if "$@" >${OUTPUT} 2>&1; then
 		if expr match "$1" '^apt.*' >/dev/null; then
-			if grep -q -E '^E: ' "$OUTPUT"; then
-				echo >&2
-				cat >&2 $OUTPUT
-				msgfail 'successful run, but output contains errors'
+			if grep -q -E ' runtime error: ' "$OUTPUT"; then
+				msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@"
+			elif grep -q -E '^E: ' "$OUTPUT"; then
+				msgfailoutput 'successful run, but output contains errors' "$OUTPUT" "$@"
 			elif ! grep -q -E '^W: ' "$OUTPUT"; then
-				echo >&2
-				cat >&2 $OUTPUT
-				msgfail 'successful run, but output contains no warnings'
+				msgfailoutput 'successful run, but output contains no warnings' "$OUTPUT" "$@"
 			else
 				msgpass
 			fi
@@ -1269,16 +1315,14 @@ testfailure() {
 	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output"
 	if "$@" >${OUTPUT} 2>&1; then
 		local EXITCODE=$?
-		echo >&2
-		cat >&2 $OUTPUT
-		msgfail "exitcode $EXITCODE"
+		msgfailoutput "exitcode $EXITCODE" "$OUTPUT" "$@"
 	else
 		local EXITCODE=$?
 		if expr match "$1" '^apt.*' >/dev/null; then
-			if ! grep -q -E '^E: ' "$OUTPUT"; then
-				echo >&2
-				cat >&2 $OUTPUT
-				msgfail "run failed with exitcode ${EXITCODE}, but with no errors"
+			if grep -q -E ' runtime error: ' "$OUTPUT"; then
+				msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@"
+			elif ! grep -q -E '^E: ' "$OUTPUT"; then
+				msgfailoutput "run failed with exitcode ${EXITCODE}, but with no errors" "$OUTPUT" "$@"
 			else
 				msgpass
 			fi
@@ -1307,11 +1351,10 @@ testaccessrights() {
 
 testwebserverlaststatuscode() {
 	local DOWNLOG='rootdir/tmp/webserverstatus-testfile.log'
-	local STATUS='rootdir/tmp/webserverstatus-statusfile.log'
+	local STATUS='downloaded/webserverstatus-statusfile.log'
 	rm -f "$DOWNLOG" "$STATUS"
 	msgtest 'Test last status code from the webserver was' "$1"
-	downloadfile "http://localhost:8080/_config/find/aptwebserver::last-status-code" "$STATUS" > "$DOWNLOG"
-	if [ "$(cat "$STATUS")" = "$1" ]; then
+	if downloadfile "http://localhost:8080/_config/find/aptwebserver::last-status-code" "$STATUS" > "$DOWNLOG" && [ "$(cat "$STATUS")" = "$1" ]; then
 		msgpass
 	else
 		echo >&2
@@ -1353,20 +1396,20 @@ aptautotest() {
 		shift 3
 		# save and restore the *.output files from other tests
 		# as we might otherwise override them in these automatic tests
-		rm -rf rootdir/tmp-before
-		mv rootdir/tmp rootdir/tmp-before
-		mkdir rootdir/tmp
+		rm -rf ${TMPWORKINGDIRECTORY}/rootdir/tmp-before
+		mv ${TMPWORKINGDIRECTORY}/rootdir/tmp ${TMPWORKINGDIRECTORY}/rootdir/tmp-before
+		mkdir ${TMPWORKINGDIRECTORY}/rootdir/tmp
 		$AUTOTEST "$TESTCALL" "$@"
-		rm -rf rootdir/tmp-aptautotest
-		mv rootdir/tmp rootdir/tmp-aptautotest
-		mv rootdir/tmp-before rootdir/tmp
+		rm -rf ${TMPWORKINGDIRECTORY}/rootdir/tmp-aptautotest
+		mv ${TMPWORKINGDIRECTORY}/rootdir/tmp ${TMPWORKINGDIRECTORY}/rootdir/tmp-aptautotest
+		mv ${TMPWORKINGDIRECTORY}/rootdir/tmp-before ${TMPWORKINGDIRECTORY}/rootdir/tmp
 	fi
 }
 
 aptautotest_aptget_update() {
 	if ! test -d "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"; then return; fi
 	# all copied files are properly chmodded
-	find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f | while read file; do
+	for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f); do
 		testfilestats "$file" '%U:%G:%a' '=' "${USER}:${USER}:644"
 	done
 }
diff --git a/test/integration/run-tests b/test/integration/run-tests
index c39a2ac68..6c6a37611 100755
--- a/test/integration/run-tests
+++ b/test/integration/run-tests
@@ -46,7 +46,11 @@ for testcase in $(run-parts --list $DIR | grep '/test-'); do
 	if ! ${testcase}; then
 		FAIL=$((FAIL+1))
 		FAILED_TESTS="$FAILED_TESTS $(basename $testcase)"
-		echo >&2 "$(basename $testcase) ... FAIL"
+		if [ "$MSGLEVEL" -le 2 ]; then
+			printf >&2 "\n${CHIGH}Running $(basename $testcase) -> FAILED${CRESET}"
+		else
+			echo >&2 "${CHIGH}Running $(basename $testcase) -> FAILED${CRESET}"
+		fi
 	else
 		PASS=$((PASS+1))
 	fi
diff --git a/test/integration/test-apt-cdrom b/test/integration/test-apt-cdrom
index 2220a290c..3a33219fe 100755
--- a/test/integration/test-apt-cdrom
+++ b/test/integration/test-apt-cdrom
@@ -21,7 +21,7 @@ echo 'Description-de: automatisch generiertes Testpaket testing=0.8.15/stable
 ' >> Translation-de
 compressfile Translation-de
 rm -f Translation-en Translation-de
-chmod -R -w .
+chmod -R 555 .
 cd - > /dev/null
 
 aptcdromlog() {
@@ -99,6 +99,7 @@ Conf testing:i386 (0.8.15 stable [i386])' aptget install testing:i386 -s
 	testsuccess aptget purge testing:i386 -y
 	testdpkgnotinstalled testing:i386
 
+	cd downloaded
 	rm -f testing_0.8.15_amd64.deb
 	testsuccess aptget download testing
 	testsuccess test -s testing_0.8.15_amd64.deb
@@ -108,6 +109,7 @@ Conf testing:i386 (0.8.15 stable [i386])' aptget install testing:i386 -s
 	testsuccess aptget source testing --dsc-only -d
 	testsuccess test -s testing_0.8.15.dsc
 	rm -f testing_0.8.15.dsc
+	cd - >/dev/null
 }
 testcdromusage
 
@@ -142,3 +144,8 @@ testcdromusage
 testsuccess aptget update
 testfileequal rootdir/tmp/testsuccess.output 'Reading package lists...'
 testcdromusage
+
+msgmsg 'Check that nothing touched our' 'CD-ROM'
+for file in $(find rootdir/media/cdrom-unmounted/dists); do
+	testfilestats "$file" '%U:%G:%a' '=' "${USER}:${USER}:555"
+done
diff --git a/test/integration/test-apt-get-changelog b/test/integration/test-apt-get-changelog
index 76a32a122..648dccf40 100755
--- a/test/integration/test-apt-get-changelog
+++ b/test/integration/test-apt-get-changelog
@@ -21,23 +21,28 @@ chmod -R -w rootdir/var/cache/apt/archives
 
 echo 'Apt::Changelogs::Server "http://localhost:8080/";' > rootdir/etc/apt/apt.conf.d/changelog.conf
 
-testequal "'http://localhost:8080//pool/apt_1.0/changelog'" aptget changelog apt --print-uris
+testequal "'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
 
-testequal "'http://localhost:8080//pool/apt_1.0/changelog'
-'http://localhost:8080//pool/apt_1.0/changelog'" aptget changelog apt apt --print-uris
+testequal "'http://localhost:8080/pool/apt_1.0/changelog'
+'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt apt --print-uris
+
+cd downloaded
 
 testsuccess aptget changelog apt -qq
-testfileequal 'rootdir/tmp/testsuccess.output' "$(cat aptarchive/pool/apt_1.0/changelog)"
+testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/apt_1.0/changelog)"
 
 testsuccess aptget changelog apt -d
-testfileequal 'apt.changelog' "$(cat  aptarchive/pool/apt_1.0/changelog)"
-rm -f apt.changelog aptarchive/pool/apt_1.0/changelog
+testfileequal 'apt.changelog' "$(cat  ../aptarchive/pool/apt_1.0/changelog)"
+testfilestats 'apt.changelog' '%U:%G:%a' '=' "${USER}:${USER}:644"
+rm -f apt.changelog ../aptarchive/pool/apt_1.0/changelog
 
-testequal "$(cat aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \
+testequal "$(cat ../aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \
 	-qq -o APT::Changelogs::Server='http://not-on-the-main-server:8080/'
 
 testsuccess aptget changelog apt -d
-testfileequal 'apt.changelog' "$(cat aptarchive/pool/apt_1.0.changelog)"
-rm -f apt.changelog aptarchive/pool/apt_1.0.changelog
+testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0.changelog)"
+testfilestats 'apt.changelog' '%U:%G:%a' '=' "${USER}:${USER}:644"
+rm -f apt.changelog ../aptarchive/pool/apt_1.0.changelog
 
-testequal 'E: changelog download failed' aptget changelog apt -qq -o APT::Changelogs::Server='http://not-on-the-main-server:8080/'
+testequal 'E: changelog download failed' aptget changelog apt -qq -d -o APT::Changelogs::Server='http://not-on-the-main-server:8080/'
+testfailure test -e apt.changelog
diff --git a/test/integration/test-apt-get-clean b/test/integration/test-apt-get-clean
index 98f7c84d0..457bff9d3 100755
--- a/test/integration/test-apt-get-clean
+++ b/test/integration/test-apt-get-clean
@@ -26,10 +26,10 @@ touch rootdir/var/cache/apt/archives/foo_4_all.deb
 
 testsuccess aptget clean
 
-testsuccess test ! -e rootdir/var/lib/apt/lists/partial/http.debian.net_debian_dists_sid_main_i18n_Translation-en
-testsuccess test ! -e rootdir/var/cache/apt/archives/foo_1_all.deb
-testsuccess test ! -e rootdir/var/cache/apt/archives/foo_2_all.deb
-testsuccess test ! -e rootdir/var/cache/apt/archives/foo_3_all.deb
-testsuccess test ! -e rootdir/var/cache/apt/archives/foo_4_all.deb
+testfailure test -e rootdir/var/lib/apt/lists/partial/http.debian.net_debian_dists_sid_main_i18n_Translation-en
+testfailure test -e rootdir/var/cache/apt/archives/foo_1_all.deb
+testfailure test -e rootdir/var/cache/apt/archives/foo_2_all.deb
+testfailure test -e rootdir/var/cache/apt/archives/foo_3_all.deb
+testfailure test -e rootdir/var/cache/apt/archives/foo_4_all.deb
 
 
diff --git a/test/integration/test-apt-get-download b/test/integration/test-apt-get-download
index 48086d808..6503bbd1c 100755
--- a/test/integration/test-apt-get-download
+++ b/test/integration/test-apt-get-download
@@ -44,7 +44,10 @@ testdownload() {
 }
 
 # normal case as "root"
+OLDPWD="$(pwd)"
+cd downloaded
 testdownload apt_2.0_all.deb apt
+cd "$OLDPWD"
 
 # simulate normal user with non-existent root-owned directories
 rm -rf rootdir/var/cache/apt/archives/
@@ -52,14 +55,18 @@ mkdir rootdir/var/cache/apt/archives/
 addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/cache/apt/archives || true;"
 chmod -R -w rootdir/var/cache/apt/archives
 
+OLDPWD="$(pwd)"
+cd downloaded
+
 # normal case(es)
 testdownload apt_1.0_all.deb apt stable
 testdownload apt_2.0_all.deb apt
 
-DEBFILE="$(readlink -f aptarchive)/pool/apt_2.0_all.deb"
+DEBFILE="$(readlink -f ../aptarchive)/pool/apt_2.0_all.deb"
 testequal "'file://${DEBFILE}' apt_2.0_all.deb $(stat -c%s $DEBFILE) SHA512:$(sha512sum $DEBFILE | cut -d' ' -f 1)" aptget download apt --print-uris
 
 # deb:677887
+testequal "E: Can't find a source to download version '1.0' of 'vrms:i386'" aptget download vrms --print-uris
 testequal "E: Can't find a source to download version '1.0' of 'vrms:i386'" aptget download vrms
 
 # deb:736962
@@ -74,6 +81,7 @@ testsuccess aptget download apt apt apt/unstable apt=2.0
 testsuccess test -s apt_2.0_all.deb
 
 # restore "root" rights
+cd "$OLDPWD"
 chmod -f -R +w $PWD/rootdir/var/cache/apt/archives
 rm -rf rootdir/var/cache/apt/archives/
 
@@ -86,5 +94,7 @@ testsuccess aptget install -d apt
 testsuccess test -s rootdir/var/cache/apt/archives/apt_2.0_all.deb
 testaccessrights 'aptarchive/pool/apt_2.0_all.deb' '644'
 mv aptarchive/pool/apt_2.0_all.deb aptarchive/pool/apt_2.0_all.deb.gone
+cd downloaded
 testdownload apt_2.0_all.deb apt
+cd "$OLDPWD"
 mv aptarchive/pool/apt_2.0_all.deb.gone aptarchive/pool/apt_2.0_all.deb
diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated
index fedfc8308..f68c32386 100755
--- a/test/integration/test-apt-get-source-authenticated
+++ b/test/integration/test-apt-get-source-authenticated
@@ -25,7 +25,10 @@ testwarning aptget update --allow-insecure-repositories
 
 # this all should fail
 testfailure aptget install -y foo
+cd downloaded
 testfailure aptget source foo
-
+testfailure test -e foo_2.0.dsc
 # allow overriding the warning
-testsuccess aptget source --allow-unauthenticated foo
+testsuccess aptget source --allow-unauthenticated foo -o Debug::pkgAcquire::Worker=1
+testsuccess test -s foo_2.0.dsc -a -L foo_2.0.dsc
+testaccessrights 'foo_2.0.dsc' '777'
diff --git a/test/integration/test-apt-get-upgrade b/test/integration/test-apt-get-upgrade
index 5335c243a..d042e4fb7 100755
--- a/test/integration/test-apt-get-upgrade
+++ b/test/integration/test-apt-get-upgrade
@@ -8,23 +8,18 @@ setupenvironment
 configarchitecture "i386"
 
 # simple case
-insertpackage 'stable' 'upgrade-simple' 'all' '1.0'
+insertpackage 'stable,installed' 'upgrade-simple' 'all' '1.0'
 insertpackage 'unstable' 'upgrade-simple' 'all' '2.0'
-insertinstalledpackage 'upgrade-simple' 'all' '1.0'
 
 # upgrade with a new dependency
-insertpackage 'stable' 'upgrade-with-new-dep' 'all' '1.0' 
+insertpackage 'stable,installed' 'upgrade-with-new-dep' 'all' '1.0'
 insertpackage 'unstable' 'upgrade-with-new-dep' 'all' '2.0' 'Depends: new-dep'
 insertpackage 'stable' 'new-dep' 'all' '1.0'
-insertinstalledpackage 'upgrade-with-new-dep' 'all' '1.0'
 
 # upgrade with conflict and a new pkg with higher priority than conflict
-insertpackage 'stable' 'upgrade-with-conflict' 'all' '1.0'
+insertpackage 'stable,installed' 'upgrade-with-conflict' 'all' '1.0'
 insertpackage 'unstable' 'upgrade-with-conflict' 'all' '2.0' 'Conflicts: conflicting-dep' 'standard'
-insertpackage 'stable' 'conflicting-dep' 'all' '1.0'
-insertinstalledpackage 'upgrade-with-conflict' 'all' '1.0'
-insertinstalledpackage 'conflicting-dep' 'all' '1.0'
-
+insertpackage 'stable,installed' 'conflicting-dep' 'all' '1.0'
 
 setupaptarchive
 
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index 42c40bb9e..06e497ff7 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -13,23 +13,23 @@ test_apt_helper_download() {
     echo 'foo' > aptarchive/foo
 
     msgtest 'apt-file download-file' 'md5sum'
-    apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail
-    testfileequal foo2 'foo'
+    apthelper -qq download-file http://localhost:8080/foo downloaded/foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail
+    testfileequal downloaded/foo2 'foo'
 
     msgtest 'apt-file download-file' 'sha1'
-    apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail
-    testfileequal foo1 'foo'
+    apthelper -qq download-file http://localhost:8080/foo downloaded/foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail
+    testfileequal downloaded/foo1 'foo'
 
     msgtest 'apt-file download-file' 'sha256'
-    apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail
-    testfileequal foo3 'foo'
+    apthelper -qq download-file http://localhost:8080/foo downloaded/foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail
+    testfileequal downloaded/foo3 'foo'
 
     msgtest 'apt-file download-file' 'no-hash'
-    apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail
-    testfileequal foo4 'foo'
+    apthelper -qq download-file http://localhost:8080/foo downloaded/foo4 && msgpass || msgfail
+    testfileequal downloaded/foo4 'foo'
     
     msgtest 'apt-file download-file' 'wrong hash'
-    if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then
+    if ! apthelper -qq download-file http://localhost:8080/foo downloaded/foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then
         msgpass
     else
         msgfail
@@ -37,7 +37,8 @@ test_apt_helper_download() {
     testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
 
 E: Download Failed'
-    testfileequal foo5.FAILED 'foo'
+    testfileequal downloaded/foo5.FAILED 'foo'
+    testfailure test -e downloaded/foo5
 }
 
 test_apt_helper_detect_proxy() {
diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key
index e6ac530a6..b6b7b7909 100755
--- a/test/integration/test-apt-key
+++ b/test/integration/test-apt-key
@@ -41,7 +41,7 @@ gpg:              unchanged: 1' aptkey --fakeroot update
 
 	testaptkeys 'pub   2048R/DBAC8DAE 2010-08-18'
 
-	testsuccess test ! -e rootdir/etc/apt/trusted.gpg
+	testfailure test -e rootdir/etc/apt/trusted.gpg
 	testsuccess aptkey --fakeroot add ./keys/rexexpired.pub
 	msgtest 'Check if trusted.gpg is created with permissions set to' '0644'
 	if [ "$(stat -c '%a' rootdir/etc/apt/trusted.gpg )" = '644' ]; then
@@ -78,7 +78,7 @@ pub   2048R/DBAC8DAE 2010-08-18'
 	cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess --nomsg aptkey --fakeroot del DBAC8DAE
 	testempty aptkey list
-	testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
+	testfailure test -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~
 
 	msgtest 'Test key removal with' 'long key ID'
@@ -86,7 +86,7 @@ pub   2048R/DBAC8DAE 2010-08-18'
 	cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess --nomsg aptkey --fakeroot del 5A90D141DBAC8DAE
 	testempty aptkey list
-	testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
+	testfailure test -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~
 
 	msgtest 'Test key removal with' 'fingerprint'
@@ -94,7 +94,7 @@ pub   2048R/DBAC8DAE 2010-08-18'
 	cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess --nomsg aptkey --fakeroot del 34A8E9D18DB320F367E8EAA05A90D141DBAC8DAE
 	testempty aptkey list
-	testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
+	testfailure test -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~
 
 	msgtest 'Test key removal with' 'single key in softlink'
@@ -102,7 +102,7 @@ pub   2048R/DBAC8DAE 2010-08-18'
 	ln -s $(readlink -f ./keys/joesixpack.pub) rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess --nomsg aptkey --fakeroot del DBAC8DAE
 	testempty aptkey list
-	testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
+	testfailure test -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess test -L rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~
 
 	cleanplate
@@ -125,7 +125,7 @@ pub   2048R/528144E2 2011-01-16'
 	testsuccess --nomsg aptkey --fakeroot del DBAC8DAE
 	testaptkeys 'pub   2048R/528144E2 2011-01-16'
 	testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~
-	testsuccess test ! -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg
+	testfailure test -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg
 	testsuccess test -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg~
 
 	msgtest 'Test key removal with' 'multiple files including key'
@@ -134,7 +134,7 @@ pub   2048R/528144E2 2011-01-16'
 	cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg
 	testsuccess --nomsg aptkey --fakeroot del DBAC8DAE
 	testaptkeys 'pub   2048R/528144E2 2011-01-16'
-	testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
+	testfailure test -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 	testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~
 	testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~
 
diff --git a/test/integration/test-apt-update-filesize-mismatch b/test/integration/test-apt-update-filesize-mismatch
new file mode 100755
index 000000000..f78b83b5f
--- /dev/null
+++ b/test/integration/test-apt-update-filesize-mismatch
@@ -0,0 +1,50 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'i386'
+configcompression 'gz'
+
+insertpackage 'testing' 'foo' 'all' '1'
+insertpackage 'testing' 'foo2' 'all' '1'
+insertsource 'testing' 'foo' 'all' '1'
+insertsource 'testing' 'foo2' 'all' '1'
+
+setupaptarchive --no-update
+changetowebserver
+
+find aptarchive \( -name 'Packages' -o -name 'Sources' -o -name 'Translation-en' \) -delete
+for release in $(find aptarchive -name 'Release'); do
+	cp "$release" "${release}.backup"
+done
+
+testsuccess aptget update
+testsuccess aptcache show foo
+testsuccess aptget install foo -s
+
+for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver.log); do
+	for ext in '' '.gz'; do
+		COMPRESSFILE="$get"
+		get="${get}${ext}"
+		FILE="$(basename "$get" '.gz')"
+		msgmsg 'Test filesize mismatch with file' "$FILE"
+		rm -rf rootdir/var/lib/apt/lists
+
+		for release in $(find aptarchive -name 'Release'); do
+			SIZE="$(awk "/$FILE\$/ { print \$2; exit }" "${release}.backup")"
+			sed "s# $SIZE # $(($SIZE + 111)) #" "${release}.backup" > "$release"
+		done
+		signreleasefiles
+
+		testfailure aptget update -o Debug::pkgAcquire::Worker=1
+		cp rootdir/tmp/testfailure.output rootdir/tmp/update.output
+		testsuccess grep -E "$(basename -s '.gz' "$COMPRESSFILE").*Hash Sum mismatch" rootdir/tmp/update.output
+		testfailure aptcache show foo
+		testfailure aptget install foo -s
+
+		testfailure aptcache show bar
+		testfailure aptget install bar -s
+	done
+done
diff --git a/test/integration/test-apt-update-hashsum-mismatch b/test/integration/test-apt-update-hashsum-mismatch
new file mode 100755
index 000000000..c2c5b3887
--- /dev/null
+++ b/test/integration/test-apt-update-hashsum-mismatch
@@ -0,0 +1,44 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'i386'
+configcompression 'gz'
+
+insertpackage 'testing' 'foo' 'all' '1'
+insertpackage 'testing' 'foo2' 'all' '1'
+insertsource 'testing' 'foo' 'all' '1'
+insertsource 'testing' 'foo2' 'all' '1'
+
+setupaptarchive --no-update
+changetowebserver
+
+echo 'Package: bar
+Maintainer: Doctor Evil <evil@example.com>
+Description: come to the dark side
+' > aptarchive/DoctorEvil
+compressfile aptarchive/DoctorEvil
+
+find aptarchive \( -name 'Packages' -o -name 'Sources' -o -name 'Translation-en' \) -delete
+
+testsuccess aptget update
+testsuccess aptcache show foo
+testsuccess aptget install foo -s
+
+for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver.log); do
+	msgmsg 'Test hashsum mismatch with file' "$get"
+	rm -rf rootdir/var/lib/apt/lists
+	webserverconfig 'aptwebserver::overwrite' ''
+	webserverconfig "aptwebserver::overwrite::$(printf '%s' "${get}" | sed 's#/#%2F#g' )::filename" '%2FDoctorEvil.gz'
+
+	testfailure aptget update
+	cp rootdir/tmp/testfailure.output rootdir/tmp/update.output
+	testsuccess grep -E "$(basename -s '.gz' "$get").*Hash Sum mismatch" rootdir/tmp/update.output
+	testfailure aptcache show foo
+	testfailure aptget install foo -s
+
+	testfailure aptcache show bar
+	testfailure aptget install bar -s
+done
diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims
index afae99563..5394a9f30 100755
--- a/test/integration/test-apt-update-ims
+++ b/test/integration/test-apt-update-ims
@@ -30,7 +30,7 @@ runtest() {
     
     # ensure that we still do a hash check on ims hit
     msgtest 'Test I-M-S' 'reverify'
-    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A1 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
+    aptget update -o Debug::pkgAcquire::Auth=1 2>&1 | grep -A2 'RecivedHash:' | grep -q -- '- SHA' && msgpass || msgfail
 
     # ensure no leftovers in partial
     testfailure ls "rootdir/var/lib/apt/lists/partial/*"
diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback
index 831fc67eb..e82a976a6 100755
--- a/test/integration/test-apt-update-nofallback
+++ b/test/integration/test-apt-update-nofallback
@@ -195,7 +195,7 @@ test_release_gpg_to_invalid_release_release_gpg()
 
     testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release.gpg: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) <joe@example.org>
 
-W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg  
+W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg  The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) <joe@example.org>
 
 W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
 
diff --git a/test/integration/test-authentication-basic b/test/integration/test-authentication-basic
new file mode 100755
index 000000000..21b024970
--- /dev/null
+++ b/test/integration/test-authentication-basic
@@ -0,0 +1,106 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'i386'
+
+insertpackage 'unstable' 'foo' 'all' '1'
+setupaptarchive --no-update
+
+changetohttpswebserver --authorization="$(printf '%s' 'star:hunter2' | base64 )"
+
+echo 'See, when YOU type hunter2, it shows to us as *******' > aptarchive/bash
+
+testauthfailure() {
+	testfailure apthelper download-file "${1}/bash" ./downloaded/bash
+	# crappy test, but http and https output are wastely different…
+	testsuccess grep 401 rootdir/tmp/testfailure.output
+	testfailure test -s ./downloaded/bash
+}
+
+testauthsuccess() {
+	testsuccess apthelper download-file "${1}/bash" ./downloaded/bash
+	testfileequal ./downloaded/bash "$(cat aptarchive/bash)"
+	testfilestats ./downloaded/bash '%U:%G:%a' '=' "${USER}:${USER}:644"
+	rm -f ./downloaded/bash
+
+	# lets see if got/retains acceptable permissions
+	if [ -n "$AUTHCONF" ]; then
+		if [ "$(id -u)" = '0' ]; then
+			testfilestats "$AUTHCONF" '%U:%G:%a' '=' "_apt:root:600"
+		else
+			testfilestats "$AUTHCONF" '%U:%G:%a' '=' "${USER}:${USER}:600"
+		fi
+	fi
+
+	rm -rf rootdir/var/lib/apt/lists
+	testsuccess aptget update
+	testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  foo
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+Inst foo (1 unstable [all])
+Conf foo (1 unstable [all])' aptget install foo -s
+}
+
+authfile() {
+	local AUTHCONF='rootdir/etc/apt/auth.conf'
+	rm -f "$AUTHCONF"
+	printf '%s' "$1" > "$AUTHCONF"
+	chmod 600 "$AUTHCONF"
+}
+
+runtest() {
+	# unauthorized fails
+	authfile ''
+	testauthfailure "$1"
+
+	# good auth
+	authfile 'machine localhost
+login star
+password hunter2'
+	testauthsuccess "$1"
+
+	# bad auth
+	authfile 'machine localhost
+login anonymous
+password hunter2'
+	testauthfailure "$1"
+
+	# 2 stanzas: unmatching + good auth
+	authfile 'machine debian.org
+login debian
+password jessie
+
+machine localhost
+login star
+password hunter2'
+	testauthsuccess "$1"
+}
+
+msgmsg 'server basic auth'
+rewritesourceslist 'http://localhost:8080'
+runtest 'http://localhost:8080'
+rewritesourceslist 'https://localhost:4433'
+runtest 'https://localhost:4433'
+rewritesourceslist 'http://localhost:8080'
+
+msgmsg 'proxy to server basic auth'
+webserverconfig 'aptwebserver::request::absolute' 'uri'
+export http_proxy='http://localhost:8080'
+runtest 'http://localhost:8080'
+unset http_proxy
+
+msgmsg 'proxy basic auth to server basic auth'
+webserverconfig 'aptwebserver::proxy-authorization' "$(printf 'moon:deer2' | base64)"
+export http_proxy='http://moon:deer2@localhost:8080'
+runtest 'http://localhost:8080'
+
+msgmsg 'proxy basic auth to server'
+authfile ''
+webserverconfig 'aptwebserver::authorization' ''
+testauthsuccess 'http://localhost:8080'
diff --git a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted
index 164dca070..582e1bf5e 100755
--- a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted
+++ b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted
@@ -24,6 +24,7 @@ testfilemissing() {
 
 testrun() {
 	rm -rf rootdir/var/lib/apt
+	cd downloaded
 
 	if [ "$1" = 'trusted' ]; then
 	        testsuccess aptget update
@@ -43,18 +44,19 @@ testrun() {
 		testfileexists 'cool_1.0_i386.deb'
 	fi
 
-	mv aptarchive/pool/cool_1.0_i386.deb aptarchive/pool/cool_1.0_i386.deb.bak
-	echo 'this is not a good package' > aptarchive/pool/cool_1.0_i386.deb
+	mv ../aptarchive/pool/cool_1.0_i386.deb ../aptarchive/pool/cool_1.0_i386.deb.bak
+	echo 'this is not a good package' > ../aptarchive/pool/cool_1.0_i386.deb
 	testfailure aptget download cool
 	testfilemissing cool_1.0_i386.deb
 
 	testfailure aptget download cool --allow-unauthenticated  # unauthenticated doesn't mean unchecked
 	testfilemissing cool_1.0_i386.deb
 
-	rm -f aptarchive/pool/cool_1.0_i386.deb
-	mv aptarchive/pool/cool_1.0_i386.deb.bak aptarchive/pool/cool_1.0_i386.deb
+	rm -f ../aptarchive/pool/cool_1.0_i386.deb
+	mv ../aptarchive/pool/cool_1.0_i386.deb.bak ../aptarchive/pool/cool_1.0_i386.deb
 	testsuccess aptget download cool --allow-unauthenticated
 	testfileexists 'cool_1.0_i386.deb'
+	cd - >/dev/null
 }
 
 testrun 'trusted'
diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index f450e5e5a..0f458e099 100755
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -25,6 +25,7 @@ downloadfile 'http://localhost:8080/pool/apt_1.0/changelog' changelog >/dev/null
 echo 'Apt::Changelogs::Server "http://localhost:8080/redirectme";' > rootdir/etc/apt/apt.conf.d/changelog.conf
 testequal "'http://localhost:8080/redirectme/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
 
+cd downloaded
 testsuccess aptget changelog apt -d
 testsuccess test -s apt.changelog
 rm -f apt.changelog
@@ -32,6 +33,7 @@ rm -f apt.changelog
 testsuccess aptget download apt
 testsuccess test -s apt_1.0_all.deb
 rm apt_1.0_all.deb
+cd - >/dev/null
 
 testsuccess aptget install apt -y
 testdpkginstalled 'apt'
@@ -49,15 +51,11 @@ rm https
 cd - >/dev/null
 echo "Dir::Bin::Methods \"${COPYMETHODS}\";" >> aptconfig.conf
 
-if [ "$(id -u)" = '0' ]; then
-	testequal "Can't drop privileges for downloading as file '$(pwd)/apt_1.0_all.deb' couldn't be accessed by user '_apt'.
-E: The method driver $(pwd)/rootdir/usr/lib/apt/methods/https could not be found.
-N: Is the package apt-transport-https installed?" aptget download apt -q=0
-else
-	testequal "E: The method driver $(pwd)/rootdir/usr/lib/apt/methods/https could not be found.
+cd downloaded
+testequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found.
 N: Is the package apt-transport-https installed?" aptget download apt -q=0
-fi
-testsuccess test ! -e apt_1.0_all.deb
+testfailure test -e apt_1.0_all.deb
+cd - >/dev/null
 
 # revert to all methods
 rm -rf rootdir/$COPYMETHODS
diff --git a/test/integration/test-bug-745036-new-foreign-invalidates-cache b/test/integration/test-bug-745036-new-foreign-invalidates-cache
index 490cbecdd..2b7ee06ad 100755
--- a/test/integration/test-bug-745036-new-foreign-invalidates-cache
+++ b/test/integration/test-bug-745036-new-foreign-invalidates-cache
@@ -6,10 +6,8 @@ TESTDIR=$(readlink -f $(dirname $0))
 setupenvironment
 configarchitecture 'amd64'
 
-insertpackage 'unstable' 'cool-foo' 'amd64' '1.0' 'Depends: foo'
-insertpackage 'unstable' 'foo' 'amd64' '1.0' 'Multi-Arch: foreign'
-insertinstalledpackage 'cool-foo' 'amd64' '1.0' 'Depends: foo'
-insertinstalledpackage 'foo' 'amd64' '1.0' 'Multi-Arch: foreign'
+insertpackage 'unstable,installed' 'cool-foo' 'amd64' '1.0' 'Depends: foo'
+insertpackage 'unstable,installed' 'foo' 'amd64' '1.0' 'Multi-Arch: foreign'
 
 setupaptarchive
 
diff --git a/test/integration/test-bug-753297-upgradable b/test/integration/test-bug-753297-upgradable
index 068704b3e..01395a095 100755
--- a/test/integration/test-bug-753297-upgradable
+++ b/test/integration/test-bug-753297-upgradable
@@ -16,8 +16,7 @@ Pin: release unstable
 Pin-Priority: 1
 EOF
 
-insertinstalledpackage 'foo' 'all' '1'
-insertpackage 'testing' 'foo' 'all' '1'
+insertpackage 'testing,installed' 'foo' 'all' '1'
 insertpackage 'testing-updates' 'foo' 'all' '2'
 insertpackage 'unstable' 'foo' 'all' '3'
 
diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
index 72b262e84..92e7c0e84 100755
--- a/test/integration/test-compressed-indexes
+++ b/test/integration/test-compressed-indexes
@@ -58,9 +58,11 @@ testrun() {
 		msgpass
 	fi
 	msgtest 'Check if package is downloadable'
+	cd downloaded
 	testsuccess --nomsg aptget download testpkg
 	msgtest '\tdeb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
 	rm testpkg_1.0_i386.deb
+	cd - >/dev/null
 	testequal 'Reading package lists...
 Building dependency tree...
 The following NEW packages will be installed:
@@ -79,10 +81,12 @@ Conf testpkg (1.0 unstable [i386])' aptget install testpkg -s
 	testequal "$GOODSHOWSRC" aptcache showsrc testpkg
 	aptget clean
 	msgtest 'Check if the source is aptgetable'
+	cd downloaded
 	testsuccess --nomsg aptget source testpkg
-	msgtest '\tdsc file is present'; testsuccess --nomsg test -f testpkg_1.0.dsc
-	msgtest '\tdirectory is present'; testsuccess --nomsg test -d testpkg-1.0
+	testsuccess test -s testpkg_1.0.dsc
+	testsuccess test -d testpkg-1.0
 	rm -rf testpkg-1.0*
+	cd - >/dev/null
 	testequal "$(aptcache show testpkg -o Acquire::Languages=none)
 " aptcache dumpavail
 }
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 5ab326def..98b2f242a 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -52,36 +52,37 @@ cp -a ${TESTDIR}/framework $TESTFILE
 
 testrun() {
 	webserverconfig 'aptwebserver::support::range' 'true'
+	local DOWN='./downloaded/testfile'
 
-	copysource $TESTFILE 0 ./testfile
-	testdownloadfile 'no data' "${1}/testfile" './testfile' '='
+	copysource $TESTFILE 0 $DOWN
+	testdownloadfile 'no data' "${1}/testfile" "$DOWN" '='
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 
-	copysource $TESTFILE 20 ./testfile
-	testdownloadfile 'valid partial data' "${1}/testfile" './testfile' '='
+	copysource $TESTFILE 20 $DOWN
+	testdownloadfile 'valid partial data' "${1}/testfile" "$DOWN" '='
 	testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 
-	copysource /dev/zero 20 ./testfile
-	testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
+	copysource /dev/zero 20 $DOWN
+	testdownloadfile 'invalid partial data' "${1}/testfile" "$DOWN" '!='
 	testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 
-	copysource $TESTFILE 1M ./testfile
-	testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '='
+	copysource $TESTFILE 1M $DOWN
+	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
 	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
 
-	copysource /dev/zero 1M ./testfile
-	testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='
+	copysource /dev/zero 1M $DOWN
+	testdownloadfile 'too-big partial file' "${1}/testfile" "$DOWN" '='
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 
-	copysource /dev/zero 20 ./testfile
-	touch ./testfile
-	testdownloadfile 'old data' "${1}/testfile" './testfile' '='
+	copysource /dev/zero 20 $DOWN
+	touch $DOWN
+	testdownloadfile 'old data' "${1}/testfile" "$DOWN" '='
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 
 	webserverconfig 'aptwebserver::support::range' 'false'
 
-	copysource $TESTFILE 20 ./testfile
-	testdownloadfile 'no server support' "${1}/testfile" './testfile' '='
+	copysource $TESTFILE 20 $DOWN
+	testdownloadfile 'no server support' "${1}/testfile" "$DOWN" '='
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 
diff --git a/test/integration/test-prevent-markinstall-multiarch-same-versionscrew b/test/integration/test-prevent-markinstall-multiarch-same-versionscrew
index 9d2ea2d5d..db97687ce 100755
--- a/test/integration/test-prevent-markinstall-multiarch-same-versionscrew
+++ b/test/integration/test-prevent-markinstall-multiarch-same-versionscrew
@@ -9,20 +9,17 @@ configarchitecture 'amd64' 'i386' 'armel'
 insertpackage 'stable' 'allarchs' 'all' '1'
 insertpackage 'unstable' 'allarchs' 'all' '2'
 
-insertinstalledpackage 'fine' 'i386,amd64' '1' 'Multi-Arch: same'
-insertpackage 'stable' 'fine' 'i386,amd64' '1' 'Multi-Arch: same'
+insertpackage 'stable,installed' 'fine' 'i386,amd64' '1' 'Multi-Arch: same'
 insertpackage 'unstable' 'fine' 'amd64,i386' '2' 'Multi-Arch: same'
 
 insertinstalledpackage 'fine-installed' 'i386,amd64' '1' 'Multi-Arch: same'
 insertpackage 'stable' 'fine-installed' 'i386,amd64,armel' '1' 'Multi-Arch: same'
 insertpackage 'unstable' 'fine-installed' 'i386,amd64' '2' 'Multi-Arch: same'
 
-insertinstalledpackage 'out-of-sync-native' 'i386,amd64' '1' 'Multi-Arch: same'
-insertpackage 'stable' 'out-of-sync-native' 'i386,amd64' '1' 'Multi-Arch: same'
+insertpackage 'stable,installed' 'out-of-sync-native' 'i386,amd64' '1' 'Multi-Arch: same'
 insertpackage 'unstable' 'out-of-sync-native' 'amd64' '2' 'Multi-Arch: same'
 
-insertinstalledpackage 'out-of-sync-foreign' 'i386,amd64' '1' 'Multi-Arch: same'
-insertpackage 'stable' 'out-of-sync-foreign' 'i386,amd64' '1' 'Multi-Arch: same'
+insertpackage 'stable,installed' 'out-of-sync-foreign' 'i386,amd64' '1' 'Multi-Arch: same'
 insertpackage 'unstable' 'out-of-sync-foreign' 'i386' '2' 'Multi-Arch: same'
 
 insertinstalledpackage 'out-of-sync-gone-native' 'i386,amd64' '1' 'Multi-Arch: same'
diff --git a/test/integration/test-sourceslist-trusted-options b/test/integration/test-sourceslist-trusted-options
index 28415dd62..55d4e0233 100755
--- a/test/integration/test-sourceslist-trusted-options
+++ b/test/integration/test-sourceslist-trusted-options
@@ -20,6 +20,7 @@ foo/testing 2 amd64
 foo/stable 1 amd64
 ' apt list foo -a
 
+	cd downloaded
 	rm -f foo_1_amd64.deb foo_2_amd64.deb
 	testsuccess aptget download foo "$@"
 	testsuccess test -s foo_1_amd64.deb -o -s foo_2_amd64.deb
@@ -27,6 +28,7 @@ foo/stable 1 amd64
 	rm -f foo_1.dsc foo_2.dsc
 	testsuccess aptget source foo --dsc-only -d "$@"
 	testsuccess test -s foo_1.dsc -o -s foo_2.dsc
+	cd - >/dev/null
 }
 
 everythingfails() {
@@ -39,18 +41,22 @@ foo/stable 1 amd64
   foo
 E: Some packages could not be authenticated'
 
+	cd downloaded
 	rm -f foo_1_amd64.deb foo_2_amd64.deb
 	testfailure aptget download foo "$@"
-	testequal "$WARNING" tail -n 3 rootdir/tmp/testfailure.output
+	testequal "$WARNING" tail -n 3 ../rootdir/tmp/testfailure.output
 	testfailure test -s foo_1_amd64.deb -o -s foo_2_amd64.deb
 
 	rm -f foo_1.dsc foo_2.dsc
 	testfailure aptget source foo --dsc-only -d "$@"
-	testequal "$WARNING" tail -n 3 rootdir/tmp/testfailure.output
+	testequal "$WARNING" tail -n 3 ../rootdir/tmp/testfailure.output
 	testfailure test -s foo_1.dsc -o -s foo_2.dsc
+	cd - >/dev/null
 }
 
 cp -a rootdir/etc/apt/sources.list.d/ rootdir/etc/apt/sources.list.d.bak/
+echo 'Debug::Acquire::Transaction "true";
+Debug::pkgAcquire::Worker "true";' > rootdir/etc/apt/apt.conf.d/00debugging
 
 aptgetupdate() {
 	rm -rf rootdir/var/lib/apt/lists
diff --git a/test/integration/test-suggest-installed-multiarch-silbing b/test/integration/test-suggest-installed-multiarch-silbing
index d55d250aa..89640a30c 100755
--- a/test/integration/test-suggest-installed-multiarch-silbing
+++ b/test/integration/test-suggest-installed-multiarch-silbing
@@ -9,8 +9,7 @@ configarchitecture 'amd64' 'i386' 'armel'
 insertinstalledpackage 'foo' 'i386' '1'
 insertpackage 'unstable' 'foo' 'amd64,i386' '1'
 
-insertinstalledpackage 'foo2' 'i386' '1'
-insertpackage 'unstable' 'foo2' 'i386' '1'
+insertpackage 'unstable,installed' 'foo2' 'i386' '1'
 
 insertinstalledpackage 'foo3' 'amd64' '1'
 insertpackage 'unstable' 'foo3' 'amd64,i386' '1'
diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
index 8c9c9c767..574183b0a 100755
--- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
+++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum
@@ -141,6 +141,8 @@ setupaptarchive
 changetowebserver
 testsuccess aptget update
 
+cd downloaded
+
 testok() {
 	rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz
 	testequal "Reading package lists...
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 34476e1af..00004a524 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -157,9 +157,8 @@ static bool sendData(int const client, std::string const &data)		/*{{{*/
 }
 									/*}}}*/
 static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/
-	       bool content, std::string const &error = "")
+	       bool content, std::string const &error = "", std::list<std::string> headers = std::list<std::string>())
 {
-   std::list<std::string> headers;
    std::string response("<html><head><title>");
    response.append(httpcodeToStr(httpcode)).append("</title></head>");
    response.append("<body><h1>").append(httpcodeToStr(httpcode)).append("</h1>");
@@ -367,22 +366,88 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 
    // Proxies require absolute uris, so this is a simple proxy-fake option
    std::string const absolute = _config->Find("aptwebserver::request::absolute", "uri,path");
-   if (strncmp(host.c_str(), filename.c_str(), host.length()) == 0)
+   if (strncmp(host.c_str(), filename.c_str(), host.length()) == 0 && APT::String::Startswith(filename, "/_config/") == false)
    {
       if (absolute.find("uri") == std::string::npos)
       {
 	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
 	 return false;
       }
+
       // strip the host from the request to make it an absolute path
       filename.erase(0, host.length());
+
+      std::string const authConf = _config->Find("aptwebserver::proxy-authorization", "");
+      std::string auth = LookupTag(request, "Proxy-Authorization", "");
+      if (authConf.empty() != auth.empty())
+      {
+	 if (auth.empty())
+	    sendError(client, 407, request, sendContent, "Proxy requires authentication");
+	 else
+	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it");
+	return false;
+      }
+      if (authConf.empty() == false)
+      {
+	 char const * const basic = "Basic ";
+	 if (strncmp(auth.c_str(), basic, strlen(basic)) == 0)
+	 {
+	    auth.erase(0, strlen(basic));
+	    if (auth != authConf)
+	    {
+	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match");
+	       return false;
+	    }
+	 }
+	 else
+	 {
+	    std::list<std::string> headers;
+	    headers.push_back("Proxy-Authenticate: Basic");
+	    sendError(client, 407, request, sendContent, "Unsupported Proxy-Authentication Scheme", headers);
+	    return false;
+	 }
+      }
    }
-   else if (absolute.find("path") == std::string::npos)
+   else if (absolute.find("path") == std::string::npos && APT::String::Startswith(filename, "/_config/") == false)
    {
       sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
       return false;
    }
 
+   if (APT::String::Startswith(filename, "/_config/") == false)
+   {
+      std::string const authConf = _config->Find("aptwebserver::authorization", "");
+      std::string auth = LookupTag(request, "Authorization", "");
+      if (authConf.empty() != auth.empty())
+      {
+	 if (auth.empty())
+	    sendError(client, 401, request, sendContent, "Server requires authentication");
+	 else
+	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it");
+	 return false;
+      }
+      if (authConf.empty() == false)
+      {
+	 char const * const basic = "Basic ";
+	 if (strncmp(auth.c_str(), basic, strlen(basic)) == 0)
+	 {
+	    auth.erase(0, strlen(basic));
+	    if (auth != authConf)
+	    {
+	       sendError(client, 401, request, sendContent, "Authentication doesn't match");
+	       return false;
+	    }
+	 }
+	 else
+	 {
+	    std::list<std::string> headers;
+	    headers.push_back("WWW-Authenticate: Basic");
+	    sendError(client, 401, request, sendContent, "Unsupported Authentication Scheme", headers);
+	    return false;
+	 }
+      }
+   }
+
    size_t paramspos = filename.find('?');
    if (paramspos != std::string::npos)
    {
@@ -434,9 +499,11 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector<std::string> const &parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector<std::string> parts)/*{{{*/
 {
    size_t const pcount = parts.size();
+   for (size_t i = 0; i < pcount; ++i)
+      parts[i] = DeQuoteString(parts[i]);
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
@@ -656,6 +723,8 @@ int main(int const argc, const char * argv[])
    CommandLine::Args Args[] = {
       {0, "port", "aptwebserver::port", CommandLine::HasArg},
       {0, "request-absolute", "aptwebserver::request::absolute", CommandLine::HasArg},
+      {0, "authorization", "aptwebserver::authorization", CommandLine::HasArg},
+      {0, "proxy-authorization", "aptwebserver::proxy-authorization", CommandLine::HasArg},
       {'c',"config-file",0,CommandLine::ConfigFile},
       {'o',"option",0,CommandLine::ArbItem},
       {0,0,0,0}
diff --git a/test/libapt/hashsums_test.cc b/test/libapt/hashsums_test.cc
index 2159996ff..a19a0befd 100644
--- a/test/libapt/hashsums_test.cc
+++ b/test/libapt/hashsums_test.cc
@@ -163,30 +163,34 @@ TEST(HashSumsTest, FileBased)
 
    FileFd fd(__FILE__, FileFd::ReadOnly);
    EXPECT_TRUE(fd.IsOpen());
+   std::string FileSize;
+   strprintf(FileSize, "%llu", fd.FileSize());
 
    {
       Hashes hashes;
       hashes.AddFD(fd.Fd());
       HashStringList list = hashes.GetHashStringList();
       EXPECT_FALSE(list.empty());
-      EXPECT_EQ(4, list.size());
+      EXPECT_EQ(5, list.size());
       EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue());
       EXPECT_EQ(sha1.Value(), list.find("SHA1")->HashValue());
       EXPECT_EQ(sha256.Value(), list.find("SHA256")->HashValue());
       EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue());
+      EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
    }
-   unsigned long sz = fd.FileSize();
+   unsigned long long sz = fd.FileSize();
    fd.Seek(0);
    {
       Hashes hashes;
       hashes.AddFD(fd.Fd(), sz);
       HashStringList list = hashes.GetHashStringList();
       EXPECT_FALSE(list.empty());
-      EXPECT_EQ(4, list.size());
+      EXPECT_EQ(5, list.size());
       EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue());
       EXPECT_EQ(sha1.Value(), list.find("SHA1")->HashValue());
       EXPECT_EQ(sha256.Value(), list.find("SHA256")->HashValue());
       EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue());
+      EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
    }
    fd.Seek(0);
    {
@@ -279,37 +283,46 @@ TEST(HashSumsTest, HashStringList)
    EXPECT_EQ(NULL, list.find(""));
    EXPECT_EQ(NULL, list.find("MD5Sum"));
 
+   // empty lists aren't equal
    HashStringList list2;
    EXPECT_FALSE(list == list2);
    EXPECT_TRUE(list != list2);
 
+   // some hashes don't really contribute to usability
+   list.push_back(HashString("Checksum-FileSize", "29"));
+   EXPECT_FALSE(list.empty());
+   EXPECT_FALSE(list.usable());
+
    Hashes hashes;
    hashes.Add("The quick brown fox jumps over the lazy dog");
    list = hashes.GetHashStringList();
    EXPECT_FALSE(list.empty());
    EXPECT_TRUE(list.usable());
-   EXPECT_EQ(4, list.size());
+   EXPECT_EQ(5, list.size());
    EXPECT_TRUE(NULL != list.find(NULL));
    EXPECT_TRUE(NULL != list.find(""));
    EXPECT_TRUE(NULL != list.find("MD5Sum"));
+   EXPECT_TRUE(NULL != list.find("Checksum-FileSize"));
    EXPECT_TRUE(NULL == list.find("ROT26"));
 
    _config->Set("Acquire::ForceHash", "MD5Sum");
    EXPECT_FALSE(list.empty());
    EXPECT_TRUE(list.usable());
-   EXPECT_EQ(4, list.size());
+   EXPECT_EQ(5, list.size());
    EXPECT_TRUE(NULL != list.find(NULL));
    EXPECT_TRUE(NULL != list.find(""));
    EXPECT_TRUE(NULL != list.find("MD5Sum"));
+   EXPECT_TRUE(NULL != list.find("Checksum-FileSize"));
    EXPECT_TRUE(NULL == list.find("ROT26"));
 
    _config->Set("Acquire::ForceHash", "ROT26");
    EXPECT_FALSE(list.empty());
    EXPECT_FALSE(list.usable());
-   EXPECT_EQ(4, list.size());
+   EXPECT_EQ(5, list.size());
    EXPECT_TRUE(NULL == list.find(NULL));
    EXPECT_TRUE(NULL == list.find(""));
    EXPECT_TRUE(NULL != list.find("MD5Sum"));
+   EXPECT_TRUE(NULL != list.find("Checksum-FileSize"));
    EXPECT_TRUE(NULL == list.find("ROT26"));
 
    _config->Clear("Acquire::ForceHash");
diff --git a/test/libapt/strutil_test.cc b/test/libapt/strutil_test.cc
index 8dd9114ec..9bc3c76fd 100644
--- a/test/libapt/strutil_test.cc
+++ b/test/libapt/strutil_test.cc
@@ -1,10 +1,13 @@
 #include <config.h>
 #include <apt-pkg/strutl.h>
+#include <apt-pkg/fileutl.h>
 #include <string>
 #include <vector>
 
 #include <gtest/gtest.h>
 
+#include "file-helpers.h"
+
 TEST(StrUtilTest,DeEscapeString)
 {
    // nothing special
@@ -85,7 +88,7 @@ TEST(StrUtilTest,EndsWith)
    EXPECT_FALSE(Endswith("abcd", "x"));
    EXPECT_FALSE(Endswith("abcd", "abcndefg"));
 }
-TEST(StrUtilTest,StartWith)
+TEST(StrUtilTest,StartsWith)
 {
    using APT::String::Startswith;
    EXPECT_TRUE(Startswith("abcd", "a"));
@@ -129,3 +132,84 @@ TEST(StrUtilTest,SubstVar)
    EXPECT_EQ(" bb a bb a bb a bb ", SubstVar(" aaa a aaa a aaa a aaa ", "aaa", "bb"));
 
 }
+TEST(StrUtilTest,Base64Encode)
+{
+   EXPECT_EQ("QWxhZGRpbjpvcGVuIHNlc2FtZQ==", Base64Encode("Aladdin:open sesame"));
+   EXPECT_EQ("cGxlYXN1cmUu", Base64Encode("pleasure."));
+   EXPECT_EQ("bGVhc3VyZS4=", Base64Encode("leasure."));
+   EXPECT_EQ("ZWFzdXJlLg==", Base64Encode("easure."));
+   EXPECT_EQ("YXN1cmUu", Base64Encode("asure."));
+   EXPECT_EQ("c3VyZS4=", Base64Encode("sure."));
+   EXPECT_EQ("dXJlLg==", Base64Encode("ure."));
+   EXPECT_EQ("cmUu", Base64Encode("re."));
+   EXPECT_EQ("ZS4=", Base64Encode("e."));
+   EXPECT_EQ("Lg==", Base64Encode("."));
+   EXPECT_EQ("", Base64Encode(""));
+}
+void ReadMessagesTestWithNewLine(char const * const nl, char const * const ab)
+{
+   SCOPED_TRACE(SubstVar(SubstVar(nl, "\n", "n"), "\r", "r") + " # " + ab);
+   FileFd fd;
+   std::string pkgA = "Package: pkgA\n"
+      "Version: 1\n"
+      "Size: 100\n"
+      "Description: aaa\n"
+      " aaa";
+   std::string pkgB = "Package: pkgB\n"
+      "Version: 1\n"
+      "Flag: no\n"
+      "Description: bbb";
+   std::string pkgC = "Package: pkgC\n"
+      "Version: 2\n"
+      "Flag: yes\n"
+      "Description:\n"
+      " ccc";
+
+   createTemporaryFile("readmessage", fd, NULL, (pkgA + nl + pkgB + nl + pkgC + nl).c_str());
+   std::vector<std::string> list;
+   EXPECT_TRUE(ReadMessages(fd.Fd(), list));
+   EXPECT_EQ(3, list.size());
+   EXPECT_EQ(pkgA, list[0]);
+   EXPECT_EQ(pkgB, list[1]);
+   EXPECT_EQ(pkgC, list[2]);
+
+   size_t const msgsize = 63990;
+   createTemporaryFile("readmessage", fd, NULL, NULL);
+   for (size_t j = 0; j < msgsize; ++j)
+      fd.Write(ab, strlen(ab));
+   for (size_t i = 0; i < 21; ++i)
+   {
+      std::string msg;
+      strprintf(msg, "msgsize=%zu  i=%zu", msgsize, i);
+      SCOPED_TRACE(msg);
+      fd.Seek((msgsize + (i - 1)) * strlen(ab));
+      fd.Write(ab, strlen(ab));
+      fd.Write(nl, strlen(nl));
+      fd.Seek(0);
+      list.clear();
+      EXPECT_TRUE(ReadMessages(fd.Fd(), list));
+      EXPECT_EQ(1, list.size());
+      EXPECT_EQ((msgsize + i) * strlen(ab), list[0].length());
+      EXPECT_EQ(std::string::npos, list[0].find_first_not_of(ab));
+   }
+
+   list.clear();
+   fd.Write(pkgA.c_str(), pkgA.length());
+   fd.Write(nl, strlen(nl));
+   fd.Seek(0);
+   EXPECT_TRUE(ReadMessages(fd.Fd(), list));
+   EXPECT_EQ(2, list.size());
+   EXPECT_EQ((msgsize + 20) * strlen(ab), list[0].length());
+   EXPECT_EQ(std::string::npos, list[0].find_first_not_of(ab));
+   EXPECT_EQ(pkgA, list[1]);
+
+
+   fd.Close();
+}
+TEST(StrUtilTest,ReadMessages)
+{
+   ReadMessagesTestWithNewLine("\n\n", "a");
+   ReadMessagesTestWithNewLine("\r\n\r\n", "a");
+   ReadMessagesTestWithNewLine("\n\n", "ab");
+   ReadMessagesTestWithNewLine("\r\n\r\n", "ab");
+}