Merge branch 'feature/amtshilfe' into 'master'

reimplement mirror method

See merge request apt-team/apt!1

36 files changed, 1484 insertions, 959 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 9b163081b..611876dd2 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -16,6 +16,7 @@
 #include <config.h>
 
 #include <apt-pkg/acquire-item.h>
+#include <apt-pkg/acquire-worker.h>
 #include <apt-pkg/acquire.h>
 #include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/configuration.h>
@@ -34,6 +35,7 @@
 #include <algorithm>
 #include <ctime>
 #include <iostream>
+#include <memory>
 #include <numeric>
 #include <random>
 #include <sstream>
@@ -1070,6 +1072,12 @@ bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI)	/*{{{*/
 									/*}}}*/
 int pkgAcquire::Item::Priority()					/*{{{*/
 {
+   // Stage 0: Files requested by methods
+   // - they will usually not end up here, but if they do we make sure
+   //   to get them as soon as possible as they are probably blocking
+   //   the processing of files by the requesting method
+   if (dynamic_cast<pkgAcqAuxFile *>(this) != nullptr)
+      return 5000;
    // Stage 1: Meta indices and diff indices
    // - those need to be fetched first to have progress reporting working
    //   for the rest
@@ -3892,3 +3900,114 @@ string pkgAcqFile::Custom600Headers() const				/*{{{*/
 }
 									/*}}}*/
 pkgAcqFile::~pkgAcqFile() {}
+
+void pkgAcqAuxFile::Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) /*{{{*/
+{
+   pkgAcqFile::Failed(Message, Cnf);
+   if (Status == StatIdle)
+      return;
+   if (RealFileExists(DestFile))
+      Rename(DestFile, DestFile + ".FAILED");
+   Worker->ReplyAux(Desc);
+}
+									/*}}}*/
+void pkgAcqAuxFile::Done(std::string const &Message, HashStringList const &CalcHashes, /*{{{*/
+			 pkgAcquire::MethodConfig const *const Cnf)
+{
+   pkgAcqFile::Done(Message, CalcHashes, Cnf);
+   if (Status == StatDone)
+      Worker->ReplyAux(Desc);
+   else if (Status == StatAuthError || Status == StatError)
+      Worker->ReplyAux(Desc);
+}
+									/*}}}*/
+std::string pkgAcqAuxFile::Custom600Headers() const /*{{{*/
+{
+   if (MaximumSize == 0)
+      return pkgAcqFile::Custom600Headers();
+   std::string maxsize;
+   strprintf(maxsize, "\nMaximum-Size: %llu", MaximumSize);
+   return pkgAcqFile::Custom600Headers().append(maxsize);
+}
+									/*}}}*/
+void pkgAcqAuxFile::Finished() /*{{{*/
+{
+   auto dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/");
+   if (APT::String::Startswith(DestFile, dirname))
+   {
+      // the file is never returned by method requesting it, so fix up the permission now
+      if (FileExists(DestFile))
+      {
+	 ChangeOwnerAndPermissionOfFile("pkgAcqAuxFile", DestFile.c_str(), "root", ROOT_GROUP, 0644);
+	 if (Status == StatDone)
+	    return;
+      }
+   }
+   else
+   {
+      dirname = flNotFile(DestFile);
+      RemoveFile("pkgAcqAuxFile::Finished", DestFile);
+      RemoveFile("pkgAcqAuxFile::Finished", DestFile + ".FAILED");
+      rmdir(dirname.c_str());
+   }
+   DestFile.clear();
+}
+									/*}}}*/
+// GetAuxFileNameFromURI						/*{{{*/
+static std::string GetAuxFileNameFromURIInLists(std::string const &uri)
+{
+   // check if we have write permission for our usual location.
+   auto const dirname = flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/");
+   char const * const filetag = ".apt-acquire-privs-test.XXXXXX";
+   std::string const tmpfile_tpl = flCombine(dirname, filetag);
+   std::unique_ptr<char, decltype(std::free) *> tmpfile { strdup(tmpfile_tpl.c_str()), std::free };
+   int const fd = mkstemp(tmpfile.get());
+   if (fd == -1 && errno == EACCES)
+      return "";
+   RemoveFile("GetAuxFileNameFromURI", tmpfile.get());
+   close(fd);
+   return flCombine(dirname, URItoFileName(uri));
+}
+static std::string GetAuxFileNameFromURI(std::string const &uri)
+{
+   auto const lists = GetAuxFileNameFromURIInLists(uri);
+   if (lists.empty() == false)
+      return lists;
+
+   std::string tmpdir_tpl;
+   strprintf(tmpdir_tpl, "%s/apt-auxfiles-XXXXXX", GetTempDir().c_str());
+   std::unique_ptr<char, decltype(std::free) *> tmpdir { strndup(tmpdir_tpl.data(), tmpdir_tpl.length()), std::free };
+   if (mkdtemp(tmpdir.get()) == nullptr)
+   {
+      _error->Errno("GetAuxFileNameFromURI", "mkdtemp of %s failed", tmpdir.get());
+      return flCombine("/nonexistent/auxfiles/", URItoFileName(uri));
+   }
+   chmod(tmpdir.get(), 0755);
+   auto const filename = flCombine(tmpdir.get(), URItoFileName(uri));
+   _error->PushToStack();
+   FileFd in(flCombine(flCombine(_config->FindDir("Dir::State::lists"), "auxfiles/"), URItoFileName(uri)), FileFd::ReadOnly);
+   if (in.IsOpen())
+   {
+      FileFd out(filename, FileFd::WriteOnly | FileFd::Create | FileFd::Exclusive);
+      CopyFile(in, out);
+   }
+   _error->RevertToStack();
+   return filename;
+}
+									/*}}}*/
+pkgAcqAuxFile::pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker,
+			     std::string const &ShortDesc, std::string const &Desc, std::string const &URI,
+			     HashStringList const &Hashes, unsigned long long const MaximumSize) : pkgAcqFile(Owner->GetOwner(), URI, Hashes, Hashes.FileSize(), Desc, ShortDesc, "", GetAuxFileNameFromURI(URI), false),
+												   Owner(Owner), Worker(Worker), MaximumSize(MaximumSize)
+{
+   /* very bad failures can happen while constructing which causes
+      us to hang as the aux request is never answered (e.g. method not available)
+      Ideally we catch failures earlier, but a safe guard can't hurt. */
+   if (Status == pkgAcquire::Item::StatIdle || Status == pkgAcquire::Item::StatFetching)
+      return;
+   Failed(std::string("400 URI Failure\n") +
+	     "URI: " + URI + "\n" +
+	     "Filename: " + DestFile,
+	  nullptr);
+}
+pkgAcqAuxFile::~pkgAcqAuxFile() {}
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index cf227d1b5..46d79df92 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -1238,6 +1238,25 @@ class pkgAcqFile : public pkgAcquire::Item
    virtual ~pkgAcqFile();
 };
 									/*}}}*/
+class APT_HIDDEN pkgAcqAuxFile : public pkgAcqFile /*{{{*/
+{
+   pkgAcquire::Item *const Owner;
+   pkgAcquire::Worker *const Worker;
+   unsigned long long MaximumSize;
+
+   public:
+   virtual void Failed(std::string const &Message, pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE;
+   virtual void Done(std::string const &Message, HashStringList const &CalcHashes,
+		     pkgAcquire::MethodConfig const *const Cnf) APT_OVERRIDE;
+   virtual std::string Custom600Headers() const APT_OVERRIDE;
+   virtual void Finished() APT_OVERRIDE;
+
+   pkgAcqAuxFile(pkgAcquire::Item *const Owner, pkgAcquire::Worker *const Worker,
+		 std::string const &ShortDesc, std::string const &Desc, std::string const &URI,
+		 HashStringList const &Hashes, unsigned long long const MaximumSize);
+   virtual ~pkgAcqAuxFile();
+};
+									/*}}}*/
 /** @} */
 
 #endif
diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc
index 309b5dcf9..8934d87a0 100644
--- a/apt-pkg/acquire-method.cc
+++ b/apt-pkg/acquire-method.cc
@@ -27,7 +27,10 @@
 #include <apt-pkg/sha2.h>
 #include <apt-pkg/strutl.h>
 
+#include <algorithm>
 #include <iostream>
+#include <iterator>
+#include <sstream>
 #include <string>
 #include <vector>
 #include <stdarg.h>
@@ -39,33 +42,44 @@
 
 using namespace std;
 
+// poor mans unordered_map::try_emplace for C++11 as it is a C++17 feature /*{{{*/
+template <typename Arg>
+static void try_emplace(std::unordered_map<std::string, std::string> &fields, std::string &&name, Arg &&value)
+{
+   if (fields.find(name) == fields.end())
+      fields.emplace(std::move(name), std::forward<Arg>(value));
+}
+									/*}}}*/
+
 // AcqMethod::pkgAcqMethod - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* This constructs the initialization text */
 pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags)
 {
-   std::cout << "100 Capabilities\n"
-	     << "Version: " << Ver << "\n";
-
+   std::unordered_map<std::string, std::string> fields;
+   try_emplace(fields, "Version", Ver);
    if ((Flags & SingleInstance) == SingleInstance)
-      std::cout << "Single-Instance: true\n";
+      try_emplace(fields, "Single-Instance", "true");
 
    if ((Flags & Pipeline) == Pipeline)
-      std::cout << "Pipeline: true\n";
+      try_emplace(fields, "Pipeline", "true");
 
    if ((Flags & SendConfig) == SendConfig)
-      std::cout << "Send-Config: true\n";
+      try_emplace(fields, "Send-Config", "true");
 
    if ((Flags & LocalOnly) == LocalOnly)
-      std::cout <<"Local-Only: true\n";
+      try_emplace(fields, "Local-Only", "true");
 
    if ((Flags & NeedsCleanup) == NeedsCleanup)
-      std::cout << "Needs-Cleanup: true\n";
+      try_emplace(fields, "Needs-Cleanup", "true");
 
    if ((Flags & Removable) == Removable)
-      std::cout << "Removable: true\n";
+      try_emplace(fields, "Removable", "true");
 
-   std::cout << "\n" << std::flush;
+   if ((Flags & AuxRequests) == AuxRequests)
+      try_emplace(fields, "AuxRequests", "true");
+
+   SendMessage("100 Capabilities", std::move(fields));
 
    SetNonBlock(STDIN_FILENO,true);
 
@@ -73,6 +87,26 @@ pkgAcqMethod::pkgAcqMethod(const char *Ver,unsigned long Flags)
    QueueBack = 0;
 }
 									/*}}}*/
+void pkgAcqMethod::SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields) /*{{{*/
+{
+   std::cout << header << '\n';
+   for (auto const &f : fields)
+   {
+      if (f.second.empty())
+	 continue;
+      std::cout << f.first << ": ";
+      auto const lines = VectorizeString(f.second, '\n');
+      if (likely(lines.empty() == false))
+      {
+	 std::copy(lines.begin(), std::prev(lines.end()), std::ostream_iterator<std::string>(std::cout, "\n "));
+	 std::cout << *lines.rbegin();
+      }
+      std::cout << '\n';
+   }
+   std::cout << '\n'
+	     << std::flush;
+}
+									/*}}}*/
 // AcqMethod::Fail - A fetch has failed					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -97,40 +131,35 @@ void pkgAcqMethod::Fail(bool Transient)
 }
 									/*}}}*/
 // AcqMethod::Fail - A fetch has failed					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void pkgAcqMethod::Fail(string Err,bool Transient)
 {
    // Strip out junk from the error messages
-   for (string::iterator I = Err.begin(); I != Err.end(); ++I)
-   {
-      if (*I == '\r') 
-	 *I = ' ';
-      if (*I == '\n') 
-	 *I = ' ';
-   }
-
-   if (Queue != 0)
-   {
-      std::cout << "400 URI Failure\nURI: " << Queue->Uri << "\n"
-		<< "Message: " << Err;
-      if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true)
-	 std::cout << " " << IP;
-      std::cout << "\n";
-      Dequeue();
-   }
+   std::transform(Err.begin(), Err.end(), Err.begin(), [](char const c) {
+      if (c == '\r' || c == '\n')
+	 return ' ';
+      return c;
+   });
+   if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true)
+      Err.append(" ").append(IP);
+
+   std::unordered_map<std::string, std::string> fields;
+   if (Queue != nullptr)
+      try_emplace(fields, "URI", Queue->Uri);
    else
-      std::cout << "400 URI Failure\nURI: <UNKNOWN>\nMessage: " << Err << "\n";
+      try_emplace(fields, "URI", "<UNKNOWN>");
+   try_emplace(fields, "Message", Err);
 
    if(FailReason.empty() == false)
-      std::cout << "FailReason: " << FailReason << "\n";
+      try_emplace(fields, "FailReason", FailReason);
    if (UsedMirror.empty() == false)
-      std::cout << "UsedMirror: " << UsedMirror << "\n";
-   // Set the transient flag
+      try_emplace(fields, "UsedMirror", UsedMirror);
    if (Transient == true)
-      std::cout << "Transient-Failure: true\n";
+      try_emplace(fields, "Transient-Failure", "true");
 
-   std::cout << "\n" << std::flush;
+   SendMessage("400 URI Failure", std::move(fields));
+
+   if (Queue != nullptr)
+      Dequeue();
 }
 									/*}}}*/
 // AcqMethod::DropPrivsOrDie - Drop privileges or die		/*{{{*/
@@ -146,96 +175,79 @@ void pkgAcqMethod::DropPrivsOrDie()
 
 									/*}}}*/
 // AcqMethod::URIStart - Indicate a download is starting		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 void pkgAcqMethod::URIStart(FetchResult &Res)
 {
    if (Queue == 0)
       abort();
 
-   std::cout << "200 URI Start\n"
-	     << "URI: " << Queue->Uri << "\n";
+   std::unordered_map<std::string, std::string> fields;
+   try_emplace(fields, "URI", Queue->Uri);
    if (Res.Size != 0)
-      std::cout << "Size: " << std::to_string(Res.Size) << "\n";
-
+      try_emplace(fields, "Size", std::to_string(Res.Size));
    if (Res.LastModified != 0)
-      std::cout << "Last-Modified: " << TimeRFC1123(Res.LastModified, true) << "\n";
-
+      try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true));
    if (Res.ResumePoint != 0)
-      std::cout << "Resume-Point: " << std::to_string(Res.ResumePoint) << "\n";
-
+      try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint));
    if (UsedMirror.empty() == false)
-      std::cout << "UsedMirror: " << UsedMirror << "\n";
+      try_emplace(fields, "UsedMirror", UsedMirror);
 
-   std::cout << "\n" << std::flush;
+   SendMessage("200 URI Start", std::move(fields));
 }
 									/*}}}*/
 // AcqMethod::URIDone - A URI is finished				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-static void printHashStringList(char const *const Prefix, HashStringList const *const list)
+static void printHashStringList(std::unordered_map<std::string, std::string> &fields, std::string const &Prefix, HashStringList const &list)
 {
-      for (HashStringList::const_iterator hash = list->begin(); hash != list->end(); ++hash)
-      {
-	 // very old compatibility name for MD5Sum
-	 if (hash->HashType() == "MD5Sum")
-	    std::cout << Prefix << "MD5-Hash: " << hash->HashValue() << "\n";
-	 std::cout << hash->HashType() << "-Hash: " << hash->HashValue() << "\n";
-      }
+   for (auto const &hash : list)
+   {
+      // very old compatibility name for MD5Sum
+      if (hash.HashType() == "MD5Sum")
+	 try_emplace(fields, Prefix + "MD5-Hash", hash.HashValue());
+      try_emplace(fields, Prefix + hash.HashType() + "-Hash", hash.HashValue());
+   }
 }
 void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt)
 {
    if (Queue == 0)
       abort();
 
-   std::cout << "201 URI Done\n"
-	     << "URI: " << Queue->Uri << "\n";
-
+   std::unordered_map<std::string, std::string> fields;
+   try_emplace(fields, "URI", Queue->Uri);
    if (Res.Filename.empty() == false)
-      std::cout << "Filename: " << Res.Filename << "\n";
-
+      try_emplace(fields, "Filename", Res.Filename);
    if (Res.Size != 0)
-      std::cout << "Size: " << std::to_string(Res.Size) << "\n";
-
+      try_emplace(fields, "Size", std::to_string(Res.Size));
    if (Res.LastModified != 0)
-      std::cout << "Last-Modified: " << TimeRFC1123(Res.LastModified, true) << "\n";
-
-   printHashStringList("", &Res.Hashes);
+      try_emplace(fields, "Last-Modified", TimeRFC1123(Res.LastModified, true));
+   printHashStringList(fields, "", Res.Hashes);
 
    if (UsedMirror.empty() == false)
-      std::cout << "UsedMirror: " << UsedMirror << "\n";
+      try_emplace(fields, "UsedMirror", UsedMirror);
    if (Res.GPGVOutput.empty() == false)
    {
-      std::cout << "GPGVOutput:\n";
-      for (vector<string>::const_iterator I = Res.GPGVOutput.begin();
-	   I != Res.GPGVOutput.end(); ++I)
-	 std::cout << " " << *I << "\n";
+      std::ostringstream os;
+      std::copy(Res.GPGVOutput.begin(), Res.GPGVOutput.end() - 1, std::ostream_iterator<std::string>(os, "\n"));
+      os << *Res.GPGVOutput.rbegin();
+      try_emplace(fields, "GPGVOutput", os.str());
    }
-
    if (Res.ResumePoint != 0)
-      std::cout << "Resume-Point: " << std::to_string(Res.ResumePoint) << "\n";
-
+      try_emplace(fields, "Resume-Point", std::to_string(Res.ResumePoint));
    if (Res.IMSHit == true)
-      std::cout << "IMS-Hit: true\n";
+      try_emplace(fields, "IMS-Hit", "true");
 
-   if (Alt != 0)
+   if (Alt != nullptr)
    {
       if (Alt->Filename.empty() == false)
-	 std::cout << "Alt-Filename: " << Alt->Filename << "\n";
-
+	 try_emplace(fields, "Alt-Filename", Alt->Filename);
       if (Alt->Size != 0)
-	 std::cout << "Alt-Size: " << std::to_string(Alt->Size) << "\n";
-
+	 try_emplace(fields, "Alt-Size", std::to_string(Alt->Size));
       if (Alt->LastModified != 0)
-	 std::cout << "Alt-Last-Modified: " << TimeRFC1123(Alt->LastModified, true) << "\n";
-
-      printHashStringList("Alt-", &Alt->Hashes);
-
+	 try_emplace(fields, "Alt-Last-Modified", TimeRFC1123(Alt->LastModified, true));
       if (Alt->IMSHit == true)
-	 std::cout << "Alt-IMS-Hit: true\n";
+	 try_emplace(fields, "Alt-IMS-Hit", "true");
+      printHashStringList(fields, "Alt-", Alt->Hashes);
    }
 
-   std::cout << "\n" << std::flush;
+   SendMessage("201 URI Done", std::move(fields));
    Dequeue();
 }
 									/*}}}*/
@@ -459,9 +471,10 @@ void pkgAcqMethod::Status(const char *Format,...)
  * the worker will enqueue again later on to the right queue */
 void pkgAcqMethod::Redirect(const string &NewURI)
 {
-   std::cout << "103 Redirect\nURI: " << Queue->Uri << "\n"
-	     << "New-URI: " << NewURI << "\n"
-	     << "\n" << std::flush;
+   std::unordered_map<std::string, std::string> fields;
+   try_emplace(fields, "URI", Queue->Uri);
+   try_emplace(fields, "New-URI", NewURI);
+   SendMessage("103 Redirect", std::move(fields));
    Dequeue();
 }
                                                                         /*}}}*/
diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h
index 2de9cf5c2..664b95c18 100644
--- a/apt-pkg/acquire-method.h
+++ b/apt-pkg/acquire-method.h
@@ -26,6 +26,7 @@
 #include <time.h>
 
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #ifndef APT_8_CLEANER_HEADERS
@@ -99,6 +100,7 @@ class pkgAcqMethod
    virtual void Fail(std::string Why, bool Transient = false);
    virtual void URIStart(FetchResult &Res);
    virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0);
+   void SendMessage(std::string const &header, std::unordered_map<std::string, std::string> &&fields);
 
    bool MediaFail(std::string Required,std::string Drive);
    virtual void Exit() {};
@@ -106,10 +108,16 @@ class pkgAcqMethod
    void PrintStatus(char const * const header, const char* Format, va_list &args) const;
 
    public:
-   enum CnfFlags {SingleInstance = (1<<0),
-                  Pipeline = (1<<1), SendConfig = (1<<2),
-                  LocalOnly = (1<<3), NeedsCleanup = (1<<4), 
-                  Removable = (1<<5)};
+   enum CnfFlags
+   {
+      SingleInstance = (1 << 0),
+      Pipeline = (1 << 1),
+      SendConfig = (1 << 2),
+      LocalOnly = (1 << 3),
+      NeedsCleanup = (1 << 4),
+      Removable = (1 << 5),
+      AuxRequests = (1 << 6)
+   };
 
    void Log(const char *Format,...);
    void Status(const char *Format,...);
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 995750dea..6cbf8b7aa 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -192,7 +192,8 @@ bool pkgAcquire::Worker::ReadMessages()
 // ---------------------------------------------------------------------
 /* This takes the messages from the message queue and runs them through
    the parsers in order. */
-enum class APT_HIDDEN MessageType {
+enum class APT_HIDDEN MessageType
+{
    CAPABILITIES = 100,
    LOG = 101,
    STATUS = 102,
@@ -200,6 +201,7 @@ enum class APT_HIDDEN MessageType {
    WARNING = 104,
    URI_START = 200,
    URI_DONE = 201,
+   AUX_REQUEST = 351,
    URI_FAILURE = 400,
    GENERAL_FAILURE = 401,
    MEDIA_CHANGE = 403
@@ -306,6 +308,7 @@ bool pkgAcquire::Worker::RunMessages()
 
 	    std::string const NewURI = LookupTag(Message,"New-URI",URI.c_str());
             Itm->URI = NewURI;
+	    auto const AltUris = VectorizeString(LookupTag(Message, "Alternate-URIs"), '\n');
 
 	    ItemDone();
 
@@ -333,28 +336,14 @@ bool pkgAcquire::Worker::RunMessages()
 	       if (Log != nullptr)
 		  Log->Done(desc);
 
-	       // if we change site, treat it as a mirror change
-	       if (URI::SiteOnly(NewURI) != URI::SiteOnly(desc.URI))
-	       {
-		  auto const firstSpace = desc.Description.find(" ");
-		  if (firstSpace != std::string::npos)
-		  {
-		     std::string const OldSite = desc.Description.substr(0, firstSpace);
-		     if (likely(APT::String::Startswith(desc.URI, OldSite)))
-		     {
-			std::string const OldExtra = desc.URI.substr(OldSite.length() + 1);
-			if (likely(APT::String::Endswith(NewURI, OldExtra)))
-			{
-			   std::string const NewSite = NewURI.substr(0, NewURI.length() - OldExtra.length());
-			   Owner->UsedMirror = URI::ArchiveOnly(NewSite);
-			   desc.Description.replace(0, firstSpace, Owner->UsedMirror);
-			}
-		     }
-		  }
-	       }
+	       ChangeSiteIsMirrorChange(NewURI, desc, Owner);
 	       desc.URI = NewURI;
 	       if (isDoomedItem(Owner) == false)
+	       {
+		  for (auto alt = AltUris.crbegin(); alt != AltUris.crend(); ++alt)
+		     Owner->PushAlternativeURI(std::string(*alt), {}, false);
 		  OwnerQ->Owner->Enqueue(desc);
+	       }
 	    }
             break;
          }
@@ -512,6 +501,42 @@ bool pkgAcquire::Worker::RunMessages()
 	    break;
 	 }
 
+	 case MessageType::AUX_REQUEST:
+	 {
+	    if (Itm == nullptr)
+	    {
+	       _error->Error("Method gave invalid Aux Request message");
+	       break;
+	    }
+	    else if (Config->GetAuxRequests() == false)
+	    {
+	       std::vector<Item *> const ItmOwners = Itm->Owners;
+	       Message.append("\nMessage: Method tried to make an Aux Request while not being allowed to do them");
+	       OwnerQ->ItemDone(Itm);
+	       Itm = nullptr;
+	       HandleFailure(ItmOwners, Config, Log, Message, false, false);
+	       ItemDone();
+
+	       std::string Msg = "600 URI Acquire\n";
+	       Msg.reserve(200);
+	       Msg += "URI: " + LookupTag(Message, "Aux-URI", "");
+	       Msg += "\nFilename: /nonexistent/auxrequest.blocked";
+	       Msg += "\n\n";
+	       if (Debug == true)
+		  clog << " -> " << Access << ':' << QuoteString(Msg, "\n") << endl;
+	       OutQueue += Msg;
+	       OutReady = true;
+	       break;
+	    }
+
+	    auto maxsizestr = LookupTag(Message, "MaximumSize", "");
+	    unsigned long long const MaxSize = maxsizestr.empty() ? 0 : strtoull(maxsizestr.c_str(), nullptr, 10);
+	    new pkgAcqAuxFile(Itm->Owner, this, LookupTag(Message, "Aux-ShortDesc", ""),
+			      LookupTag(Message, "Aux-Description", ""), LookupTag(Message, "Aux-URI", ""),
+			      GetHashesFromMessage("Aux-", Message), MaxSize);
+	    break;
+	 }
+
 	 case MessageType::URI_FAILURE:
 	 {
 	    if (Itm == nullptr)
@@ -549,44 +574,7 @@ bool pkgAcquire::Worker::RunMessages()
 		  errAuthErr = std::find(std::begin(reasons), std::end(reasons), failReason) != std::end(reasons);
 	       }
 	    }
-
-	    for (auto const Owner: ItmOwners)
-	    {
-	       std::string NewURI;
-	       if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0)
-	       {
-		  --Owner->ModifyRetries();
-		  Owner->FailMessage(Message);
-		  auto SavedDesc = Owner->GetItemDesc();
-		  if (Log != nullptr)
-		     Log->Fail(SavedDesc);
-		  if (isDoomedItem(Owner) == false)
-		     OwnerQ->Owner->Enqueue(SavedDesc);
-	       }
-	       else if (Owner->PopAlternativeURI(NewURI))
-	       {
-		  Owner->FailMessage(Message);
-		  auto &desc = Owner->GetItemDesc();
-		  if (Log != nullptr)
-		     Log->Fail(desc);
-		  ChangeSiteIsMirrorChange(NewURI, desc, Owner);
-		  desc.URI = NewURI;
-		  if (isDoomedItem(Owner) == false)
-		     OwnerQ->Owner->Enqueue(desc);
-	       }
-	       else
-	       {
-		  if (errAuthErr && Owner->GetExpectedHashes().empty() == false)
-		     Owner->Status = pkgAcquire::Item::StatAuthError;
-		  else if (errTransient)
-		     Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
-		  auto SavedDesc = Owner->GetItemDesc();
-		  if (isDoomedItem(Owner) == false)
-		     Owner->Failed(Message, Config);
-		  if (Log != nullptr)
-		     Log->Fail(SavedDesc);
-	       }
-	    }
+	    HandleFailure(ItmOwners, Config, Log, Message, errTransient, errAuthErr);
 	    ItemDone();
 
 	    break;
@@ -604,6 +592,49 @@ bool pkgAcquire::Worker::RunMessages()
    return true;
 }
 									/*}}}*/
+void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &ItmOwners, /*{{{*/
+				       pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log,
+				       std::string const &Message, bool const errTransient, bool const errAuthErr)
+{
+   for (auto const Owner : ItmOwners)
+   {
+      std::string NewURI;
+      if (errTransient == true && Config->LocalOnly == false && Owner->ModifyRetries() != 0)
+      {
+	 --Owner->ModifyRetries();
+	 Owner->FailMessage(Message);
+	 auto SavedDesc = Owner->GetItemDesc();
+	 if (Log != nullptr)
+	    Log->Fail(SavedDesc);
+	 if (isDoomedItem(Owner) == false)
+	    OwnerQ->Owner->Enqueue(SavedDesc);
+      }
+      else if (Owner->PopAlternativeURI(NewURI))
+      {
+	 Owner->FailMessage(Message);
+	 auto &desc = Owner->GetItemDesc();
+	 if (Log != nullptr)
+	    Log->Fail(desc);
+	 ChangeSiteIsMirrorChange(NewURI, desc, Owner);
+	 desc.URI = NewURI;
+	 if (isDoomedItem(Owner) == false)
+	    OwnerQ->Owner->Enqueue(desc);
+      }
+      else
+      {
+	 if (errAuthErr && Owner->GetExpectedHashes().empty() == false)
+	    Owner->Status = pkgAcquire::Item::StatAuthError;
+	 else if (errTransient)
+	    Owner->Status = pkgAcquire::Item::StatTransientNetworkError;
+	 auto SavedDesc = Owner->GetItemDesc();
+	 if (isDoomedItem(Owner) == false)
+	    Owner->Failed(Message, Config);
+	 if (Log != nullptr)
+	    Log->Fail(SavedDesc);
+      }
+   }
+}
+									/*}}}*/
 // Worker::Capabilities - 100 Capabilities handler			/*{{{*/
 // ---------------------------------------------------------------------
 /* This parses the capabilities message and dumps it into the configuration
@@ -620,18 +651,13 @@ bool pkgAcquire::Worker::Capabilities(string Message)
    Config->LocalOnly = StringToBool(LookupTag(Message,"Local-Only"),false);
    Config->NeedsCleanup = StringToBool(LookupTag(Message,"Needs-Cleanup"),false);
    Config->Removable = StringToBool(LookupTag(Message,"Removable"),false);
+   Config->SetAuxRequests(StringToBool(LookupTag(Message, "AuxRequests"), false));
 
    // Some debug text
    if (Debug == true)
    {
       clog << "Configured access method " << Config->Access << endl;
-      clog << "Version:" << Config->Version <<
-	      " SingleInstance:" << Config->SingleInstance <<
-	      " Pipeline:" << Config->Pipeline <<
-	      " SendConfig:" << Config->SendConfig <<
-	      " LocalOnly: " << Config->LocalOnly <<
-	      " NeedsCleanup: " << Config->NeedsCleanup <<
-	      " Removable: " << Config->Removable << endl;
+      clog << "Version:" << Config->Version << " SingleInstance:" << Config->SingleInstance << " Pipeline:" << Config->Pipeline << " SendConfig:" << Config->SendConfig << " LocalOnly: " << Config->LocalOnly << " NeedsCleanup: " << Config->NeedsCleanup << " Removable: " << Config->Removable << " AuxRequests: " << Config->GetAuxRequests() << endl;
    }
 
    return true;
@@ -772,6 +798,46 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
    return true;
 }
 									/*}}}*/
+// Worker::ReplyAux - reply to an aux request from this worker		/*{{{*/
+bool pkgAcquire::Worker::ReplyAux(pkgAcquire::ItemDesc const &Item)
+{
+   if (OutFd == -1)
+      return false;
+
+   if (isDoomedItem(Item.Owner))
+      return true;
+
+   string Message = "600 URI Acquire\n";
+   Message.reserve(200);
+   Message += "URI: " + Item.URI;
+   if (RealFileExists(Item.Owner->DestFile))
+   {
+      if (Item.Owner->Status == pkgAcquire::Item::StatDone)
+      {
+	 std::string const SandboxUser = _config->Find("APT::Sandbox::User");
+	 ChangeOwnerAndPermissionOfFile("Worker::ReplyAux", Item.Owner->DestFile.c_str(),
+					SandboxUser.c_str(), ROOT_GROUP, 0600);
+	 Message += "\nFilename: " + Item.Owner->DestFile;
+      }
+      else
+      {
+	 // we end up here in case we would need root-rights to delete a file,
+	 // but we run the command as non-root… (yes, it is unlikely)
+	 Message += "\nFilename: " + flCombine("/nonexistent", Item.Owner->DestFile);
+      }
+   }
+   else
+      Message += "\nFilename: " + Item.Owner->DestFile;
+   Message += "\n\n";
+
+   if (Debug == true)
+      clog << " -> " << Access << ':' << QuoteString(Message, "\n") << endl;
+   OutQueue += Message;
+   OutReady = true;
+
+   return true;
+}
+									/*}}}*/
 // Worker::OutFdRead - Out bound FD is ready				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h
index 8fc686880..f04db2b3c 100644
--- a/apt-pkg/acquire-worker.h
+++ b/apt-pkg/acquire-worker.h
@@ -276,6 +276,7 @@ class pkgAcquire::Worker : public WeakPointable
     *  queue.
     */
    bool QueueItem(pkgAcquire::Queue::QItem *Item);
+   APT_HIDDEN bool ReplyAux(pkgAcquire::ItemDesc const &Item);
 
    /** \brief Start up the worker and fill in #Config.
     *
@@ -328,6 +329,9 @@ class pkgAcquire::Worker : public WeakPointable
 
 private:
    APT_HIDDEN void PrepareFiles(char const * const caller, pkgAcquire::Queue::QItem const * const Itm);
+   APT_HIDDEN void HandleFailure(std::vector<pkgAcquire::Item *> const &ItmOwners,
+				 pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log,
+				 std::string const &Message, bool const errTransient, bool const errAuthErr);
 };
 
 /** @} */
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index aabcb0aba..b25a4434a 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -78,13 +78,13 @@ void pkgAcquire::Initialize()
 }
 									/*}}}*/
 // Acquire::GetLock - lock directory and prepare for action		/*{{{*/
-static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent)
+static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent, std::string const &postfix, mode_t const mode)
 {
-   std::string const partial = parent + "partial";
-   mode_t const mode = umask(S_IWGRP | S_IWOTH);
+   std::string const partial = parent + postfix;
+   mode_t const old_umask = umask(S_IWGRP | S_IWOTH);
    bool const creation_fail = (CreateAPTDirectoryIfNeeded(grand, partial) == false &&
 	 CreateAPTDirectoryIfNeeded(parent, partial) == false);
-   umask(mode);
+   umask(old_umask);
    if (creation_fail == true)
       return false;
 
@@ -100,7 +100,7 @@ static bool SetupAPTPartialDirectory(std::string const &grand, std::string const
             _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str());
       }
    }
-   if (chmod(partial.c_str(), 0700) != 0)
+   if (chmod(partial.c_str(), mode) != 0)
       _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str());
 
    _error->PushToStack();
@@ -117,10 +117,12 @@ bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock)
    if (Lock.empty())
    {
       string const listDir = _config->FindDir("Dir::State::lists");
-      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false)
+	 return _error->Errno("Acquire", _("List directory %sauxfiles is missing."), listDir.c_str());
       string const archivesDir = _config->FindDir("Dir::Cache::Archives");
-      if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
       return true;
    }
@@ -137,14 +139,19 @@ bool pkgAcquire::GetLock(std::string const &Lock)
 
    if (Lock == listDir)
    {
-      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false)
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str());
    }
    if (Lock == archivesDir)
    {
-      if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false)
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir, "partial", 0700) == false)
 	 return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str());
    }
+   if (Lock == listDir || Lock == archivesDir)
+   {
+      if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir, "auxfiles", 0755) == false)
+	 return _error->Errno("Acquire", _("List directory %sauxfiles is missing."), listDir.c_str());
+   }
 
    if (_config->FindB("Debug::NoLocking", false) == true)
       return true;
@@ -288,7 +295,6 @@ static bool CheckForBadItemAndFailIt(pkgAcquire::Item * const Item,
 	 "\nFilename: " + Item->DestFile +
 	 "\nFailReason: WeakHashSums";
 
-      auto SavedDesc = Item->GetItemDesc();
       Item->Status = pkgAcquire::Item::StatAuthError;
       Item->Failed(Message, Config);
       if (Log != nullptr)
@@ -303,7 +309,10 @@ void pkgAcquire::Enqueue(ItemDesc &Item)
    const MethodConfig *Config;
    string Name = QueueName(Item.URI,Config);
    if (Name.empty() == true)
+   {
+      Item.Owner->Status = pkgAcquire::Item::StatError;
       return;
+   }
 
    /* the check for running avoids that we produce errors
       in logging before we actually have started, which would
@@ -769,11 +778,12 @@ bool pkgAcquire::Clean(string Dir)
    for (struct dirent *E = readdir(D); E != nullptr; E = readdir(D))
    {
       // Skip some entries
-      if (strcmp(E->d_name,"lock") == 0 ||
-	  strcmp(E->d_name,"partial") == 0 ||
-	  strcmp(E->d_name,"lost+found") == 0 ||
-	  strcmp(E->d_name,".") == 0 ||
-	  strcmp(E->d_name,"..") == 0)
+      if (strcmp(E->d_name, "lock") == 0 ||
+	  strcmp(E->d_name, "partial") == 0 ||
+	  strcmp(E->d_name, "auxfiles") == 0 ||
+	  strcmp(E->d_name, "lost+found") == 0 ||
+	  strcmp(E->d_name, ".") == 0 ||
+	  strcmp(E->d_name, "..") == 0)
 	 continue;
 
       // Look in the get list and if not found nuke
@@ -845,12 +855,25 @@ pkgAcquire::UriIterator pkgAcquire::UriEnd()
 }
 									/*}}}*/
 // Acquire::MethodConfig::MethodConfig - Constructor			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false),
-   Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
-   Removable(false)
+class pkgAcquire::MethodConfig::Private
+{
+   public:
+   bool AuxRequests = false;
+};
+pkgAcquire::MethodConfig::MethodConfig() : d(new Private()), Next(0), SingleInstance(false),
+					   Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false),
+					   Removable(false)
+{
+}
+									/*}}}*/
+bool pkgAcquire::MethodConfig::GetAuxRequests() const /*{{{*/
+{
+   return d->AuxRequests;
+}
+									/*}}}*/
+void pkgAcquire::MethodConfig::SetAuxRequests(bool const value) /*{{{*/
 {
+   d->AuxRequests = value;
 }
 									/*}}}*/
 // Queue::Queue - Constructor						/*{{{*/
diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index e58aeef65..1cf4da5bf 100644
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -636,9 +636,10 @@ class pkgAcquire::UriIterator
 /** \brief Information about the properties of a single acquire method.	{{{*/
 struct pkgAcquire::MethodConfig
 {
+   class Private;
    /** \brief dpointer placeholder (for later in case we need it) */
-   void * const d;
-   
+   Private *const d;
+
    /** \brief The next link on the acquire method list.
     *
     *  \todo Why not an STL container?
@@ -688,6 +689,9 @@ struct pkgAcquire::MethodConfig
     */
    MethodConfig();
 
+   APT_HIDDEN bool GetAuxRequests() const;
+   APT_HIDDEN void SetAuxRequests(bool const value);
+
    virtual ~MethodConfig();
 };
 									/*}}}*/
diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc
index 04a7c4910..1abc638ee 100644
--- a/apt-pkg/clean.cc
+++ b/apt-pkg/clean.cc
@@ -62,11 +62,12 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache)
    for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
    {
       // Skip some files..
-      if (strcmp(Dir->d_name,"lock") == 0 ||
-	  strcmp(Dir->d_name,"partial") == 0 ||
-	  strcmp(Dir->d_name,"lost+found") == 0 ||
-	  strcmp(Dir->d_name,".") == 0 ||
-	  strcmp(Dir->d_name,"..") == 0)
+      if (strcmp(Dir->d_name, "lock") == 0 ||
+	  strcmp(Dir->d_name, "partial") == 0 ||
+	  strcmp(Dir->d_name, "auxfiles") == 0 ||
+	  strcmp(Dir->d_name, "lost+found") == 0 ||
+	  strcmp(Dir->d_name, ".") == 0 ||
+	  strcmp(Dir->d_name, "..") == 0)
 	 continue;
 
       struct stat St;
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index 207f0d902..5c34113e7 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -138,7 +138,6 @@ bool pkgInitConfig(Configuration &Cnf)
    Cnf.CndSet("Dir::State", STATE_DIR + 1);
    Cnf.CndSet("Dir::State::lists","lists/");
    Cnf.CndSet("Dir::State::cdroms","cdroms.list");
-   Cnf.CndSet("Dir::State::mirrors","mirrors/");
 
    // Cache
    Cnf.CndSet("Dir::Cache", CACHE_DIR + 1);
diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc
index 77c35f4de..8bd7e33e0 100644
--- a/apt-private/private-download.cc
+++ b/apt-private/private-download.cc
@@ -225,7 +225,8 @@ bool DoDownload(CommandLine &CmdL)
       std::string const filename = cwd + flNotDir((*I)->DestFile);
       if ((*I)->Local == true &&
           filename != (*I)->DestFile &&
-          (*I)->Status == pkgAcquire::Item::StatDone)
+          (*I)->Status == pkgAcquire::Item::StatDone &&
+	  dynamic_cast<pkgAcqArchive*>(*I) != nullptr)
       {
 	 std::ifstream src((*I)->DestFile.c_str(), std::ios::binary);
 	 std::ofstream dst(filename.c_str(), std::ios::binary);
diff --git a/debian/apt.install b/debian/apt.install
index d30f429b0..705ea12d2 100644
--- a/debian/apt.install
+++ b/debian/apt.install
@@ -35,6 +35,9 @@ usr/share/man/*/apt-get.*
 usr/share/man/*/apt-key.*
 usr/share/man/*/apt-mark.*
 usr/share/man/*/apt-secure.*
+usr/share/man/*/apt-transport-http.*
+usr/share/man/*/apt-transport-https.*
+usr/share/man/*/apt-transport-mirror.*
 usr/share/man/*/apt.*
 usr/share/man/*/apt_auth.*
 usr/share/man/*/apt_preferences.*
diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt
index 73d808c64..7cca4cf81 100644
--- a/doc/CMakeLists.txt
+++ b/doc/CMakeLists.txt
@@ -84,6 +84,9 @@ add_docbook(apt-man MANPAGE ALL
         apt_preferences.5.xml
         apt-secure.8.xml
         apt-sortpkgs.1.xml
+        apt-transport-http.1.xml
+        apt-transport-https.1.xml
+        apt-transport-mirror.1.xml
         sources.list.5.xml
     DEPENDS ${ENTITIES}
     TRANSLATED_ENTITIES ${TRANSLATED_ENTITIES}
diff --git a/doc/apt-transport-http.1.xml b/doc/apt-transport-http.1.xml
new file mode 100644
index 000000000..546e47761
--- /dev/null
+++ b/doc/apt-transport-http.1.xml
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+  "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % aptent SYSTEM "apt.ent"> %aptent;
+<!ENTITY % aptverbatiment SYSTEM "apt-verbatim.ent"> %aptverbatiment;
+<!ENTITY % aptvendor SYSTEM "apt-vendor.ent"> %aptvendor;
+]>
+
+<refentry>
+
+ <refentryinfo>
+   &apt-author.team;
+   &apt-email;
+   &apt-product;
+   <!-- The last update date -->
+   <date>2017-11-22T00:00:00Z</date>
+ </refentryinfo>
+
+ <refmeta>
+   <refentrytitle>apt-transport-http</refentrytitle>
+   <manvolnum>1</manvolnum>
+   <refmiscinfo class="manual">APT</refmiscinfo>
+ </refmeta>
+
+ <!-- Man page title -->
+ <refnamediv>
+    <refname>apt-transport-http</refname>
+    <refpurpose>APT transport for downloading via the Hypertext Transfer Protocol (HTTP)</refpurpose>
+ </refnamediv>
+
+<refsect1><title>Description</title>
+<para>This APT transport allows the use of repositories accessed via the
+Hypertext Transfer Protocol (HTTP). It is available by default and probably the
+most used of all transports. Note that a transport is never called directly by
+a user but used by APT tools based on user configuration.</para>
+<para>HTTP is an unencrypted transport protocol meaning that the
+whole communication with the remote server (or proxy) can be observed by a
+sufficiently capable attacker referred to commonly as man in the middle (MITM).
+Such an attacker can <emphasis>not</emphasis> modify the communication to compromise
+the security of your system through as APTs data security model is independent of the
+chosen transport method. This is explained in detail in &apt-secure;. An overview over
+available transport methods is given in &sources-list;.</para>
+</refsect1>
+
+<refsect1><title>Options</title>
+<para>Various options are available to modify its behaviour which can be set in
+an &apt-conf; file ranging from proxy configuration to workaround for specific
+server insufficiencies.</para>
+
+<refsect2><title>Proxy Configuration</title>
+<para>The environment variable <envar>http_proxy</envar> is supported for system wide configuration.
+Proxies specific to apt can be configured via the option <literal>Acquire::http::Proxy</literal>.
+Proxies which should be used only for certain hosts can be specified via
+<literal>Acquire::http::Proxy::<replaceable>host</replaceable></literal>. Even more finegrained control
+can be achieved via proxy autodetection detailed further below.
+All these options use the URI format <literal><replaceable>scheme</replaceable>://[[<replaceable>user</replaceable>][:<replaceable>pass</replaceable>]@]<replaceable>host</replaceable>[:<replaceable>port</replaceable>]/</literal>.
+Supported URI schemes are <literal>socks5h</literal> (SOCKS5 with remote DNS resolution), <literal>http</literal> and <literal>https</literal>.
+Authentification details can be supplied via &apt-authconf; instead of including it in the URI directly.</para>
+<para>The various APT configuration options support the special value <literal>DIRECT</literal> meaning that
+no proxy should be used. The environment variable <envar>no_proxy</envar> with the same propose is also supported.</para>
+<para>Further more there are three settings provided for cache control with HTTP/1.1 compliant proxy caches:
+<literal>Acquire::http::No-Cache</literal> tells the proxy not to use its
+cached response under any circumstances.
+<literal>Acquire::http::Max-Age</literal> sets the allowed maximum age (in
+seconds) of an index file in the cache of the proxy.
+<literal>Acquire::http::No-Store</literal> specifies that the proxy should not
+store the requested archive files in its cache, which can be used to prevent
+the proxy from polluting its cache with (big) .deb files.</para>
+</refsect2>
+
+<refsect2><title>Automatic Proxy Configuration</title>
+<para><literal>Acquire::http::Proxy-Auto-Detect</literal> can be used to
+specify an external command to discover the http proxy to use. The first
+and only parameter is an URI denoting the host to be contacted to allow
+for host-specific configuration. APT expects the command to output the
+proxy on stdout as a single line in the previously specified URI format
+or the word <literal>DIRECT</literal> if no proxy should be used. No output
+indicates that the generic proxy settings should be used.</para>
+<para>Note that auto-detection will not be used for a host if a host-specific proxy
+configuration is already set via <literal>Acquire::http::Proxy::<replaceable>host</replaceable></literal>.</para>
+<para>See the &squid-deb-proxy-client; and &auto-apt-proxy; packages for example implementations.</para>
+<para>This option takes precedence over the legacy option name <literal>Acquire::http::ProxyAutoDetect</literal>.</para>
+</refsect2>
+
+<refsect2><title>Connection Configuration</title>
+<para>The option <literal>Acquire::http::Timeout</literal> sets the timeout timer used by the method;
+this value applies to the connection as well as the data timeout.</para>
+<para>The used bandwidth can be limited with
+<literal>Acquire::http::Dl-Limit</literal> which accepts integer values in
+kilobytes per second. The default value is 0 which deactivates the limit and
+tries to use all available bandwidth. Note that this option implicitly
+disables downloading from multiple servers at the same time.</para>
+<para>The setting <literal>Acquire::http::Pipeline-Depth</literal> can be used to
+enable HTTP pipelining (RFC 2616 section 8.1.2.2) which can be beneficial e.g. on
+high-latency connections. It specifies how many requests are sent in a pipeline.
+APT tries to detect and workaround misbehaving webservers and proxies at runtime, but
+if you know that yours does not conform to the HTTP/1.1 specification pipelining can
+be disabled by setting the value to 0. It is enabled by default with the value 10.</para>
+<para><literal>Acquire::http::AllowRedirect</literal> controls whether APT will follow
+redirects, which is enabled by default.</para>
+<para><literal>Acquire::http::User-Agent</literal> can be used to set a different
+User-Agent for the http download method as some proxies allow access for clients
+only if the client uses a known identifier.</para>
+<para><literal>Acquire::http::SendAccept</literal> is enabled by default and
+sends a <literal>Accept: text/*</literal> header field to the server for
+requests without file extensions to prevent the server from attempting content
+negotiation.</para>
+</refsect2>
+</refsect1>
+
+<refsect1><title>Examples</title>
+<literallayout>
+Acquire::http {
+	Proxy::example.org "DIRECT";
+	Proxy "socks5h://apt:pass@localhost:9050";
+	Proxy-Auto-Detect "/usr/local/bin/apt-http-proxy-auto-detect";
+	No-Cache "true";
+	Max-Age "3600";
+	No-Store "true";
+	Timeout "10";
+	Dl-Limit "42";
+	Pipeline-Depth "0";
+	AllowRedirect "false";
+	User-Agent "My APT-HTTP";
+	SendAccept "false";
+};
+</literallayout>
+</refsect1>
+
+<refsect1>
+<title>See Also</title>
+<para>&apt-conf; &apt-authconf; &sources-list;
+</para>
+</refsect1>
+
+ &manbugs;
+
+</refentry>
diff --git a/doc/apt-transport-https.1.xml b/doc/apt-transport-https.1.xml
new file mode 100644
index 000000000..97137fc2c
--- /dev/null
+++ b/doc/apt-transport-https.1.xml
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+  "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % aptent SYSTEM "apt.ent"> %aptent;
+<!ENTITY % aptverbatiment SYSTEM "apt-verbatim.ent"> %aptverbatiment;
+<!ENTITY % aptvendor SYSTEM "apt-vendor.ent"> %aptvendor;
+]>
+
+<refentry>
+
+ <refentryinfo>
+   &apt-author.team;
+   &apt-email;
+   &apt-product;
+   <!-- The last update date -->
+   <date>2017-11-22T00:00:00Z</date>
+ </refentryinfo>
+
+ <refmeta>
+   <refentrytitle>apt-transport-https</refentrytitle>
+   <manvolnum>1</manvolnum>
+   <refmiscinfo class="manual">APT</refmiscinfo>
+ </refmeta>
+
+ <!-- Man page title -->
+ <refnamediv>
+    <refname>apt-transport-https</refname>
+    <refpurpose>APT transport for downloading via the HTTP Secure protocol (HTTPS)</refpurpose>
+ </refnamediv>
+
+<refsect1><title>Description</title>
+<para>This APT transport allows the use of repositories accessed via the
+HTTP Secure protocol (HTTPS) also referred to as HTTP over TLS. It is available
+by default since apt 1.5 and was before that available in a <package>apt-transport-https</package>
+package. Note that a transport is never called directly by
+a user but used by APT tools based on user configuration.</para>
+<para>HTTP is by itself an unencrypted transport protocol (compare &apt-transport-http;),
+which, as indicated by the appended S is wrapped in an encrypted layer known as
+Transport Layer Security (TLS) which provides end-to-end encryption.
+A sufficiently capable attacker can still observe the communication partners
+and deeper analyse of the encrypted communcation might still reveal important details.
+An overview over available alternative transport methods is given in &sources-list;.</para>
+</refsect1>
+
+<refsect1><title>Options</title>
+<para>The HTTPS protocol is based on the HTTP protocol and as such this implementation
+has the same relation meaning that all options supported by &apt-transport-http; are also
+available via <literal>Acquire::https</literal> and will default to the same values specified
+for <literal>Acquire::http</literal>. This manpage will only document the options
+<emphasis>unique to https</emphasis>.</para>
+
+<refsect2><title>Server credentials</title>
+<para>By default all certificates trusted by the system (see
+<package>ca-certificates</package> package) are used for the verification of
+the server certificate. An alternative certificate authority (CA) can be
+configured with the <literal>Acquire::https::CAInfo</literal> option and its
+host-specific option <literal>Acquire::https::CAInfo::<replaceable>host</replaceable></literal>.
+The option specifies a file is made of the concatenation of the CA certificates
+(in PEM format) creating the chain used for the verification of the path
+from the root (self signed one). If the remote server provides the
+whole chain during the exchange, the file need only contain the root
+certificate. Otherwise, the whole chain is required. If you need to support
+multiple authorities, the only way is to concatenate everything.</para>
+<para>A custom certificate revocation list (CRL) can be configured with the options
+<literal>Acquire::https::CRLFile</literal> and
+<literal>Acquire::https::CRLFile::<replaceable>host</replaceable></literal> respectively.
+Like the previous option a file in PEM format needs to be specified.</para>
+</refsect2>
+
+<refsect2><title>Disabling security</title>
+<para>When authenticating the server, if the certificate verification fails
+for some reason (expired, revoked, man in the middle, …), the connection fails.
+This is obviously what you want in all cases and what the default value (true)
+of the option <literal>Acquire::https::Verify-Peer</literal> and its host-specific
+variant provides. If you know <emphasis>exactly</emphasis> what you are doing,
+setting this option to "false" allows you to skip peer certificate verification and
+make the exchange succeed. Again, this option is for debugging or testing purpose
+only as it removes all security provided by the use of HTTPS.</para>
+<para>Similarly the option <literal>Acquire::https::Verify-Host</literal> and its
+host-specific variant can be used to deactivate a security feature: The certificate
+provided by the server includes the identity of the server which should match the
+DNS name used to access it. By default, as requested by RFC 2818, the name of the
+mirror is checked against the identity found in the certificate. This default behavior
+is safe and should not be changed, but if you know that the server you are using has a
+DNS name which does not match the identity in its certificate, you can set the option to
+"false", which will prevent the comparison to be done.</para>
+</refsect2>
+
+<refsect2><title>Client authentication</title>
+<para>Beside the password based authentication available (see &apt-authconf;) HTTPS supports
+authentication based on client certificates as well via <literal>Acquire::https::SSLCert</literal>
+and <literal>Acquire::https::SSLKey</literal>. They respectively should be set to the filename of
+the X.509 client certificate and the associated (unencrypted) private key, both in PEM format.
+In practice the use of the host-specific variants of both options is highly recommended.</para>
+</refsect2>
+
+</refsect1>
+
+<refsect1><title>Examples</title>
+<literallayout>
+Acquire::https {
+	Proxy::example.org "DIRECT";
+	Proxy "socks5h://apt:pass@localhost:9050";
+	Proxy-Auto-Detect "/usr/local/bin/apt-https-proxy-auto-detect";
+	No-Cache "true";
+	Max-Age "3600";
+	No-Store "true";
+	Timeout "10";
+	Dl-Limit "42";
+	Pipeline-Depth "0";
+	AllowRedirect "false";
+	User-Agent "My APT-HTTPS";
+	SendAccept "false";
+
+	CAInfo "/path/to/ca/certs.pem";
+	CRLFile "/path/to/all/crl.pem";
+	Verify-Peer "true";
+	Verify-Host::broken.example.org "false";
+	SSLCert::example.org "/path/to/client/cert.pem";
+	SSLKey::example.org "/path/to/client/key.pem"
+};
+</literallayout>
+</refsect1>
+
+<refsect1>
+<title>See Also</title>
+<para>&apt-transport-http; &apt-conf; &apt-authconf; &sources-list;
+</para>
+</refsect1>
+
+ &manbugs;
+
+</refentry>
diff --git a/doc/apt-transport-mirror.1.xml b/doc/apt-transport-mirror.1.xml
new file mode 100644
index 000000000..72001fad5
--- /dev/null
+++ b/doc/apt-transport-mirror.1.xml
@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+  "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [
+<!ENTITY % aptent SYSTEM "apt.ent"> %aptent;
+<!ENTITY % aptverbatiment SYSTEM "apt-verbatim.ent"> %aptverbatiment;
+<!ENTITY % aptvendor SYSTEM "apt-vendor.ent"> %aptvendor;
+]>
+
+<refentry>
+
+ <refentryinfo>
+   &apt-author.team;
+   &apt-email;
+   &apt-product;
+   <!-- The last update date -->
+   <date>2017-12-09T00:00:00Z</date>
+ </refentryinfo>
+
+ <refmeta>
+   <refentrytitle>apt-transport-mirror</refentrytitle>
+   <manvolnum>1</manvolnum>
+   <refmiscinfo class="manual">APT</refmiscinfo>
+ </refmeta>
+
+ <!-- Man page title -->
+ <refnamediv>
+    <refname>apt-transport-mirror</refname>
+    <refpurpose>APT transport for more automated mirror selection</refpurpose>
+ </refnamediv>
+
+<refsect1><title>Description</title>
+<para>This APT transport isn't implementing a protocol to access local or remote repositories
+on its own, but acquires a mirrorlist and redirects all requests to the mirror(s) picked from
+this list access via other transports like &apt-transport-http;. The basic functionality is
+available since apt 0.7.24, but was undocumented until apt 1.6 which contained a complete
+rework of the transport and its supported features. Note that a transport is never called directly
+by a user but used by APT tools based on user configuration.</para>
+<para>If the acquisition of a file via a mirror fails the method ensures that automatically
+another possible mirror of the list is tried until either the file is retrieved or no mirror is
+left in the list handling transparently server downtimes and similar problems.</para>
+<para>The security implications of the transport are based on the security considerations
+associated with the transport used to acquire the mirrorlist and the transports involved in
+accessing the chosen mirror(s) by the transport.</para>
+</refsect1>
+
+<refsect1><title>Options</title>
+<para>This transport has no configuration options at present. The mirror selection is
+based entirely on the mirrors offered in the mirrorlist and the files apt needs to
+acquire.</para>
+
+<refsect2><title>Mirrorlist format</title>
+<para>A mirrorlist contains at least one line each specifying an URI for a mirror.
+Empty lines and those starting with a hash key (<literal>#</literal>) are ignored.
+An URI always starts with a URI scheme which defines the transport used for this
+mirror. If the URI e.g. starts with <literal>http:</literal> the responsible transport
+is &apt-transport-http; which might have specific requirements for the format of
+the remaining part of the URI.</para>
+<para>An URI can optionally be separated from metadata about the mirror by a tab.
+Multiple datapoints in the provided metadata can itself be separated by spaces for tabs.
+(This is an advanced feature only available with apt >= 1.6. Earlier apt versions will
+fail to parse mirrorlists using this feature)</para>
+<para>Since apt 1.6 the usage of compressed mirrorlists is also supported.
+Note that the filename of the mirrorlist must specify the compression algorithm used,
+there is no auto-detection based on file content performed.</para>
+</refsect2>
+
+<refsect2><title>Mirror selection by metadata</title>
+<para>As specified in the format a mirror can have additional metadata attached to
+prevent a mirror from being selected for acquiring a file not matching this metadata.
+This way the mirrorlist can e.g. contain partial mirrors serving only certain
+architectures and apt will automatically choose a different mirror for files requiring
+an unlisted architecture. Supported are limits for the architecture (<literal>arch</literal>),
+codename of the release (<literal>codename</literal>), component of the repository
+the file is in (<literal>component</literal>), language the file applies to (<literal>lang</literal>),
+suite name of the release (<literal>suite</literal>) and type of the file (<literal>type</literal>).</para>
+</refsect2>
+
+<refsect2><title>Fallback order for mirrors</title>
+<para>If no priority is given via the metadata key <literal>priority</literal> for a
+mirror the order in which mirrors are contacted is random. If a certain set of mirrors
+should be tried first before any of another set is tried a priority can be explicitly
+set. The mirrors with the lowest number are tried first. Mirrors which have no explicit
+priority set default to the highest possible number and are therefore tried last. The
+choice between mirrors with the same priority is again random.</para>
+</refsect2>
+
+<refsect2><title>Allowed transports in a mirrorlist</title>
+<para>The availability and choice of transports in a mirrorlist is limited by how the apt
+client is accessing the mirrorlist. If a local transport like <literal>file</literal>
+or <literal>copy</literal> is used the mirrorlist can also include local sources while a
+mirrorlist accessed via <literal>http</literal> can not. Additionally, a mirrorlist can
+not contain a mirrorlist or other wrapping transports (like <package>apt-transport-tor</package>).
+See the documentation of these transports on how to use them with the mirror method.</para>
+<para>Note that apt versions before 1.6 do not support any other transport than <literal>http</literal>.</para>
+</refsect2>
+</refsect1>
+
+<refsect1><title>Examples</title>
+<refsect2><title>Basic example</title>
+<para>A basic mirrorlist example supported by all apt versions with a mirror method
+(>= 0.7.24) in which the client will pick any of the three mirrors:</para>
+<literallayout>
+http://ftp.de.debian.org/debian/
+http://ftp.us.debian.org/debian/
+http://deb.debian.org/debian/
+</literallayout>
+<para>Assuming a file with this content is stored as <filename>/etc/apt/mirrorlist.txt</filename>
+on your machine it can be used like this in &sources-list; (since apt 1.6):</para>
+<literallayout>
+deb mirror+file:/etc/apt/mirrorlist.txt &debian-stable-codename; main
+</literallayout>
+<para>All versions of the mirror method support a mirrorlist accessible via http, so assuming
+it is available at <literal>http://apt.example.org/mirror.lst</literal> the sources.list entry
+from above could be written instead as:</para>
+<literallayout>
+deb mirror://apt.example.org/mirror.lst &debian-stable-codename; main
+</literallayout>
+<para>Note that since apt 1.6 the use of <literal>mirror+http</literal> should
+be preferred over <literal>mirror</literal> for uniformity. The functionality is the same.</para>
+</refsect2>
+<refsect2><title>Example with metadata-enhanced mirror selection</title>
+<para>As explained in the format definition apt versions before 1.6 do not support this and
+will fail parsing the mirrorlist. The example mirrorlist is proposefully complicated to show some
+aspects of the selection. The following setup is assumed: The first mirror is local mirror
+accessible via the file method, but potentially incomplete. The second mirror has a great
+connection, but is a partial mirror in sofar as it only contains files related
+to the architectures <literal>amd64</literal> and <literal>all</literal>. The remaining mirrors
+are average mirrors which should be contacted only if the earlier ones didn't work.
+</para>
+<literallayout>
+file:/srv/local/debian/mirror/	priority:1 type:index
+http://partial.example.org/mirror/	priority:2 arch:amd64 arch:all type:deb
+http://ftp.us.debian.org/debian/	type:deb
+http://ftp.de.debian.org/debian/	type:deb
+https://deb.debian.org/debian/
+</literallayout>
+<para>In this setup with this mirrorlist the first mirror will be used to download all
+index files assuming the mirrorlist itself is accessed via a local transport like
+<literal>file</literal>. If it isn't, the mirror is otherwise inaccessible or does not
+contain the requested file another mirror will be used to acquire the file, which one
+depending on the type of the file: An index file will be served by the last
+mirror in the list, while a package of architecture <literal>amd64</literal> is served by
+the second and those of e.g. architecture <literal>i386</literal> by one of the last three.</para>
+
+</refsect2>
+</refsect1>
+
+ &manbugs;
+
+</refentry>
diff --git a/doc/apt-verbatim.ent b/doc/apt-verbatim.ent
index 946c7cc7d..8d0a56b6c 100644
--- a/doc/apt-verbatim.ent
+++ b/doc/apt-verbatim.ent
@@ -81,6 +81,17 @@
   </citerefentry>"
 >
 
+<!ENTITY apt-transport-http "<citerefentry>
+    <refentrytitle><command>apt-transport-http</command></refentrytitle>
+    <manvolnum>1</manvolnum>
+  </citerefentry>"
+>
+
+<!ENTITY apt-transport-https "<citerefentry>
+    <refentrytitle><command>apt-transport-https</command></refentrytitle>
+    <manvolnum>1</manvolnum>
+  </citerefentry>"
+>
 
 <!ENTITY sources-list "<citerefentry>
     <refentrytitle><filename>sources.list</filename></refentrytitle>
@@ -148,6 +159,12 @@
   </citerefentry>"
 >
 
+<!ENTITY auto-apt-proxy "<citerefentry>
+    <refentrytitle><command>auto-apt-proxy</command></refentrytitle>
+    <manvolnum>1</manvolnum>
+  </citerefentry>"
+>
+
 <!ENTITY debsign "<citerefentry>
     <refentrytitle><command>debsign</command></refentrytitle>
     <manvolnum>1</manvolnum>
diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml
index 6f47bb029..e0be6a37d 100644
--- a/doc/apt.conf.5.xml
+++ b/doc/apt.conf.5.xml
@@ -409,99 +409,12 @@ APT::Compressor::rev {
      be symlinked when possible instead of copying. True is the default.</para></listitem>
      </varlistentry>
 
-     <varlistentry><term><option>http</option></term>
-     <listitem><para><literal>http::Proxy</literal> sets the default proxy to use for HTTP
-     URIs. It is in the standard form of <literal>http://[[user][:pass]@]host[:port]/</literal>.
-     Per host proxies can also be specified by using the form
-     <literal>http::Proxy::&lt;host&gt;</literal> with the special keyword <literal>DIRECT</literal> 
-	 meaning to use no proxies. If no one of the above settings is specified,
-	 <envar>http_proxy</envar> environment variable
-	 will be used.</para>
-
-     <para>Three settings are provided for cache control with HTTP/1.1 compliant
-     proxy caches.
-     <literal>No-Cache</literal> tells the proxy not to use its cached
-     response under any circumstances.
-     <literal>Max-Age</literal> sets the allowed maximum age (in seconds) of
-     an index file in the cache of the proxy.
-     <literal>No-Store</literal> specifies that the proxy should not store
-     the requested archive files in its cache, which can be used to prevent
-     the proxy from polluting its cache with (big) .deb files.</para>
-
-     <para>The option <literal>timeout</literal> sets the timeout timer used by the method;
-     this value applies to the connection as well as the data timeout.</para>
-
-     <para>The setting <literal>Acquire::http::Pipeline-Depth</literal> can be used to
-     enable HTTP pipelining (RFC 2616 section 8.1.2.2) which can be beneficial e.g. on
-     high-latency connections. It specifies how many requests are sent in a pipeline.
-     APT tries to detect and workaround misbehaving webservers and proxies at runtime, but
-     if you know that yours does not conform to the HTTP/1.1 specification pipelining can
-     be disabled by setting the value to 0. It is enabled by default with the value 10.</para>
-
-     <para><literal>Acquire::http::AllowRedirect</literal> controls whether APT will follow
-     redirects, which is enabled by default.</para>
-
-     <para>The used bandwidth can be limited with
-     <literal>Acquire::http::Dl-Limit</literal> which accepts integer
-     values in kilobytes per second. The default value is 0 which
-     deactivates the limit and tries to use all available bandwidth.
-     Note that this option implicitly disables downloading from
-     multiple servers at the same time.</para>
-
-     <para><literal>Acquire::http::User-Agent</literal> can be used to set a different
-     User-Agent for the http download method as some proxies allow access for clients
-     only if the client uses a known identifier.</para>
-
-     <para><literal>Acquire::http::Proxy-Auto-Detect</literal> can be used to
-     specify an external command to discover the http proxy to use. The first
-     and only parameter is an URI denoting the host to be contacted to allow
-     for host-specific configuration. APT expects the command to output the
-     proxy on stdout as a single line in the style <literal>http://proxy:port/</literal>
-     or the word <literal>DIRECT</literal> if no proxy should be used. No output
-     indicates that the generic proxy settings should be used.
-
-     Note that auto-detection will not be used for a host if a host-specific proxy
-     configuration is already set via <literal>Acquire::http::Proxy::<replaceable>HOST</replaceable></literal>.
-
-     See the &squid-deb-proxy-client; package for an example implementation that
-     uses avahi.
-
-     This option takes precedence over the legacy option name
-     <literal>ProxyAutoDetect</literal>.
-     </para>
-
-     </listitem>
+     <varlistentry><term><option>http</option> <option>https</option></term>
+     <listitem><para>The options in these scopes configure APTs acquire transports for the protocols
+     HTTP and HTTPS and are documented in the &apt-transport-http; and &apt-transport-https;
+     manpages respectively.</para></listitem>
      </varlistentry>
 
-     <varlistentry><term><option>https</option></term>
-	 <listitem><para>
-	 The <literal>Cache-control</literal>, <literal>Timeout</literal>,
-	 <literal>AllowRedirect</literal>, <literal>Dl-Limit</literal> and
-	 <literal>proxy</literal> options work for HTTPS URIs in the same way
-	 as for the <literal>http</literal> method, and default to the same
-	 values if they are not explicitly set. The
-	 <literal>Pipeline-Depth</literal> option is not yet supported.
-	 </para>
-
-	 <para><literal>CaInfo</literal> suboption specifies place of file that
-	 holds info about trusted certificates.
-	 <literal>&lt;host&gt;::CaInfo</literal> is the corresponding per-host option.
-	 <literal>Verify-Peer</literal> boolean suboption determines whether or not the
-	 server's host certificate should be verified against trusted certificates.
-	 <literal>&lt;host&gt;::Verify-Peer</literal> is the corresponding per-host option.
-	 <literal>Verify-Host</literal> boolean suboption determines whether or not the
-	 server's hostname should be verified.
-	 <literal>&lt;host&gt;::Verify-Host</literal> is the corresponding per-host option.
-	 <literal>SslCert</literal> determines what certificate to use for client
-	 authentication. <literal>&lt;host&gt;::SslCert</literal> is the corresponding per-host option.
-	 <literal>SslKey</literal> determines what private key to use for client
-	 authentication. <literal>&lt;host&gt;::SslKey</literal> is the corresponding per-host option.
-	 <literal>SslForceVersion</literal> overrides default SSL version to use.
-	 It can contain either of the strings '<literal>TLSv1</literal>' or
-	 '<literal>SSLv3</literal>'.
-	 <literal>&lt;host&gt;::SslForceVersion</literal> is the corresponding per-host option.
-	 </para></listitem></varlistentry>
-
      <varlistentry><term><option>ftp</option></term>
      <listitem><para>
      <literal>ftp::Proxy</literal> sets the default proxy to use for FTP URIs.
diff --git a/doc/examples/CMakeLists.txt b/doc/examples/CMakeLists.txt
index 1998867db..8d9ea068f 100644
--- a/doc/examples/CMakeLists.txt
+++ b/doc/examples/CMakeLists.txt
@@ -1,4 +1,4 @@
-install(FILES apt.conf apt-https-method-example.conf configure-index preferences
+install(FILES apt.conf configure-index preferences
         DESTINATION ${CMAKE_INSTALL_DOCDIR}/examples)
 install(FILES apt-ftparchive.conf ftp-archive.conf
         DESTINATION ${CMAKE_INSTALL_DOCDIR}/../apt-utils/examples)
diff --git a/doc/examples/apt-https-method-example.conf b/doc/examples/apt-https-method-example.conf
deleted file mode 100644
index 3152ef1e2..000000000
--- a/doc/examples/apt-https-method-example.conf
+++ /dev/null
@@ -1,186 +0,0 @@
-/* This file is a sample configuration for apt https method. Configuration
-   parameters found in this example file are expected to be used in main
-   apt.conf file, just like other configuration parameters for different
-   methods (ftp, file, ...).
-
-   This example file starts with a common setup that voluntarily exhibits
-   all available configurations knobs with simple comments. Extended
-   comments on the behavior of the option is provided at the end for
-   better readability. As a matter of fact, a common configuration file
-   will certainly contain far less elements and benefit of default values
-   for many parameters.
-
-   Because some configuration parameters for apt https method in following
-   examples apply to specific (fictional) repositories, the associated
-   sources.list file is provided here:
-
-   ...
-
-   deb     https://secure.dom1.tld/debian unstable main contrib non-free
-   deb-src https://secure.dom1.tld/debian unstable main contrib non-free
-
-   deb     https://secure.dom2.tld/debian unstable main contrib non-free
-   deb-src https://secure.dom2.tld/debian unstable main contrib non-free
-
-   ...
-
-
-   Some notes on the servers:
-
-    - secure.dom1.tld is freely accessible using https (no client
-      authentication is required).
-    - secure.dom1.tld certificate is part of a multi level PKI, and we
-      want to specifically check the issuer of its certificate. We do
-      not have the constraint for secure.dom2.tld
-    - secure.dom2.tld requires client authentication by certificate
-      to access its content.
-    - The certificate presented by both server have (as expected) a CN that
-      matches their respective DNS names.
-    - We have CRL available for both dom1.tld and dom2.tld PKI, and intend
-      to use them.
-    - It sometimes happens that we had other more generic https available
-      repository to our list. We want the checks to be performed against
-      a common list of anchors (like the one provided by ca-certificates
-      package for instance)
-
-   The sample configuration below basically covers those simple needs.
-*/
-
-
-// Verify peer certificate and also matching between certificate name
-// and server name as provided in sources.list (default values)
-Acquire::https::Verify-Peer "true";
-Acquire::https::Verify-Host "true";
-
-// Except otherwise specified, use that list of anchors
-Acquire::https::CaInfo     "/etc/ssl/certs/ca-certificates.pem";
-
-// Use a specific anchor and associated CRL. Enforce issuer of
-// server certificate using its cert.
-Acquire::https::secure.dom1.tld::CaInfo     "/etc/apt/certs/ca-dom1-crt.pem";
-Acquire::https::secure.dom1.tld::CrlFile    "/etc/apt/certs/ca-dom1-crl.pem";
-Acquire::https::secure.dom1.tld::IssuerCert "/etc/apt/certs/secure.dom1-issuer-crt.pem";
-
-// Like previous for anchor and CRL, but also provide our
-// certificate and keys for client authentication.
-Acquire::https::secure.dom2.tld::CaInfo  "/etc/apt/certs/ca-dom2-crt.pem";
-Acquire::https::secure.dom2.tld::CrlFile "/etc/apt/certs/ca-dom2-crl.pem";
-Acquire::https::secure.dom2.tld::SslCert "/etc/apt/certs/my-crt.pem";
-Acquire::https::secure.dom2.tld::SslKey  "/etc/apt/certs/my-key.pem";
-
-// No need to downgrade, TLS will be proposed by default. Uncomment
-// to have SSLv3 proposed.
-// Acquire::https::mirror.ipv6.ssi.corp::SslForceVersion "SSLv3";
-
-// No need for more debug if every is fine (default). Uncomment
-// me to get additional information.
-// Debug::Acquire::https "true";
-
-
-/*
-  Options with extended comments:
-
-  Acquire::https[::repo.domain.tld]::CaInfo  "/path/to/ca/certs.pem";
-
-    A string providing the path of a file containing the list of trusted
-    CA certificates used to verify the server certificate. The pointed
-    file is made of the concatenation of the CA certificates (in
-    PEM format) creating the chain used for the verification of the path
-    from the root (self signed one). If the remote server provides the
-    whole chain during the exchange, the file need only contain the root
-    certificate. Otherwise, the whole chain is required.
-
-    If you need to support multiple authorities, the only way is to
-    concatenate everything.
-
-    If None is provided, the default CA bundle used by GnuTLS (apt https
-    method is linked against libcurl-gnutls) is used. At the time of
-    writing, /etc/ssl/certs/ca-certificates.crt.
-
-    If no specific hostname is provided, the file is used by default
-    for all https targets. If a specific mirror is provided, it is
-    used for the https entries in the sources.list file that use that
-    repository (with the same name).
-
-  Acquire::https[::repo.domain.tld]::CrlFile  "/path/to/all/crl.pem";
-
-    Like previous knob but for passing the list of CRL files (in PEM
-    format) to be used to verify revocation status. Again, if the
-    option is defined with no specific mirror (probably makes little
-    sense), this CRL information is used for all defined https entries
-    in sources.list file. In a mirror specific context, it only applies
-    to that mirror.
-
-  Acquire::https[::repo.domain.tld]::IssuerCert "/path/to/issuer/cert.pem";
-
-    Allows one to constrain the issuer of the server certificate (for all
-    https mirrors or a specific one) to a specific issuer. If the
-    server certificate has not been issued by this certificate,
-    connection fails.
-
-  Acquire::https[::repo.domain.tld]::Verify-Peer "true";
-
-    When authenticating the server, if the certificate verification fails
-    for some reason (expired, revoked, man in the middle, lack of anchor,
-    ...), the connection fails. This is obviously what you want in all
-    cases and what the default value (true) of this option provides.
-
-    If you know EXACTLY what you are doing, setting this option to "false"
-    allow you to skip peer certificate verification and make the exchange
-    succeed. Again, this option is for debugging or testing purpose only.
-    It removes ALL the security provided by the use of SSL.TLS to secure
-    the HTTP exchanges.
-
-  Acquire::https[::repo.domain.tld]::Verify-Host "true";
-
-    The certificate provided by the server during the TLS/SSL exchange
-    provides the identity of the server which should match the DNS name
-    used to access it. By default, as requested by RFC 2818, the name
-    of the mirror is checked against the identity found in the
-    certificate. This default behavior is safe and should not be
-    changed. If you know that the server you are using has a DNS name
-    which does not match the identity in its certificate, you can
-    [report that issue to its administrator or] set the option to
-    "false", which will prevent the comparison to be done.
-
-    The options can be set globally or on a per-mirror basis. If set
-    globally, the DNS name used is the one found in the sources.list
-    file in the https URI.
-
-  Acquire::https[::repo.domain.tld]::SslCert "/path/to/client/cert.pem";
-  Acquire::https[::repo.domain.tld]::SslKey  "/path/to/client/key.pem";
-
-    These two options provides support for client authentication using
-    certificates. They respectively accept the X.509 client certificate
-    in PEM format and the associated client key in PEM format (non
-    encrypted form).
-
-    The options can be set globally (which rarely makes sense) or on a
-    per-mirror basis.
-
-  Acquire::https[::repo.domain.tld]::SslForceVersion "TLSv1";
-
-    This option can be use to select the version which will be proposed
-    to the server. "SSLv3" and "TLSv1" are supported. SSLv2, which is
-    considered insecure anyway is not supported (by gnutls, which is
-    used by libcurl against which apt https method is linked).
-
-    When the option is set to "SSLv3" to have apt propose SSLv3 (and
-    associated sets of ciphersuites) instead of TLSv1 (the default)
-    when performing the exchange. This prevents the server to select
-    TLSv1 and use associated ciphersuites. You should probably not use
-    this option except if you know exactly what you are doing.
-
-    Note that the default setting does not guarantee that the server
-    will not select SSLv3 (for ciphersuites and SSL/TLS version as
-    selection is always done by the server, in the end). It only means
-    that apt will not advertise TLS support.
-
-  Debug::Acquire::https "true";
-
-    This option can be used to show debug information. Because it is
-    quite verbose, it is mainly useful to debug problems in case of
-    failure to connect to a server for some reason. The default value
-    is "false".
-
-*/
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index 153637ebc..9088bd844 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -274,9 +274,6 @@ Acquire
   // - cache-control values
   // - Dl-Limit, Timeout, ... values
   // if not set explicit for https
-  //
-  // see /usr/share/doc/apt/examples/apt-https-method-example.conf.gz
-  // for more examples
   https
   {
 	Verify-Peer "false";
diff --git a/doc/method.dbk b/doc/method.dbk
index e5e035a2b..410d6898c 100644
--- a/doc/method.dbk
+++ b/doc/method.dbk
@@ -267,6 +267,11 @@ an informational string provided for visual debugging.
 </listitem>
 <listitem>
 <para>
+351 Aux Request - Method requests an auxiliary file
+</para>
+</listitem>
+<listitem>
+<para>
 400 URI Failure - URI has failed to acquire
 </para>
 </listitem>
@@ -309,9 +314,9 @@ Authorization is User/Pass
 </itemizedlist>
 <para>
 Only the 6xx series of status codes is sent TO the method. Furthermore the
-method may not emit status codes in the 6xx range. The Codes 402 and 403
+method may not emit status codes in the 6xx range. The Codes 351, 402 and 403
 require that the method continue reading all other 6xx codes until the proper
-602/603 code is received. This means the method must be capable of handling an
+600/602/603 code is received. This means the method must be capable of handling an
 unlimited number of 600 messages.
 </para>
 <para>
@@ -567,6 +572,20 @@ Size, Last-Modified, Filename, MD5-Hash
 </listitem>
 </varlistentry>
 <varlistentry>
+<term>351 Aux Request</term>
+<listitem>
+<para>
+Indicates a request for an auxiliary file to be downloaded by the acquire system
+(via another method) and made available for the requesting method. The requestor
+will get a <emphasis>600 URI Acquire</emphasis> with the URI it requested and the
+filename will either be an existing file if the request was success or if the
+acquire failed for some reason the file will not exist.
+Fields: URI (of the file causing the need for the auxiliary file), MaximumSize,
+Aux-ShortDesc, Aux-Description, Aux-URI
+</para>
+</listitem>
+</varlistentry>
+<varlistentry>
 <term>400 URI Failure</term>
 <listitem>
 <para>
diff --git a/doc/po4a.conf b/doc/po4a.conf
index 955ebbc76..587215abc 100644
--- a/doc/po4a.conf
+++ b/doc/po4a.conf
@@ -27,6 +27,9 @@
 [type: manpage] apt-sortpkgs.1.xml $lang:$lang/apt-sortpkgs.$lang.1.xml add_$lang:xml.add
 [type: manpage] apt-ftparchive.1.xml $lang:$lang/apt-ftparchive.$lang.1.xml add_$lang:xml.add
 [type: manpage] apt_auth.conf.5.xml $lang:$lang/apt_auth.conf.$lang.5.xml add_$lang:xml.add
+[type: manpage] apt-transport-http.1.xml $lang:$lang/apt-transport-http.$lang.1.xml add_$lang:xml.add
+[type: manpage] apt-transport-https.1.xml $lang:$lang/apt-transport-https.$lang.1.xml add_$lang:xml.add
+[type: manpage] apt-transport-mirror.1.xml $lang:$lang/apt-transport-mirror.$lang.1.xml add_$lang:xml.add
 
 [type: docbook] guide.dbk $lang:$lang/guide.$lang.dbk
 #                add_$lang::$lang/addendum/docbook_$lang.add
diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml
index 694082bea..5572b8da3 100644
--- a/doc/sources.list.5.xml
+++ b/doc/sources.list.5.xml
@@ -352,17 +352,15 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [.
 
     <para>The currently recognized URI types are:
    <variablelist>
-    <varlistentry><term><command>http</command></term>
+    <varlistentry><term><command>http</command> (&apt-transport-http;)</term>
     <listitem><para>
     The http scheme specifies an HTTP server for an archive and is the most
-    commonly used method, with many options in the
-    <literal>Acquire::http</literal> scope detailed in &apt-conf;. The URI can
-    directly include login information if the archive requires it, but the use
-    of &apt-authconf; should be preferred. The method also supports SOCKS5 and
-    HTTP(S) proxies either configured via apt-specific configuration or
-    specified by the environment variable <envar>http_proxy</envar> in the
-    format (assuming an HTTP proxy requiring authentication)
-    <replaceable>http://user:pass@server:port/</replaceable>.
+    commonly used method. The URI can directly include login information if the
+    archive requires it, but the use of &apt-authconf; should be preferred.
+    The method also supports SOCKS5 and HTTP(S) proxies either configured via
+    apt-specific configuration or specified by the environment variable
+    <envar>http_proxy</envar> in the format (assuming an HTTP proxy requiring
+    authentication) <replaceable>http://user:pass@server:port/</replaceable>.
     The authentication details for proxies can also be supplied via
     &apt-authconf;.</para>
     <para>Note that these forms of authentication are insecure as the whole
@@ -373,7 +371,7 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [.
     chosen transport method. See &apt-secure; for details.</para></listitem>
     </varlistentry>
 
-    <varlistentry><term><command>https</command></term>
+    <varlistentry><term><command>https</command> (&apt-transport-https;)</term>
     <listitem><para>
     The https scheme specifies an HTTPS server for an archive and is very
     similar in use and available options to the http scheme. The main
diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt
index a25d4b525..c4a32b4f5 100644
--- a/methods/CMakeLists.txt
+++ b/methods/CMakeLists.txt
@@ -2,7 +2,6 @@
 include_directories($<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_INCLUDE_DIR}>)
 link_libraries(apt-pkg $<$<BOOL:${SECCOMP_FOUND}>:${SECCOMP_LIBRARIES}>)
 
-add_library(httplib OBJECT http.cc basehttp.cc)
 add_library(connectlib OBJECT connect.cc rfc2553emu.cc)
 
 add_executable(file file.cc)
@@ -10,8 +9,8 @@ add_executable(copy copy.cc)
 add_executable(store store.cc)
 add_executable(gpgv gpgv.cc)
 add_executable(cdrom cdrom.cc)
-add_executable(http http_main.cc $<TARGET_OBJECTS:httplib> $<TARGET_OBJECTS:connectlib>)
-add_executable(mirror mirror.cc $<TARGET_OBJECTS:httplib> $<TARGET_OBJECTS:connectlib>)
+add_executable(http http.cc basehttp.cc $<TARGET_OBJECTS:connectlib>)
+add_executable(mirror mirror.cc)
 add_executable(ftp ftp.cc $<TARGET_OBJECTS:connectlib>)
 add_executable(rred rred.cc)
 add_executable(rsh rsh.cc)
@@ -21,14 +20,13 @@ target_include_directories(connectlib PRIVATE ${GNUTLS_INCLUDE_DIR})
 
 # Additional libraries to link against for networked stuff
 target_link_libraries(http ${GNUTLS_LIBRARIES})
-target_link_libraries(mirror ${RESOLV_LIBRARIES} ${GNUTLS_LIBRARIES})
 target_link_libraries(ftp ${GNUTLS_LIBRARIES})
 
 # Install the library
 install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror
         RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods)
 
-add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods store)
+add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods mirror mirror+ftp mirror+http mirror+https mirror+file mirror+copy)
 add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh)
 
 
diff --git a/methods/aptmethod.h b/methods/aptmethod.h
index 88d325cba..331411571 100644
--- a/methods/aptmethod.h
+++ b/methods/aptmethod.h
@@ -448,6 +448,16 @@ protected:
       return true;
    }
 
+   // This is a copy of #pkgAcqMethod::Dequeue which is private & hidden
+   void Dequeue()
+   {
+      FetchItem const *const Tmp = Queue;
+      Queue = Queue->Next;
+      if (Tmp == QueueBack)
+	 QueueBack = Queue;
+      delete Tmp;
+   }
+
    aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3)
        : pkgAcqMethod(Ver, Flags), Binary(Binary), SeccompFlags(0), methodNames({Binary})
    {
diff --git a/methods/http.cc b/methods/http.cc
index 2d23b1646..5d286bcb4 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -31,6 +31,7 @@
 #include <sstream>
 #include <arpa/inet.h>
 #include <errno.h>
+#include <signal.h>
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -1034,7 +1035,7 @@ BaseHttpMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &R
    return FILE_IS_OPEN;
 }
 									/*}}}*/
-HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/
+HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg), "1.2", Pipeline | SendConfig) /*{{{*/
 {
    SeccompFlags = aptMethod::BASE | aptMethod::NETWORK;
 
@@ -1051,3 +1052,14 @@ HttpMethod::HttpMethod(std::string &&pProg) : BaseHttpMethod(pProg.c_str(), "1.2
    }
 }
 									/*}}}*/
+
+int main(int, const char *argv[])
+{
+   // ignore SIGPIPE, this can happen on write() if the socket
+   // closes the connection (this is dealt with via ServerDie())
+   signal(SIGPIPE, SIG_IGN);
+   std::string Binary = flNotDir(argv[0]);
+   if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http")
+      Binary.append("+http");
+   return HttpMethod(std::move(Binary)).Loop();
+}
diff --git a/methods/http_main.cc b/methods/http_main.cc
deleted file mode 100644
index 792b5e22f..000000000
--- a/methods/http_main.cc
+++ /dev/null
@@ -1,17 +0,0 @@
-#include <config.h>
-#include <apt-pkg/error.h>
-#include <apt-pkg/fileutl.h>
-#include <signal.h>
-
-#include "http.h"
-
-int main(int, const char *argv[])
-{
-   // ignore SIGPIPE, this can happen on write() if the socket
-   // closes the connection (this is dealt with via ServerDie())
-   signal(SIGPIPE, SIG_IGN);
-   std::string Binary = flNotDir(argv[0]);
-   if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http")
-      Binary.append("+http");
-   return HttpMethod(std::move(Binary)).Loop();
-}
diff --git a/methods/mirror.cc b/methods/mirror.cc
index b551802e4..add9f0875 100644
--- a/methods/mirror.cc
+++ b/methods/mirror.cc
@@ -1,18 +1,17 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: mirror.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
 /* ######################################################################
 
-   Mirror Acquire Method - This is the Mirror acquire method for APT.
-   
+   Mirror URI – This method helps avoiding hardcoding of mirrors in the
+   sources.lists by looking up a list of mirrors first to which the
+   following requests are redirected.
+
    ##################################################################### */
 									/*}}}*/
 // Include Files							/*{{{*/
 #include <config.h>
 
-#include <apt-pkg/acquire-item.h>
-#include <apt-pkg/acquire.h>
-#include <apt-pkg/aptconfiguration.h>
+#include "aptmethod.h"
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
@@ -20,447 +19,388 @@
 #include <apt-pkg/sourcelist.h>
 #include <apt-pkg/strutl.h>
 
-#include <algorithm>
-#include <fstream>
-#include <iostream>
+#include <functional>
+#include <random>
+#include <string>
+#include <unordered_map>
 
-#include <dirent.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
 #include <sys/utsname.h>
-#include <unistd.h>
-
-using namespace std;
-
-#include <sstream>
 
-#include "http.h"
-#include "mirror.h"
 #include <apti18n.h>
 									/*}}}*/
+constexpr char const *const disallowLocal[] = {"ftp", "http", "https"};
 
-/* Done:
- * - works with http (only!)
- * - always picks the first mirror from the list
- * - call out to problem reporting script
- * - supports "deb mirror://host/path/to/mirror-list/// dist component"
- * - uses pkgAcqMethod::FailReason() to have a string representation
- *   of the failure that is also send to LP
- * 
- * TODO: 
- * - deal with running as non-root because we can't write to the lists
-     dir then -> use the cached mirror file
- * - better method to download than having a pkgAcquire interface here
- *   and better error handling there!
- * - support more than http
- * - testing :)
- */
-
-MirrorMethod::MirrorMethod()
-   : HttpMethod("mirror"), DownloadedMirrorFile(false), Debug(false)
+static void sortByLength(std::vector<std::string> &vec)			/*{{{*/
 {
-}
-
-// HttpMethod::Configuration - Handle a configuration message		/*{{{*/
-// ---------------------------------------------------------------------
-/* We stash the desired pipeline depth */
-bool MirrorMethod::Configuration(string Message)
-{
-   if (HttpMethod::Configuration(Message) == false)
-      return false;
-   Debug = DebugEnabled();
-   
-   return true;
+   // this ensures having mirror://foo/ and mirror://foo/bar/ works as expected
+   // by checking for the longest matches first
+   std::sort(vec.begin(), vec.end(), [](std::string const &a, std::string const &b) {
+      return a.length() > b.length();
+   });
 }
 									/*}}}*/
-
-// clean the mirrors dir based on ttl information
-bool MirrorMethod::Clean(string Dir)
+class MirrorMethod : public aptMethod /*{{{*/
 {
-   vector<metaIndex *>::const_iterator I;
-
-   if(Debug)
-      clog << "MirrorMethod::Clean(): " << Dir << endl;
-
-   if(Dir == "/")
-      return _error->Error("will not clean: '/'");
-
-   // read sources.list
-   pkgSourceList list;
-   list.ReadMainList();
-
-   int const dirfd = open(Dir.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
-   if (dirfd == -1)
-      return _error->Errno("open",_("Unable to read %s"), Dir.c_str());
-   DIR * const D = fdopendir(dirfd);
-   if (D == nullptr)
-      return _error->Errno("fdopendir",_("Unable to read %s"),Dir.c_str());
-
-   for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
+   std::mt19937 genrng;
+   std::vector<std::string> sourceslist;
+   std::unordered_map<std::string, std::string> msgCache;
+   enum MirrorFileState
    {
-      // Skip some files..
-      if (strcmp(Dir->d_name,"lock") == 0 ||
-	  strcmp(Dir->d_name,"partial") == 0 ||
-	  strcmp(Dir->d_name,"lost+found") == 0 ||
-	  strcmp(Dir->d_name,".") == 0 ||
-	  strcmp(Dir->d_name,"..") == 0)
-	 continue;
-
-      // see if we have that uri
-      for(I=list.begin(); I != list.end(); ++I)
+      REQUESTED,
+      FAILED,
+      AVAILABLE
+   };
+   struct MirrorInfo
+   {
+      std::string uri;
+      unsigned long priority = std::numeric_limits<decltype(priority)>::max();
+      decltype(genrng)::result_type seed = 0;
+      std::unordered_map<std::string, std::vector<std::string>> tags;
+      MirrorInfo(std::string const &u, std::vector<std::string> &&ptags = {}) : uri(u)
       {
-	 string uri = (*I)->GetURI();
-	 if(uri.compare(0, strlen("mirror://"), "mirror://") != 0)
-	    continue;
-	 string BaseUri = uri.substr(0,uri.size()-1);
-	 if (URItoFileName(BaseUri) == Dir->d_name)
-	    break;
+	 for (auto &&tag : ptags)
+	 {
+	    auto const colonfound = tag.find(':');
+	    if (unlikely(colonfound == std::string::npos))
+	       continue;
+	    auto name = tag.substr(0, colonfound);
+	    auto value = tag.substr(colonfound + 1);
+	    if (name == "arch")
+	       tags["Architecture"].emplace_back(std::move(value));
+	    else if (name == "lang")
+	       tags["Language"].emplace_back(std::move(value));
+	    else if (name == "priority")
+	       priority = std::strtoul(value.c_str(), nullptr, 10);
+	    else if (likely(name.empty() == false))
+	    {
+	       if (name == "codename" || name == "suite")
+		  tags["Release"].push_back(value);
+	       name[0] = std::toupper(name[0]);
+	       tags[std::move(name)].emplace_back(std::move(value));
+	    }
+	 }
       }
-      // nothing found, nuke it
-      if (I == list.end())
-	 RemoveFileAt("mirror", dirfd, Dir->d_name);
-   }
-   closedir(D);
-   return true;
-}
-
-
-bool MirrorMethod::DownloadMirrorFile(string /*mirror_uri_str*/)
-{
-   // not that great to use pkgAcquire here, but we do not have
-   // any other way right now
-   string fetch = BaseUri;
-   fetch.replace(0,strlen("mirror://"),"http://");
-
-#if 0 // no need for this, the getArchitectures() will also include the main
-      // arch
-   // append main architecture
-   fetch += "?arch=" + _config->Find("Apt::Architecture");
-#endif
-
-   // append all architectures
-   std::vector<std::string> vec = APT::Configuration::getArchitectures();
-   for (std::vector<std::string>::const_iterator I = vec.begin();
-        I != vec.end(); ++I)
-      if (I == vec.begin())
-         fetch += "?arch=" + (*I);
-      else
-         fetch += "&arch=" + (*I);
+   };
+   struct MirrorListInfo
+   {
+      MirrorFileState state;
+      std::string baseuri;
+      std::vector<MirrorInfo> list;
+   };
+   std::unordered_map<std::string, MirrorListInfo> mirrorfilestate;
 
-   // append the dist as a query string
-   if (Dist != "")
-      fetch += "&dist=" + Dist;
+   virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
 
-   if(Debug)
-      clog << "MirrorMethod::DownloadMirrorFile(): '" << fetch << "'"
-           << " to " << MirrorFile << endl;
+   void RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message);
+   bool MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm);
+   std::string GetMirrorFileURI(std::string const &Message, FetchItem *const Itm);
+   void DealWithPendingItems(std::vector<std::string> const &baseuris, MirrorListInfo const &info, FetchItem *const Itm, std::function<void()> handler);
 
-   pkgAcquire Fetcher;
-   new pkgAcqFile(&Fetcher, fetch, "", 0, "", "", "", MirrorFile);
-   bool res = (Fetcher.Run() == pkgAcquire::Continue);
-   if(res) {
-      DownloadedMirrorFile = true;
-      chmod(MirrorFile.c_str(), 0644);
+   public:
+   MirrorMethod(std::string &&pProg) : aptMethod(std::move(pProg), "2.0", SingleInstance | Pipeline | SendConfig | AuxRequests), genrng(clock())
+   {
+      SeccompFlags = aptMethod::BASE | aptMethod::DIRECTORY;
    }
-   Fetcher.Shutdown();
-
-   if(Debug)
-      clog << "MirrorMethod::DownloadMirrorFile() success: " << res << endl;
-
-   return res;
-}
-
-// Randomizes the lines in the mirror file, this is used so that
-// we spread the load on the mirrors evenly
-bool MirrorMethod::RandomizeMirrorFile(string mirror_file)
+};
+									/*}}}*/
+void MirrorMethod::RedirectItem(MirrorListInfo const &info, FetchItem *const Itm, std::string const &Message) /*{{{*/
 {
-   vector<string> content;
-   string line;
-
-   if (!FileExists(mirror_file))
-      return false;
-
-   // read
-   ifstream in(mirror_file.c_str());
-   while ( !in.eof() ) {
-      getline(in, line);
-      content.push_back(line);
-   }
-
-   // we want the file to be random for each different machine, but also
-   // "stable" on the same machine. this is to avoid running into out-of-sync
-   // issues (i.e. Release/Release.gpg different on each mirror)
-   struct utsname buf;
-   int seed=1;
-   if(uname(&buf) == 0) {
-      for(int i=0,seed=1; buf.nodename[i] != 0; ++i) {
-         seed = seed * 31 + buf.nodename[i];
+   std::unordered_map<std::string, std::string> matchers;
+   matchers.emplace("Architecture", LookupTag(Message, "Target-Architecture"));
+   matchers.emplace("Codename", LookupTag(Message, "Target-Codename"));
+   matchers.emplace("Component", LookupTag(Message, "Target-Component"));
+   matchers.emplace("Language", LookupTag(Message, "Target-Language"));
+   matchers.emplace("Release", LookupTag(Message, "Target-Release"));
+   matchers.emplace("Suite", LookupTag(Message, "Target-Suite"));
+   matchers.emplace("Type", LookupTag(Message, "Target-Type"));
+   decltype(info.list) possMirrors;
+   for (auto const &mirror : info.list)
+   {
+      bool failedMatch = false;
+      for (auto const &m : matchers)
+      {
+	 if (m.second.empty())
+	    continue;
+	 auto const tagsetiter = mirror.tags.find(m.first);
+	 if (tagsetiter == mirror.tags.end())
+	    continue;
+	 auto const tagset = tagsetiter->second;
+	 if (tagset.empty() == false && std::find(tagset.begin(), tagset.end(), m.second) == tagset.end())
+	 {
+	    failedMatch = true;
+	    break;
+	 }
       }
+      if (failedMatch)
+	 continue;
+      possMirrors.push_back(mirror);
    }
-   srand( seed );
-   random_shuffle(content.begin(), content.end());
-
-   // write
-   ofstream out(mirror_file.c_str());
-   while ( !content.empty()) {
-      line = content.back();
-      content.pop_back();
-      out << line << "\n";
+   for (auto &&mirror : possMirrors)
+      mirror.seed = genrng();
+   std::sort(possMirrors.begin(), possMirrors.end(), [](MirrorInfo const &a, MirrorInfo const &b) {
+      if (a.priority != b.priority)
+	 return a.priority < b.priority;
+      return a.seed < b.seed;
+   });
+   std::string const path = Itm->Uri.substr(info.baseuri.length());
+   std::string altMirrors;
+   std::unordered_map<std::string, std::string> fields;
+   fields.emplace("URI", Queue->Uri);
+   for (auto curMirror = possMirrors.cbegin(); curMirror != possMirrors.cend(); ++curMirror)
+   {
+      std::string mirror = curMirror->uri;
+      if (APT::String::Endswith(mirror, "/") == false)
+	 mirror.append("/");
+      mirror.append(path);
+      if (curMirror == possMirrors.cbegin())
+	 fields.emplace("New-URI", mirror);
+      else if (altMirrors.empty())
+	 altMirrors.append(mirror);
+      else
+	 altMirrors.append("\n").append(mirror);
    }
-
-   return true;
+   fields.emplace("Alternate-URIs", altMirrors);
+   SendMessage("103 Redirect", std::move(fields));
+   Dequeue();
 }
-
-/* convert a the Queue->Uri back to the mirror base uri and look
- * at all mirrors we have for this, this is needed as queue->uri
- * may point to different mirrors (if TryNextMirror() was run)
- */
-void MirrorMethod::CurrentQueueUriToMirror()
+									/*}}}*/
+void MirrorMethod::DealWithPendingItems(std::vector<std::string> const &baseuris, /*{{{*/
+					MirrorListInfo const &info, FetchItem *const Itm,
+					std::function<void()> handler)
 {
-   // already in mirror:// style so nothing to do
-   if(Queue->Uri.find("mirror://") == 0)
-      return;
-
-   // find current mirror and select next one
-   for (vector<string>::const_iterator mirror = AllMirrors.begin();
-	mirror != AllMirrors.end(); ++mirror)
+   FetchItem **LastItm = &Itm->Next;
+   while (*LastItm != nullptr)
+      LastItm = &((*LastItm)->Next);
+   while (Queue != Itm)
    {
-      if (Queue->Uri.find(*mirror) == 0)
+      if (APT::String::Startswith(Queue->Uri, info.baseuri) == false ||
+	  std::any_of(baseuris.cbegin(), baseuris.cend(), [&](std::string const &b) { return APT::String::Startswith(Queue->Uri, b); }))
       {
-	 Queue->Uri.replace(0, mirror->length(), BaseUri);
-	 return;
+	 // move the item behind the aux file not related to it
+	 *LastItm = Queue;
+	 Queue = QueueBack = Queue->Next;
+	 (*LastItm)->Next = nullptr;
+	 LastItm = &((*LastItm)->Next);
+      }
+      else
+      {
+	 handler();
       }
    }
-   _error->Error("Internal error: Failed to convert %s back to %s",
-		 Queue->Uri.c_str(), BaseUri.c_str());
+   // now remove out trigger
+   QueueBack = Queue = Queue->Next;
+   delete Itm;
 }
-
-bool MirrorMethod::TryNextMirror()
+									/*}}}*/
+bool MirrorMethod::MirrorListFileRecieved(MirrorListInfo &info, FetchItem *const Itm) /*{{{*/
 {
-   // find current mirror and select next one
-   for (vector<string>::const_iterator mirror = AllMirrors.begin();
-	mirror != AllMirrors.end(); ++mirror)
+   std::vector<std::string> baseuris;
+   for (auto const &i : mirrorfilestate)
+      if (info.baseuri.length() < i.second.baseuri.length() &&
+	  i.second.state == REQUESTED &&
+	  APT::String::Startswith(i.second.baseuri, info.baseuri))
+	 baseuris.push_back(i.second.baseuri);
+   sortByLength(baseuris);
+
+   FileFd mirrorlist;
+   if (FileExists(Itm->DestFile) && mirrorlist.Open(Itm->DestFile, FileFd::ReadOnly, FileFd::Extension))
    {
-      if (Queue->Uri.find(*mirror) != 0)
-	 continue;
-
-      vector<string>::const_iterator nextmirror = mirror + 1;
-      if (nextmirror == AllMirrors.end())
-	 break;
-      Queue->Uri.replace(0, mirror->length(), *nextmirror);
-      if (Debug)
-	 clog << "TryNextMirror: " << Queue->Uri << endl;
-
-      // inform parent
-      UsedMirror = *nextmirror;
-      Log("Switching mirror");
-      return true;
-   }
-
-   if (Debug)
-      clog << "TryNextMirror could not find another mirror to try" << endl;
-
-   return false;
-}
+      auto const accessColon = info.baseuri.find(':');
+      auto access = info.baseuri.substr(0, accessColon);
+      std::string prefixAccess;
+      if (APT::String::Startswith(access, "mirror") == false)
+      {
+	 auto const plus = info.baseuri.find('+');
+	 prefixAccess = info.baseuri.substr(0, plus);
+	 access.erase(0, plus + 1);
+      }
+      std::vector<std::string> limitAccess;
+      // If the mirror file comes from an online source, allow only other online
+      // sources, not e.g. file:///. If the mirrorlist comes from there we can assume
+      // the admin knows what (s)he is doing through and not limit the options.
+      if (std::any_of(std::begin(disallowLocal), std::end(disallowLocal),
+		      [&access](char const *const a) { return APT::String::Endswith(access, std::string("+") + a); }) ||
+	  access == "mirror")
+      {
+	 for (auto const &a : disallowLocal)
+	    limitAccess.emplace_back(a);
+      }
+      std::string line;
+      while (mirrorlist.ReadLine(line))
+      {
+	 if (line.empty() || line[0] == '#')
+	    continue;
+	 auto const access = line.substr(0, line.find(':'));
+	 if (limitAccess.empty() == false && std::find(limitAccess.begin(), limitAccess.end(), access) == limitAccess.end())
+	    continue;
+	 auto const tab = line.find('\t');
+	 if (tab == std::string::npos)
+	 {
+	    if (prefixAccess.empty())
+	       info.list.emplace_back(std::move(line));
+	    else
+	       info.list.emplace_back(prefixAccess + '+' + line);
+	 }
+	 else
+	 {
+	    auto uri = line.substr(0, tab);
+	    if (prefixAccess.empty() == false)
+	       uri = prefixAccess + '+' + uri;
+	    auto tagline = line.substr(tab + 1);
+	    std::replace_if(tagline.begin(), tagline.end(), isspace_ascii, ' ');
+	    auto tags = VectorizeString(tagline, ' ');
+	    tags.erase(std::remove_if(tags.begin(), tags.end(), [](std::string const &a) { return a.empty(); }), tags.end());
+	    info.list.emplace_back(std::move(uri), std::move(tags));
+	 }
+      }
+      mirrorlist.Close();
 
-bool MirrorMethod::InitMirrors()
-{
-   // if we do not have a MirrorFile, fallback
-   if(!FileExists(MirrorFile))
-   {
-      // FIXME: fallback to a default mirror here instead 
-      //        and provide a config option to define that default
-      return _error->Error(_("No mirror file '%s' found "), MirrorFile.c_str());
+      if (info.list.empty())
+      {
+	 info.state = FAILED;
+	 DealWithPendingItems(baseuris, info, Itm, [&]() {
+	    std::string msg;
+	    strprintf(msg, "Mirror list %s is empty for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+	    Fail(msg, false);
+	 });
+      }
+      else
+      {
+	 info.state = AVAILABLE;
+	 DealWithPendingItems(baseuris, info, Itm, [&]() {
+	    RedirectItem(info, Queue, msgCache[Queue->Uri]);
+	 });
+	 msgCache.clear();
+      }
    }
-
-   if (access(MirrorFile.c_str(), R_OK) != 0)
-   {
-      // FIXME: fallback to a default mirror here instead 
-      //        and provide a config option to define that default
-      return _error->Error(_("Can not read mirror file '%s'"), MirrorFile.c_str());
-   }  
-
-   // FIXME: make the mirror selection more clever, do not 
-   //        just use the first one!
-   // BUT: we can not make this random, the mirror has to be
-   //      stable across session, because otherwise we can
-   //      get into sync issues (got indexfiles from mirror A,
-   //      but packages from mirror B - one might be out of date etc)
-   ifstream in(MirrorFile.c_str());
-   string s;
-   while (!in.eof()) 
+   else
    {
-      getline(in, s);
-
-      // ignore lines that start with #
-      if (s.find("#") == 0)
-         continue;
-      // ignore empty lines
-      if (s.size() == 0)
-         continue;
-      // ignore non http lines
-      if (s.compare(0, strlen("http://"), "http://") != 0)
-         continue;
-
-      AllMirrors.push_back(s);
+      info.state = FAILED;
+      DealWithPendingItems(baseuris, info, Itm, [&]() {
+	 std::string msg;
+	 strprintf(msg, "Downloading mirror file %s failed for %s", Itm->DestFile.c_str(), Queue->Uri.c_str());
+	 Fail(msg, false);
+      });
    }
-   if (AllMirrors.empty()) {
-	return _error->Error(_("No entry found in mirror file '%s'"), MirrorFile.c_str());
-   }
-   Mirror = AllMirrors[0];
-   UsedMirror = Mirror;
    return true;
 }
-
-string MirrorMethod::GetMirrorFileName(string mirror_uri_str)
+									/*}}}*/
+std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem *const Itm) /*{{{*/
 {
-   /* 
-    - a mirror_uri_str looks like this:
-    mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors/dists/feisty/Release.gpg
-   
-    - the matching source.list entry
-    deb mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors feisty main
-   
-    - we actually want to go after:
-    http://people.ubuntu.com/~mvo/apt/mirror/mirrors
-
-    And we need to save the BaseUri for later:
-    - mirror://people.ubuntu.com/~mvo/apt/mirror/mirrors
-
-   FIXME: what if we have two similar prefixes?
-     mirror://people.ubuntu.com/~mvo/mirror
-     mirror://people.ubuntu.com/~mvo/mirror2
-   then mirror_uri_str looks like:
-     mirror://people.ubuntu.com/~mvo/apt/mirror/dists/feisty/Release.gpg
-     mirror://people.ubuntu.com/~mvo/apt/mirror2/dists/feisty/Release.gpg
-   we search sources.list and find:
-     mirror://people.ubuntu.com/~mvo/apt/mirror
-   in both cases! So we need to apply some domain knowledge here :( and
-   check for /dists/ or /Release.gpg as suffixes
-   */
-   string name;
-   if(Debug)
-      std::cerr << "GetMirrorFileName: " << mirror_uri_str << std::endl;
-
-   // read sources.list and find match
-   vector<metaIndex *>::const_iterator I;
-   pkgSourceList list;
-   list.ReadMainList();
-   for(I=list.begin(); I != list.end(); ++I)
+   if (APT::String::Startswith(Itm->Uri, Binary))
    {
-      string uristr = (*I)->GetURI();
-      if(Debug)
-	 std::cerr << "Checking: " << uristr << std::endl;
-      if(uristr.substr(0,strlen("mirror://")) != string("mirror://"))
-	 continue;
-      // find matching uri in sources.list
-      if(mirror_uri_str.substr(0,uristr.size()) == uristr)
+      std::string const repouri = LookupTag(Message, "Target-Repo-Uri");
+      if (repouri.empty() == false && std::find(sourceslist.cbegin(), sourceslist.cend(), repouri) == sourceslist.cend())
+	 sourceslist.push_back(repouri);
+   }
+   if (sourceslist.empty())
+   {
+      // read sources.list and find the matching base uri
+      pkgSourceList sl;
+      if (sl.ReadMainList() == false)
+      {
+	 _error->Error(_("The list of sources could not be read."));
+	 return "";
+      }
+      std::string const needle = Binary + ":";
+      for (auto const &SL : sl)
       {
-	 if(Debug)
-	    std::cerr << "found BaseURI: " << uristr << std::endl;
-	 BaseUri = uristr.substr(0,uristr.size()-1);
-         Dist = (*I)->GetDist();
+	 std::string uristr = SL->GetURI();
+	 if (APT::String::Startswith(uristr, needle))
+	    sourceslist.push_back(uristr);
       }
+      sortByLength(sourceslist);
    }
-   // get new file
-   name = _config->FindDir("Dir::State::mirrors") + URItoFileName(BaseUri);
-
-   if(Debug) 
+   for (auto uristr : sourceslist)
    {
-      cerr << "base-uri: " << BaseUri << endl;
-      cerr << "mirror-file: " << name << endl;
+      if (APT::String::Startswith(Itm->Uri, uristr))
+      {
+	 uristr.erase(uristr.length() - 1); // remove the ending '/'
+	 auto const colon = uristr.find(':');
+	 if (unlikely(colon == std::string::npos))
+	    continue;
+	 auto const plus = uristr.find("+");
+	 if (plus < colon)
+	 {
+	    // started as tor+mirror+http we want to get the file via tor+http
+	    auto access = uristr.substr(0, colon);
+	    std::string prefixAccess;
+	    if (APT::String::Startswith(access, "mirror") == false)
+	    {
+	       prefixAccess = uristr.substr(0, plus);
+	       access.erase(0, plus + 1);
+	       uristr.erase(plus, strlen("mirror") + 1);
+	       return uristr;
+	    }
+	    else
+	       return uristr.substr(plus + 1);
+	 }
+	 else
+	 {
+	    uristr.replace(0, strlen("mirror"), "http");
+	    return uristr;
+	 }
+      }
    }
-   return name;
+   return "";
 }
-
-// MirrorMethod::Fetch - Fetch an item					/*{{{*/
-// ---------------------------------------------------------------------
-/* This adds an item to the pipeline. We keep the pipeline at a fixed
-   depth. */
-bool MirrorMethod::Fetch(FetchItem *Itm)
+									/*}}}*/
+bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*/
 {
-   if(Debug)
-      clog << "MirrorMethod::Fetch()" << endl;
-
-   // the http method uses Fetch(0) as a way to update the pipeline,
-   // just let it do its work in this case - Fetch() with a valid
-   // Itm will always run before the first Fetch(0)
-   if(Itm == NULL) 
-      return HttpMethod::Fetch(Itm);
-
-   // if we don't have the name of the mirror file on disk yet,
-   // calculate it now (can be derived from the uri)
-   if(MirrorFile.empty())
-      MirrorFile = GetMirrorFileName(Itm->Uri);
+   auto mirrorinfo = mirrorfilestate.find(Itm->Uri);
+   if (mirrorinfo != mirrorfilestate.end())
+      return MirrorListFileRecieved(mirrorinfo->second, Itm);
 
-  // download mirror file once (if we are after index files)
-   if(Itm->IndexFile && !DownloadedMirrorFile)
+   std::string const mirrorfileuri = GetMirrorFileURI(Message, Itm);
+   if (mirrorfileuri.empty())
    {
-      Clean(_config->FindDir("Dir::State::mirrors"));
-      if (DownloadMirrorFile(Itm->Uri))
-         RandomizeMirrorFile(MirrorFile);
+      _error->Error("Couldn't determine mirror list to query for %s", Itm->Uri.c_str());
+      return false;
    }
+   if (DebugEnabled())
+      std::clog << "Mirror-URI: " << mirrorfileuri << " for " << Itm->Uri << std::endl;
 
-   if(AllMirrors.empty()) {
-      if(!InitMirrors()) {
-	 // no valid mirror selected, something went wrong downloading
-	 // from the master mirror site most likely and there is
-	 // no old mirror file availalbe
+   // have we requested this mirror file already?
+   auto const state = mirrorfilestate.find(mirrorfileuri);
+   if (state == mirrorfilestate.end())
+   {
+      msgCache[Itm->Uri] = Message;
+      MirrorListInfo info;
+      info.state = REQUESTED;
+      info.baseuri = mirrorfileuri + '/';
+      auto const colon = info.baseuri.find(':');
+      if (unlikely(colon == std::string::npos))
 	 return false;
-      }
+      info.baseuri.replace(0, colon, Binary);
+      mirrorfilestate[mirrorfileuri] = info;
+      std::unordered_map<std::string, std::string> fields;
+      fields.emplace("URI", Itm->Uri);
+      fields.emplace("MaximumSize", std::to_string(1 * 1024 * 1024)); //FIXME: 1 MB is enough for everyone
+      fields.emplace("Aux-ShortDesc", "Mirrorlist");
+      fields.emplace("Aux-Description", mirrorfileuri + " Mirrorlist");
+      fields.emplace("Aux-Uri", mirrorfileuri);
+      SendMessage("351 Aux Request", std::move(fields));
+      return true;
    }
 
-   if(Itm->Uri.find("mirror://") != string::npos)
-      Itm->Uri.replace(0,BaseUri.size(), Mirror);
-
-   if(Debug)
-      clog << "Fetch: " << Itm->Uri << endl << endl;
-
-   // now run the real fetcher
-   return HttpMethod::Fetch(Itm);
-}
-
-void MirrorMethod::Fail(string Err,bool Transient)
-{
-   // FIXME: TryNextMirror is not ideal for indexfile as we may
-   //        run into auth issues
-
-   if (Debug)
-      clog << "Failure to get " << Queue->Uri << endl;
-
-   // try the next mirror on fail (if its not a expected failure,
-   // e.g. translations are ok to ignore)
-   if (!Queue->FailIgnore && TryNextMirror())
-      return;
-
-   // all mirrors failed, so bail out
-   string s;
-   strprintf(s, _("[Mirror: %s]"), Mirror.c_str());
-   SetIP(s);
-
-   CurrentQueueUriToMirror();
-   pkgAcqMethod::Fail(Err, Transient);
-}
-
-void MirrorMethod::URIStart(FetchResult &Res)
-{
-   CurrentQueueUriToMirror();
-   pkgAcqMethod::URIStart(Res);
-}
-
-void MirrorMethod::URIDone(FetchResult &Res,FetchResult *Alt)
-{
-   CurrentQueueUriToMirror();
-   pkgAcqMethod::URIDone(Res, Alt);
+   switch (state->second.state)
+   {
+   case REQUESTED:
+      // lets wait for the requested mirror file
+      msgCache[Itm->Uri] = Message;
+      return true;
+   case FAILED:
+      Fail("Downloading mirror file failed", false);
+      return true;
+   case AVAILABLE:
+      RedirectItem(state->second, Itm, Message);
+      return true;
+   }
+   return false;
 }
+									/*}}}*/
 
-
-int main()
+int main(int, const char *argv[])
 {
-   return MirrorMethod().Loop();
+   return MirrorMethod(flNotDir(argv[0])).Run();
 }
-
-
diff --git a/methods/mirror.h b/methods/mirror.h
deleted file mode 100644
index 6ebe08e6b..000000000
--- a/methods/mirror.h
+++ /dev/null
@@ -1,57 +0,0 @@
-// -*- mode: cpp; mode: fold -*-
-// Description								/*{{{*/
-/* ######################################################################
-
-   MIRROR Acquire Method - This is the MIRROR acquire method for APT.
-
-   ##################################################################### */
-									/*}}}*/
-
-#ifndef APT_MIRROR_H
-#define APT_MIRROR_H
-
-#include <iostream>
-#include <string>
-#include <vector>
-
-using std::cout;
-using std::cerr;
-using std::endl;
-
-#include "http.h"
-
-class MirrorMethod : public HttpMethod
-{
-   FetchResult Res;
-   // we simply transform between BaseUri and Mirror
-   std::string BaseUri;    // the original mirror://... url
-   std::string Mirror;     // the selected mirror uri (http://...)
-   std::vector<std::string> AllMirrors; // all available mirrors
-   std::string MirrorFile; // the file that contains the list of mirrors
-   bool DownloadedMirrorFile; // already downloaded this session
-   std::string Dist;       // the target distrubtion (e.g. sid, oneiric)
-
-   bool Debug;
-
- protected:
-   bool DownloadMirrorFile(std::string uri);
-   bool RandomizeMirrorFile(std::string file);
-   std::string GetMirrorFileName(std::string uri);
-   bool InitMirrors();
-   bool TryNextMirror();
-   void CurrentQueueUriToMirror();
-   bool Clean(std::string dir);
-   
-   // we need to overwrite those to transform the url back
-   virtual void Fail(std::string Why, bool Transient = false) APT_OVERRIDE;
-   virtual void URIStart(FetchResult &Res) APT_OVERRIDE;
-   virtual void URIDone(FetchResult &Res,FetchResult *Alt = 0) APT_OVERRIDE;
-   virtual bool Configuration(std::string Message) APT_OVERRIDE;
-
- public:
-   MirrorMethod();
-   virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
-};
-
-
-#endif
diff --git a/po/CMakeLists.txt b/po/CMakeLists.txt
index f69123a26..a8893e8ba 100644
--- a/po/CMakeLists.txt
+++ b/po/CMakeLists.txt
@@ -15,7 +15,7 @@ apt_add_translation_domain(
     TARGETS apt apt-cache apt-get apt-config apt-cdrom apt-helper apt-mark
             apt-private
             # Methods
-            connectlib httplib file copy store gpgv cdrom http ftp rred rsh mirror
+            connectlib file copy store gpgv cdrom http ftp rred rsh mirror
     SCRIPTS ../dselect/install ../dselect/update
     EXCLUDE_LANGUAGES ${languages_excluded}
 )
diff --git a/test/integration/framework b/test/integration/framework
index f9d98835c..ff7a7c514 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -2031,9 +2031,11 @@ mkdir() {
 		command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt"
 		command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"
 		command mkdir -m 700 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial"
+		command mkdir -m 755 -p "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/auxfiles"
 		touch "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/lock"
 		if [ "$(id -u)" = '0' ]; then
 			chown _apt:$(id -gn) "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial"
+			chown _apt:$(id -gn) "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/auxfiles"
 		fi
 	else
 		command mkdir "$@"
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 616e0234c..a0d7a59d9 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -40,7 +40,8 @@ N: See apt-secure(8) manpage for repository creation and user configuration deta
 
 # no package foo
 testsuccessequal 'Listing...' apt list foo
-testequal 'lock
+testequal 'auxfiles
+lock
 partial' ls rootdir/var/lib/apt/lists
 
 filesize() {
diff --git a/test/integration/test-method-mirror b/test/integration/test-method-mirror
new file mode 100755
index 000000000..d00118405
--- /dev/null
+++ b/test/integration/test-method-mirror
@@ -0,0 +1,190 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture "i386"
+
+buildsimplenativepackage 'foo' 'all' '1' 'stable'
+buildsimplenativepackage 'foo' 'all' '2' 'unstable'
+setupaptarchive --no-update
+changetowebserver
+webserverconfig 'aptwebserver::redirect::replace::/redirectme/' "http://localhost:${APTHTTPPORT}/"
+addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/lib/apt/lists || true;"
+
+testrundownload_internal() {
+	cd downloaded
+	testsuccess apt download "$@"
+	while [ -n "$1" ]; do
+		local fndeb="$(echo "$1" | tr '=' '_')_all.deb"
+		testsuccess test -e "$fndeb"
+		rm -f "$fndeb"
+		shift
+	done
+	testempty find . -name '*mirror*'
+	cd ..
+}
+testrundownload() {
+	if [ "$(id -u)" = '0' ]; then
+		testrundownload_internal "$@"
+	else
+		chmod -f -R -w rootdir/var/lib/apt/lists
+		testrundownload_internal "$@"
+		chmod -f -R +w rootdir/var/lib/apt/lists/auxfiles
+		rm -f rootdir/var/lib/apt/lists/auxfiles/*
+		chmod -f -R -w rootdir/var/lib/apt/lists/auxfiles
+		testrundownload_internal "$@"
+		chmod -f -R +w rootdir/var/lib/apt/lists
+	fi
+}
+testrun() {
+	rm -rf rootdir/var/lib/apt/lists
+	testsuccess apt update #-o Debug::Acquire::mirror=1 -o Debug::Acquire::http=1 -o Debug::pkgAcquire::Worker=1
+	cp -a rootdir/tmp/testsuccess.output aptupdate.output
+	test -z "$1" || testempty find rootdir/var/lib/apt/lists -maxdepth 1 -name "$1" -type f
+	test -z "$2" || testnotempty find rootdir/var/lib/apt/lists -maxdepth 1 -name "$2" -type f
+	testsuccess apt show foo=1
+	testrundownload 'foo=1' 'foo=2'
+}
+
+msgmsg 'basic setup'
+testrun '' ''
+
+msgmsg 'redirect setup'
+sed -i -e 's#/ stable#/redirectme stable#' rootdir/etc/apt/sources.list.d/*-stable-*
+testrun '' '*_redirectme_*'
+
+msgmsg 'mirror file does not exist'
+sed -i -e 's# http:# mirror:#' -e 's#/redirectme stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/*-stable-*
+testfailure apt update
+
+echo "http://localhost:${APTHTTPPORT}" > aptarchive/mirror.txt
+
+msgmsg 'stable mirror setup'
+testrun '*_redirectme_stable_*' '*_mirror.txt_*'
+
+msgmsg 'all mirror setup'
+sed -i -e 's# http:# mirror:#' -e 's#/ unstable#/mirror.txt unstable#' rootdir/etc/apt/sources.list.d/*
+testrun '*_redirectme_stable_*' '*_mirror.txt_*'
+
+msgmsg 'all mirror+http setup'
+sed -i -e 's# mirror:# mirror+http:#' rootdir/etc/apt/sources.list.d/*
+testrun '*_redirectme_*' '*_mirror.txt_*'
+
+msgmsg 'stable gzipped redirect setup'
+echo "http://localhost:${APTHTTPPORT}/redirectme" > aptarchive/mirror.txt
+compressfile aptarchive/mirror.txt
+sed -i -e 's#/mirror\.txt stable#/mirror.txt.gz stable#' rootdir/etc/apt/sources.list.d/*-stable-*
+testrun '*_redirectme_*' '*_mirror.txt.gz_*'
+
+msgmsg 'all mirrored via file'
+APTARCHIVE="$(readlink -f ./aptarchive)"
+sed -i -e "s#mirror+http://localhost:${APTHTTPPORT}#mirror+file:${APTARCHIVE}#" rootdir/etc/apt/sources.list.d/*
+testrun '*_localhost_*' '*_aptarchive_mirror.txt.gz_*'
+
+msgmsg 'fallback mirrors are used if needed' 'as usual'
+sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/*
+echo "http://localhost:${APTHTTPPORT}/failure2	priority:3
+http://localhost:${APTHTTPPORT}/redirectme	priority:2
+http://localhost:${APTHTTPPORT}/failure	priority:1" > aptarchive/mirror.txt
+testrun '*_localhost_*' '*_aptarchive_mirror.txt_*'
+testsuccessequal "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Ign:2 http://localhost:${APTHTTPPORT}/failure stable InRelease
+  404  Not Found
+Ign:3 http://localhost:${APTHTTPPORT}/failure unstable InRelease
+  404  Not Found" head -n 5 aptupdate.output
+
+msgmsg 'fallback mirrors are used if needed' 'by tags'
+echo "http://localhost:${APTHTTPPORT}/failure2	priority:1	release:stable
+http://localhost:${APTHTTPPORT}/redirectme	priority:2
+http://localhost:${APTHTTPPORT}/failure	priority:1 release:unstable" > aptarchive/mirror.txt
+testrun '*_localhost_*' '*_aptarchive_mirror.txt_*'
+testsuccessequal "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Ign:2 http://localhost:${APTHTTPPORT}/failure2 stable InRelease
+  404  Not Found
+Ign:3 http://localhost:${APTHTTPPORT}/failure unstable InRelease
+  404  Not Found" head -n 5 aptupdate.output
+
+changetohttpswebserver
+rm -f rootdir/etc/apt/sources.list.d/*-stable-*
+msgmsg 'fallback mirrors are used if needed' 'random'
+echo "http://localhost:${APTHTTPPORT}/failure2	priority:1
+http://localhost:${APTHTTPPORT}/redirectme	priority:2
+https://localhost:${APTHTTPSPORT}/failure	priority:1
+http://localhost:${APTHTTPPORT}/unused-failure1
+http://localhost:${APTHTTPPORT}/unused-failure2
+http://localhost:${APTHTTPPORT}/unused-failure3
+http://localhost:${APTHTTPPORT}/unused-failure4
+http://localhost:${APTHTTPPORT}/unused-failure5
+http://localhost:${APTHTTPPORT}/unused-failure6
+http://localhost:${APTHTTPPORT}/unused-failure7
+http://localhost:${APTHTTPPORT}/unused-failure8
+http://localhost:${APTHTTPPORT}/unused-failure9
+" > aptarchive/mirror.txt
+testequalor2 "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Ign:2 http://localhost:${APTHTTPPORT}/failure2 unstable InRelease
+  404  Not Found
+Ign:2 https://localhost:${APTHTTPSPORT}/failure unstable InRelease
+  404  Not Found
+Hit:2 http://localhost:${APTHTTPPORT}/redirectme unstable InRelease
+Reading package lists...
+Building dependency tree...
+All packages are up to date." "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Ign:2 https://localhost:${APTHTTPSPORT}/failure unstable InRelease
+  404  Not Found
+Ign:2 http://localhost:${APTHTTPPORT}/failure2 unstable InRelease
+  404  Not Found
+Hit:2 http://localhost:${APTHTTPPORT}/failure2 unstable InRelease
+Reading package lists...
+Building dependency tree...
+All packages are up to date." apt update
+testfailure grep '/unused-failure' aptarchive/webserver.log
+
+msgmsg 'Mirrors can be filtered by' 'type'
+echo "http://localhost:${APTHTTPPORT}/failure
+http://localhost:${APTHTTPPORT}/redirectme	type:deb
+" > aptarchive/mirror.txt
+testfailure apt update
+testrundownload 'foo=2'
+
+msgmsg 'The prefix for the mirrorlist is' 'passed on'
+echo 'Dir::Bin::Methods::foo+mirror+file "mirror";
+Dir::Bin::Methods::foo+mirror+http "mirror";
+Dir::Bin::Methods::foo+http "http";
+' > rootdir/etc/apt/apt.conf.d/99add-foo-method
+echo "http://localhost:${APTHTTPPORT}/redirectme
+" > aptarchive/mirror.txt
+testsuccessequal "Get:1 file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Hit:2 http://localhost:${APTHTTPPORT}/redirectme unstable InRelease
+Reading package lists...
+Building dependency tree...
+All packages are up to date." apt update
+sed -i -e 's# mirror+# foo+mirror+#' rootdir/etc/apt/sources.list.d/*
+testfailure apt update
+testsuccess grep 'package apt-transport-foo installed' rootdir/tmp/testfailure.output
+echo 'Dir::Bin::Methods::foo+file "file";' >> rootdir/etc/apt/apt.conf.d/99add-foo-method
+testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease
+Reading package lists...
+Building dependency tree...
+All packages are up to date." apt update
+echo "file:/nonexistent/apt/archive	priority:1
+http://localhost:${APTHTTPPORT}/redirectme
+" > aptarchive/mirror.txt
+testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Get:2 foo+file:/nonexistent/apt/archive unstable InRelease
+Ign:2 foo+file:/nonexistent/apt/archive unstable InRelease
+  File not found - /nonexistent/apt/archive/dists/unstable/InRelease (2: No such file or directory)
+Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease
+Reading package lists...
+Building dependency tree...
+All packages are up to date." apt update
+sed -i -e "s#+file:${APTARCHIVE}#+http://localhost:${APTHTTPPORT}#" rootdir/etc/apt/sources.list.d/*
+testsuccess apt update
+testsuccessequal "Get:1 foo+http://localhost:${APTHTTPPORT}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
+Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease
+Reading package lists...
+Building dependency tree...
+All packages are up to date." apt update
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 53586ef32..5a650ad83 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -364,7 +364,7 @@ SHA256-Download:
 	# we let it fail by removing the files so the webserver reports 404
 	rm -f "$PATCHINDEX" "$PATCHFILE" "${PATCHFILE}.gz"
 	wasmergeused "$@" -o test::cannot-use-pdiff=1
-	testsuccess grep '400%20URI%20Failure.*Packages\.diff/Index.*FailReason.*HttpError404' rootdir/tmp/aptupdate.output
+	testsuccess grep '400%20URI%20Failure.*FailReason.*HttpError404.*Packages\.diff/Index' rootdir/tmp/aptupdate.output
 	testnopackage oldstuff
 	testsuccessequal "$(cat "${PKGFILE}-new")
 " aptcache show apt newstuff
diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall
index 3571a9f25..5f2109db9 100755
--- a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall
+++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall
@@ -41,7 +41,8 @@ runtests() {
 	testsuccess grep "$1" rootdir/tmp/testfailure.output
 
 	ensure_n_canary_strings_in_dir "$LISTS" 'ni ni ni' 0
-	testequal 'lock
+	testequal 'auxfiles
+lock
 partial' ls "$LISTS"
 
 	# and again with pre-existing files with "valid data" which should remain