Merge branch 'fix/partialfilemirror' into 'main'

Support uncompressed indexes from partial file:/ mirrors

See merge request apt-team/apt!235

17 files changed, 602 insertions, 286 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 73e9940ee..12524778e 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -32,7 +32,9 @@
 #include <apt-pkg/tagfile.h>
 
 #include <algorithm>
+#include <array>
 #include <cerrno>
+#include <ctime>
 #include <chrono>
 #include <cstddef>
 #include <cstdio>
@@ -471,15 +473,16 @@ bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
       // now add the actual by-hash uris
       auto const Expected = GetExpectedHashes();
       auto const TargetHash = Expected.find(nullptr);
-      auto const PushByHashURI = [&](std::string U) {
+      auto const PushByHashURI = [&](std::string const &U) {
 	 if (unlikely(TargetHash == nullptr))
 	    return false;
-	 auto const trailing_slash = U.find_last_of("/");
+	 ::URI uri{U};
+	 auto const trailing_slash = uri.Path.find_last_of("/");
 	 if (unlikely(trailing_slash == std::string::npos))
 	    return false;
-	 auto byhashSuffix = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
-	 U.replace(trailing_slash, U.length() - trailing_slash, std::move(byhashSuffix));
-	 PushAlternativeURI(std::move(U), {}, false);
+	 auto altPath = uri.Path.substr(0, trailing_slash) + "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
+	 std::swap(uri.Path, altPath);
+	 PushAlternativeURI(uri, {{"Alternate-Paths", "../../" + flNotDir(altPath)}}, false);
 	 return true;
       };
       PushByHashURI(Item.URI);
@@ -843,8 +846,17 @@ void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_m
       d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields));
 }
 									/*}}}*/
-void pkgAcquire::Item::RemoveAlternativeSite(std::string &&OldSite) /*{{{*/
+void pkgAcquire::Item::RemoveAlternativeSite(std::string const &AltUriStr)/*{{{*/
 {
+   ::URI AltUri{AltUriStr};
+   // the hostnames for these methods are empty for absolute paths which would result
+   // in the elimination of all sites accessed via those methods. On the upside, those
+   // methods are local and fast to reply with failure, so it doesn't hurt that much to
+   // keep them in the loop – so we just exit here rather than trying to guess the site.
+   std::array const badhosts{"file", "copy", "cdrom"};
+   if (std::find(badhosts.begin(), badhosts.end(), AltUri.Access) != badhosts.end())
+      return;
+   auto const OldSite = URI::SiteOnly(AltUriStr);
    d->AlternativeURIs.erase(std::remove_if(d->AlternativeURIs.begin(), d->AlternativeURIs.end(),
 					   [&](decltype(*d->AlternativeURIs.cbegin()) AltUri) {
 					      return URI::SiteOnly(AltUri.URI) == OldSite;
@@ -939,7 +951,7 @@ void pkgAcquire::Item::FailMessage(string const &Message)
       failreason = WEAK_HASHSUMS;
    else if (FailReason == "RedirectionLoop")
       failreason = REDIRECTION_LOOP;
-   else if (Status == StatAuthError)
+   else if (Status == StatAuthError || FailReason == "HashSumMismatch")
       failreason = HASHSUM_MISMATCH;
 
    if(ErrorText.empty())
@@ -964,7 +976,7 @@ void pkgAcquire::Item::FailMessage(string const &Message)
 	    break;
       }
 
-      if (Status == StatAuthError)
+      if (Status == StatAuthError || failreason == HASHSUM_MISMATCH)
       {
 	 auto const ExpectedHashes = GetExpectedHashes();
 	 if (ExpectedHashes.empty() == false)
@@ -976,13 +988,25 @@ void pkgAcquire::Item::FailMessage(string const &Message)
 	 if (failreason == HASHSUM_MISMATCH)
 	 {
 	    out << "Hashes of received file:" << std::endl;
+	    size_t hashes = 0;
 	    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 	    {
 	       std::string const tagname = std::string(*type) + "-Hash";
 	       std::string const hashsum = LookupTag(Message, tagname.c_str());
-	       if (hashsum.empty() == false)
+	       if (not hashsum.empty())
+	       {
 		  formatHashsum(out, HashString(*type, hashsum));
+		  ++hashes;
+	       }
 	    }
+	    if (hashes == 0)
+	       for (char const *const *type = HashString::SupportedHashes(); *type != nullptr; ++type)
+	       {
+		  std::string const tagname = std::string("Alt-") + *type + "-Hash";
+		  std::string const hashsum = LookupTag(Message, tagname.c_str());
+		  if (not hashsum.empty())
+		     formatHashsum(out, HashString(*type, hashsum));
+	       }
 	 }
 	 auto const lastmod = LookupTag(Message, "Last-Modified", "");
 	 if (lastmod.empty() == false)
@@ -1024,8 +1048,7 @@ void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const
 bool pkgAcquire::Item::VerifyDone(std::string const &Message,
 	 pkgAcquire::MethodConfig const * const /*Cnf*/)
 {
-   std::string const FileName = LookupTag(Message,"Filename");
-   if (FileName.empty() == true)
+   if (LookupTag(Message,"Filename").empty() && LookupTag(Message, "Alt-Filename").empty())
    {
       Status = StatError;
       ErrorText = "Method gave a blank filename";
@@ -1413,7 +1436,7 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st
    // verified yet)
 
    // Save the final base URI we got this Release file from
-   if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
+   if (not I->Local && not I->UsedMirror.empty() && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
    {
       auto InReleasePath = Target.Option(IndexTarget::INRELEASE_PATH);
       if (InReleasePath.empty())
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index cfea61cf6..70620163a 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -240,7 +240,7 @@ class APT_PUBLIC pkgAcquire::Item : public WeakPointable				/*{{{*/
    APT_HIDDEN bool PopAlternativeURI(std::string &NewURI);
    APT_HIDDEN bool IsGoodAlternativeURI(std::string const &AltUri) const;
    APT_HIDDEN void PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back);
-   APT_HIDDEN void RemoveAlternativeSite(std::string &&OldSite);
+   APT_HIDDEN void RemoveAlternativeSite(std::string const &AltUri);
 
    /** \brief A "descriptive" URI-like string.
     *
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 0f22f71e8..55897945c 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -20,6 +20,7 @@
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/hashes.h>
+#include <apt-pkg/metaindex.h>
 #include <apt-pkg/proxy.h>
 #include <apt-pkg/strutl.h>
 
@@ -434,11 +435,18 @@ bool pkgAcquire::Worker::RunMessages()
 		  Log->Pulse((*O)->GetOwner());
 
 	    HashStringList ReceivedHashes;
+	    bool AltFile = false;
 	    {
-	       std::string const givenfilename = LookupTag(Message, "Filename");
-	       std::string const filename = givenfilename.empty() ? Itm->Owner->DestFile : givenfilename;
+	       std::string filename;
+	       if (filename = LookupTag(Message, "Filename"); filename.empty())
+	       {
+		  if (filename = LookupTag(Message, "Alt-Filename"); not filename.empty())
+		     AltFile = true;
+		  else
+		     filename = Itm->Owner->DestFile;
+	       }
 	       // see if we got hashes to verify
-	       ReceivedHashes = GetHashesFromMessage("", Message);
+	       ReceivedHashes = GetHashesFromMessage(AltFile ? "Alt-" : "", Message);
 	       // not all methods always sent Hashes our way
 	       if (ReceivedHashes.usable() == false)
 	       {
@@ -449,64 +457,107 @@ bool pkgAcquire::Worker::RunMessages()
 		     FileFd file(filename, FileFd::ReadOnly, FileFd::None);
 		     calc.AddFD(file);
 		     ReceivedHashes = calc.GetHashStringList();
+		     for (auto const &h : ReceivedHashes)
+		     {
+			std::string tagname;
+			if (AltFile)
+			   tagname = "Alt-";
+			tagname.append(h.HashType()).append("-Hash");
+			if (not LookupTag(Message, tagname.c_str()).empty())
+			   continue;
+			if (not APT::String::Endswith(Message, "\n"))
+			   Message.append("\n");
+			Message.append(tagname).append(": ").append(h.HashValue());
+		     }
 		  }
 	       }
 
 	       // only local files can refer other filenames and counting them as fetched would be unfair
-	       if (Log != NULL && Itm->Owner->Complete == false && Itm->Owner->Local == false && givenfilename == filename)
+	       if (Log != nullptr && not Itm->Owner->Complete && not Itm->Owner->Local && not AltFile && Itm->Owner->DestFile == filename)
 		  Log->Fetched(ReceivedHashes.FileSize(),atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 	    }
 
 	    std::vector<Item*> const ItmOwners = Itm->Owners;
+	    for (auto const Owner : ItmOwners)
+	       Owner->ErrorText.clear();
 	    OwnerQ->ItemDone(Itm);
 	    Itm = NULL;
 
 	    bool const isIMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) ||
 	       StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false);
 	    auto const forcedHash = _config->Find("Acquire::ForceHash");
+
+	    bool consideredOkay = true;
+	    HashStringList ExpectedHashes;
+	    bool const DebugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
+	    if (DebugAuth)
+	    {
+	       std::clog << "201 URI Done: " << URI << endl
+			 << "ReceivedHash:" << endl;
+	       for (auto const &hs : ReceivedHashes)
+		  std::clog << "\t- " << hs.toStr() << std::endl;
+	       std::clog << "ExpectedHash:" << endl;
+	    }
 	    for (auto const Owner: ItmOwners)
 	    {
-	       HashStringList const ExpectedHashes = Owner->GetExpectedHashes();
-	       if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
-	       {
-		  std::clog << "201 URI Done: " << Owner->DescURI() << endl
-		     << "ReceivedHash:" << endl;
-		  for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs)
-		     std::clog <<  "\t- " << hs->toStr() << std::endl;
-		  std::clog << "ExpectedHash:" << endl;
-		  for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
-		     std::clog <<  "\t- " << hs->toStr() << std::endl;
-		  std::clog << endl;
-	       }
-
-	       // decide if what we got is what we expected
-	       bool consideredOkay = false;
-	       if ((forcedHash.empty() && ExpectedHashes.empty() == false) ||
-		     (forcedHash.empty() == false && ExpectedHashes.usable()))
-	       {
-		  if (ReceivedHashes.empty())
+	       HashStringList const OwnerExpectedHashes = [&]() {
+		  if (AltFile)
 		  {
-		     /* IMS-Hits can't be checked here as we will have uncompressed file,
-			but the hashes for the compressed file. What we have was good through
-			so all we have to ensure later is that we are not stalled. */
-		     consideredOkay = isIMSHit;
+		     auto const * const transOwner = dynamic_cast<pkgAcqTransactionItem const * const>(Owner);
+		     if (transOwner != nullptr && transOwner->TransactionManager != nullptr && transOwner->TransactionManager->MetaIndexParser != nullptr)
+		     {
+			auto const * const hashes = transOwner->TransactionManager->MetaIndexParser->Lookup(transOwner->Target.MetaKey);
+			if (hashes != nullptr)
+			   return hashes->Hashes;
+		     }
 		  }
-		  else if (ReceivedHashes == ExpectedHashes)
-		     consideredOkay = true;
-		  else
+		  return Owner->GetExpectedHashes();
+	       }();
+	       for (auto const &h : OwnerExpectedHashes)
+	       {
+		  if (not ExpectedHashes.push_back(h))
+		  {
 		     consideredOkay = false;
-
+		     std::clog << "\t- " << h.toStr() << " [conflict]" << std::endl;
+		  }
+		  else if (DebugAuth)
+		     std::clog << "\t- " << h.toStr() << std::endl;
 	       }
+	    }
+	    if (DebugAuth)
+	       std::clog << endl;
+
+	    // decide if what we got is what we expected
+	    if (not consideredOkay)
+	       ;
+	    else if ((forcedHash.empty() && not ExpectedHashes.empty()) ||
+		     (not forcedHash.empty() && ExpectedHashes.usable()))
+	    {
+	       if (ReceivedHashes.empty())
+	       {
+		  /* IMS-Hits can't be checked here as we will have uncompressed file,
+		     but the hashes for the compressed file. What we have was good through
+		     so all we have to ensure later is that we are not stalled. */
+		  consideredOkay = isIMSHit;
+	       }
+	       else if (ReceivedHashes == ExpectedHashes)
+		  consideredOkay = true;
 	       else
-		  consideredOkay = !Owner->HashesRequired();
-
-	       if (consideredOkay == true)
-		  consideredOkay = Owner->VerifyDone(Message, Config);
-	       else // hashsum mismatch
-		  Owner->Status = pkgAcquire::Item::StatAuthError;
+		  consideredOkay = false;
+	    }
+	    else
+	       consideredOkay = std::none_of(ItmOwners.begin(), ItmOwners.end(), [](auto const * const O) { return O->HashesRequired(); });
 
+	    bool otherReasons = false;
+	    if (consideredOkay && not std::all_of(ItmOwners.begin(), ItmOwners.end(), [&](auto * const O) { return O->VerifyDone(Message, Config); }))
+	    {
+	       consideredOkay = false;
+	       otherReasons = true;
+	    }
 
-	       if (consideredOkay == true)
+	    if (consideredOkay)
+	    {
+	       for (auto const Owner : ItmOwners)
 	       {
 		  if (isDoomedItem(Owner) == false)
 		     Owner->Done(Message, ReceivedHashes, Config);
@@ -518,22 +569,21 @@ bool pkgAcquire::Worker::RunMessages()
 			Log->Done(Owner->GetItemDesc());
 		  }
 	       }
+	    }
+	    else
+	    {
+	       if (otherReasons)
+		  HandleFailure(ItmOwners, Config, Log, Message, false, false);
 	       else
 	       {
-		  auto SavedDesc = Owner->GetItemDesc();
-		  if (isDoomedItem(Owner) == false)
+		  if (Message.find("\nFailReason:") == std::string::npos)
 		  {
-		     if (Message.find("\nFailReason:") == std::string::npos)
-		     {
-			if (ReceivedHashes != ExpectedHashes)
-			   Message.append("\nFailReason: HashSumMismatch");
-			else
-			   Message.append("\nFailReason: WeakHashSums");
-		     }
-		     Owner->Failed(Message,Config);
+		     if (ReceivedHashes != ExpectedHashes)
+			Message.append("\nFailReason: HashSumMismatch");
+		     else
+			Message.append("\nFailReason: WeakHashSums");
 		  }
-		  if (Log != nullptr)
-		     Log->Fail(SavedDesc);
+		  HandleFailure(ItmOwners, Config, Log, Message, false, true);
 	       }
 	    }
 	    ItemDone();
@@ -593,6 +643,8 @@ bool pkgAcquire::Worker::RunMessages()
 		  Log->Pulse((*O)->GetOwner());
 
 	    std::vector<Item*> const ItmOwners = Itm->Owners;
+	    for (auto const Owner : ItmOwners)
+	       Owner->ErrorText.clear();
 	    OwnerQ->ItemDone(Itm);
 	    Itm = nullptr;
 
@@ -684,7 +736,7 @@ void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &It
       else
       {
 	 if (errAuthErr)
-	    Owner->RemoveAlternativeSite(URI::SiteOnly(Owner->GetItemDesc().URI));
+	    Owner->RemoveAlternativeSite(Owner->GetItemDesc().URI);
 	 if (Owner->PopAlternativeURI(NewURI))
 	 {
 	    Owner->FailMessage(Message);
@@ -845,14 +897,12 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
    }
    Message += "\nFilename: " + Item->Owner->DestFile;
 
-   // FIXME: We should not hard code proxy protocols here.
-   if (URL.Access == "http" || URL.Access == "https")
+   // AutoDetectProxy() checks this already by itself, but we don't want to access unknown configs
+   if (CanURIBeAccessedViaProxy(URL))
    {
       AutoDetectProxy(URL);
-      if (_config->Exists("Acquire::" + URL.Access + "::proxy::" + URL.Host))
-      {
-	 Message += "\nProxy: " + _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host);
-      }
+      if (auto const proxy = _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host); not proxy.empty())
+	 Message.append("\nProxy: ").append(proxy);
    }
 
    HashStringList const hsl = Item->GetExpectedHashes();
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index f367e4310..07f64cc69 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -524,7 +524,7 @@ pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
    Configs = Conf;
 
    /* if a method uses DownloadLimit, we switch to SingleInstance mode */
-   if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
+   if (not Conf->SingleInstance && _config->FindI("Acquire::" + Access + "::Dl-Limit", 0) > 0)
       Conf->SingleInstance = true;
     
    return Conf;
diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index c9b2dc0cc..fff10fb17 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -55,12 +55,14 @@ Configuration *_config = new Configuration;
    but a Cnf-member – but that would need ABI breaks and stuff and for now
    that really is an apt-dev-only tool, so it isn't that bad that it is
    unusable and allaround a bit strange */
-enum class APT_HIDDEN ConfigType { UNDEFINED, INT, BOOL, STRING, STRING_OR_BOOL, STRING_OR_LIST, FILE, DIR, LIST, PROGRAM_PATH = FILE };
+enum class APT_HIDDEN ConfigType { INVALID, UNDEFINED, INT, BOOL, STRING, STRING_OR_BOOL, STRING_OR_LIST, FILE, DIR, LIST, PROGRAM_PATH = FILE };
 APT_HIDDEN std::unordered_map<std::string, ConfigType> apt_known_config {};
+APT_HIDDEN std::vector<std::pair<std::string, ConfigType>> apt_known_config_patterns {};
 static std::string getConfigTypeString(ConfigType const type)		/*{{{*/
 {
    switch (type)
    {
+      case ConfigType::INVALID: return "INVALID";
       case ConfigType::UNDEFINED: return "UNDEFINED";
       case ConfigType::INT: return "INT";
       case ConfigType::BOOL: return "BOOL";
@@ -94,6 +96,8 @@ static ConfigType getConfigType(std::string const &type)		/*{{{*/
       return ConfigType::STRING_OR_LIST;
    else if (type == "<PROGRAM_PATH>")
       return ConfigType::PROGRAM_PATH;
+   else if (type == "<INVALID>")
+      return ConfigType::INVALID;
    return ConfigType::UNDEFINED;
 }
 									/*}}}*/
@@ -101,59 +105,86 @@ static ConfigType getConfigType(std::string const &type)		/*{{{*/
 static void checkFindConfigOptionTypeInternal(std::string name, ConfigType const type)
 {
    std::transform(name.begin(), name.end(), name.begin(), ::tolower);
-   auto known = apt_known_config.find(name);
-   if (known == apt_known_config.cend())
+   bool found = false;
+   ConfigType expectedType = ConfigType::INVALID;
+   if (auto const known = apt_known_config.find(name); known != apt_known_config.cend())
    {
-      auto const rcolon = name.rfind(':');
-      if (rcolon != std::string::npos)
+      found = true;
+      expectedType = known->second;
+   }
+   else
+      for (auto const &known : apt_known_config_patterns)
       {
-	 known = apt_known_config.find(name.substr(0, rcolon) + ":*");
-	 if (known == apt_known_config.cend())
+	 std::string_view n{name};
+	 std::string_view k{known.first};
+	 do
 	 {
-	    auto const parts = StringSplit(name, "::");
-	    size_t psize = parts.size();
-	    if (psize > 1)
+	    if (auto star = k.find('*'); star != std::string_view::npos)
 	    {
-	       for (size_t max = psize; max != 1; --max)
-	       {
-		  std::ostringstream os;
-		  std::copy(parts.begin(), parts.begin() + max, std::ostream_iterator<std::string>(os, "::"));
-		  os << "**";
-		  known = apt_known_config.find(os.str());
-		  if (known != apt_known_config.cend() && known->second == ConfigType::UNDEFINED)
-		     return;
-	       }
-	       for (size_t max = psize - 1; max != 1; --max)
-	       {
-		  std::ostringstream os;
-		  std::copy(parts.begin(), parts.begin() + max - 1, std::ostream_iterator<std::string>(os, "::"));
-		  os << "*::";
-		  std::copy(parts.begin() + max + 1, parts.end() - 1, std::ostream_iterator<std::string>(os, "::"));
-		  os << *(parts.end() - 1);
-		  known = apt_known_config.find(os.str());
-		  if (known != apt_known_config.cend())
-		     break;
-	       }
+	       if (auto nextstar = star + 1; k.length() > nextstar && k[nextstar] == '*')
+		  star -= 3;
+	       if (n.compare(0, star, k, 0, star) != 0)
+		  break;
+	       n.remove_prefix(star);
+	       k.remove_prefix(star + 1);
 	    }
-	 }
+	    else if (k == n)
+	    {
+	       n = {};
+	       break;
+	    }
+	    else
+	       break;
+
+	    if (k.empty())
+	    {
+	       if (n.find("::") == std::string_view::npos)
+		  n = {};
+	       break;
+	    }
+	    if (k == "::**")
+	    {
+	       if (known.second == ConfigType::UNDEFINED)
+		  return;
+	       n = {};
+	       break;
+	    }
+	    if (k.compare(0, 2, "::") == 0)
+	    {
+	       auto const colons = n.find("::");
+	       if (colons == std::string_view::npos)
+		  break;
+	       n.remove_prefix(colons);
+	    }
+	    else
+	       break;
+	 } while (not n.empty());
+	 if (not n.empty())
+	    continue;
+	 found = true;
+	 expectedType = known.second;
+	 break;
       }
-   }
-   if (known == apt_known_config.cend())
+
+   if (not found)
       _error->Warning("Using unknown config option »%s« of type %s",
 	    name.c_str(), getConfigTypeString(type).c_str());
-   else if (known->second != type)
+   else if (expectedType != type)
    {
-      if (known->second == ConfigType::DIR && type == ConfigType::FILE)
+      if (expectedType == ConfigType::INVALID)
+	 _error->Warning("Using invalid config option »%s« as a type %s",
+	       name.c_str(), getConfigTypeString(type).c_str());
+      if (expectedType == ConfigType::DIR && type == ConfigType::FILE)
 	 ; // implementation detail
-      else if (type == ConfigType::STRING && (known->second == ConfigType::FILE || known->second == ConfigType::DIR))
+      else if (type == ConfigType::STRING && (expectedType == ConfigType::FILE || expectedType == ConfigType::DIR))
 	 ; // TODO: that might be an error or not, we will figure this out later
-      else if (known->second == ConfigType::STRING_OR_BOOL && (type == ConfigType::BOOL || type == ConfigType::STRING))
+      else if (expectedType == ConfigType::STRING_OR_BOOL && (type == ConfigType::BOOL || type == ConfigType::STRING))
 	 ;
-      else if (known->second == ConfigType::STRING_OR_LIST && (type == ConfigType::LIST || type == ConfigType::STRING))
+      else if (expectedType == ConfigType::STRING_OR_LIST && (type == ConfigType::LIST || type == ConfigType::STRING))
 	 ;
       else
 	 _error->Warning("Using config option »%s« of type %s as a type %s",
-	       name.c_str(), getConfigTypeString(known->second).c_str(), getConfigTypeString(type).c_str());
+	       name.c_str(), getConfigTypeString(expectedType).c_str(), getConfigTypeString(type).c_str());
    }
 }
 static void checkFindConfigOptionType(char const * const name, ConfigType const type)
@@ -166,6 +197,7 @@ static void checkFindConfigOptionType(char const * const name, ConfigType const
 static bool LoadConfigurationIndex(std::string const &filename)		/*{{{*/
 {
    apt_known_config.clear();
+   apt_known_config_patterns.clear();
    if (filename.empty())
       return true;
    Configuration Idx;
@@ -181,7 +213,10 @@ static bool LoadConfigurationIndex(std::string const &filename)		/*{{{*/
       {
 	 std::string fulltag = Top->FullTag();
 	 std::transform(fulltag.begin(), fulltag.end(), fulltag.begin(), ::tolower);
-	 apt_known_config.emplace(std::move(fulltag), getConfigType(Top->Value));
+	 if (fulltag.find("::*") == std::string::npos)
+	    apt_known_config.emplace(std::move(fulltag), getConfigType(Top->Value));
+	 else
+	    apt_known_config_patterns.emplace_back(std::move(fulltag), getConfigType(Top->Value));
       }
 
       if (Top->Child != nullptr)
diff --git a/apt-pkg/contrib/proxy.cc b/apt-pkg/contrib/proxy.cc
index a99f44f49..9a9a9f29c 100644
--- a/apt-pkg/contrib/proxy.cc
+++ b/apt-pkg/contrib/proxy.cc
@@ -12,9 +12,11 @@
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
+#include <apt-pkg/string_view.h>
 #include <apt-pkg/strutl.h>
 
 #include <algorithm>
+#include <array>
 #include <iostream>
 #include <fcntl.h>
 #include <unistd.h>
@@ -22,6 +24,16 @@
 #include "proxy.h"
 									/*}}}*/
 
+bool CanURIBeAccessedViaProxy(URI const &URL)				/*{{{*/
+{
+   // for some methods a proxy doesn't make sense, so we don't have to fork
+   if (URL.Host.empty() ||
+       APT::String::Startswith(URL.Access, "mirror+") || URL.Access.find("+mirror+") != std::string::npos || APT::String::Endswith(URL.Access, "+mirror"))
+      return false;
+   std::array const noproxy{"file", "copy", "store", "gpgv", "rred", "cdrom", "mirror"};
+   return std::find(noproxy.begin(), noproxy.end(), URL.Access) == noproxy.end();
+}
+									/*}}}*/
 // AutoDetectProxy - auto detect proxy					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -29,16 +41,19 @@ static std::vector<std::string> CompatibleProxies(URI const &URL)
 {
    if (URL.Access == "http" || URL.Access == "https")
       return {"http", "https", "socks5h"};
-   return {URL.Access};
+   if (URL.Access == "tor" || URL.Access == "tor+http" || URL.Access == "tor+https")
+      return {"socks5h"};
+   if (URL.Access == "ftp")
+      return {"ftp"};
+   return {};
 }
-
 bool AutoDetectProxy(URI &URL)
 {
-   // we support both http/https debug options
-   bool Debug = _config->FindB("Debug::Acquire::"+URL.Access,false);
+   if (not CanURIBeAccessedViaProxy(URL))
+      return true;
 
    // the user already explicitly set a proxy for this host
-   if(_config->Find("Acquire::"+URL.Access+"::proxy::"+URL.Host, "") != "")
+   if (not _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host, "").empty())
       return true;
 
    // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old
@@ -49,6 +64,7 @@ bool AutoDetectProxy(URI &URL)
    if (AutoDetectProxyCmd.empty())
       return true;
 
+   bool const Debug = _config->FindB("Debug::Acquire::" + URL.Access, false);
    if (Debug)
       std::clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << std::endl;
 
@@ -73,7 +89,7 @@ bool AutoDetectProxy(URI &URL)
    // and apt will use the generic proxy settings
    if (goodread == false)
       return true;
-   auto const cleanedbuf = _strstrip(buf);
+   APT::StringView const cleanedbuf = _strstrip(buf);
    // We warn about this as the implementor probably meant to use DIRECT instead
    if (cleanedbuf[0] == '\0')
    {
@@ -82,17 +98,25 @@ bool AutoDetectProxy(URI &URL)
    }
 
    if (Debug)
-      std::clog << "auto detect command returned: '" << cleanedbuf << "'" << std::endl;
+      std::clog << "auto detect command returned: '" << cleanedbuf.data() << "'" << std::endl;
 
-   auto compatibleTypes = CompatibleProxies(URL);
-   bool compatible = strcmp(cleanedbuf, "DIRECT") == 0 ||
-		     compatibleTypes.end() != std::find_if(compatibleTypes.begin(),
-							   compatibleTypes.end(), [cleanedbuf](std::string &compat) {
-							      return strstr(cleanedbuf, compat.c_str()) == cleanedbuf;
-							   });
+   bool compatible = true;
+   if (cleanedbuf != "DIRECT")
+   {
+      if (auto const compatibleTypes = CompatibleProxies(URL); not compatibleTypes.empty())
+	 compatible = std::any_of(compatibleTypes.begin(), compatibleTypes.end(),
+				  [cleanedbuf](std::string const &compat)
+				  {
+				     return cleanedbuf.substr(0, compat.size()) == compat;
+				  });
+   }
+   else if (URL.Access == "tor" || URL.Access == "tor+http" || URL.Access == "tor+https")
+      compatible = false; // Accepting DIRECT would silently disable tor
 
    if (compatible)
-      _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf);
+      _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf.data());
+   else
+      _error->Warning("ProxyAutoDetect command returned incompatible proxy '%s' for access type %s", cleanedbuf.data(), URL.Access.c_str());
 
    return true;
 }
diff --git a/apt-pkg/contrib/proxy.h b/apt-pkg/contrib/proxy.h
index f6d70ea8b..2a4c19fc8 100644
--- a/apt-pkg/contrib/proxy.h
+++ b/apt-pkg/contrib/proxy.h
@@ -9,8 +9,10 @@
 #ifndef PKGLIB_PROXY_H
 #define PKGLIB_PROXY_H
 
+#include <apt-pkg/macros.h>
+
 class URI;
 APT_PUBLIC bool AutoDetectProxy(URI &URL);
-
+APT_HIDDEN bool CanURIBeAccessedViaProxy(URI const &URL);
 
 #endif
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index 6723a48da..9623514b8 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -184,18 +184,16 @@ APT
      Error-Mode "<STRING>";
   };
 
-  /* define a new supported compressor on the fly
-  Compressor::rev {
-     Name "rev";
-     Extension ".reversed";
-     Binary "rev";
-     CompressArg {};
-     UncompressArg {};
-     Cost "10";
-  };
-  */
   Compressor "<LIST>";
-  Compressor::** "<UNDEFINED>";
+  // define a new compressor on the fly, like the super-compressor: /usr/bin/rev
+  Compressor::* {
+     Name "<STRING>"; // rev
+     Extension "<STRING>"; // .reversed
+     Binary "<STRING>"; // rev
+     CompressArg "<LIST>"; // {}
+     UncompressArg "<LIST>"; // {}
+     Cost "<INT>"; // 10
+  };
 
   Authentication
   {
@@ -584,6 +582,9 @@ Debug
   Acquire::Ftp "<BOOL>";    // Show ftp command traffic
   Acquire::Http "<BOOL>";   // Show http command traffic
   Acquire::Https "<BOOL>";   // Show https debug
+  Acquire::tor "<BOOL>";
+  Acquire::tor+http "<BOOL>";
+  Acquire::tor+https "<BOOL>";
   Acquire::gpgv "<BOOL>";   // Show the gpgv traffic
   Acquire::cdrom "<BOOL>";   // Show cdrom debug output
   Acquire::Transaction "<BOOL>";
@@ -819,17 +820,44 @@ acquire::max-pipeline-depth "<INT>";
 acquire::progress::diffpercent "<BOOL>";
 acquire::gzipindexes "<BOOL>";
 acquire::indextargets::randomized "<BOOL>";
-acquire::indextargets::deb::** "<UNDEFINED>";
-acquire::indextargets::deb-src::** "<UNDEFINED>";
+acquire::indextargets::* "<LIST>" {
+	* {
+		MetaKey "<STRING>";
+		ShortDescription "<STRING>";
+		Description "<STRING>";
+		flatMetaKey "<STRING>";
+		flatDescription "<STRING>";
+		Identifier "<STRING>";
+		DefaultEnabled "<BOOL>";
+		Optional "<BOOL>";
+		KeepCompressed "<BOOL>";
+		PDiffs "<BOOL>";
+		By-Hash "<STRING>";
+		Fallback-Of "<STRING>";
+		CompressionTypes "<STRING>";
+		KeepCompressedAs "<STRING>";
+	};
+};
 acquire::progress::ignore::showerrortext "<BOOL>";
 acquire::*::dl-limit "<INT>"; // catches file: and co which do not have these
+acquire::file::dl-limit "<INVALID>";
+acquire::copy::dl-limit "<INVALID>";
+acquire::gpgv::dl-limit "<INVALID>";
+acquire::store::dl-limit "<INVALID>";
+acquire::mirror::dl-limit "<INVALID>";
 methods::mirror::problemreporting "<STRING>";
-acquire::http::proxyautodetect "<STRING>";
-acquire::http::proxy-auto-detect "<STRING>";
-acquire::http::proxy::* "<STRING>";
-acquire::https::proxyautodetect "<STRING>";
-acquire::https::proxy-auto-detect "<STRING>";
-acquire::https::proxy::* "<STRING>";
+acquire::*::proxyautodetect "<STRING>";
+acquire::file::proxyautodetect "<INVALID>";
+acquire::copy::proxyautodetect "<INVALID>";
+acquire::store::proxyautodetect "<INVALID>";
+acquire::*::proxy-auto-detect "<STRING>";
+acquire::file::proxy-auto-detect "<INVALID>";
+acquire::copy::proxy-auto-detect "<INVALID>";
+acquire::store::proxy-auto-detect "<INVALID>";
+acquire::file::proxy::* "<INVALID>";
+acquire::copy::proxy::* "<INVALID>";
+acquire::store::proxy::* "<INVALID>";
+acquire::*::proxy::* "<STRING>";
 
 // Options used by apt-ftparchive
 dir::archivedir "<DIR>";
@@ -900,6 +928,7 @@ APT::Internal::OpProgress::EraseLines "<BOOL>";
 APT::Color "<BOOL>";
 APT::Color::Show::Field "<STRING>";
 APT::Color::Show::Package "<STRING>";
+APT::Color::* "<STRING>";
 
 update-manager::always-include-phased-updates "<BOOL>";
 update-manager::never-include-phased-updates "<BOOL>";
diff --git a/methods/aptmethod.h b/methods/aptmethod.h
index 1c24f3a98..1ea173075 100644
--- a/methods/aptmethod.h
+++ b/methods/aptmethod.h
@@ -514,6 +514,17 @@ protected:
       return part;
    }
 
+   static std::string CombineWithAlternatePath(std::string Path, std::string Change)
+   {
+      while (APT::String::Startswith(Change, "../"))
+      {
+	 Path.erase(Path.find_last_not_of('/'));
+	 Path = flNotFile(Path);
+	 Change.erase(0, 3);
+      }
+      return flCombine(Path, Change);
+   }
+
    aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3)
        : pkgAcqMethod(Ver, Flags), aptConfigWrapperForMethods(Binary), Binary(std::move(Binary)), SeccompFlags(0)
    {
diff --git a/methods/copy.cc b/methods/copy.cc
index 44205e4cd..9c9c6572d 100644
--- a/methods/copy.cc
+++ b/methods/copy.cc
@@ -26,7 +26,7 @@
 
 class CopyMethod final : public aptMethod
 {
-   virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+   bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
 
    public:
    CopyMethod() : aptMethod("copy", "1.0", SingleInstance | SendConfig | SendURIEncoded)
@@ -36,55 +36,71 @@ class CopyMethod final : public aptMethod
 };
 
 // CopyMethod::Fetch - Fetch a file					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool CopyMethod::Fetch(FetchItem *Itm)
+bool CopyMethod::URIAcquire(std::string const &Message, FetchItem *Itm)
 {
-   // this ensures that relative paths work in copy
-   std::string const File = DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1));
-
-   // Stat the file and send a start message
-   struct stat Buf;
-   if (stat(File.c_str(),&Buf) != 0)
-      return _error->Errno("stat",_("Failed to stat"));
+   struct FileCopyType {
+      std::string name;
+      struct stat stat{};
+      explicit FileCopyType(std::string &&file) : name{std::move(file)} {}
+   };
+   std::vector<FileCopyType> files;
+   // this ensures that relative paths work
+   files.emplace_back(DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1)));
+   for (auto const &AltPath : VectorizeString(LookupTag(Message, "Alternate-Paths"), '\n'))
+      files.emplace_back(CombineWithAlternatePath(flNotFile(files[0].name), DecodeSendURI(AltPath)));
+   files.erase(std::remove_if(files.begin(), files.end(), [](auto &file)
+			      { return stat(file.name.c_str(), &file.stat) != 0; }),
+	       files.end());
+   if (files.empty())
+      return _error->Errno("copy-stat", _("Failed to stat"));
 
    // Forumulate a result and send a start message
    FetchResult Res;
-   Res.Size = Buf.st_size;
    Res.Filename = Itm->DestFile;
-   Res.LastModified = Buf.st_mtime;
    Res.IMSHit = false;
-   URIStart(Res);
 
-   // just calc the hashes if the source and destination are identical
-   if (File == Itm->DestFile || Itm->DestFile == "/dev/null")
+   for (auto const &File : files)
    {
+      Res.Size = File.stat.st_size;
+      Res.LastModified = File.stat.st_mtime;
+
+      // just calc the hashes if the source and destination are identical
+      if (Itm->DestFile == "/dev/null" || File.name == Itm->DestFile)
+      {
+	 URIStart(Res);
+	 CalculateHashes(Itm, Res);
+	 URIDone(Res);
+	 return true;
+      }
+
+      FileFd From(File.name, FileFd::ReadOnly);
+      FileFd To(Itm->DestFile, FileFd::WriteAtomic);
+      To.EraseOnFailure();
+      if (not From.IsOpen() || not To.IsOpen())
+	 continue;
+
+      // Copy the file
+      URIStart(Res);
+      if (not CopyFile(From, To))
+      {
+	 To.OpFail();
+	 continue;
+      }
+      From.Close();
+      To.Close();
+
       CalculateHashes(Itm, Res);
-      URIDone(Res);
-      return true;
-   }
+      if (not Itm->ExpectedHashes.empty() && Itm->ExpectedHashes != Res.Hashes)
+	 continue;
 
-   // See if the file exists
-   FileFd From(File,FileFd::ReadOnly);
-   FileFd To(Itm->DestFile,FileFd::WriteAtomic);
-   To.EraseOnFailure();
+      if (not TransferModificationTimes(File.name.c_str(), Res.Filename.c_str(), Res.LastModified))
+	 continue;
 
-   // Copy the file
-   if (CopyFile(From,To) == false)
-   {
-      To.OpFail();
-      return false;
+      URIDone(Res);
+      return true;
    }
 
-   From.Close();
-   To.Close();
-
-   if (TransferModificationTimes(File.c_str(), Res.Filename.c_str(), Res.LastModified) == false)
-      return false;
-
-   CalculateHashes(Itm, Res);
-   URIDone(Res);
-   return true;
+   return false;
 }
 									/*}}}*/
 
diff --git a/methods/file.cc b/methods/file.cc
index 0a76c0c94..111440ce1 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -29,7 +29,7 @@
 
 class FileMethod final : public aptMethod
 {
-   virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+   bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
 
    public:
    FileMethod() : aptMethod("file", "1.0", SingleInstance | SendConfig | LocalOnly | SendURIEncoded)
@@ -41,86 +41,101 @@ class FileMethod final : public aptMethod
 // FileMethod::Fetch - Fetch a file					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool FileMethod::Fetch(FetchItem *Itm)
+bool FileMethod::URIAcquire(std::string const &Message, FetchItem *Itm)
 {
    URI Get(Itm->Uri);
-   std::string const File = DecodeSendURI(Get.Path);
-   FetchResult Res;
    if (Get.Host.empty() == false)
       return _error->Error(_("Invalid URI, local URIS must not start with //"));
 
-   struct stat Buf;
+   std::vector<std::string> Files;
+   Files.emplace_back(DecodeSendURI(Get.Path));
+   for (auto const &AltPath : VectorizeString(LookupTag(Message, "Alternate-Paths"), '\n'))
+      Files.emplace_back(CombineWithAlternatePath(flNotFile(Files[0]), DecodeSendURI(AltPath)));
+
+   FetchResult Res;
    // deal with destination files which might linger around
-   if (lstat(Itm->DestFile.c_str(), &Buf) == 0)
+   struct stat Buf;
+   if (lstat(Itm->DestFile.c_str(), &Buf) == 0 && S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
+   {
+      char name[Buf.st_size + 1];
+      if (ssize_t const sp = readlink(Itm->DestFile.c_str(), name, Buf.st_size); sp == -1)
+      {
+	 Itm->LastModified = 0;
+	 RemoveFile("file", Itm->DestFile);
+      }
+   }
+
+   int olderrno = 0;
+   // See if the file exists
+   for (auto const &File : Files)
    {
-      if ((Buf.st_mode & S_IFREG) != 0)
+      if (stat(File.c_str(), &Buf) == 0)
       {
+	 Res.Size = Buf.st_size;
+	 Res.Filename = File;
+	 Res.LastModified = Buf.st_mtime;
+	 Res.IMSHit = false;
 	 if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
 	 {
-	    if (Itm->ExpectedHashes.VerifyFile(File))
-	    {
-	       Res.Filename = Itm->DestFile;
+	    auto const filesize = Itm->ExpectedHashes.FileSize();
+	    if (filesize != 0 && filesize == Res.Size)
 	       Res.IMSHit = true;
-	    }
 	 }
+	 break;
       }
+      if (olderrno == 0)
+	 olderrno = errno;
    }
-   if (Res.IMSHit != true)
-      RemoveFile("file", Itm->DestFile);
 
-   int olderrno = 0;
-   // See if the file exists
-   if (stat(File.c_str(),&Buf) == 0)
+   if (not Res.IMSHit)
    {
-      Res.Size = Buf.st_size;
-      Res.Filename = File;
-      Res.LastModified = Buf.st_mtime;
-      Res.IMSHit = false;
-      if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+      RemoveFile("file", Itm->DestFile);
+      if (not Res.Filename.empty())
       {
-	 unsigned long long const filesize = Itm->ExpectedHashes.FileSize();
-	 if (filesize != 0 && filesize == Res.Size)
-	    Res.IMSHit = true;
+	 URIStart(Res);
+	 CalculateHashes(Itm, Res);
       }
-
-      CalculateHashes(Itm, Res);
    }
-   else
-      olderrno = errno;
-   if (Res.IMSHit == false)
-      URIStart(Res);
 
    // See if the uncompressed file exists and reuse it
    FetchResult AltRes;
-   AltRes.Filename.clear();
-   std::vector<std::string> extensions = APT::Configuration::getCompressorExtensions();
-   for (std::vector<std::string>::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext)
+   for (auto const &File : Files)
    {
-      if (APT::String::Endswith(File, *ext) == true)
+      for (const auto &ext : APT::Configuration::getCompressorExtensions())
       {
-	 std::string const unfile = File.substr(0, File.length() - ext->length());
-	 if (stat(unfile.c_str(),&Buf) == 0)
+	 if (APT::String::Endswith(File, ext))
 	 {
-	    AltRes.Size = Buf.st_size;
-	    AltRes.Filename = unfile;
-	    AltRes.LastModified = Buf.st_mtime;
-	    AltRes.IMSHit = false;
-	    if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
-	       AltRes.IMSHit = true;
-	    break;
+	    std::string const unfile = File.substr(0, File.length() - ext.length());
+	    if (stat(unfile.c_str(), &Buf) == 0)
+	    {
+	       AltRes.Size = Buf.st_size;
+	       AltRes.Filename = unfile;
+	       AltRes.LastModified = Buf.st_mtime;
+	       AltRes.IMSHit = false;
+	       if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+		  AltRes.IMSHit = true;
+	       if (Res.Filename.empty())
+	       {
+		  URIStart(Res);
+		  CalculateHashes(Itm, AltRes);
+	       }
+	       break;
+	    }
+	    // no break here as we could have situations similar to '.gz' vs '.tar.gz' here
 	 }
-	 // no break here as we could have situations similar to '.gz' vs '.tar.gz' here
       }
+      if (not AltRes.Filename.empty())
+	 break;
    }
 
-   if (AltRes.Filename.empty() == false)
+   if (not AltRes.Filename.empty())
       URIDone(Res,&AltRes);
-   else if (Res.Filename.empty() == false)
+   else if (not Res.Filename.empty())
       URIDone(Res);
    else
    {
       errno = olderrno;
-      return _error->Errno(File.c_str(), _("File not found"));
+      return _error->Errno(Files[0].c_str(), _("File not found"));
    }
 
    return true;
diff --git a/methods/mirror.cc b/methods/mirror.cc
index 787e4c7d5..5781ebcf0 100644
--- a/methods/mirror.cc
+++ b/methods/mirror.cc
@@ -326,7 +326,11 @@ std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem
    {
       if (APT::String::Startswith(Itm->Uri, uristr))
       {
-	 uristr.erase(uristr.length() - 1); // remove the ending '/'
+	 if (::URI uri{uristr}; uri.Path.length() > 1 && APT::String::Endswith(uri.Path, "/"))
+	 {
+	    uri.Path.erase(uri.Path.length() - 1); // remove the ending '/'
+	    uristr = uri;
+	 }
 	 auto const colon = uristr.find(':');
 	 if (unlikely(colon == std::string::npos))
 	    continue;
@@ -375,7 +379,10 @@ bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*
       msgCache[Itm->Uri] = Message;
       MirrorListInfo info;
       info.state = REQUESTED;
-      info.baseuri = mirrorfileuri + '/';
+      if (not APT::String::Endswith(mirrorfileuri, "/"))
+	 info.baseuri = mirrorfileuri + '/';
+      else
+	 info.baseuri = mirrorfileuri;
       auto const colon = info.baseuri.find(':');
       if (unlikely(colon == std::string::npos))
 	 return false;
diff --git a/test/integration/test-apt-by-hash-update b/test/integration/test-apt-by-hash-update
index 65c3766d0..a33eb9117 100755
--- a/test/integration/test-apt-by-hash-update
+++ b/test/integration/test-apt-by-hash-update
@@ -13,6 +13,7 @@ insertpackage 'unstable' 'foo' 'all' '1.0'
 insertpackage 'unstable' 'bar' 'i386' '1.0'
 
 setupaptarchive --no-update
+changetowebserver
 
 # make Packages *only* accessible by-hash for this test
 makebyhashonly() {
@@ -100,8 +101,7 @@ msgmsg 'Test InRelease by-hash with' 'no fallback'
 rm -rf aptarchive/dists
 cp -a aptarchive/dists.bak aptarchive/dists
 
-testfailureequal "Get:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease
-Err:1 file:${TMPWORKINGDIRECTORY}/aptarchive unstable InRelease
-  File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)
+testfailureequal "Err:1 http://localhost:${APTHTTPPORT} unstable InRelease
+  404  Not Found
 Reading package lists...
-E: Failed to fetch file:${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/InRelease  File not found - ${TMPWORKINGDIRECTORY}/aptarchive/dists/unstable/by-hash/SHA256/${inrelease_hash} (2: No such file or directory)" aptget update
+E: Failed to fetch http://localhost:${APTHTTPPORT}/dists/unstable/InRelease  404  Not Found" aptget update
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 42c4e5a9d..9f252ad9d 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -28,9 +28,7 @@ testwarning aptget update --allow-insecure-repositories
 rm -rf rootdir/var/lib/apt/lists
 find "$APTARCHIVE/dists/unstable" -name '*Release*' -delete
 # update without authenticated files leads to warning
-testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease
-Ign:1 file:$APTARCHIVE unstable InRelease
-Get:2 file:$APTARCHIVE unstable Release
+testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease
 Err:2 file:$APTARCHIVE unstable Release
   File not found - ${APTARCHIVE}/dists/unstable/Release (2: No such file or directory)
 Reading package lists...
@@ -44,46 +42,13 @@ testequal 'auxfiles
 lock
 partial' ls rootdir/var/lib/apt/lists
 
-filesize() {
-	local CREATEDBY="$1"
-	shift
-	stat -c%s "/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 2- ).gz"
-}
 # allow override
-#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1
-#exit
-testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease
-Ign:1 file:$APTARCHIVE unstable InRelease
-Get:2 file:$APTARCHIVE unstable Release
+testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease
 Ign:2 file:$APTARCHIVE unstable Release
 Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
 Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
 Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
 Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
-Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
-Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
-Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
-Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
-Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
-Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
-Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B]
-Get:5 file:$APTARCHIVE unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B]
-Get:6 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
 Reading package lists...
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file.
 N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use.
@@ -92,3 +57,38 @@ N: See apt-secure(8) manpage for repository creation and user configuration deta
 testfailureequal "WARNING: The following packages cannot be authenticated!
   foo
 E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo
+
+rm -rf rootdir/var/lib/apt/lists
+changetowebserver
+
+filesize() {
+	local CREATEDBY="$1"
+	shift
+	stat -c%s "${APTARCHIVE}/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 4- ).gz"
+}
+testwarningequal "Ign:1 http://localhost:$APTHTTPPORT unstable InRelease
+Ign:2 http://localhost:$APTHTTPPORT unstable Release
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Get:3 http://localhost:$APTHTTPPORT unstable/main Sources [$(filesize 'Sources') B]
+Get:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B]
+Get:5 http://localhost:$APTHTTPPORT unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B]
+Get:6 http://localhost:$APTHTTPPORT unstable/main Translation-en [$(filesize 'Translations') B]
+Reading package lists...
+W: The repository 'http://localhost:$APTHTTPPORT unstable Release' does not have a Release file.
+N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use.
+N: See apt-secure(8) manpage for repository creation and user configuration details." aptget update --allow-insecure-repositories
+# ensure we can not install the package
+testfailureequal "WARNING: The following packages cannot be authenticated!
+  foo
+E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index ae1ca7456..4f883b440 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -74,10 +74,11 @@ E: Download Failed"
 setupproxydetect() {
 	local METH="$1"
 	shift
-	{
-		echo '#!/bin/sh -e'
-		echo "$@"
-	} > "${TMPWORKINGDIRECTORY}/apt-proxy-detect"
+	cat >"${TMPWORKINGDIRECTORY}/apt-proxy-detect" <<EOF
+#!/bin/sh
+set -e
+$*
+EOF
 	chmod 755 "${TMPWORKINGDIRECTORY}/apt-proxy-detect"
 	echo "Acquire::${METH}::${CONFNAME} \"${TMPWORKINGDIRECTORY}/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect
 }
@@ -97,15 +98,38 @@ W: ProxyAutoDetect command returned an empty line" apthelper auto-detect-proxy h
     chmod -x "${TMPWORKINGDIRECTORY}/apt-proxy-detect"
     testfailureequal "E: ProxyAutoDetect command '${TMPWORKINGDIRECTORY}/apt-proxy-detect' can not be executed! - access (13: Permission denied)" apthelper auto-detect-proxy http://example.com/
 
-    msgmsg "apt-helper $CONFNAME" 'http proxy'
-    setupproxydetect 'http' 'echo "http://some-proxy"'
-    testsuccessequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com
-    testsuccessequal "Using proxy '' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com
+    for meth in 'http' 'https'; do
+	msgmsg "apt-helper $CONFNAME" "${meth} proxy"
+	for type in 'http' 'https' 'socks5h'; do
+		setupproxydetect "$meth" "echo '${type}://some-proxy'"
+		testsuccessequal "Using proxy '${type}://some-proxy' for URL '${meth}://www.example.com/'" apthelper auto-detect-proxy "${meth}://www.example.com"
+		if [ "$meth" = 'http' ]; then
+			testsuccessequal "Using proxy '' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy 'https://ssl.example.com'
+		else
+			testsuccessequal "Using proxy '' for URL 'http://no-ssl.example.com/'" apthelper auto-detect-proxy 'http://no-ssl.example.com'
+		fi
+	done
+    done
+
+    msgmsg "apt-helper $CONFNAME" 'no strange proxy'
+    for meth in gpgv copy store file; do
+	setupproxydetect "$meth" "echo '${meth}://some-proxy'"
+	testsuccessequal "Using proxy '' for URL '${meth}:/foo/bar'" apthelper auto-detect-proxy "${meth}:/foo/bar"
+	testsuccessequal "Using proxy '' for URL '${meth}://./foo/bar'" apthelper auto-detect-proxy "${meth}://./foo/bar"
+    done
+    for url in 'cdrom://Moo Cow Rom/debian' 'cdrom://[The Debian 1.2 disk, 1/2 R16]/debian/'; do
+	setupproxydetect "${url%%:*}" "echo 'cdrom://some-proxy'"
+	testsuccessequal "Using proxy '' for URL '${url}'" apthelper auto-detect-proxy "$url"
+    done
+    for meth in tor tor+http tor+https; do
+	setupproxydetect "$meth" "echo 'socks5h://some-proxy'"
+	testsuccessequal "Using proxy 'socks5h://some-proxy' for URL '${meth}://example.org/'" apthelper auto-detect-proxy "${meth}://example.org"
+
+	setupproxydetect "$meth" "echo 'DIRECT'"
+	testwarningequal "Using proxy '' for URL '${meth}://example.org/'
+W: ProxyAutoDetect command returned incompatible proxy 'DIRECT' for access type ${meth}" apthelper auto-detect-proxy "${meth}://example.org"
+    done
 
-    msgmsg "apt-helper $CONFNAME" 'https proxy'
-    setupproxydetect 'https' 'echo "https://https-proxy"'
-    testsuccessequal "Using proxy '' for URL 'http://no-ssl.example.com/'" apthelper auto-detect-proxy http://no-ssl.example.com
-    testsuccessequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com
     rm -f rootdir/etc/apt/apt.conf.d/02proxy-detect "${TMPWORKINGDIRECTORY}/apt-proxy-detect"
 }
 
diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror
new file mode 100755
index 000000000..dec9383c9
--- /dev/null
+++ b/test/integration/test-apt-update-incomplete-file-mirror
@@ -0,0 +1,68 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture "amd64"
+configcompression 'gz'
+
+insertpackage 'unstable' 'foo' 'amd64' '1'
+
+setupaptarchive --no-update
+
+testfailure aptcache show foo
+
+# the Release file says gz available, but the mirror has only uncompressed files
+find aptarchive/dists -name '*.gz' -delete
+
+testsuccess apt update
+testsuccess aptcache show foo
+testfailure aptcache show bar
+
+testsuccess apt update
+
+rm -rf rootdir/var/lib/apt/lists
+
+
+msgmsg 'File mirror was hacked'
+mkdir aptarchive2
+cp -a aptarchive/dists aptarchive2/
+rm -rf rootdir/var/lib/apt/lists
+find aptarchive/dists -name 'Packages' | while read FILE; do
+	echo 'hacked' > $FILE
+done
+testfailure apt update -o Debug::pkgAcquire::Worker=1
+testsuccessequal '4' grep -c -- '- Filesize:' rootdir/tmp/testfailure.output
+testsuccessequal '2' grep -c '%0aAlt-Checksum-FileSize-Hash:%20' rootdir/tmp/testfailure.output
+
+echo 'Acquire::By-Hash "force";' > rootdir/etc/apt/apt.conf.d/99force-by-hash.conf
+
+msgmsg 'Fallback over hashsum errors'
+rm -f rootdir/etc/apt/sources.list rootdir/etc/apt/sources.list.d/*
+echo "deb mirror+file:${TMPWORKINGDIRECTORY}/mirror.list unstable main" > rootdir/etc/apt/sources.list
+rm -rf rootdir/var/lib/apt/lists
+cat > mirror.list <<EOF
+copy:${TMPWORKINGDIRECTORY}/aptarchive	priority:1
+file:${TMPWORKINGDIRECTORY}/aptarchive2	priority:2
+EOF
+testsuccess apt update
+cp -a rootdir/tmp/testsuccess.output aptupdate.log
+testsuccessequal '2' grep -c -- '^Ign:' aptupdate.log
+
+rm -rf rootdir/var/lib/apt/lists
+cat > mirror.list <<EOF
+file:${TMPWORKINGDIRECTORY}/aptarchive	priority:1
+copy:${TMPWORKINGDIRECTORY}/aptarchive2	priority:2
+EOF
+testsuccess apt update
+
+rm -rf rootdir/var/lib/apt/lists
+cat > mirror.list <<EOF
+file:${TMPWORKINGDIRECTORY}/aptarchive	priority:1
+file:${TMPWORKINGDIRECTORY}/aptarchive2	priority:2
+EOF
+testsuccess apt update
+cp -a rootdir/tmp/testsuccess.output aptupdate.log
+testsuccessequal '1' grep -c -- '^Ign:' aptupdate.log
diff --git a/test/integration/test-method-mirror b/test/integration/test-method-mirror
index ce44b86e3..79c675455 100755
--- a/test/integration/test-method-mirror
+++ b/test/integration/test-method-mirror
@@ -124,9 +124,22 @@ msgmsg 'all mirrored via file'
 APTARCHIVE="$(readlink -f ./aptarchive)"
 sed -i -e "s#mirror+http://localhost:${APTHTTPPORT}#mirror+file:${APTARCHIVE}#" rootdir/etc/apt/sources.list.d/*
 testrun '*_localhost_*' '*_aptarchive_mirror.txt.gz_*'
+sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/*
+
+mv rootdir/etc/apt/sources.list.d rootdir/etc/apt/sources.list.d.bak
+mkdir rootdir/etc/apt/sources.list.d
+msgmsg 'fail gracefully if mirror uri has no filename'
+echo "deb mirror://localhost:${APTHTTPPORT}/ stable main" > rootdir/etc/apt/sources.list.d/mirrordir.list
+testfailure apt update
+
+msgmsg 'but succeed if it is indeed a mirror list'
+ln -s mirror.txt aptarchive/index.html
+testsuccess apt update
+rm aptarchive/index.html rootdir/etc/apt/sources.list.d/mirrordir.list
+rmdir rootdir/etc/apt/sources.list.d
+mv rootdir/etc/apt/sources.list.d.bak rootdir/etc/apt/sources.list.d
 
 msgmsg 'fallback mirrors are used if needed' 'as usual'
-sed -i -e 's#/mirror\.txt\.gz stable#/mirror.txt stable#' rootdir/etc/apt/sources.list.d/*
 echo "http://localhost:${APTHTTPPORT}/failure2	priority:3
 http://localhost:${APTHTTPPORT}/redirectme	priority:2
 http://localhost:${APTHTTPPORT}/failure	priority:1" > aptarchive/mirror.txt
@@ -242,7 +255,6 @@ echo "file:/nonexistent/apt/archive	priority:1
 http://localhost:${APTHTTPPORT}/redirectme
 " > aptarchive/mirror.txt
 testsuccessequal "Get:1 foo+file:${APTARCHIVE}/mirror.txt Mirrorlist [$(stat -c%s 'aptarchive/mirror.txt') B]
-Get:2 foo+file:/nonexistent/apt/archive unstable InRelease
 Ign:2 foo+file:/nonexistent/apt/archive unstable InRelease
   File not found - /nonexistent/apt/archive/dists/unstable/InRelease (2: No such file or directory)
 Hit:2 foo+http://localhost:${APTHTTPPORT}/redirectme unstable InRelease