Merge branch 'fix/partialfilemirror' into 'main'

Support uncompressed indexes from partial file:/ mirrors

See merge request apt-team/apt!235

7 files changed, 260 insertions, 126 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 73e9940ee..12524778e 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -32,7 +32,9 @@
 #include <apt-pkg/tagfile.h>
 
 #include <algorithm>
+#include <array>
 #include <cerrno>
+#include <ctime>
 #include <chrono>
 #include <cstddef>
 #include <cstdio>
@@ -471,15 +473,16 @@ bool pkgAcqTransactionItem::QueueURI(pkgAcquire::ItemDesc &Item)
       // now add the actual by-hash uris
       auto const Expected = GetExpectedHashes();
       auto const TargetHash = Expected.find(nullptr);
-      auto const PushByHashURI = [&](std::string U) {
+      auto const PushByHashURI = [&](std::string const &U) {
 	 if (unlikely(TargetHash == nullptr))
 	    return false;
-	 auto const trailing_slash = U.find_last_of("/");
+	 ::URI uri{U};
+	 auto const trailing_slash = uri.Path.find_last_of("/");
 	 if (unlikely(trailing_slash == std::string::npos))
 	    return false;
-	 auto byhashSuffix = "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
-	 U.replace(trailing_slash, U.length() - trailing_slash, std::move(byhashSuffix));
-	 PushAlternativeURI(std::move(U), {}, false);
+	 auto altPath = uri.Path.substr(0, trailing_slash) + "/by-hash/" + TargetHash->HashType() + "/" + TargetHash->HashValue();
+	 std::swap(uri.Path, altPath);
+	 PushAlternativeURI(uri, {{"Alternate-Paths", "../../" + flNotDir(altPath)}}, false);
 	 return true;
       };
       PushByHashURI(Item.URI);
@@ -843,8 +846,17 @@ void pkgAcquire::Item::PushAlternativeURI(std::string &&NewURI, std::unordered_m
       d->AlternativeURIs.emplace_front(std::move(NewURI), std::move(fields));
 }
 									/*}}}*/
-void pkgAcquire::Item::RemoveAlternativeSite(std::string &&OldSite) /*{{{*/
+void pkgAcquire::Item::RemoveAlternativeSite(std::string const &AltUriStr)/*{{{*/
 {
+   ::URI AltUri{AltUriStr};
+   // the hostnames for these methods are empty for absolute paths which would result
+   // in the elimination of all sites accessed via those methods. On the upside, those
+   // methods are local and fast to reply with failure, so it doesn't hurt that much to
+   // keep them in the loop – so we just exit here rather than trying to guess the site.
+   std::array const badhosts{"file", "copy", "cdrom"};
+   if (std::find(badhosts.begin(), badhosts.end(), AltUri.Access) != badhosts.end())
+      return;
+   auto const OldSite = URI::SiteOnly(AltUriStr);
    d->AlternativeURIs.erase(std::remove_if(d->AlternativeURIs.begin(), d->AlternativeURIs.end(),
 					   [&](decltype(*d->AlternativeURIs.cbegin()) AltUri) {
 					      return URI::SiteOnly(AltUri.URI) == OldSite;
@@ -939,7 +951,7 @@ void pkgAcquire::Item::FailMessage(string const &Message)
       failreason = WEAK_HASHSUMS;
    else if (FailReason == "RedirectionLoop")
       failreason = REDIRECTION_LOOP;
-   else if (Status == StatAuthError)
+   else if (Status == StatAuthError || FailReason == "HashSumMismatch")
       failreason = HASHSUM_MISMATCH;
 
    if(ErrorText.empty())
@@ -964,7 +976,7 @@ void pkgAcquire::Item::FailMessage(string const &Message)
 	    break;
       }
 
-      if (Status == StatAuthError)
+      if (Status == StatAuthError || failreason == HASHSUM_MISMATCH)
       {
 	 auto const ExpectedHashes = GetExpectedHashes();
 	 if (ExpectedHashes.empty() == false)
@@ -976,13 +988,25 @@ void pkgAcquire::Item::FailMessage(string const &Message)
 	 if (failreason == HASHSUM_MISMATCH)
 	 {
 	    out << "Hashes of received file:" << std::endl;
+	    size_t hashes = 0;
 	    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 	    {
 	       std::string const tagname = std::string(*type) + "-Hash";
 	       std::string const hashsum = LookupTag(Message, tagname.c_str());
-	       if (hashsum.empty() == false)
+	       if (not hashsum.empty())
+	       {
 		  formatHashsum(out, HashString(*type, hashsum));
+		  ++hashes;
+	       }
 	    }
+	    if (hashes == 0)
+	       for (char const *const *type = HashString::SupportedHashes(); *type != nullptr; ++type)
+	       {
+		  std::string const tagname = std::string("Alt-") + *type + "-Hash";
+		  std::string const hashsum = LookupTag(Message, tagname.c_str());
+		  if (not hashsum.empty())
+		     formatHashsum(out, HashString(*type, hashsum));
+	       }
 	 }
 	 auto const lastmod = LookupTag(Message, "Last-Modified", "");
 	 if (lastmod.empty() == false)
@@ -1024,8 +1048,7 @@ void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const
 bool pkgAcquire::Item::VerifyDone(std::string const &Message,
 	 pkgAcquire::MethodConfig const * const /*Cnf*/)
 {
-   std::string const FileName = LookupTag(Message,"Filename");
-   if (FileName.empty() == true)
+   if (LookupTag(Message,"Filename").empty() && LookupTag(Message, "Alt-Filename").empty())
    {
       Status = StatError;
       ErrorText = "Method gave a blank filename";
@@ -1413,7 +1436,7 @@ bool pkgAcqMetaBase::CheckDownloadDone(pkgAcqTransactionItem * const I, const st
    // verified yet)
 
    // Save the final base URI we got this Release file from
-   if (I->UsedMirror.empty() == false && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
+   if (not I->Local && not I->UsedMirror.empty() && _config->FindB("Acquire::SameMirrorForAllIndexes", true))
    {
       auto InReleasePath = Target.Option(IndexTarget::INRELEASE_PATH);
       if (InReleasePath.empty())
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index cfea61cf6..70620163a 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -240,7 +240,7 @@ class APT_PUBLIC pkgAcquire::Item : public WeakPointable				/*{{{*/
    APT_HIDDEN bool PopAlternativeURI(std::string &NewURI);
    APT_HIDDEN bool IsGoodAlternativeURI(std::string const &AltUri) const;
    APT_HIDDEN void PushAlternativeURI(std::string &&NewURI, std::unordered_map<std::string, std::string> &&fields, bool const at_the_back);
-   APT_HIDDEN void RemoveAlternativeSite(std::string &&OldSite);
+   APT_HIDDEN void RemoveAlternativeSite(std::string const &AltUri);
 
    /** \brief A "descriptive" URI-like string.
     *
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 0f22f71e8..55897945c 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -20,6 +20,7 @@
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/hashes.h>
+#include <apt-pkg/metaindex.h>
 #include <apt-pkg/proxy.h>
 #include <apt-pkg/strutl.h>
 
@@ -434,11 +435,18 @@ bool pkgAcquire::Worker::RunMessages()
 		  Log->Pulse((*O)->GetOwner());
 
 	    HashStringList ReceivedHashes;
+	    bool AltFile = false;
 	    {
-	       std::string const givenfilename = LookupTag(Message, "Filename");
-	       std::string const filename = givenfilename.empty() ? Itm->Owner->DestFile : givenfilename;
+	       std::string filename;
+	       if (filename = LookupTag(Message, "Filename"); filename.empty())
+	       {
+		  if (filename = LookupTag(Message, "Alt-Filename"); not filename.empty())
+		     AltFile = true;
+		  else
+		     filename = Itm->Owner->DestFile;
+	       }
 	       // see if we got hashes to verify
-	       ReceivedHashes = GetHashesFromMessage("", Message);
+	       ReceivedHashes = GetHashesFromMessage(AltFile ? "Alt-" : "", Message);
 	       // not all methods always sent Hashes our way
 	       if (ReceivedHashes.usable() == false)
 	       {
@@ -449,64 +457,107 @@ bool pkgAcquire::Worker::RunMessages()
 		     FileFd file(filename, FileFd::ReadOnly, FileFd::None);
 		     calc.AddFD(file);
 		     ReceivedHashes = calc.GetHashStringList();
+		     for (auto const &h : ReceivedHashes)
+		     {
+			std::string tagname;
+			if (AltFile)
+			   tagname = "Alt-";
+			tagname.append(h.HashType()).append("-Hash");
+			if (not LookupTag(Message, tagname.c_str()).empty())
+			   continue;
+			if (not APT::String::Endswith(Message, "\n"))
+			   Message.append("\n");
+			Message.append(tagname).append(": ").append(h.HashValue());
+		     }
 		  }
 	       }
 
 	       // only local files can refer other filenames and counting them as fetched would be unfair
-	       if (Log != NULL && Itm->Owner->Complete == false && Itm->Owner->Local == false && givenfilename == filename)
+	       if (Log != nullptr && not Itm->Owner->Complete && not Itm->Owner->Local && not AltFile && Itm->Owner->DestFile == filename)
 		  Log->Fetched(ReceivedHashes.FileSize(),atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 	    }
 
 	    std::vector<Item*> const ItmOwners = Itm->Owners;
+	    for (auto const Owner : ItmOwners)
+	       Owner->ErrorText.clear();
 	    OwnerQ->ItemDone(Itm);
 	    Itm = NULL;
 
 	    bool const isIMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false) ||
 	       StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false);
 	    auto const forcedHash = _config->Find("Acquire::ForceHash");
+
+	    bool consideredOkay = true;
+	    HashStringList ExpectedHashes;
+	    bool const DebugAuth = _config->FindB("Debug::pkgAcquire::Auth", false);
+	    if (DebugAuth)
+	    {
+	       std::clog << "201 URI Done: " << URI << endl
+			 << "ReceivedHash:" << endl;
+	       for (auto const &hs : ReceivedHashes)
+		  std::clog << "\t- " << hs.toStr() << std::endl;
+	       std::clog << "ExpectedHash:" << endl;
+	    }
 	    for (auto const Owner: ItmOwners)
 	    {
-	       HashStringList const ExpectedHashes = Owner->GetExpectedHashes();
-	       if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
-	       {
-		  std::clog << "201 URI Done: " << Owner->DescURI() << endl
-		     << "ReceivedHash:" << endl;
-		  for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs)
-		     std::clog <<  "\t- " << hs->toStr() << std::endl;
-		  std::clog << "ExpectedHash:" << endl;
-		  for (HashStringList::const_iterator hs = ExpectedHashes.begin(); hs != ExpectedHashes.end(); ++hs)
-		     std::clog <<  "\t- " << hs->toStr() << std::endl;
-		  std::clog << endl;
-	       }
-
-	       // decide if what we got is what we expected
-	       bool consideredOkay = false;
-	       if ((forcedHash.empty() && ExpectedHashes.empty() == false) ||
-		     (forcedHash.empty() == false && ExpectedHashes.usable()))
-	       {
-		  if (ReceivedHashes.empty())
+	       HashStringList const OwnerExpectedHashes = [&]() {
+		  if (AltFile)
 		  {
-		     /* IMS-Hits can't be checked here as we will have uncompressed file,
-			but the hashes for the compressed file. What we have was good through
-			so all we have to ensure later is that we are not stalled. */
-		     consideredOkay = isIMSHit;
+		     auto const * const transOwner = dynamic_cast<pkgAcqTransactionItem const * const>(Owner);
+		     if (transOwner != nullptr && transOwner->TransactionManager != nullptr && transOwner->TransactionManager->MetaIndexParser != nullptr)
+		     {
+			auto const * const hashes = transOwner->TransactionManager->MetaIndexParser->Lookup(transOwner->Target.MetaKey);
+			if (hashes != nullptr)
+			   return hashes->Hashes;
+		     }
 		  }
-		  else if (ReceivedHashes == ExpectedHashes)
-		     consideredOkay = true;
-		  else
+		  return Owner->GetExpectedHashes();
+	       }();
+	       for (auto const &h : OwnerExpectedHashes)
+	       {
+		  if (not ExpectedHashes.push_back(h))
+		  {
 		     consideredOkay = false;
-
+		     std::clog << "\t- " << h.toStr() << " [conflict]" << std::endl;
+		  }
+		  else if (DebugAuth)
+		     std::clog << "\t- " << h.toStr() << std::endl;
 	       }
+	    }
+	    if (DebugAuth)
+	       std::clog << endl;
+
+	    // decide if what we got is what we expected
+	    if (not consideredOkay)
+	       ;
+	    else if ((forcedHash.empty() && not ExpectedHashes.empty()) ||
+		     (not forcedHash.empty() && ExpectedHashes.usable()))
+	    {
+	       if (ReceivedHashes.empty())
+	       {
+		  /* IMS-Hits can't be checked here as we will have uncompressed file,
+		     but the hashes for the compressed file. What we have was good through
+		     so all we have to ensure later is that we are not stalled. */
+		  consideredOkay = isIMSHit;
+	       }
+	       else if (ReceivedHashes == ExpectedHashes)
+		  consideredOkay = true;
 	       else
-		  consideredOkay = !Owner->HashesRequired();
-
-	       if (consideredOkay == true)
-		  consideredOkay = Owner->VerifyDone(Message, Config);
-	       else // hashsum mismatch
-		  Owner->Status = pkgAcquire::Item::StatAuthError;
+		  consideredOkay = false;
+	    }
+	    else
+	       consideredOkay = std::none_of(ItmOwners.begin(), ItmOwners.end(), [](auto const * const O) { return O->HashesRequired(); });
 
+	    bool otherReasons = false;
+	    if (consideredOkay && not std::all_of(ItmOwners.begin(), ItmOwners.end(), [&](auto * const O) { return O->VerifyDone(Message, Config); }))
+	    {
+	       consideredOkay = false;
+	       otherReasons = true;
+	    }
 
-	       if (consideredOkay == true)
+	    if (consideredOkay)
+	    {
+	       for (auto const Owner : ItmOwners)
 	       {
 		  if (isDoomedItem(Owner) == false)
 		     Owner->Done(Message, ReceivedHashes, Config);
@@ -518,22 +569,21 @@ bool pkgAcquire::Worker::RunMessages()
 			Log->Done(Owner->GetItemDesc());
 		  }
 	       }
+	    }
+	    else
+	    {
+	       if (otherReasons)
+		  HandleFailure(ItmOwners, Config, Log, Message, false, false);
 	       else
 	       {
-		  auto SavedDesc = Owner->GetItemDesc();
-		  if (isDoomedItem(Owner) == false)
+		  if (Message.find("\nFailReason:") == std::string::npos)
 		  {
-		     if (Message.find("\nFailReason:") == std::string::npos)
-		     {
-			if (ReceivedHashes != ExpectedHashes)
-			   Message.append("\nFailReason: HashSumMismatch");
-			else
-			   Message.append("\nFailReason: WeakHashSums");
-		     }
-		     Owner->Failed(Message,Config);
+		     if (ReceivedHashes != ExpectedHashes)
+			Message.append("\nFailReason: HashSumMismatch");
+		     else
+			Message.append("\nFailReason: WeakHashSums");
 		  }
-		  if (Log != nullptr)
-		     Log->Fail(SavedDesc);
+		  HandleFailure(ItmOwners, Config, Log, Message, false, true);
 	       }
 	    }
 	    ItemDone();
@@ -593,6 +643,8 @@ bool pkgAcquire::Worker::RunMessages()
 		  Log->Pulse((*O)->GetOwner());
 
 	    std::vector<Item*> const ItmOwners = Itm->Owners;
+	    for (auto const Owner : ItmOwners)
+	       Owner->ErrorText.clear();
 	    OwnerQ->ItemDone(Itm);
 	    Itm = nullptr;
 
@@ -684,7 +736,7 @@ void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &It
       else
       {
 	 if (errAuthErr)
-	    Owner->RemoveAlternativeSite(URI::SiteOnly(Owner->GetItemDesc().URI));
+	    Owner->RemoveAlternativeSite(Owner->GetItemDesc().URI);
 	 if (Owner->PopAlternativeURI(NewURI))
 	 {
 	    Owner->FailMessage(Message);
@@ -845,14 +897,12 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item)
    }
    Message += "\nFilename: " + Item->Owner->DestFile;
 
-   // FIXME: We should not hard code proxy protocols here.
-   if (URL.Access == "http" || URL.Access == "https")
+   // AutoDetectProxy() checks this already by itself, but we don't want to access unknown configs
+   if (CanURIBeAccessedViaProxy(URL))
    {
       AutoDetectProxy(URL);
-      if (_config->Exists("Acquire::" + URL.Access + "::proxy::" + URL.Host))
-      {
-	 Message += "\nProxy: " + _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host);
-      }
+      if (auto const proxy = _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host); not proxy.empty())
+	 Message.append("\nProxy: ").append(proxy);
    }
 
    HashStringList const hsl = Item->GetExpectedHashes();
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index f367e4310..07f64cc69 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -524,7 +524,7 @@ pkgAcquire::MethodConfig *pkgAcquire::GetConfig(string Access)
    Configs = Conf;
 
    /* if a method uses DownloadLimit, we switch to SingleInstance mode */
-   if(_config->FindI("Acquire::"+Access+"::Dl-Limit",0) > 0)
+   if (not Conf->SingleInstance && _config->FindI("Acquire::" + Access + "::Dl-Limit", 0) > 0)
       Conf->SingleInstance = true;
     
    return Conf;
diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index c9b2dc0cc..fff10fb17 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -55,12 +55,14 @@ Configuration *_config = new Configuration;
    but a Cnf-member – but that would need ABI breaks and stuff and for now
    that really is an apt-dev-only tool, so it isn't that bad that it is
    unusable and allaround a bit strange */
-enum class APT_HIDDEN ConfigType { UNDEFINED, INT, BOOL, STRING, STRING_OR_BOOL, STRING_OR_LIST, FILE, DIR, LIST, PROGRAM_PATH = FILE };
+enum class APT_HIDDEN ConfigType { INVALID, UNDEFINED, INT, BOOL, STRING, STRING_OR_BOOL, STRING_OR_LIST, FILE, DIR, LIST, PROGRAM_PATH = FILE };
 APT_HIDDEN std::unordered_map<std::string, ConfigType> apt_known_config {};
+APT_HIDDEN std::vector<std::pair<std::string, ConfigType>> apt_known_config_patterns {};
 static std::string getConfigTypeString(ConfigType const type)		/*{{{*/
 {
    switch (type)
    {
+      case ConfigType::INVALID: return "INVALID";
       case ConfigType::UNDEFINED: return "UNDEFINED";
       case ConfigType::INT: return "INT";
       case ConfigType::BOOL: return "BOOL";
@@ -94,6 +96,8 @@ static ConfigType getConfigType(std::string const &type)		/*{{{*/
       return ConfigType::STRING_OR_LIST;
    else if (type == "<PROGRAM_PATH>")
       return ConfigType::PROGRAM_PATH;
+   else if (type == "<INVALID>")
+      return ConfigType::INVALID;
    return ConfigType::UNDEFINED;
 }
 									/*}}}*/
@@ -101,59 +105,86 @@ static ConfigType getConfigType(std::string const &type)		/*{{{*/
 static void checkFindConfigOptionTypeInternal(std::string name, ConfigType const type)
 {
    std::transform(name.begin(), name.end(), name.begin(), ::tolower);
-   auto known = apt_known_config.find(name);
-   if (known == apt_known_config.cend())
+   bool found = false;
+   ConfigType expectedType = ConfigType::INVALID;
+   if (auto const known = apt_known_config.find(name); known != apt_known_config.cend())
    {
-      auto const rcolon = name.rfind(':');
-      if (rcolon != std::string::npos)
+      found = true;
+      expectedType = known->second;
+   }
+   else
+      for (auto const &known : apt_known_config_patterns)
       {
-	 known = apt_known_config.find(name.substr(0, rcolon) + ":*");
-	 if (known == apt_known_config.cend())
+	 std::string_view n{name};
+	 std::string_view k{known.first};
+	 do
 	 {
-	    auto const parts = StringSplit(name, "::");
-	    size_t psize = parts.size();
-	    if (psize > 1)
+	    if (auto star = k.find('*'); star != std::string_view::npos)
 	    {
-	       for (size_t max = psize; max != 1; --max)
-	       {
-		  std::ostringstream os;
-		  std::copy(parts.begin(), parts.begin() + max, std::ostream_iterator<std::string>(os, "::"));
-		  os << "**";
-		  known = apt_known_config.find(os.str());
-		  if (known != apt_known_config.cend() && known->second == ConfigType::UNDEFINED)
-		     return;
-	       }
-	       for (size_t max = psize - 1; max != 1; --max)
-	       {
-		  std::ostringstream os;
-		  std::copy(parts.begin(), parts.begin() + max - 1, std::ostream_iterator<std::string>(os, "::"));
-		  os << "*::";
-		  std::copy(parts.begin() + max + 1, parts.end() - 1, std::ostream_iterator<std::string>(os, "::"));
-		  os << *(parts.end() - 1);
-		  known = apt_known_config.find(os.str());
-		  if (known != apt_known_config.cend())
-		     break;
-	       }
+	       if (auto nextstar = star + 1; k.length() > nextstar && k[nextstar] == '*')
+		  star -= 3;
+	       if (n.compare(0, star, k, 0, star) != 0)
+		  break;
+	       n.remove_prefix(star);
+	       k.remove_prefix(star + 1);
 	    }
-	 }
+	    else if (k == n)
+	    {
+	       n = {};
+	       break;
+	    }
+	    else
+	       break;
+
+	    if (k.empty())
+	    {
+	       if (n.find("::") == std::string_view::npos)
+		  n = {};
+	       break;
+	    }
+	    if (k == "::**")
+	    {
+	       if (known.second == ConfigType::UNDEFINED)
+		  return;
+	       n = {};
+	       break;
+	    }
+	    if (k.compare(0, 2, "::") == 0)
+	    {
+	       auto const colons = n.find("::");
+	       if (colons == std::string_view::npos)
+		  break;
+	       n.remove_prefix(colons);
+	    }
+	    else
+	       break;
+	 } while (not n.empty());
+	 if (not n.empty())
+	    continue;
+	 found = true;
+	 expectedType = known.second;
+	 break;
       }
-   }
-   if (known == apt_known_config.cend())
+
+   if (not found)
       _error->Warning("Using unknown config option »%s« of type %s",
 	    name.c_str(), getConfigTypeString(type).c_str());
-   else if (known->second != type)
+   else if (expectedType != type)
    {
-      if (known->second == ConfigType::DIR && type == ConfigType::FILE)
+      if (expectedType == ConfigType::INVALID)
+	 _error->Warning("Using invalid config option »%s« as a type %s",
+	       name.c_str(), getConfigTypeString(type).c_str());
+      if (expectedType == ConfigType::DIR && type == ConfigType::FILE)
 	 ; // implementation detail
-      else if (type == ConfigType::STRING && (known->second == ConfigType::FILE || known->second == ConfigType::DIR))
+      else if (type == ConfigType::STRING && (expectedType == ConfigType::FILE || expectedType == ConfigType::DIR))
 	 ; // TODO: that might be an error or not, we will figure this out later
-      else if (known->second == ConfigType::STRING_OR_BOOL && (type == ConfigType::BOOL || type == ConfigType::STRING))
+      else if (expectedType == ConfigType::STRING_OR_BOOL && (type == ConfigType::BOOL || type == ConfigType::STRING))
 	 ;
-      else if (known->second == ConfigType::STRING_OR_LIST && (type == ConfigType::LIST || type == ConfigType::STRING))
+      else if (expectedType == ConfigType::STRING_OR_LIST && (type == ConfigType::LIST || type == ConfigType::STRING))
 	 ;
       else
 	 _error->Warning("Using config option »%s« of type %s as a type %s",
-	       name.c_str(), getConfigTypeString(known->second).c_str(), getConfigTypeString(type).c_str());
+	       name.c_str(), getConfigTypeString(expectedType).c_str(), getConfigTypeString(type).c_str());
    }
 }
 static void checkFindConfigOptionType(char const * const name, ConfigType const type)
@@ -166,6 +197,7 @@ static void checkFindConfigOptionType(char const * const name, ConfigType const
 static bool LoadConfigurationIndex(std::string const &filename)		/*{{{*/
 {
    apt_known_config.clear();
+   apt_known_config_patterns.clear();
    if (filename.empty())
       return true;
    Configuration Idx;
@@ -181,7 +213,10 @@ static bool LoadConfigurationIndex(std::string const &filename)		/*{{{*/
       {
 	 std::string fulltag = Top->FullTag();
 	 std::transform(fulltag.begin(), fulltag.end(), fulltag.begin(), ::tolower);
-	 apt_known_config.emplace(std::move(fulltag), getConfigType(Top->Value));
+	 if (fulltag.find("::*") == std::string::npos)
+	    apt_known_config.emplace(std::move(fulltag), getConfigType(Top->Value));
+	 else
+	    apt_known_config_patterns.emplace_back(std::move(fulltag), getConfigType(Top->Value));
       }
 
       if (Top->Child != nullptr)
diff --git a/apt-pkg/contrib/proxy.cc b/apt-pkg/contrib/proxy.cc
index a99f44f49..9a9a9f29c 100644
--- a/apt-pkg/contrib/proxy.cc
+++ b/apt-pkg/contrib/proxy.cc
@@ -12,9 +12,11 @@
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
+#include <apt-pkg/string_view.h>
 #include <apt-pkg/strutl.h>
 
 #include <algorithm>
+#include <array>
 #include <iostream>
 #include <fcntl.h>
 #include <unistd.h>
@@ -22,6 +24,16 @@
 #include "proxy.h"
 									/*}}}*/
 
+bool CanURIBeAccessedViaProxy(URI const &URL)				/*{{{*/
+{
+   // for some methods a proxy doesn't make sense, so we don't have to fork
+   if (URL.Host.empty() ||
+       APT::String::Startswith(URL.Access, "mirror+") || URL.Access.find("+mirror+") != std::string::npos || APT::String::Endswith(URL.Access, "+mirror"))
+      return false;
+   std::array const noproxy{"file", "copy", "store", "gpgv", "rred", "cdrom", "mirror"};
+   return std::find(noproxy.begin(), noproxy.end(), URL.Access) == noproxy.end();
+}
+									/*}}}*/
 // AutoDetectProxy - auto detect proxy					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -29,16 +41,19 @@ static std::vector<std::string> CompatibleProxies(URI const &URL)
 {
    if (URL.Access == "http" || URL.Access == "https")
       return {"http", "https", "socks5h"};
-   return {URL.Access};
+   if (URL.Access == "tor" || URL.Access == "tor+http" || URL.Access == "tor+https")
+      return {"socks5h"};
+   if (URL.Access == "ftp")
+      return {"ftp"};
+   return {};
 }
-
 bool AutoDetectProxy(URI &URL)
 {
-   // we support both http/https debug options
-   bool Debug = _config->FindB("Debug::Acquire::"+URL.Access,false);
+   if (not CanURIBeAccessedViaProxy(URL))
+      return true;
 
    // the user already explicitly set a proxy for this host
-   if(_config->Find("Acquire::"+URL.Access+"::proxy::"+URL.Host, "") != "")
+   if (not _config->Find("Acquire::" + URL.Access + "::proxy::" + URL.Host, "").empty())
       return true;
 
    // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old
@@ -49,6 +64,7 @@ bool AutoDetectProxy(URI &URL)
    if (AutoDetectProxyCmd.empty())
       return true;
 
+   bool const Debug = _config->FindB("Debug::Acquire::" + URL.Access, false);
    if (Debug)
       std::clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << std::endl;
 
@@ -73,7 +89,7 @@ bool AutoDetectProxy(URI &URL)
    // and apt will use the generic proxy settings
    if (goodread == false)
       return true;
-   auto const cleanedbuf = _strstrip(buf);
+   APT::StringView const cleanedbuf = _strstrip(buf);
    // We warn about this as the implementor probably meant to use DIRECT instead
    if (cleanedbuf[0] == '\0')
    {
@@ -82,17 +98,25 @@ bool AutoDetectProxy(URI &URL)
    }
 
    if (Debug)
-      std::clog << "auto detect command returned: '" << cleanedbuf << "'" << std::endl;
+      std::clog << "auto detect command returned: '" << cleanedbuf.data() << "'" << std::endl;
 
-   auto compatibleTypes = CompatibleProxies(URL);
-   bool compatible = strcmp(cleanedbuf, "DIRECT") == 0 ||
-		     compatibleTypes.end() != std::find_if(compatibleTypes.begin(),
-							   compatibleTypes.end(), [cleanedbuf](std::string &compat) {
-							      return strstr(cleanedbuf, compat.c_str()) == cleanedbuf;
-							   });
+   bool compatible = true;
+   if (cleanedbuf != "DIRECT")
+   {
+      if (auto const compatibleTypes = CompatibleProxies(URL); not compatibleTypes.empty())
+	 compatible = std::any_of(compatibleTypes.begin(), compatibleTypes.end(),
+				  [cleanedbuf](std::string const &compat)
+				  {
+				     return cleanedbuf.substr(0, compat.size()) == compat;
+				  });
+   }
+   else if (URL.Access == "tor" || URL.Access == "tor+http" || URL.Access == "tor+https")
+      compatible = false; // Accepting DIRECT would silently disable tor
 
    if (compatible)
-      _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf);
+      _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, cleanedbuf.data());
+   else
+      _error->Warning("ProxyAutoDetect command returned incompatible proxy '%s' for access type %s", cleanedbuf.data(), URL.Access.c_str());
 
    return true;
 }
diff --git a/apt-pkg/contrib/proxy.h b/apt-pkg/contrib/proxy.h
index f6d70ea8b..2a4c19fc8 100644
--- a/apt-pkg/contrib/proxy.h
+++ b/apt-pkg/contrib/proxy.h
@@ -9,8 +9,10 @@
 #ifndef PKGLIB_PROXY_H
 #define PKGLIB_PROXY_H
 
+#include <apt-pkg/macros.h>
+
 class URI;
 APT_PUBLIC bool AutoDetectProxy(URI &URL);
-
+APT_HIDDEN bool CanURIBeAccessedViaProxy(URI const &URL);
 
 #endif