Merge branch 'pu/fuzzerpatches' into 'master'

Various patches uplifted from unfinished fuzzer branches

See merge request apt-team/apt!158

17 files changed, 310 insertions, 294 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index a4ad6b854..c9e81070b 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -2604,14 +2604,32 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       return false;
    }
 
+   /* decide if we should download patches one by one or in one go:
+      The first is good if the server merges patches, but many don't so client
+      based merging can be attempt in which case the second is better.
+      "bad things" will happen if patches are merged on the server,
+      but client side merging is attempt as well */
+   pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
+   if (pdiff_merge == true)
+   {
+      // reprepro and dak add this flag if they merge patches on the server
+      std::string const precedence = Tags.FindS("X-Patch-Precedence");
+      pdiff_merge = (precedence != "merged");
+   }
+
    // calculate the size of all patches we have to get
    unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
    if (sizeLimitPercent > 0)
    {
-      unsigned long long downloadSize = std::accumulate(available_patches.begin(),
-	    available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
-	    return T + I.download_hashes.FileSize();
-	    });
+      unsigned long long downloadSize = 0;
+      if (pdiff_merge)
+	 downloadSize = std::accumulate(available_patches.begin(), available_patches.end(), 0llu,
+					[](unsigned long long const T, DiffInfo const &I) {
+					   return T + I.download_hashes.FileSize();
+					});
+      // if server-side merging, assume we will need only the first patch
+      else if (not available_patches.empty())
+	 downloadSize = available_patches.front().download_hashes.FileSize();
       if (downloadSize != 0)
       {
 	 unsigned long long downloadSizeIdx = 0;
@@ -2636,19 +2654,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       }
    }
 
-   /* decide if we should download patches one by one or in one go:
-      The first is good if the server merges patches, but many don't so client
-      based merging can be attempt in which case the second is better.
-      "bad things" will happen if patches are merged on the server,
-      but client side merging is attempt as well */
-   pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
-   if (pdiff_merge == true)
-   {
-      // reprepro adds this flag if it has merged patches on the server
-      std::string const precedence = Tags.FindS("X-Patch-Precedence");
-      pdiff_merge = (precedence != "merged");
-   }
-
    // clean the plate
    {
       std::string const Final = GetExistingFilename(CurrentPackagesFile);
diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc
index 6e1674f7f..089582561 100644
--- a/apt-pkg/acquire-method.cc
+++ b/apt-pkg/acquire-method.cc
@@ -482,9 +482,25 @@ void pkgAcqMethod::PrintStatus(char const * const header, const char* Format,
 void pkgAcqMethod::Log(const char *Format,...)
 {
    va_list args;
-   va_start(args,Format);
-   PrintStatus("101 Log", Format, args);
-   va_end(args);
+   ssize_t size = 400;
+   std::ostringstream outstr;
+   while (true) {
+      bool ret;
+      va_start(args,Format);
+      ret = iovprintf(outstr, Format, args, size);
+      va_end(args);
+      if (ret == true)
+	 break;
+   }
+   std::unordered_map<std::string, std::string> fields;
+   if (Queue != 0)
+      try_emplace(fields, "URI", Queue->Uri);
+   else
+      try_emplace(fields, "URI", "<UNKNOWN>");
+   if (not UsedMirror.empty())
+      try_emplace(fields, "UsedMirror", UsedMirror);
+   try_emplace(fields, "Message", outstr.str());
+   SendMessage("101 Log", std::move(fields));
 }
 									/*}}}*/
 // AcqMethod::Status - Send a status message				/*{{{*/
@@ -493,9 +509,25 @@ void pkgAcqMethod::Log(const char *Format,...)
 void pkgAcqMethod::Status(const char *Format,...)
 {
    va_list args;
-   va_start(args,Format);
-   PrintStatus("102 Status", Format, args);
-   va_end(args);
+   ssize_t size = 400;
+   std::ostringstream outstr;
+   while (true) {
+      bool ret;
+      va_start(args,Format);
+      ret = iovprintf(outstr, Format, args, size);
+      va_end(args);
+      if (ret == true)
+	 break;
+   }
+   std::unordered_map<std::string, std::string> fields;
+   if (Queue != 0)
+      try_emplace(fields, "URI", Queue->Uri);
+   else
+      try_emplace(fields, "URI", "<UNKNOWN>");
+   if (not UsedMirror.empty())
+      try_emplace(fields, "UsedMirror", UsedMirror);
+   try_emplace(fields, "Message", outstr.str());
+   SendMessage("102 Status", std::move(fields));
 }
 									/*}}}*/
 // AcqMethod::Redirect - Send a redirect message                       /*{{{*/
diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h
index e854f5ff1..b4b238c4c 100644
--- a/apt-pkg/acquire-method.h
+++ b/apt-pkg/acquire-method.h
@@ -101,7 +101,7 @@ class APT_PUBLIC pkgAcqMethod
    bool MediaFail(std::string Required,std::string Drive);
    virtual void Exit() {};
 
-   void PrintStatus(char const * const header, const char* Format, va_list &args) const;
+   APT_DEPRECATED_MSG("Use SendMessage instead") void PrintStatus(char const * const header, const char* Format, va_list &args) const;
 
    public:
    enum CnfFlags
diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc
index 8b86fa3e4..5355994d3 100644
--- a/apt-pkg/cachefile.cc
+++ b/apt-pkg/cachefile.cc
@@ -27,10 +27,13 @@
 #include <apt-pkg/progress.h>
 #include <apt-pkg/sourcelist.h>
 
+#include <limits>
 #include <memory>
 #include <string>
 #include <vector>
 #include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
 #include <unistd.h>
 
 #include <apti18n.h>
@@ -288,15 +291,27 @@ bool pkgCacheFile::AddIndexFile(pkgIndexFile * const File)		/*{{{*/
 // CacheFile::RemoveCaches - remove all cache files from disk		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
+static void SetCacheStartBeforeRemovingCache(std::string const &cache)
+{
+   if (cache.empty())
+      return;
+   auto const CacheStart = _config->FindI("APT::Cache-Start", 0);
+   constexpr auto CacheStartDefault = 24 * 1024 * 1024;
+   struct stat Buf;
+   if (stat(cache.c_str(), &Buf) == 0 && (Buf.st_mode & S_IFREG) != 0)
+   {
+      RemoveFile("RemoveCaches", cache);
+      if (CacheStart == 0 && std::numeric_limits<decltype(CacheStart)>::max() >= Buf.st_size && Buf.st_size > CacheStartDefault)
+	 _config->Set("APT::Cache-Start", Buf.st_size);
+   }
+}
 void pkgCacheFile::RemoveCaches()
 {
    std::string const pkgcache = _config->FindFile("Dir::cache::pkgcache");
+   SetCacheStartBeforeRemovingCache(pkgcache);
    std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache");
+   SetCacheStartBeforeRemovingCache(srcpkgcache);
 
-   if (pkgcache.empty() == false && RealFileExists(pkgcache) == true)
-      RemoveFile("RemoveCaches", pkgcache);
-   if (srcpkgcache.empty() == false && RealFileExists(srcpkgcache) == true)
-      RemoveFile("RemoveCaches", srcpkgcache);
    if (pkgcache.empty() == false)
    {
       std::string cachedir = flNotFile(pkgcache);
diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h
index 6261b5089..466492442 100644
--- a/apt-pkg/cacheiterators.h
+++ b/apt-pkg/cacheiterators.h
@@ -82,10 +82,10 @@ template<typename Str, typename Itr> class APT_PUBLIC pkgCache::Iterator :
 	inline unsigned long Index() const {return S - OwnerPointer();}
 	inline map_pointer<Str> MapPointer() const {return map_pointer<Str>(Index()) ;}
 
-	void ReMap(void const * const oldMap, void const * const newMap) {
+	void ReMap(void const * const oldMap, void * const newMap) {
 		if (Owner == 0 || S == 0)
 			return;
-		S += static_cast<Str const *>(newMap) - static_cast<Str const *>(oldMap);
+		S = static_cast<Str *>(newMap) + (S - static_cast<Str const *>(oldMap));
 	}
 
 	// Constructors - look out for the variable assigning
@@ -350,12 +350,12 @@ class APT_PUBLIC pkgCache::DepIterator : public Iterator<Dependency, DepIterator
 	};
 	inline DependencyProxy operator->() const {return (DependencyProxy) { S2->Version, S2->Package, S->ID, S2->Type, S2->CompareOp, S->ParentVer, S->DependencyData, S->NextRevDepends, S->NextDepends, S2->NextData };}
 	inline DependencyProxy operator->() {return (DependencyProxy) { S2->Version, S2->Package, S->ID, S2->Type, S2->CompareOp, S->ParentVer, S->DependencyData, S->NextRevDepends, S->NextDepends, S2->NextData };}
-	void ReMap(void const * const oldMap, void const * const newMap)
+	void ReMap(void const * const oldMap, void * const newMap)
 	{
 		Iterator<Dependency, DepIterator>::ReMap(oldMap, newMap);
 		if (Owner == 0 || S == 0 || S2 == 0)
 			return;
-		S2 += static_cast<DependencyData const *>(newMap) - static_cast<DependencyData const *>(oldMap);
+		S2 = static_cast<DependencyData *>(newMap) + (S2 - static_cast<DependencyData const *>(oldMap));
 	}
 
 	//Nice printable representation
diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc
index 5df03a99c..aacb78bba 100644
--- a/apt-pkg/cdrom.cc
+++ b/apt-pkg/cdrom.cc
@@ -464,14 +464,12 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
 
    string File = _config->FindFile("Dir::Etc::sourcelist");
 
-   // Open the stream for reading
-   ifstream F((FileExists(File)?File.c_str():"/dev/null"),
-	      ios::in );
-   if (F.fail() == true)
-      return _error->Errno("ifstream::ifstream","Opening %s",File.c_str());
+   FileFd F(FileExists(File) ? File : "/dev/null", FileFd::ReadOnly);
+   if (not F.IsOpen() || F.Failed())
+      return _error->Errno("WriteSourceList", "Opening %s failed", File.c_str());
 
    string NewFile = File + ".new";
-   RemoveFile("WriteDatabase", NewFile);
+   RemoveFile("WriteSourceList", NewFile);
    ofstream Out(NewFile.c_str());
    if (!Out)
       return _error->Errno("ofstream::ofstream",
@@ -487,21 +485,16 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
    else
       Type = "deb";
 
-   char Buffer[300];
    int CurLine = 0;
    bool First = true;
-   while (F.eof() == false)
-   {
-      F.getline(Buffer,sizeof(Buffer));
-      CurLine++;
-      if (F.fail() && !F.eof())
-	 return _error->Error(_("Line %u too long in source list %s."),
-			      CurLine,File.c_str());
-      _strtabexpand(Buffer,sizeof(Buffer));
-      _strstrip(Buffer);
-            
+   std::string Buffer;
+   while (F.ReadLine(Buffer))
+   {
+      ++CurLine;
+      auto const Cleaned = APT::String::Strip(SubstVar(Buffer, "\t", "        "));
+
       // Comment or blank
-      if (Buffer[0] == '#' || Buffer[0] == 0)
+      if (Cleaned.empty() || Cleaned[0] == '#')
       {
 	 Out << Buffer << endl;
 	 continue;
@@ -523,7 +516,7 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
       // Grok it
       string cType;
       string URI;
-      const char *C = Buffer;
+      const char *C = Cleaned.c_str();
       if (ParseQuoteWord(C,cType) == false ||
 	  ParseQuoteWord(C,URI) == false)
       {
diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index 931df9f6c..a8fced724 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -21,6 +21,7 @@
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/macros.h>
 #include <apt-pkg/strutl.h>
+#include <apt-pkg/string_view.h>
 
 #include <ctype.h>
 #include <regex.h>
@@ -861,107 +862,82 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
       // The input line with comments stripped.
       std::string Fragment;
 
-      // Expand tabs in the input line and remove leading and trailing
-      // whitespace.
-      {
-	const int BufferSize = Input.size() * 8 + 1;
-	char *Buffer = new char[BufferSize];
-	try
-	  {
-	    memcpy(Buffer, Input.c_str(), Input.size() + 1);
-
-	    _strtabexpand(Buffer, BufferSize);
-	    _strstrip(Buffer);
-	    Input = Buffer;
-	  }
-	catch(...)
-	  {
-	    delete[] Buffer;
-	    throw;
-	  }
-	delete[] Buffer;
-      }
+      // Expand tabs in the input line and remove leading and trailing whitespace.
+      Input = APT::String::Strip(SubstVar(Input, "\t", "        "));
       CurLine++;
 
       // Now strip comments; if the whole line is contained in a
       // comment, skip this line.
+      APT::StringView Line{Input.data(), Input.size()};
 
-      // The first meaningful character in the current fragment; will
-      // be adjusted below as we remove bytes from the front.
-      std::string::const_iterator Start = Input.begin();
-      // The last meaningful character in the current fragment.
-      std::string::const_iterator End = Input.end();
-
-      // Multi line comment
-      if (InComment == true)
+      // continued Multi line comment
+      if (InComment)
       {
-	for (std::string::const_iterator I = Start;
-	     I != End; ++I)
+	 size_t end = Line.find("*/");
+	 if (end != APT::StringView::npos)
 	 {
-	    if (*I == '*' && I + 1 != End && I[1] == '/')
-	    {
-	       Start = I + 2;
-	       InComment = false;
-	       break;
-	    }
+	    Line.remove_prefix(end + 2);
+	    InComment = false;
 	 }
-	 if (InComment == true)
+	 else
 	    continue;
       }
 
       // Discard single line comments
-      bool InQuote = false;
-      for (std::string::const_iterator I = Start;
-	   I != End; ++I)
       {
-	 if (*I == '"')
-	    InQuote = !InQuote;
-	 if (InQuote == true)
-	    continue;
-
-	 if ((*I == '/' && I + 1 != End && I[1] == '/') ||
-	     (*I == '#' && strcmp(string(I,I+6).c_str(),"#clear") != 0 &&
-	      strcmp(string(I,I+8).c_str(),"#include") != 0 &&
-	      strcmp(string(I,I+strlen("#x-apt-configure-index")).c_str(), "#x-apt-configure-index") != 0))
+	 size_t start = 0;
+	 while ((start = Line.find("//", start)) != APT::StringView::npos)
 	 {
-	    End = I;
+	    if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0)
+	    {
+	       ++start;
+	       continue;
+	    }
+	    Line.remove_suffix(Line.length() - start);
+	    break;
+	 }
+	 using APT::operator""_sv;
+	 constexpr std::array<APT::StringView, 3> magicComments { "clear"_sv, "include"_sv, "x-apt-configure-index"_sv };
+	 start = 0;
+	 while ((start = Line.find('#', start)) != APT::StringView::npos)
+	 {
+	    if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0 ||
+		std::any_of(magicComments.begin(), magicComments.end(), [&](auto const m) { return Line.compare(start+1, m.length(), m) == 0; }))
+	    {
+	       ++start;
+	       continue;
+	    }
+	    Line.remove_suffix(Line.length() - start);
 	    break;
 	 }
       }
 
       // Look for multi line comments and build up the
       // fragment.
-      Fragment.reserve(End - Start);
-      InQuote = false;
-      for (std::string::const_iterator I = Start;
-	   I != End; ++I)
+      Fragment.reserve(Line.length());
       {
-	 if (*I == '"')
-	    InQuote = !InQuote;
-	 if (InQuote == true)
-	   Fragment.push_back(*I);
-	 else if (*I == '/' && I + 1 != End && I[1] == '*')
-         {
-	    InComment = true;
-	    for (std::string::const_iterator J = I;
-		 J != End; ++J)
+	 size_t start = 0;
+	 while ((start = Line.find("/*", start)) != APT::StringView::npos)
+	 {
+	    if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0)
 	    {
-	       if (*J == '*' && J + 1 != End && J[1] == '/')
-	       {
-		  // Pretend we just finished walking over the
-		  // comment, and don't add anything to the output
-		  // fragment.
-		  I = J + 1;
-		  InComment = false;
-		  break;
-	       }
+	       start += 2;
+	       continue;
 	    }
-
-	    if (InComment == true)
-	      break;
+	    Fragment.append(Line.data(), start);
+	    auto const end = Line.find("*/", start + 2);
+	    if (end == APT::StringView::npos)
+	    {
+	       Line.clear();
+	       InComment = true;
+	       break;
+	    }
+	    else
+	       Line.remove_prefix(end + 2);
+	    start = 0;
 	 }
-	 else
-	   Fragment.push_back(*I);
+	 if (not Line.empty())
+	    Fragment.append(Line.data(), Line.length());
       }
 
       // Skip blank lines.
@@ -969,9 +945,9 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
 	 continue;
 
       // The line has actual content; interpret what it means.
-      InQuote = false;
-      Start = Fragment.begin();
-      End = Fragment.end();
+      bool InQuote = false;
+      auto Start = Fragment.cbegin();
+      auto End = Fragment.cend();
       for (std::string::const_iterator I = Start;
 	   I != End; ++I)
       {
@@ -1150,10 +1126,13 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
 bool ReadConfigDir(Configuration &Conf,const string &Dir,
 		   bool const &AsSectional, unsigned const &Depth)
 {
+   _error->PushToStack();
    auto const files = GetListOfFilesInDir(Dir, "conf", true, true);
+   auto const successfulList = not _error->PendingError();
+   _error->MergeWithStack();
    return std::accumulate(files.cbegin(), files.cend(), true, [&](bool good, auto const &file) {
       return ReadConfigFile(Conf, file, AsSectional, Depth) && good;
-   });
+   }) && successfulList;
 }
 									/*}}}*/
 // MatchAgainstConfig Constructor					/*{{{*/
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index e91c1acc3..091def3d4 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -1774,9 +1774,8 @@ public:
 #endif
 };
 									/*}}}*/
-
-class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate
-{ /*{{{*/
+class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate		/*{{{*/
+{
 #ifdef HAVE_ZSTD
    ZSTD_DStream *dctx;
    ZSTD_CStream *cctx;
@@ -1986,7 +1985,7 @@ class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate
 #endif
 };
 									/*}}}*/
-class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate {				/*{{{*/
+class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate {		/*{{{*/
 #ifdef HAVE_LZMA
    struct LZMAFILE {
       FILE* file;
@@ -2092,17 +2091,9 @@ public:
       }
       else
       {
-	 uint64_t const memlimit = UINT64_MAX;
-	 if (compressor.Name == "xz")
-	 {
-	    if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
-	       return false;
-	 }
-	 else
-	 {
-	    if (lzma_alone_decoder(&lzma->stream, memlimit) != LZMA_OK)
-	       return false;
-	 }
+	 uint64_t constexpr memlimit = 1024 * 1024 * 500;
+	 if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
+	    return false;
 	 lzma->compressing = false;
       }
       return true;
diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc
index 28f3150c3..3368ece84 100644
--- a/apt-pkg/contrib/gpgv.cc
+++ b/apt-pkg/contrib/gpgv.cc
@@ -121,21 +121,6 @@ static bool operator!=(LineBuffer const &buf, APT::StringView const exp) noexcep
    And as a cherry on the cake, we use our apt-key wrapper to do part
    of the lifting in regards to merging keyrings. Fun for the whole family.
 */
-static bool iovprintf(std::ostream &out, const char *format,
-		      va_list &args, ssize_t &size) {
-   auto S = make_unique_char(malloc(size));
-   ssize_t const n = vsnprintf(S.get(), size, format, args);
-   if (n > -1 && n < size) {
-      out << S.get();
-      return true;
-   } else {
-      if (n > -1)
-	 size = n + 1;
-      else
-	 size *= 2;
-   }
-   return false;
-}
 static void APT_PRINTF(4) apt_error(std::ostream &outterm, int const statusfd, int fd[2], const char *format, ...)
 {
    std::ostringstream outstr;
diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc
index 020491172..642e20473 100644
--- a/apt-pkg/contrib/mmap.cc
+++ b/apt-pkg/contrib/mmap.cc
@@ -393,7 +393,7 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize)
       bool const newError = _error->PendingError();
       _error->MergeWithStack();
       if (Pools != oldPools)
-	 I += Pools - oldPools;
+	 I = Pools + (I - oldPools);
 
       // Does the allocation failed ?
       if (Result == 0 && newError)
diff --git a/apt-pkg/contrib/string_view.h b/apt-pkg/contrib/string_view.h
index 05aad3327..04f6ff115 100644
--- a/apt-pkg/contrib/string_view.h
+++ b/apt-pkg/contrib/string_view.h
@@ -39,6 +39,10 @@ public:
     StringView(const char *data) : data_(data), size_(strlen(data)) {}
     StringView(std::string const & str): data_(str.data()), size_(str.size()) {}
 
+    /* Modifiers */
+    void remove_prefix(size_t n) { data_ += n; size_ -= n; }
+    void remove_suffix(size_t n) { size_ -= n; }
+    void clear() { size_ = 0; }
 
     /* Viewers */
     constexpr StringView substr(size_t pos, size_t n = npos) const {
@@ -76,6 +80,28 @@ public:
         return found - data_;
     }
 
+    size_t find(APT::StringView const needle) const {
+       if (needle.empty())
+	  return npos;
+       if (needle.length() == 1)
+	  return find(*needle.data());
+       size_t found = 0;
+       while ((found = find(*needle.data(), found)) != npos) {
+	  if (compare(found, needle.length(), needle) == 0)
+	     return found;
+	  ++found;
+       }
+       return found;
+    }
+    size_t find(APT::StringView const needle, size_t pos) const {
+       if (pos == 0)
+	  return find(needle);
+       size_t const found = substr(pos).find(needle);
+       if (found == npos)
+	  return npos;
+       return pos + found;
+    }
+
     /* Conversions */
     std::string to_string() const {
         return std::string(data_, size_);
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index bd4856526..3a0a6eaa3 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -23,8 +23,10 @@
 #include <algorithm>
 #include <array>
 #include <iomanip>
+#include <limits>
 #include <locale>
 #include <sstream>
+#include <memory>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -310,11 +312,11 @@ bool ParseQuoteWord(const char *&String,string &Res)
    }
 
    // Now de-quote characters
-   char Buffer[1024];
+   Res.clear();
+   Res.reserve(C - String);
    char Tmp[3];
    const char *Start = String;
-   char *I;
-   for (I = Buffer; I < Buffer + sizeof(Buffer) && Start != C; I++)
+   while (Start != C)
    {
       if (*Start == '%' && Start + 2 < C &&
 	  isxdigit(Start[1]) && isxdigit(Start[2]))
@@ -322,19 +324,15 @@ bool ParseQuoteWord(const char *&String,string &Res)
 	 Tmp[0] = Start[1];
 	 Tmp[1] = Start[2];
 	 Tmp[2] = 0;
-	 *I = (char)strtol(Tmp,0,16);
+	 Res.push_back(static_cast<char>(strtol(Tmp, 0, 16)));
 	 Start += 3;
 	 continue;
       }
       if (*Start != '"')
-	 *I = *Start;
-      else
-	 I--;
-      Start++;
+	 Res.push_back(*Start);
+      ++Start;
    }
-   *I = 0;
-   Res = Buffer;
-   
+
    // Skip ending white space
    for (; isspace(*C) != 0; C++)
       ;
@@ -354,33 +352,28 @@ bool ParseCWord(const char *&String,string &Res)
       ;
    if (*C == 0)
       return false;
-   
-   char Buffer[1024];
-   char *Buf = Buffer;
-   if (strlen(String) >= sizeof(Buffer))
-       return false;
-       
-   for (; *C != 0; C++)
+
+   Res.clear();
+   Res.reserve(strlen(String));
+   for (; *C != 0; ++C)
    {
       if (*C == '"')
       {
 	 for (C++; *C != 0 && *C != '"'; C++)
-	    *Buf++ = *C;
-	 
+	    Res.push_back(*C);
+
 	 if (*C == 0)
 	    return false;
-	 
+
 	 continue;
-      }      
-      
+      }
+
       if (C != String && isspace(*C) != 0 && isspace(C[-1]) != 0)
 	 continue;
       if (isspace(*C) == 0)
 	 return false;
-      *Buf++ = ' ';
+      Res.push_back(' ');
    }
-   *Buf = 0;
-   Res = Buffer;
    String = C;
    return true;
 }
@@ -597,7 +590,7 @@ string Base64Encode(const string &S)
       base64.  */
    for (string::const_iterator I = S.begin(); I < S.end(); I += 3)
    {
-      char Bits[3] = {0,0,0};
+      uint8_t Bits[3] = {0,0,0};
       Bits[0] = I[0];
       if (I + 1 < S.end())
 	 Bits[1] = I[1];
@@ -1148,34 +1141,24 @@ bool FTPMDTMStrToTime(const char* const str,time_t &time)
 									/*}}}*/
 // StrToNum - Convert a fixed length string to a number			/*{{{*/
 // ---------------------------------------------------------------------
-/* This is used in decoding the crazy fixed length string headers in 
+/* This is used in decoding the crazy fixed length string headers in
    tar and ar files. */
 bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base)
 {
-   char S[30];
-   if (Len >= sizeof(S))
+   unsigned long long BigRes;
+   if (not StrToNum(Str, BigRes, Len, Base))
       return false;
-   memcpy(S,Str,Len);
-   S[Len] = 0;
-   
-   // All spaces is a zero
-   Res = 0;
-   unsigned I;
-   for (I = 0; S[I] == ' '; I++);
-   if (S[I] == 0)
-      return true;
-   
-   char *End;
-   Res = strtoul(S,&End,Base);
-   if (End == S)
+
+   if (std::numeric_limits<unsigned long>::max() < BigRes)
       return false;
-   
+
+   Res = BigRes;
    return true;
 }
 									/*}}}*/
 // StrToNum - Convert a fixed length string to a number			/*{{{*/
 // ---------------------------------------------------------------------
-/* This is used in decoding the crazy fixed length string headers in 
+/* This is used in decoding the crazy fixed length string headers in
    tar and ar files. */
 bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base)
 {
@@ -1184,20 +1167,20 @@ bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base
       return false;
    memcpy(S,Str,Len);
    S[Len] = 0;
-   
+
    // All spaces is a zero
    Res = 0;
    unsigned I;
-   for (I = 0; S[I] == ' '; I++);
+   for (I = 0; S[I] == ' '; ++I);
    if (S[I] == 0)
       return true;
-   
+   if (S[I] == '-')
+      return false;
+
    char *End;
+   errno = 0;
    Res = strtoull(S,&End,Base);
-   if (End == S)
-      return false;
-   
-   return true;
+   return not (End == S || errno != 0);
 }
 									/*}}}*/
 
@@ -1432,13 +1415,12 @@ unsigned long RegexChoice(RxChoiceList *Rxs,const char **ListBegin,
 // ---------------------------------------------------------------------
 /* This is used to make the internationalization strings easier to translate
    and to allow reordering of parameters */
-static bool iovprintf(ostream &out, const char *format,
+bool iovprintf(std::ostream &out, const char *format,
 		      va_list &args, ssize_t &size) {
-   char *S = (char*)malloc(size);
-   ssize_t const n = vsnprintf(S, size, format, args);
+   auto S = std::unique_ptr<char,decltype(&free)>{static_cast<char*>(malloc(size)), &free};
+   ssize_t const n = vsnprintf(S.get(), size, format, args);
    if (n > -1 && n < size) {
-      out << S;
-      free(S);
+      out << S.get();
       return true;
    } else {
       if (n > -1)
@@ -1446,7 +1428,6 @@ static bool iovprintf(ostream &out, const char *format,
       else
 	 size *= 2;
    }
-   free(S);
    return false;
 }
 void ioprintf(ostream &out,const char *format,...)
@@ -1611,22 +1592,26 @@ string DeEscapeString(const string &input)
       switch (*it)
       {
          case '0':
-            if (it + 2 <= input.end()) {
+            if (it + 2 < input.end()) {
                tmp[0] = it[1];
                tmp[1] = it[2];
                tmp[2] = 0;
                output += (char)strtol(tmp, 0, 8);
                it += 2;
-            }
+            } else {
+	       // FIXME: raise exception here?
+	    }
             break;
          case 'x':
-            if (it + 2 <= input.end()) {
+            if (it + 2 < input.end()) {
                tmp[0] = it[1];
                tmp[1] = it[2];
                tmp[2] = 0;
                output += (char)strtol(tmp, 0, 16);
                it += 2;
-            }
+            } else {
+	       // FIXME: raise exception here?
+	    }
             break;
          default:
             // FIXME: raise exception here?
diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h
index c25c6208a..b6e6bfdce 100644
--- a/apt-pkg/contrib/strutl.h
+++ b/apt-pkg/contrib/strutl.h
@@ -42,7 +42,7 @@ namespace APT {
 APT_PUBLIC bool UTF8ToCodeset(const char *codeset, const std::string &orig, std::string *dest);
 APT_PUBLIC char *_strstrip(char *String);
 APT_PUBLIC char *_strrstrip(char *String); // right strip only
-APT_PUBLIC char *_strtabexpand(char *String,size_t Len);
+APT_DEPRECATED_MSG("Use SubstVar to avoid memory headaches") APT_PUBLIC char *_strtabexpand(char *String,size_t Len);
 APT_PUBLIC bool ParseQuoteWord(const char *&String,std::string &Res);
 APT_PUBLIC bool ParseCWord(const char *&String,std::string &Res);
 APT_PUBLIC std::string QuoteString(const std::string &Str,const char *Bad);
@@ -117,6 +117,8 @@ APT_PUBLIC std::vector<std::string> StringSplit(std::string const &input,
                                      std::string const &sep, 
                                      unsigned int maxsplit=std::numeric_limits<unsigned int>::max()) APT_PURE;
 
+
+APT_HIDDEN bool iovprintf(std::ostream &out, const char *format, va_list &args, ssize_t &size);
 APT_PUBLIC void ioprintf(std::ostream &out,const char *format,...) APT_PRINTF(2);
 APT_PUBLIC void strprintf(std::string &out,const char *format,...) APT_PRINTF(2);
 APT_PUBLIC char *safe_snprintf(char *Buffer,char *End,const char *Format,...) APT_PRINTF(3);
diff --git a/apt-pkg/deb/debfile.cc b/apt-pkg/deb/debfile.cc
index bef0cd0d8..645a579ef 100644
--- a/apt-pkg/deb/debfile.cc
+++ b/apt-pkg/deb/debfile.cc
@@ -24,9 +24,12 @@
 #include <apt-pkg/error.h>
 #include <apt-pkg/extracttar.h>
 #include <apt-pkg/fileutl.h>
+#include <apt-pkg/strutl.h>
 #include <apt-pkg/tagfile.h>
 
+#include <algorithm>
 #include <string>
+#include <sstream>
 #include <vector>
 #include <string.h>
 #include <sys/stat.h>
@@ -105,42 +108,48 @@ const ARArchive::Member *debDebFile::GotoMember(const char *Name)
 /* Simple wrapper around tar.. */
 bool debDebFile::ExtractTarMember(pkgDirStream &Stream,const char *Name)
 {
-   // Get the archive member
-   const ARArchive::Member *Member = NULL;
    std::string Compressor;
+   auto const Compressors = APT::Configuration::getCompressors();
 
-   std::vector<APT::Configuration::Compressor> compressor = APT::Configuration::getCompressors();
-   for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin();
-	c != compressor.end(); ++c)
+   ARArchive::Member const *Member = AR.FindMember(Name);
+   if (Member != nullptr)
    {
-      Member = AR.FindMember(std::string(Name).append(c->Extension).c_str());
-      if (Member == NULL)
-	 continue;
-      Compressor = c->Name;
-      break;
+      auto const found = std::find_if(Compressors.cbegin(), Compressors.cend(), [&](auto const &c) {
+	 return not c.Extension.empty() && APT::String::Endswith(Name, c.Extension);
+      });
+      if (found != Compressors.cend())
+	 Compressor = found->Name;
    }
-
-   if (Member == NULL)
-      Member = AR.FindMember(std::string(Name).c_str());
-
-   if (Member == NULL)
+   else
    {
-      std::string ext = std::string(Name) + ".{";
-      for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin();
-	   c != compressor.end(); ++c) {
-	 if (!c->Extension.empty())
-	    ext.append(c->Extension.substr(1));
+      for (auto const &c : Compressors)
+      {
+	 if (c.Extension.empty())
+	    continue;
+	 Member = AR.FindMember(std::string(Name).append(c.Extension).c_str());
+	 if (Member == nullptr)
+	    continue;
+	 Compressor = c.Name;
+	 break;
       }
-      ext.append("}");
-      return _error->Error(_("Internal error, could not locate member %s"), ext.c_str());
    }
 
-   if (File.Seek(Member->Start) == false)
+   if (Member == nullptr)
+   {
+      std::ostringstream ext;
+      ext << Name << '{';
+      for (auto const &c : Compressors)
+	 if (not c.Extension.empty())
+	    ext << c.Extension << ',';
+      ext << '}';
+      return _error->Error(_("Internal error, could not locate member %s"), ext.str().c_str());
+   }
+
+   if (not File.Seek(Member->Start))
       return false;
 
-   // Prepare Tar
    ExtractTar Tar(File,Member->Size,Compressor);
-   if (_error->PendingError() == true)
+   if (_error->PendingError())
       return false;
    return Tar.Go(Stream);
 }
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index f7f3537aa..cbde7c42f 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -255,7 +255,9 @@ uint32_t pkgCache::CacheHash()
 			 GetMap().Size() - sizeof(header));
    }
 
-   return XXH3_64bits_digest(state) & 0xFFFFFFFF;
+   auto const digest = XXH3_64bits_digest(state);
+   XXH3_freeState(state);
+   return digest & 0xFFFFFFFF;
 }
 									/*}}}*/
 // Cache::FindPkg - Locate a package by name				/*{{{*/
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 26cf7fc68..b4fd0641e 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -37,6 +37,8 @@
 
 #include <apti18n.h>
 									/*}}}*/
+constexpr auto APT_CACHE_START_DEFAULT = 24 * 1024 * 1024;
+
 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
@@ -156,13 +158,14 @@ pkgCacheGenerator::~pkgCacheGenerator()
    Map.Sync(0,sizeof(pkgCache::Header));
 }
 									/*}}}*/
-void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
+void pkgCacheGenerator::ReMap(void const * const oldMap, void * const newMap, size_t oldSize) {/*{{{*/
+   if (oldMap == newMap)
+      return;
+
    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
    // returns (something, true) the iterator was not yet seen and we can
    // remap it.
    std::unordered_set<void *> seen;
-   if (oldMap == newMap)
-      return;
 
    if (_config->FindB("Debug::pkgCacheGen", false))
       std::clog << "Remapping from " << oldMap << " to " << newMap << std::endl;
@@ -170,42 +173,24 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
    Cache.ReMap(false);
 
    if (CurrentFile != nullptr)
-      CurrentFile += static_cast<pkgCache::PackageFile const *>(newMap) - static_cast<pkgCache::PackageFile const *>(oldMap);
+      CurrentFile = static_cast<pkgCache::PackageFile *>(newMap) + (CurrentFile - static_cast<pkgCache::PackageFile const *>(oldMap));
    if (CurrentRlsFile != nullptr)
-      CurrentRlsFile += static_cast<pkgCache::ReleaseFile const *>(newMap) - static_cast<pkgCache::ReleaseFile const *>(oldMap);
-
-   for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
-	if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
+      CurrentRlsFile = static_cast<pkgCache::ReleaseFile *>(newMap) + (CurrentRlsFile - static_cast<pkgCache::ReleaseFile const *>(oldMap));
+
+#define APT_REMAP(TYPE) \
+   for (auto * const i : Dynamic<TYPE>::toReMap) \
+      if (seen.insert(i).second) \
+	 i->ReMap(oldMap, newMap)
+   APT_REMAP(pkgCache::GrpIterator);
+   APT_REMAP(pkgCache::PkgIterator);
+   APT_REMAP(pkgCache::VerIterator);
+   APT_REMAP(pkgCache::DepIterator);
+   APT_REMAP(pkgCache::DescIterator);
+   APT_REMAP(pkgCache::PrvIterator);
+   APT_REMAP(pkgCache::PkgFileIterator);
+   APT_REMAP(pkgCache::RlsFileIterator);
+#undef APT_REMAP
+
    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
       if (std::get<1>(seen.insert(ViewP)) == false)
 	 continue;
@@ -453,7 +438,7 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
 			   Pkg.Name(), "NewVersion", 1);
 
    if (oldMap != Map.Data())
-	 LastVer += static_cast<map_pointer<pkgCache::Version> const *>(Map.Data()) - static_cast<map_pointer<pkgCache::Version> const *>(oldMap);
+	 LastVer = static_cast<map_pointer<pkgCache::Version> *>(Map.Data()) + (LastVer - static_cast<map_pointer<pkgCache::Version> const *>(oldMap));
    *LastVer = verindex;
 
    if (unlikely(List.NewVersion(Ver) == false))
@@ -1073,7 +1058,7 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 	 OldDepLast = &D->NextDepends;
    } else if (oldMap != Map.Data())
-      OldDepLast += static_cast<map_pointer<pkgCache::Dependency> const *>(Map.Data()) - static_cast<map_pointer<pkgCache::Dependency> const *>(oldMap);
+      OldDepLast = static_cast<map_pointer<pkgCache::Dependency> *>(Map.Data()) + (OldDepLast - static_cast<map_pointer<pkgCache::Dependency> const *>(oldMap));
 
    Dep->NextDepends = *OldDepLast;
    *OldDepLast = Dependency;
@@ -1400,6 +1385,13 @@ static bool CheckValidity(FileFd &CacheFile, std::string const &CacheFileName,
       return false;
    }
 
+   if (_config->FindI("APT::Cache-Start", 0) == 0)
+   {
+      auto const size = CacheFile.FileSize();
+      if (std::numeric_limits<int>::max() >= size && size > APT_CACHE_START_DEFAULT)
+	 _config->Set("APT::Cache-Start", size);
+   }
+
    if (List.GetLastModifiedTime() > CacheFile.ModificationTime())
    {
       if (Debug == true)
@@ -1608,7 +1600,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    where it builds the cache 'fast' into a memory buffer. */
 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
 {
-   map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
+   map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", APT_CACHE_START_DEFAULT);
    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
    Flags |= MMap::Moveable;
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index 62c171be4..f4781d715 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -146,7 +146,7 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
 			MMap **OutMap,pkgCache **OutCache, bool AllowMem = false);
    APT_PUBLIC static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap);
 
-   void ReMap(void const * const oldMap, void const * const newMap, size_t oldSize);
+   void ReMap(void const * const oldMap, void * const newMap, size_t oldSize);
    bool Start();
 
    pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress);