Merge branch 'pu/fuzzerpatches' into 'master'

Various patches uplifted from unfinished fuzzer branches

See merge request apt-team/apt!158

30 files changed, 524 insertions, 426 deletions

diff --git a/CMake/CheckCxxTarget.cmake b/CMake/CheckCxxTarget.cmake
deleted file mode 100644
index 17c32bfac..000000000
--- a/CMake/CheckCxxTarget.cmake
+++ /dev/null
@@ -1,36 +0,0 @@
-# CMake support for target-based function multiversioning
-#
-# Copyright (C) 2019 Canonical Ltd
-#
-# Author: Julian Andres Klode <jak@debian.org>.
-#
-# Permission is hereby granted, free of charge, to any person
-# obtaining a copy of this software and associated documentation files
-# (the "Software"), to deal in the Software without restriction,
-# including without limitation the rights to use, copy, modify, merge,
-# publish, distribute, sublicense, and/or sell copies of the Software,
-# and to permit persons to whom the Software is furnished to do so,
-# subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-
-include(CheckCXXSourceCompiles)
-function(check_cxx_target var target code)
-	check_cxx_source_compiles(
-	"
-	__attribute__((target(\"${target}\"))) static int foo(int i) { return ${code}; }
-	__attribute__((target(\"default\"))) static int foo(int i) { return i; }
-	int main(int i, char **) { return foo(i); }
-	"  ${var})
-endfunction()
diff --git a/CMake/config.h.in b/CMake/config.h.in
index 911d42464..8fa5e8b6f 100644
--- a/CMake/config.h.in
+++ b/CMake/config.h.in
@@ -80,9 +80,5 @@
 /* Group of the root user */
 #cmakedefine ROOT_GROUP "${ROOT_GROUP}"
 
-/* defined if __builtin_ia32_crc32{s,d}i() exists in an sse4.2 target */
-#cmakedefine HAVE_FMV_SSE42_AND_CRC32
-#cmakedefine HAVE_FMV_SSE42_AND_CRC32DI
-
 /* unrolling is faster combined with an optimizing compiler */
 #define SHA2_UNROLL_TRANSFORM
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2d9afba61..9e35f6e00 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -197,11 +197,6 @@ else()
     set(RESOLV_LIBRARIES -lresolv)
 endif()
 
-# Check multiversioning
-include(CheckCxxTarget)
-check_cxx_target(HAVE_FMV_SSE42_AND_CRC32 "sse4.2" "__builtin_ia32_crc32si(0,i)|__builtin_ia32_crc32hi(0,i)|__builtin_ia32_crc32qi(0,i)")
-check_cxx_target(HAVE_FMV_SSE42_AND_CRC32DI "sse4.2" "__builtin_ia32_crc32di(0,i)")
-
 # Configure some variables like package, version and architecture.
 set(PACKAGE ${PROJECT_NAME})
 set(PACKAGE_MAIL "APT Development Team <deity@lists.debian.org>")
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index a4ad6b854..c9e81070b 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -2604,14 +2604,32 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       return false;
    }
 
+   /* decide if we should download patches one by one or in one go:
+      The first is good if the server merges patches, but many don't so client
+      based merging can be attempt in which case the second is better.
+      "bad things" will happen if patches are merged on the server,
+      but client side merging is attempt as well */
+   pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
+   if (pdiff_merge == true)
+   {
+      // reprepro and dak add this flag if they merge patches on the server
+      std::string const precedence = Tags.FindS("X-Patch-Precedence");
+      pdiff_merge = (precedence != "merged");
+   }
+
    // calculate the size of all patches we have to get
    unsigned short const sizeLimitPercent = _config->FindI("Acquire::PDiffs::SizeLimit", 100);
    if (sizeLimitPercent > 0)
    {
-      unsigned long long downloadSize = std::accumulate(available_patches.begin(),
-	    available_patches.end(), 0llu, [](unsigned long long const T, DiffInfo const &I) {
-	    return T + I.download_hashes.FileSize();
-	    });
+      unsigned long long downloadSize = 0;
+      if (pdiff_merge)
+	 downloadSize = std::accumulate(available_patches.begin(), available_patches.end(), 0llu,
+					[](unsigned long long const T, DiffInfo const &I) {
+					   return T + I.download_hashes.FileSize();
+					});
+      // if server-side merging, assume we will need only the first patch
+      else if (not available_patches.empty())
+	 downloadSize = available_patches.front().download_hashes.FileSize();
       if (downloadSize != 0)
       {
 	 unsigned long long downloadSizeIdx = 0;
@@ -2636,19 +2654,6 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       }
    }
 
-   /* decide if we should download patches one by one or in one go:
-      The first is good if the server merges patches, but many don't so client
-      based merging can be attempt in which case the second is better.
-      "bad things" will happen if patches are merged on the server,
-      but client side merging is attempt as well */
-   pdiff_merge = _config->FindB("Acquire::PDiffs::Merge", true);
-   if (pdiff_merge == true)
-   {
-      // reprepro adds this flag if it has merged patches on the server
-      std::string const precedence = Tags.FindS("X-Patch-Precedence");
-      pdiff_merge = (precedence != "merged");
-   }
-
    // clean the plate
    {
       std::string const Final = GetExistingFilename(CurrentPackagesFile);
diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc
index 6e1674f7f..089582561 100644
--- a/apt-pkg/acquire-method.cc
+++ b/apt-pkg/acquire-method.cc
@@ -482,9 +482,25 @@ void pkgAcqMethod::PrintStatus(char const * const header, const char* Format,
 void pkgAcqMethod::Log(const char *Format,...)
 {
    va_list args;
-   va_start(args,Format);
-   PrintStatus("101 Log", Format, args);
-   va_end(args);
+   ssize_t size = 400;
+   std::ostringstream outstr;
+   while (true) {
+      bool ret;
+      va_start(args,Format);
+      ret = iovprintf(outstr, Format, args, size);
+      va_end(args);
+      if (ret == true)
+	 break;
+   }
+   std::unordered_map<std::string, std::string> fields;
+   if (Queue != 0)
+      try_emplace(fields, "URI", Queue->Uri);
+   else
+      try_emplace(fields, "URI", "<UNKNOWN>");
+   if (not UsedMirror.empty())
+      try_emplace(fields, "UsedMirror", UsedMirror);
+   try_emplace(fields, "Message", outstr.str());
+   SendMessage("101 Log", std::move(fields));
 }
 									/*}}}*/
 // AcqMethod::Status - Send a status message				/*{{{*/
@@ -493,9 +509,25 @@ void pkgAcqMethod::Log(const char *Format,...)
 void pkgAcqMethod::Status(const char *Format,...)
 {
    va_list args;
-   va_start(args,Format);
-   PrintStatus("102 Status", Format, args);
-   va_end(args);
+   ssize_t size = 400;
+   std::ostringstream outstr;
+   while (true) {
+      bool ret;
+      va_start(args,Format);
+      ret = iovprintf(outstr, Format, args, size);
+      va_end(args);
+      if (ret == true)
+	 break;
+   }
+   std::unordered_map<std::string, std::string> fields;
+   if (Queue != 0)
+      try_emplace(fields, "URI", Queue->Uri);
+   else
+      try_emplace(fields, "URI", "<UNKNOWN>");
+   if (not UsedMirror.empty())
+      try_emplace(fields, "UsedMirror", UsedMirror);
+   try_emplace(fields, "Message", outstr.str());
+   SendMessage("102 Status", std::move(fields));
 }
 									/*}}}*/
 // AcqMethod::Redirect - Send a redirect message                       /*{{{*/
diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h
index e854f5ff1..b4b238c4c 100644
--- a/apt-pkg/acquire-method.h
+++ b/apt-pkg/acquire-method.h
@@ -101,7 +101,7 @@ class APT_PUBLIC pkgAcqMethod
    bool MediaFail(std::string Required,std::string Drive);
    virtual void Exit() {};
 
-   void PrintStatus(char const * const header, const char* Format, va_list &args) const;
+   APT_DEPRECATED_MSG("Use SendMessage instead") void PrintStatus(char const * const header, const char* Format, va_list &args) const;
 
    public:
    enum CnfFlags
diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc
index 8b86fa3e4..5355994d3 100644
--- a/apt-pkg/cachefile.cc
+++ b/apt-pkg/cachefile.cc
@@ -27,10 +27,13 @@
 #include <apt-pkg/progress.h>
 #include <apt-pkg/sourcelist.h>
 
+#include <limits>
 #include <memory>
 #include <string>
 #include <vector>
 #include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
 #include <unistd.h>
 
 #include <apti18n.h>
@@ -288,15 +291,27 @@ bool pkgCacheFile::AddIndexFile(pkgIndexFile * const File)		/*{{{*/
 // CacheFile::RemoveCaches - remove all cache files from disk		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
+static void SetCacheStartBeforeRemovingCache(std::string const &cache)
+{
+   if (cache.empty())
+      return;
+   auto const CacheStart = _config->FindI("APT::Cache-Start", 0);
+   constexpr auto CacheStartDefault = 24 * 1024 * 1024;
+   struct stat Buf;
+   if (stat(cache.c_str(), &Buf) == 0 && (Buf.st_mode & S_IFREG) != 0)
+   {
+      RemoveFile("RemoveCaches", cache);
+      if (CacheStart == 0 && std::numeric_limits<decltype(CacheStart)>::max() >= Buf.st_size && Buf.st_size > CacheStartDefault)
+	 _config->Set("APT::Cache-Start", Buf.st_size);
+   }
+}
 void pkgCacheFile::RemoveCaches()
 {
    std::string const pkgcache = _config->FindFile("Dir::cache::pkgcache");
+   SetCacheStartBeforeRemovingCache(pkgcache);
    std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache");
+   SetCacheStartBeforeRemovingCache(srcpkgcache);
 
-   if (pkgcache.empty() == false && RealFileExists(pkgcache) == true)
-      RemoveFile("RemoveCaches", pkgcache);
-   if (srcpkgcache.empty() == false && RealFileExists(srcpkgcache) == true)
-      RemoveFile("RemoveCaches", srcpkgcache);
    if (pkgcache.empty() == false)
    {
       std::string cachedir = flNotFile(pkgcache);
diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h
index 6261b5089..466492442 100644
--- a/apt-pkg/cacheiterators.h
+++ b/apt-pkg/cacheiterators.h
@@ -82,10 +82,10 @@ template<typename Str, typename Itr> class APT_PUBLIC pkgCache::Iterator :
 	inline unsigned long Index() const {return S - OwnerPointer();}
 	inline map_pointer<Str> MapPointer() const {return map_pointer<Str>(Index()) ;}
 
-	void ReMap(void const * const oldMap, void const * const newMap) {
+	void ReMap(void const * const oldMap, void * const newMap) {
 		if (Owner == 0 || S == 0)
 			return;
-		S += static_cast<Str const *>(newMap) - static_cast<Str const *>(oldMap);
+		S = static_cast<Str *>(newMap) + (S - static_cast<Str const *>(oldMap));
 	}
 
 	// Constructors - look out for the variable assigning
@@ -350,12 +350,12 @@ class APT_PUBLIC pkgCache::DepIterator : public Iterator<Dependency, DepIterator
 	};
 	inline DependencyProxy operator->() const {return (DependencyProxy) { S2->Version, S2->Package, S->ID, S2->Type, S2->CompareOp, S->ParentVer, S->DependencyData, S->NextRevDepends, S->NextDepends, S2->NextData };}
 	inline DependencyProxy operator->() {return (DependencyProxy) { S2->Version, S2->Package, S->ID, S2->Type, S2->CompareOp, S->ParentVer, S->DependencyData, S->NextRevDepends, S->NextDepends, S2->NextData };}
-	void ReMap(void const * const oldMap, void const * const newMap)
+	void ReMap(void const * const oldMap, void * const newMap)
 	{
 		Iterator<Dependency, DepIterator>::ReMap(oldMap, newMap);
 		if (Owner == 0 || S == 0 || S2 == 0)
 			return;
-		S2 += static_cast<DependencyData const *>(newMap) - static_cast<DependencyData const *>(oldMap);
+		S2 = static_cast<DependencyData *>(newMap) + (S2 - static_cast<DependencyData const *>(oldMap));
 	}
 
 	//Nice printable representation
diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc
index 5df03a99c..aacb78bba 100644
--- a/apt-pkg/cdrom.cc
+++ b/apt-pkg/cdrom.cc
@@ -464,14 +464,12 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
 
    string File = _config->FindFile("Dir::Etc::sourcelist");
 
-   // Open the stream for reading
-   ifstream F((FileExists(File)?File.c_str():"/dev/null"),
-	      ios::in );
-   if (F.fail() == true)
-      return _error->Errno("ifstream::ifstream","Opening %s",File.c_str());
+   FileFd F(FileExists(File) ? File : "/dev/null", FileFd::ReadOnly);
+   if (not F.IsOpen() || F.Failed())
+      return _error->Errno("WriteSourceList", "Opening %s failed", File.c_str());
 
    string NewFile = File + ".new";
-   RemoveFile("WriteDatabase", NewFile);
+   RemoveFile("WriteSourceList", NewFile);
    ofstream Out(NewFile.c_str());
    if (!Out)
       return _error->Errno("ofstream::ofstream",
@@ -487,21 +485,16 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
    else
       Type = "deb";
 
-   char Buffer[300];
    int CurLine = 0;
    bool First = true;
-   while (F.eof() == false)
-   {
-      F.getline(Buffer,sizeof(Buffer));
-      CurLine++;
-      if (F.fail() && !F.eof())
-	 return _error->Error(_("Line %u too long in source list %s."),
-			      CurLine,File.c_str());
-      _strtabexpand(Buffer,sizeof(Buffer));
-      _strstrip(Buffer);
-            
+   std::string Buffer;
+   while (F.ReadLine(Buffer))
+   {
+      ++CurLine;
+      auto const Cleaned = APT::String::Strip(SubstVar(Buffer, "\t", "        "));
+
       // Comment or blank
-      if (Buffer[0] == '#' || Buffer[0] == 0)
+      if (Cleaned.empty() || Cleaned[0] == '#')
       {
 	 Out << Buffer << endl;
 	 continue;
@@ -523,7 +516,7 @@ bool pkgCdrom::WriteSourceList(string Name,vector<string> &List,bool Source)
       // Grok it
       string cType;
       string URI;
-      const char *C = Buffer;
+      const char *C = Cleaned.c_str();
       if (ParseQuoteWord(C,cType) == false ||
 	  ParseQuoteWord(C,URI) == false)
       {
diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index 931df9f6c..a8fced724 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -21,6 +21,7 @@
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/macros.h>
 #include <apt-pkg/strutl.h>
+#include <apt-pkg/string_view.h>
 
 #include <ctype.h>
 #include <regex.h>
@@ -861,107 +862,82 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
       // The input line with comments stripped.
       std::string Fragment;
 
-      // Expand tabs in the input line and remove leading and trailing
-      // whitespace.
-      {
-	const int BufferSize = Input.size() * 8 + 1;
-	char *Buffer = new char[BufferSize];
-	try
-	  {
-	    memcpy(Buffer, Input.c_str(), Input.size() + 1);
-
-	    _strtabexpand(Buffer, BufferSize);
-	    _strstrip(Buffer);
-	    Input = Buffer;
-	  }
-	catch(...)
-	  {
-	    delete[] Buffer;
-	    throw;
-	  }
-	delete[] Buffer;
-      }
+      // Expand tabs in the input line and remove leading and trailing whitespace.
+      Input = APT::String::Strip(SubstVar(Input, "\t", "        "));
       CurLine++;
 
       // Now strip comments; if the whole line is contained in a
       // comment, skip this line.
+      APT::StringView Line{Input.data(), Input.size()};
 
-      // The first meaningful character in the current fragment; will
-      // be adjusted below as we remove bytes from the front.
-      std::string::const_iterator Start = Input.begin();
-      // The last meaningful character in the current fragment.
-      std::string::const_iterator End = Input.end();
-
-      // Multi line comment
-      if (InComment == true)
+      // continued Multi line comment
+      if (InComment)
       {
-	for (std::string::const_iterator I = Start;
-	     I != End; ++I)
+	 size_t end = Line.find("*/");
+	 if (end != APT::StringView::npos)
 	 {
-	    if (*I == '*' && I + 1 != End && I[1] == '/')
-	    {
-	       Start = I + 2;
-	       InComment = false;
-	       break;
-	    }
+	    Line.remove_prefix(end + 2);
+	    InComment = false;
 	 }
-	 if (InComment == true)
+	 else
 	    continue;
       }
 
       // Discard single line comments
-      bool InQuote = false;
-      for (std::string::const_iterator I = Start;
-	   I != End; ++I)
       {
-	 if (*I == '"')
-	    InQuote = !InQuote;
-	 if (InQuote == true)
-	    continue;
-
-	 if ((*I == '/' && I + 1 != End && I[1] == '/') ||
-	     (*I == '#' && strcmp(string(I,I+6).c_str(),"#clear") != 0 &&
-	      strcmp(string(I,I+8).c_str(),"#include") != 0 &&
-	      strcmp(string(I,I+strlen("#x-apt-configure-index")).c_str(), "#x-apt-configure-index") != 0))
+	 size_t start = 0;
+	 while ((start = Line.find("//", start)) != APT::StringView::npos)
 	 {
-	    End = I;
+	    if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0)
+	    {
+	       ++start;
+	       continue;
+	    }
+	    Line.remove_suffix(Line.length() - start);
+	    break;
+	 }
+	 using APT::operator""_sv;
+	 constexpr std::array<APT::StringView, 3> magicComments { "clear"_sv, "include"_sv, "x-apt-configure-index"_sv };
+	 start = 0;
+	 while ((start = Line.find('#', start)) != APT::StringView::npos)
+	 {
+	    if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0 ||
+		std::any_of(magicComments.begin(), magicComments.end(), [&](auto const m) { return Line.compare(start+1, m.length(), m) == 0; }))
+	    {
+	       ++start;
+	       continue;
+	    }
+	    Line.remove_suffix(Line.length() - start);
 	    break;
 	 }
       }
 
       // Look for multi line comments and build up the
       // fragment.
-      Fragment.reserve(End - Start);
-      InQuote = false;
-      for (std::string::const_iterator I = Start;
-	   I != End; ++I)
+      Fragment.reserve(Line.length());
       {
-	 if (*I == '"')
-	    InQuote = !InQuote;
-	 if (InQuote == true)
-	   Fragment.push_back(*I);
-	 else if (*I == '/' && I + 1 != End && I[1] == '*')
-         {
-	    InComment = true;
-	    for (std::string::const_iterator J = I;
-		 J != End; ++J)
+	 size_t start = 0;
+	 while ((start = Line.find("/*", start)) != APT::StringView::npos)
+	 {
+	    if (std::count(Line.begin(), Line.begin() + start, '"') % 2 != 0)
 	    {
-	       if (*J == '*' && J + 1 != End && J[1] == '/')
-	       {
-		  // Pretend we just finished walking over the
-		  // comment, and don't add anything to the output
-		  // fragment.
-		  I = J + 1;
-		  InComment = false;
-		  break;
-	       }
+	       start += 2;
+	       continue;
 	    }
-
-	    if (InComment == true)
-	      break;
+	    Fragment.append(Line.data(), start);
+	    auto const end = Line.find("*/", start + 2);
+	    if (end == APT::StringView::npos)
+	    {
+	       Line.clear();
+	       InComment = true;
+	       break;
+	    }
+	    else
+	       Line.remove_prefix(end + 2);
+	    start = 0;
 	 }
-	 else
-	   Fragment.push_back(*I);
+	 if (not Line.empty())
+	    Fragment.append(Line.data(), Line.length());
       }
 
       // Skip blank lines.
@@ -969,9 +945,9 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
 	 continue;
 
       // The line has actual content; interpret what it means.
-      InQuote = false;
-      Start = Fragment.begin();
-      End = Fragment.end();
+      bool InQuote = false;
+      auto Start = Fragment.cbegin();
+      auto End = Fragment.cend();
       for (std::string::const_iterator I = Start;
 	   I != End; ++I)
       {
@@ -1150,10 +1126,13 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool const &AsSectio
 bool ReadConfigDir(Configuration &Conf,const string &Dir,
 		   bool const &AsSectional, unsigned const &Depth)
 {
+   _error->PushToStack();
    auto const files = GetListOfFilesInDir(Dir, "conf", true, true);
+   auto const successfulList = not _error->PendingError();
+   _error->MergeWithStack();
    return std::accumulate(files.cbegin(), files.cend(), true, [&](bool good, auto const &file) {
       return ReadConfigFile(Conf, file, AsSectional, Depth) && good;
-   });
+   }) && successfulList;
 }
 									/*}}}*/
 // MatchAgainstConfig Constructor					/*{{{*/
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index e91c1acc3..091def3d4 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -1774,9 +1774,8 @@ public:
 #endif
 };
 									/*}}}*/
-
-class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate
-{ /*{{{*/
+class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate		/*{{{*/
+{
 #ifdef HAVE_ZSTD
    ZSTD_DStream *dctx;
    ZSTD_CStream *cctx;
@@ -1986,7 +1985,7 @@ class APT_HIDDEN ZstdFileFdPrivate : public FileFdPrivate
 #endif
 };
 									/*}}}*/
-class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate {				/*{{{*/
+class APT_HIDDEN LzmaFileFdPrivate: public FileFdPrivate {		/*{{{*/
 #ifdef HAVE_LZMA
    struct LZMAFILE {
       FILE* file;
@@ -2092,17 +2091,9 @@ public:
       }
       else
       {
-	 uint64_t const memlimit = UINT64_MAX;
-	 if (compressor.Name == "xz")
-	 {
-	    if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
-	       return false;
-	 }
-	 else
-	 {
-	    if (lzma_alone_decoder(&lzma->stream, memlimit) != LZMA_OK)
-	       return false;
-	 }
+	 uint64_t constexpr memlimit = 1024 * 1024 * 500;
+	 if (lzma_auto_decoder(&lzma->stream, memlimit, 0) != LZMA_OK)
+	    return false;
 	 lzma->compressing = false;
       }
       return true;
diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc
index 28f3150c3..3368ece84 100644
--- a/apt-pkg/contrib/gpgv.cc
+++ b/apt-pkg/contrib/gpgv.cc
@@ -121,21 +121,6 @@ static bool operator!=(LineBuffer const &buf, APT::StringView const exp) noexcep
    And as a cherry on the cake, we use our apt-key wrapper to do part
    of the lifting in regards to merging keyrings. Fun for the whole family.
 */
-static bool iovprintf(std::ostream &out, const char *format,
-		      va_list &args, ssize_t &size) {
-   auto S = make_unique_char(malloc(size));
-   ssize_t const n = vsnprintf(S.get(), size, format, args);
-   if (n > -1 && n < size) {
-      out << S.get();
-      return true;
-   } else {
-      if (n > -1)
-	 size = n + 1;
-      else
-	 size *= 2;
-   }
-   return false;
-}
 static void APT_PRINTF(4) apt_error(std::ostream &outterm, int const statusfd, int fd[2], const char *format, ...)
 {
    std::ostringstream outstr;
diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc
index 020491172..642e20473 100644
--- a/apt-pkg/contrib/mmap.cc
+++ b/apt-pkg/contrib/mmap.cc
@@ -393,7 +393,7 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize)
       bool const newError = _error->PendingError();
       _error->MergeWithStack();
       if (Pools != oldPools)
-	 I += Pools - oldPools;
+	 I = Pools + (I - oldPools);
 
       // Does the allocation failed ?
       if (Result == 0 && newError)
diff --git a/apt-pkg/contrib/string_view.h b/apt-pkg/contrib/string_view.h
index 05aad3327..04f6ff115 100644
--- a/apt-pkg/contrib/string_view.h
+++ b/apt-pkg/contrib/string_view.h
@@ -39,6 +39,10 @@ public:
     StringView(const char *data) : data_(data), size_(strlen(data)) {}
     StringView(std::string const & str): data_(str.data()), size_(str.size()) {}
 
+    /* Modifiers */
+    void remove_prefix(size_t n) { data_ += n; size_ -= n; }
+    void remove_suffix(size_t n) { size_ -= n; }
+    void clear() { size_ = 0; }
 
     /* Viewers */
     constexpr StringView substr(size_t pos, size_t n = npos) const {
@@ -76,6 +80,28 @@ public:
         return found - data_;
     }
 
+    size_t find(APT::StringView const needle) const {
+       if (needle.empty())
+	  return npos;
+       if (needle.length() == 1)
+	  return find(*needle.data());
+       size_t found = 0;
+       while ((found = find(*needle.data(), found)) != npos) {
+	  if (compare(found, needle.length(), needle) == 0)
+	     return found;
+	  ++found;
+       }
+       return found;
+    }
+    size_t find(APT::StringView const needle, size_t pos) const {
+       if (pos == 0)
+	  return find(needle);
+       size_t const found = substr(pos).find(needle);
+       if (found == npos)
+	  return npos;
+       return pos + found;
+    }
+
     /* Conversions */
     std::string to_string() const {
         return std::string(data_, size_);
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index bd4856526..3a0a6eaa3 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -23,8 +23,10 @@
 #include <algorithm>
 #include <array>
 #include <iomanip>
+#include <limits>
 #include <locale>
 #include <sstream>
+#include <memory>
 #include <sstream>
 #include <string>
 #include <vector>
@@ -310,11 +312,11 @@ bool ParseQuoteWord(const char *&String,string &Res)
    }
 
    // Now de-quote characters
-   char Buffer[1024];
+   Res.clear();
+   Res.reserve(C - String);
    char Tmp[3];
    const char *Start = String;
-   char *I;
-   for (I = Buffer; I < Buffer + sizeof(Buffer) && Start != C; I++)
+   while (Start != C)
    {
       if (*Start == '%' && Start + 2 < C &&
 	  isxdigit(Start[1]) && isxdigit(Start[2]))
@@ -322,19 +324,15 @@ bool ParseQuoteWord(const char *&String,string &Res)
 	 Tmp[0] = Start[1];
 	 Tmp[1] = Start[2];
 	 Tmp[2] = 0;
-	 *I = (char)strtol(Tmp,0,16);
+	 Res.push_back(static_cast<char>(strtol(Tmp, 0, 16)));
 	 Start += 3;
 	 continue;
       }
       if (*Start != '"')
-	 *I = *Start;
-      else
-	 I--;
-      Start++;
+	 Res.push_back(*Start);
+      ++Start;
    }
-   *I = 0;
-   Res = Buffer;
-   
+
    // Skip ending white space
    for (; isspace(*C) != 0; C++)
       ;
@@ -354,33 +352,28 @@ bool ParseCWord(const char *&String,string &Res)
       ;
    if (*C == 0)
       return false;
-   
-   char Buffer[1024];
-   char *Buf = Buffer;
-   if (strlen(String) >= sizeof(Buffer))
-       return false;
-       
-   for (; *C != 0; C++)
+
+   Res.clear();
+   Res.reserve(strlen(String));
+   for (; *C != 0; ++C)
    {
       if (*C == '"')
       {
 	 for (C++; *C != 0 && *C != '"'; C++)
-	    *Buf++ = *C;
-	 
+	    Res.push_back(*C);
+
 	 if (*C == 0)
 	    return false;
-	 
+
 	 continue;
-      }      
-      
+      }
+
       if (C != String && isspace(*C) != 0 && isspace(C[-1]) != 0)
 	 continue;
       if (isspace(*C) == 0)
 	 return false;
-      *Buf++ = ' ';
+      Res.push_back(' ');
    }
-   *Buf = 0;
-   Res = Buffer;
    String = C;
    return true;
 }
@@ -597,7 +590,7 @@ string Base64Encode(const string &S)
       base64.  */
    for (string::const_iterator I = S.begin(); I < S.end(); I += 3)
    {
-      char Bits[3] = {0,0,0};
+      uint8_t Bits[3] = {0,0,0};
       Bits[0] = I[0];
       if (I + 1 < S.end())
 	 Bits[1] = I[1];
@@ -1148,34 +1141,24 @@ bool FTPMDTMStrToTime(const char* const str,time_t &time)
 									/*}}}*/
 // StrToNum - Convert a fixed length string to a number			/*{{{*/
 // ---------------------------------------------------------------------
-/* This is used in decoding the crazy fixed length string headers in 
+/* This is used in decoding the crazy fixed length string headers in
    tar and ar files. */
 bool StrToNum(const char *Str,unsigned long &Res,unsigned Len,unsigned Base)
 {
-   char S[30];
-   if (Len >= sizeof(S))
+   unsigned long long BigRes;
+   if (not StrToNum(Str, BigRes, Len, Base))
       return false;
-   memcpy(S,Str,Len);
-   S[Len] = 0;
-   
-   // All spaces is a zero
-   Res = 0;
-   unsigned I;
-   for (I = 0; S[I] == ' '; I++);
-   if (S[I] == 0)
-      return true;
-   
-   char *End;
-   Res = strtoul(S,&End,Base);
-   if (End == S)
+
+   if (std::numeric_limits<unsigned long>::max() < BigRes)
       return false;
-   
+
+   Res = BigRes;
    return true;
 }
 									/*}}}*/
 // StrToNum - Convert a fixed length string to a number			/*{{{*/
 // ---------------------------------------------------------------------
-/* This is used in decoding the crazy fixed length string headers in 
+/* This is used in decoding the crazy fixed length string headers in
    tar and ar files. */
 bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base)
 {
@@ -1184,20 +1167,20 @@ bool StrToNum(const char *Str,unsigned long long &Res,unsigned Len,unsigned Base
       return false;
    memcpy(S,Str,Len);
    S[Len] = 0;
-   
+
    // All spaces is a zero
    Res = 0;
    unsigned I;
-   for (I = 0; S[I] == ' '; I++);
+   for (I = 0; S[I] == ' '; ++I);
    if (S[I] == 0)
       return true;
-   
+   if (S[I] == '-')
+      return false;
+
    char *End;
+   errno = 0;
    Res = strtoull(S,&End,Base);
-   if (End == S)
-      return false;
-   
-   return true;
+   return not (End == S || errno != 0);
 }
 									/*}}}*/
 
@@ -1432,13 +1415,12 @@ unsigned long RegexChoice(RxChoiceList *Rxs,const char **ListBegin,
 // ---------------------------------------------------------------------
 /* This is used to make the internationalization strings easier to translate
    and to allow reordering of parameters */
-static bool iovprintf(ostream &out, const char *format,
+bool iovprintf(std::ostream &out, const char *format,
 		      va_list &args, ssize_t &size) {
-   char *S = (char*)malloc(size);
-   ssize_t const n = vsnprintf(S, size, format, args);
+   auto S = std::unique_ptr<char,decltype(&free)>{static_cast<char*>(malloc(size)), &free};
+   ssize_t const n = vsnprintf(S.get(), size, format, args);
    if (n > -1 && n < size) {
-      out << S;
-      free(S);
+      out << S.get();
       return true;
    } else {
       if (n > -1)
@@ -1446,7 +1428,6 @@ static bool iovprintf(ostream &out, const char *format,
       else
 	 size *= 2;
    }
-   free(S);
    return false;
 }
 void ioprintf(ostream &out,const char *format,...)
@@ -1611,22 +1592,26 @@ string DeEscapeString(const string &input)
       switch (*it)
       {
          case '0':
-            if (it + 2 <= input.end()) {
+            if (it + 2 < input.end()) {
                tmp[0] = it[1];
                tmp[1] = it[2];
                tmp[2] = 0;
                output += (char)strtol(tmp, 0, 8);
                it += 2;
-            }
+            } else {
+	       // FIXME: raise exception here?
+	    }
             break;
          case 'x':
-            if (it + 2 <= input.end()) {
+            if (it + 2 < input.end()) {
                tmp[0] = it[1];
                tmp[1] = it[2];
                tmp[2] = 0;
                output += (char)strtol(tmp, 0, 16);
                it += 2;
-            }
+            } else {
+	       // FIXME: raise exception here?
+	    }
             break;
          default:
             // FIXME: raise exception here?
diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h
index c25c6208a..b6e6bfdce 100644
--- a/apt-pkg/contrib/strutl.h
+++ b/apt-pkg/contrib/strutl.h
@@ -42,7 +42,7 @@ namespace APT {
 APT_PUBLIC bool UTF8ToCodeset(const char *codeset, const std::string &orig, std::string *dest);
 APT_PUBLIC char *_strstrip(char *String);
 APT_PUBLIC char *_strrstrip(char *String); // right strip only
-APT_PUBLIC char *_strtabexpand(char *String,size_t Len);
+APT_DEPRECATED_MSG("Use SubstVar to avoid memory headaches") APT_PUBLIC char *_strtabexpand(char *String,size_t Len);
 APT_PUBLIC bool ParseQuoteWord(const char *&String,std::string &Res);
 APT_PUBLIC bool ParseCWord(const char *&String,std::string &Res);
 APT_PUBLIC std::string QuoteString(const std::string &Str,const char *Bad);
@@ -117,6 +117,8 @@ APT_PUBLIC std::vector<std::string> StringSplit(std::string const &input,
                                      std::string const &sep, 
                                      unsigned int maxsplit=std::numeric_limits<unsigned int>::max()) APT_PURE;
 
+
+APT_HIDDEN bool iovprintf(std::ostream &out, const char *format, va_list &args, ssize_t &size);
 APT_PUBLIC void ioprintf(std::ostream &out,const char *format,...) APT_PRINTF(2);
 APT_PUBLIC void strprintf(std::string &out,const char *format,...) APT_PRINTF(2);
 APT_PUBLIC char *safe_snprintf(char *Buffer,char *End,const char *Format,...) APT_PRINTF(3);
diff --git a/apt-pkg/deb/debfile.cc b/apt-pkg/deb/debfile.cc
index bef0cd0d8..645a579ef 100644
--- a/apt-pkg/deb/debfile.cc
+++ b/apt-pkg/deb/debfile.cc
@@ -24,9 +24,12 @@
 #include <apt-pkg/error.h>
 #include <apt-pkg/extracttar.h>
 #include <apt-pkg/fileutl.h>
+#include <apt-pkg/strutl.h>
 #include <apt-pkg/tagfile.h>
 
+#include <algorithm>
 #include <string>
+#include <sstream>
 #include <vector>
 #include <string.h>
 #include <sys/stat.h>
@@ -105,42 +108,48 @@ const ARArchive::Member *debDebFile::GotoMember(const char *Name)
 /* Simple wrapper around tar.. */
 bool debDebFile::ExtractTarMember(pkgDirStream &Stream,const char *Name)
 {
-   // Get the archive member
-   const ARArchive::Member *Member = NULL;
    std::string Compressor;
+   auto const Compressors = APT::Configuration::getCompressors();
 
-   std::vector<APT::Configuration::Compressor> compressor = APT::Configuration::getCompressors();
-   for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin();
-	c != compressor.end(); ++c)
+   ARArchive::Member const *Member = AR.FindMember(Name);
+   if (Member != nullptr)
    {
-      Member = AR.FindMember(std::string(Name).append(c->Extension).c_str());
-      if (Member == NULL)
-	 continue;
-      Compressor = c->Name;
-      break;
+      auto const found = std::find_if(Compressors.cbegin(), Compressors.cend(), [&](auto const &c) {
+	 return not c.Extension.empty() && APT::String::Endswith(Name, c.Extension);
+      });
+      if (found != Compressors.cend())
+	 Compressor = found->Name;
    }
-
-   if (Member == NULL)
-      Member = AR.FindMember(std::string(Name).c_str());
-
-   if (Member == NULL)
+   else
    {
-      std::string ext = std::string(Name) + ".{";
-      for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressor.begin();
-	   c != compressor.end(); ++c) {
-	 if (!c->Extension.empty())
-	    ext.append(c->Extension.substr(1));
+      for (auto const &c : Compressors)
+      {
+	 if (c.Extension.empty())
+	    continue;
+	 Member = AR.FindMember(std::string(Name).append(c.Extension).c_str());
+	 if (Member == nullptr)
+	    continue;
+	 Compressor = c.Name;
+	 break;
       }
-      ext.append("}");
-      return _error->Error(_("Internal error, could not locate member %s"), ext.c_str());
    }
 
-   if (File.Seek(Member->Start) == false)
+   if (Member == nullptr)
+   {
+      std::ostringstream ext;
+      ext << Name << '{';
+      for (auto const &c : Compressors)
+	 if (not c.Extension.empty())
+	    ext << c.Extension << ',';
+      ext << '}';
+      return _error->Error(_("Internal error, could not locate member %s"), ext.str().c_str());
+   }
+
+   if (not File.Seek(Member->Start))
       return false;
 
-   // Prepare Tar
    ExtractTar Tar(File,Member->Size,Compressor);
-   if (_error->PendingError() == true)
+   if (_error->PendingError())
       return false;
    return Tar.Go(Stream);
 }
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index f7f3537aa..cbde7c42f 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -255,7 +255,9 @@ uint32_t pkgCache::CacheHash()
 			 GetMap().Size() - sizeof(header));
    }
 
-   return XXH3_64bits_digest(state) & 0xFFFFFFFF;
+   auto const digest = XXH3_64bits_digest(state);
+   XXH3_freeState(state);
+   return digest & 0xFFFFFFFF;
 }
 									/*}}}*/
 // Cache::FindPkg - Locate a package by name				/*{{{*/
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 26cf7fc68..b4fd0641e 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -37,6 +37,8 @@
 
 #include <apti18n.h>
 									/*}}}*/
+constexpr auto APT_CACHE_START_DEFAULT = 24 * 1024 * 1024;
+
 template<class T> using Dynamic = pkgCacheGenerator::Dynamic<T>;
 typedef std::vector<pkgIndexFile *>::iterator FileIterator;
 template <typename Iter> std::vector<Iter*> pkgCacheGenerator::Dynamic<Iter>::toReMap;
@@ -156,13 +158,14 @@ pkgCacheGenerator::~pkgCacheGenerator()
    Map.Sync(0,sizeof(pkgCache::Header));
 }
 									/*}}}*/
-void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap, size_t oldSize) {/*{{{*/
+void pkgCacheGenerator::ReMap(void const * const oldMap, void * const newMap, size_t oldSize) {/*{{{*/
+   if (oldMap == newMap)
+      return;
+
    // Prevent multiple remaps of the same iterator. If seen.insert(iterator)
    // returns (something, true) the iterator was not yet seen and we can
    // remap it.
    std::unordered_set<void *> seen;
-   if (oldMap == newMap)
-      return;
 
    if (_config->FindB("Debug::pkgCacheGen", false))
       std::clog << "Remapping from " << oldMap << " to " << newMap << std::endl;
@@ -170,42 +173,24 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
    Cache.ReMap(false);
 
    if (CurrentFile != nullptr)
-      CurrentFile += static_cast<pkgCache::PackageFile const *>(newMap) - static_cast<pkgCache::PackageFile const *>(oldMap);
+      CurrentFile = static_cast<pkgCache::PackageFile *>(newMap) + (CurrentFile - static_cast<pkgCache::PackageFile const *>(oldMap));
    if (CurrentRlsFile != nullptr)
-      CurrentRlsFile += static_cast<pkgCache::ReleaseFile const *>(newMap) - static_cast<pkgCache::ReleaseFile const *>(oldMap);
-
-   for (std::vector<pkgCache::GrpIterator*>::const_iterator i = Dynamic<pkgCache::GrpIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::GrpIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::PkgIterator*>::const_iterator i = Dynamic<pkgCache::PkgIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::PkgIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::VerIterator*>::const_iterator i = Dynamic<pkgCache::VerIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::VerIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::DepIterator*>::const_iterator i = Dynamic<pkgCache::DepIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::DepIterator>::toReMap.end(); ++i)
-	if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::DescIterator*>::const_iterator i = Dynamic<pkgCache::DescIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::DescIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::PrvIterator*>::const_iterator i = Dynamic<pkgCache::PrvIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::PrvIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::PkgFileIterator*>::const_iterator i = Dynamic<pkgCache::PkgFileIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::PkgFileIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
-   for (std::vector<pkgCache::RlsFileIterator*>::const_iterator i = Dynamic<pkgCache::RlsFileIterator>::toReMap.begin();
-	i != Dynamic<pkgCache::RlsFileIterator>::toReMap.end(); ++i)
-      if (std::get<1>(seen.insert(*i)) == true)
-	 (*i)->ReMap(oldMap, newMap);
+      CurrentRlsFile = static_cast<pkgCache::ReleaseFile *>(newMap) + (CurrentRlsFile - static_cast<pkgCache::ReleaseFile const *>(oldMap));
+
+#define APT_REMAP(TYPE) \
+   for (auto * const i : Dynamic<TYPE>::toReMap) \
+      if (seen.insert(i).second) \
+	 i->ReMap(oldMap, newMap)
+   APT_REMAP(pkgCache::GrpIterator);
+   APT_REMAP(pkgCache::PkgIterator);
+   APT_REMAP(pkgCache::VerIterator);
+   APT_REMAP(pkgCache::DepIterator);
+   APT_REMAP(pkgCache::DescIterator);
+   APT_REMAP(pkgCache::PrvIterator);
+   APT_REMAP(pkgCache::PkgFileIterator);
+   APT_REMAP(pkgCache::RlsFileIterator);
+#undef APT_REMAP
+
    for (APT::StringView* ViewP : Dynamic<APT::StringView>::toReMap) {
       if (std::get<1>(seen.insert(ViewP)) == false)
 	 continue;
@@ -453,7 +438,7 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
 			   Pkg.Name(), "NewVersion", 1);
 
    if (oldMap != Map.Data())
-	 LastVer += static_cast<map_pointer<pkgCache::Version> const *>(Map.Data()) - static_cast<map_pointer<pkgCache::Version> const *>(oldMap);
+	 LastVer = static_cast<map_pointer<pkgCache::Version> *>(Map.Data()) + (LastVer - static_cast<map_pointer<pkgCache::Version> const *>(oldMap));
    *LastVer = verindex;
 
    if (unlikely(List.NewVersion(Ver) == false))
@@ -1073,7 +1058,7 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg,
       for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D)
 	 OldDepLast = &D->NextDepends;
    } else if (oldMap != Map.Data())
-      OldDepLast += static_cast<map_pointer<pkgCache::Dependency> const *>(Map.Data()) - static_cast<map_pointer<pkgCache::Dependency> const *>(oldMap);
+      OldDepLast = static_cast<map_pointer<pkgCache::Dependency> *>(Map.Data()) + (OldDepLast - static_cast<map_pointer<pkgCache::Dependency> const *>(oldMap));
 
    Dep->NextDepends = *OldDepLast;
    *OldDepLast = Dependency;
@@ -1400,6 +1385,13 @@ static bool CheckValidity(FileFd &CacheFile, std::string const &CacheFileName,
       return false;
    }
 
+   if (_config->FindI("APT::Cache-Start", 0) == 0)
+   {
+      auto const size = CacheFile.FileSize();
+      if (std::numeric_limits<int>::max() >= size && size > APT_CACHE_START_DEFAULT)
+	 _config->Set("APT::Cache-Start", size);
+   }
+
    if (List.GetLastModifiedTime() > CacheFile.ModificationTime())
    {
       if (Debug == true)
@@ -1608,7 +1600,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    where it builds the cache 'fast' into a memory buffer. */
 static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
 {
-   map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
+   map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", APT_CACHE_START_DEFAULT);
    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
    Flags |= MMap::Moveable;
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index 62c171be4..f4781d715 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -146,7 +146,7 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
 			MMap **OutMap,pkgCache **OutCache, bool AllowMem = false);
    APT_PUBLIC static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap);
 
-   void ReMap(void const * const oldMap, void const * const newMap, size_t oldSize);
+   void ReMap(void const * const oldMap, void * const newMap, size_t oldSize);
    bool Start();
 
    pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress);
diff --git a/methods/aptmethod.h b/methods/aptmethod.h
index 7038131cf..bd50e8078 100644
--- a/methods/aptmethod.h
+++ b/methods/aptmethod.h
@@ -325,7 +325,11 @@ protected:
 
       rc = seccomp_load(ctx);
       if (rc == -EINVAL)
-	 Warning("aptMethod::Configuration: could not load seccomp policy: %s", strerror(-rc));
+      {
+	 std::string msg;
+	 strprintf(msg, "aptMethod::Configuration: could not load seccomp policy: %s", strerror(-rc));
+	 Warning(std::move(msg));
+      }
       else if (rc != 0)
 	 return _error->FatalE("aptMethod::Configuration", "could not load seccomp policy: %s", strerror(-rc));
 
@@ -380,12 +384,17 @@ protected:
       return true;
    }
 
-   void Warning(const char *Format,...)
+   void Warning(std::string &&msg)
    {
-      va_list args;
-      va_start(args,Format);
-      PrintStatus("104 Warning", Format, args);
-      va_end(args);
+      std::unordered_map<std::string, std::string> fields;
+      if (Queue != 0)
+	 fields.emplace("URI", Queue->Uri);
+      else
+	 fields.emplace("URI", "<UNKNOWN>");
+      if (not UsedMirror.empty())
+	 fields.emplace("UsedMirror", UsedMirror);
+      fields.emplace("Message", std::move(msg));
+      SendMessage("104 Warning", std::move(fields));
    }
 
    std::vector<std::string> methodNames;
@@ -560,14 +569,11 @@ class aptAuthConfMethod : public aptMethod
 	 result &= MaybeAddAuth(*authconf, uri);
       }
 
-      if (not _error->empty())
+      while (not _error->empty())
       {
 	 std::string message;
-	 while (not _error->empty())
-	 {
-	    _error->PopMessage(message);
-	    Warning("%s", message.c_str());
-	 }
+	 _error->PopMessage(message);
+	 Warning(std::move(message));
       }
       _error->RevertToStack();
 
diff --git a/methods/basehttp.cc b/methods/basehttp.cc
index 8aac1090c..3786e2e6c 100644
--- a/methods/basehttp.cc
+++ b/methods/basehttp.cc
@@ -110,6 +110,9 @@ bool RequestState::HeaderLine(string const &Line)			/*{{{*/
 	 if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
 	    return _error->Error(_("The HTTP server sent an invalid reply header"));
       }
+      auto const CodeLen = strlen(Code);
+      auto const CodeEnd = std::remove_if(Code, Code + CodeLen, [](char c) { return isprint(c) == 0; });
+      *CodeEnd = '\0';
 
       /* Check the HTTP response header to get the default persistence
          state. */
@@ -762,7 +765,9 @@ int BaseHttpMethod::Loop()
 			// yes, he did! Disable pipelining and rewrite queue
 			if (Server->Pipeline == true)
 			{
-			   Warning(_("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
+			   std::string msg;
+			   strprintf(msg, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
+			   Warning(std::move(msg));
 			   Server->Pipeline = false;
 			   Server->PipelineAllowed = false;
 			   // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
diff --git a/methods/basehttp.h b/methods/basehttp.h
index 62c9963ea..c0d14d854 100644
--- a/methods/basehttp.h
+++ b/methods/basehttp.h
@@ -60,7 +60,7 @@ struct RequestState
    bool AddPartialFileToHashes(FileFd &File);
 
    RequestState(BaseHttpMethod * const Owner, ServerState * const Server) :
-      Owner(Owner), Server(Server) { time(&Date); }
+      Owner(Owner), Server(Server) { time(&Date); Code[0] = '\0'; }
 };
 struct ServerState
 {
diff --git a/methods/gpgv.cc b/methods/gpgv.cc
index 08d030a17..a9da456ec 100644
--- a/methods/gpgv.cc
+++ b/methods/gpgv.cc
@@ -264,7 +264,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile,
 	    SubKeyMapping[tokens[9]].emplace_back(sig);
       }
       else if (strncmp(buffer, APTKEYWARNING, sizeof(APTKEYWARNING)-1) == 0)
-         Warning("%s", buffer + sizeof(APTKEYWARNING));
+         Warning(buffer + sizeof(APTKEYWARNING));
       else if (strncmp(buffer, APTKEYERROR, sizeof(APTKEYERROR)-1) == 0)
 	 _error->Error("%s", buffer + sizeof(APTKEYERROR));
    }
@@ -442,8 +442,12 @@ bool GPGVMethod::URIAcquire(std::string const &Message, FetchItem *Itm)
 	    }))
    {
       for (auto const & Signer : Signers.SoonWorthless)
+      {
+	 std::string msg;
          // TRANSLATORS: The second %s is the reason and is untranslated for repository owners.
-         Warning(_("Signature by key %s uses weak digest algorithm (%s)"), Signer.key.c_str(), Signer.note.c_str());
+	 strprintf(msg, _("Signature by key %s uses weak digest algorithm (%s)"), Signer.key.c_str(), Signer.note.c_str());
+         Warning(std::move(msg));
+      }
    }
 
    if (Signers.Good.empty() || !Signers.Bad.empty() || !Signers.NoPubKey.empty())
diff --git a/methods/rred.cc b/methods/rred.cc
index 981364a9e..f53f05ad5 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -7,12 +7,15 @@
 
 #include <config.h>
 
+#ifndef APT_EXCLUDE_RRED_METHOD_CODE
 #include "aptmethod.h"
 #include <apt-pkg/configuration.h>
+#include <apt-pkg/init.h>
+#endif
+
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/hashes.h>
-#include <apt-pkg/init.h>
 #include <apt-pkg/strutl.h>
 
 #include <apt-private/private-cmndline.h>
@@ -23,7 +26,7 @@
 #include <vector>
 #include <stddef.h>
 
-#include <assert.h>
+#include <cassert>
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -33,7 +36,9 @@
 
 #include <apti18n.h>
 
-#define BLOCK_SIZE (512*1024)
+#ifndef APT_MEMBLOCK_SIZE
+#define APT_MEMBLOCK_SIZE (512*1024)
+#endif
 
 static bool ShowHelp(CommandLine &)
 {
@@ -66,9 +71,9 @@ class MemBlock {
    char *start;
    size_t size;
    char *free;
-   MemBlock *next;
+   MemBlock *next = nullptr;
 
-   explicit MemBlock(size_t size) : size(size), next(NULL)
+   explicit MemBlock(size_t size) : size(size)
    {
       free = start = new char[size];
    }
@@ -77,11 +82,7 @@ class MemBlock {
 
    public:
 
-   MemBlock(void) {
-      free = start = new char[BLOCK_SIZE];
-      size = BLOCK_SIZE;
-      next = NULL;
-   }
+   MemBlock() : MemBlock(APT_MEMBLOCK_SIZE) {}
 
    ~MemBlock() {
       delete [] start;
@@ -100,11 +101,9 @@ class MemBlock {
 	 for (MemBlock *k = this; k; k = k->next) {
 	    if (k->free == last) {
 	       if (len <= k->avail()) {
-		  char *n = k->add(src, len);
-		  assert(last == n);
-		  if (last == n)
-		     return NULL;
-		  return n;
+		  char * const n = k->add(src, len);
+		  assert(last == n); // we checked already that the block is big enough, so a new one shouldn't be used
+		  return (last == n) ? nullptr : n;
 	       } else {
 		  break;
 	       }
@@ -119,10 +118,10 @@ class MemBlock {
    char *add(char *src, size_t len) {
       if (len > avail()) {
 	 if (!next) {
-	    if (len > BLOCK_SIZE)  {
+	    if (len > APT_MEMBLOCK_SIZE)  {
 	       next = new MemBlock(len);
 	    } else {
-	       next = new MemBlock;
+	       next = new MemBlock();
 	    }
 	 }
 	 return next->add(src, len);
@@ -155,24 +154,27 @@ struct Change {
    }
 
    /* actually, don't write <lines> lines from <add> */
-   void skip_lines(size_t lines)
+   bool skip_lines(size_t lines)
    {
       while (lines > 0) {
 	 char *s = (char*) memchr(add, '\n', add_len);
-	 assert(s != NULL);
+	 if (s == nullptr)
+	    return _error->Error("No line left in add_len data to skip (1)");
 	 s++;
 	 add_len -= (s - add);
 	 add_cnt--;
 	 lines--;
 	 if (add_len == 0) {
-	    add = NULL;
-	    assert(add_cnt == 0);
-	    assert(lines == 0);
+	    add = nullptr;
+	    if (add_cnt != 0 || lines != 0)
+	       return _error->Error("No line left in add_len data to skip (2)");
 	 } else {
 	    add = s;
-	    assert(add_cnt > 0);
+	    if (add_cnt == 0)
+	       return _error->Error("No line left in add_len data to skip (3)");
 	 }
       }
+      return true;
    }
 };
 
@@ -183,7 +185,8 @@ class FileChanges {
 
    bool pos_is_okay(void) const
    {
-#ifdef POSDEBUG
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+      // this isn't unsafe, it is just a moderately expensive check we want to avoid normally
       size_t cpos = 0;
       std::list<struct Change>::const_iterator x;
       for (x = changes.begin(); x != where; ++x) {
@@ -208,33 +211,40 @@ class FileChanges {
    std::list<struct Change>::reverse_iterator rbegin(void) { return changes.rbegin(); }
    std::list<struct Change>::reverse_iterator rend(void) { return changes.rend(); }
 
-   void add_change(Change c) {
+   bool add_change(Change c) {
       assert(pos_is_okay());
-      go_to_change_for(c.offset);
-      assert(pos + where->offset == c.offset);
+      if (not go_to_change_for(c.offset) ||
+	  pos + where->offset != c.offset)
+	 return false;
       if (c.del_cnt > 0)
-	 delete_lines(c.del_cnt);
-      assert(pos + where->offset == c.offset);
+	 if (not delete_lines(c.del_cnt))
+	    return false;
+      if (pos + where->offset != c.offset)
+	 return false;
       if (c.add_len > 0) {
 	 assert(pos_is_okay());
 	 if (where->add_len > 0)
-	    new_change();
-	 assert(where->add_len == 0 && where->add_cnt == 0);
+	    if (not new_change())
+	       return false;
+	 if (where->add_len != 0 || where->add_cnt != 0)
+	    return false;
 
 	 where->add_len = c.add_len;
 	 where->add_cnt = c.add_cnt;
 	 where->add = c.add;
       }
       assert(pos_is_okay());
-      merge();
-      assert(pos_is_okay());
+      if (not merge())
+	 return false;
+      return pos_is_okay();
    }
 
    private:
-   void merge(void)
+   bool merge(void)
    {
       while (where->offset == 0 && where != changes.begin()) {
-	 left();
+	 if (not left())
+	    return false;
       }
       std::list<struct Change>::iterator next = where;
       ++next;
@@ -253,53 +263,56 @@ class FileChanges {
 	    ++next;
 	 }
       }
+      return true;
    }
 
-   void go_to_change_for(size_t line)
+   bool go_to_change_for(size_t line)
    {
       while(where != changes.end()) {
 	 if (line < pos) {
-	    left();
+	    if (not left())
+	       return false;
 	    continue;
 	 }
 	 if (pos + where->offset + where->add_cnt <= line) {
-	    right();
+	    if (not right())
+	       return false;
 	    continue;
 	 }
 	 // line is somewhere in this slot
 	 if (line < pos + where->offset) {
 	    break;
 	 } else if (line == pos + where->offset) {
-	    return;
+	    return true;
 	 } else {
-	    split(line - pos);
-	    right();
-	    return;
+	    if (not split(line - pos))
+	       return false;
+	    return right();
 	 }
       }
       /* it goes before this patch */
-      insert(line-pos);
+      return insert(line-pos);
    }
 
-   void new_change(void) { insert(where->offset); }
+   bool new_change(void) { return insert(where->offset); }
 
-   void insert(size_t offset)
+   bool insert(size_t offset)
    {
       assert(pos_is_okay());
-      assert(where == changes.end() || offset <= where->offset);
+      if (where != changes.end() && offset > where->offset)
+	 return false;
       if (where != changes.end())
 	 where->offset -= offset;
       changes.insert(where, Change(offset));
       --where;
-      assert(pos_is_okay());
+      return pos_is_okay();
    }
 
-   void split(size_t offset)
+   bool split(size_t offset)
    {
       assert(pos_is_okay());
-
-      assert(where->offset < offset);
-      assert(offset < where->offset + where->add_cnt);
+      if (where->offset >= offset || offset >= where->offset + where->add_cnt)
+	 return false;
 
       size_t keep_lines = offset - where->offset;
 
@@ -307,27 +320,29 @@ class FileChanges {
 
       where->del_cnt = 0;
       where->offset = 0;
-      where->skip_lines(keep_lines);
+      if (not where->skip_lines(keep_lines))
+	 return false;
 
       before.add_cnt = keep_lines;
       before.add_len -= where->add_len;
 
       changes.insert(where, before);
       --where;
-      assert(pos_is_okay());
+      return pos_is_okay();
    }
 
-   void delete_lines(size_t cnt)
+   bool delete_lines(size_t cnt)
    {
-      std::list<struct Change>::iterator x = where;
       assert(pos_is_okay());
+      std::list<struct Change>::iterator x = where;
       while (cnt > 0)
       {
 	 size_t del;
 	 del = x->add_cnt;
 	 if (del > cnt)
 	    del = cnt;
-	 x->skip_lines(del);
+	 if (not x->skip_lines(del))
+	    return false;
 	 cnt -= del;
 
 	 ++x;
@@ -342,21 +357,21 @@ class FileChanges {
 	 where->del_cnt += del;
 	 cnt -= del;
       }
-      assert(pos_is_okay());
+      return pos_is_okay();
    }
 
-   void left(void) {
+   bool left(void) {
       assert(pos_is_okay());
       --where;
       pos -= where->offset + where->add_cnt;
-      assert(pos_is_okay());
+      return pos_is_okay();
    }
 
-   void right(void) {
+   bool right(void) {
       assert(pos_is_okay());
       pos += where->offset + where->add_cnt;
       ++where;
-      assert(pos_is_okay());
+      return pos_is_okay();
    }
 };
 
@@ -378,7 +393,7 @@ class Patch {
    static void dump_rest(FileFd &o, FileFd &i,
 	 Hashes * const start_hash, Hashes * const end_hash)
    {
-      char buffer[BLOCK_SIZE];
+      char buffer[APT_MEMBLOCK_SIZE];
       unsigned long long l = 0;
       while (i.Read(buffer, sizeof(buffer), &l)) {
 	 if (l ==0  || !retry_fwrite(buffer, l, o, start_hash, end_hash))
@@ -389,7 +404,7 @@ class Patch {
    static void dump_lines(FileFd &o, FileFd &i, size_t n,
 	 Hashes * const start_hash, Hashes * const end_hash)
    {
-      char buffer[BLOCK_SIZE];
+      char buffer[APT_MEMBLOCK_SIZE];
       while (n > 0) {
 	 if (i.ReadLine(buffer, sizeof(buffer)) == NULL)
 	    buffer[0] = '\0';
@@ -402,7 +417,7 @@ class Patch {
 
    static void skip_lines(FileFd &i, int n, Hashes * const start_hash)
    {
-      char buffer[BLOCK_SIZE];
+      char buffer[APT_MEMBLOCK_SIZE];
       while (n > 0) {
 	 if (i.ReadLine(buffer, sizeof(buffer)) == NULL)
 	    buffer[0] = '\0';
@@ -422,7 +437,7 @@ class Patch {
 
    bool read_diff(FileFd &f, Hashes * const h)
    {
-      char buffer[BLOCK_SIZE];
+      char buffer[APT_MEMBLOCK_SIZE];
       bool cmdwanted = true;
 
       Change ch(std::numeric_limits<size_t>::max());
@@ -478,7 +493,8 @@ class Patch {
 		  ch.add = NULL;
 		  ch.add_cnt = 0;
 		  ch.add_len = 0;
-		  filechanges.add_change(ch);
+		  if (not filechanges.add_change(ch))
+		     return _error->Error("Parsing patchfile %s failed: Delete command could not be added to changes", f.Name().c_str());
 		  break;
 	       default:
 		  return _error->Error("Parsing patchfile %s failed: Unknown command", f.Name().c_str());
@@ -486,7 +502,8 @@ class Patch {
 	 } else { /* !cmdwanted */
 	    if (strcmp(buffer, ".\n") == 0) {
 	       cmdwanted = true;
-	       filechanges.add_change(ch);
+	       if (not filechanges.add_change(ch))
+		  return _error->Error("Parsing patchfile %s failed: Data couldn't be added for command (1)", f.Name().c_str());
 	    } else {
 	       char *last = NULL;
 	       char *add;
@@ -500,7 +517,8 @@ class Patch {
 		  ch.add_cnt++;
 	       } else {
 		  if (ch.add) {
-		     filechanges.add_change(ch);
+		     if (not filechanges.add_change(ch))
+			return _error->Error("Parsing patchfile %s failed: Data couldn't be added for command (2)", f.Name().c_str());
 		     ch.del_cnt = 0;
 		  }
 		  ch.offset += ch.add_cnt;
@@ -575,6 +593,7 @@ class Patch {
    }
 };
 
+#ifndef APT_EXCLUDE_RRED_METHOD_CODE
 class RredMethod : public aptMethod {
    private:
       bool Debug;
@@ -859,3 +878,4 @@ int main(int argc, const char *argv[])
    input.Close();
    return DispatchCommandLine(CmdL, {});
 }
+#endif
diff --git a/po/de.po b/po/de.po
index ee18e57f0..a66cdcdf7 100644
--- a/po/de.po
+++ b/po/de.po
@@ -1293,7 +1293,7 @@ msgstr ""
 
 #: apt-pkg/depcache.cc
 msgid "Building dependency tree"
-msgstr "Abhängigkeitsbaum wird aufgebaut."
+msgstr "Abhängigkeitsbaum wird aufgebaut"
 
 #: apt-pkg/depcache.cc
 msgid "Candidate versions"
@@ -1305,7 +1305,7 @@ msgstr "Abhängigkeitsgenerierung"
 
 #: apt-pkg/depcache.cc
 msgid "Reading state information"
-msgstr "Statusinformationen werden eingelesen."
+msgstr "Statusinformationen werden eingelesen"
 
 #: apt-pkg/depcache.cc
 #, c-format
diff --git a/test/integration/test-apt-get-install-deb b/test/integration/test-apt-get-install-deb
index 7fa5ca3e7..6fde00708 100755
--- a/test/integration/test-apt-get-install-deb
+++ b/test/integration/test-apt-get-install-deb
@@ -23,7 +23,7 @@ mv foo.rpm foo.deb
 for exe in apt aptget; do
 	for cmd in install remove purge upgrade full-upgrade; do
 		testfailuremsg "E: Invalid archive signature
-E: Internal error, could not locate member control.tar.{zstlz4gzxzbz2lzma}
+E: Internal error, could not locate member control.tar{.zst,.lz4,.gz,.xz,.bz2,.lzma,}
 E: Could not read meta data from ${TMPWORKINGDIRECTORY}/foo.deb
 E: The package lists or status file could not be parsed or opened." $exe $cmd ./foo.deb
 	done
diff --git a/test/integration/test-bug-733028-gpg-resource-limit b/test/integration/test-bug-733028-gpg-resource-limit
index 69baf4e5c..b44facda3 100755
--- a/test/integration/test-bug-733028-gpg-resource-limit
+++ b/test/integration/test-bug-733028-gpg-resource-limit
@@ -17,6 +17,7 @@ testaptkeys 'Joe Sixpack'
 
 testsuccess aptget update
 msgtest 'Test for no gpg errors/warnings in' 'apt-get update'
+sed -i -e "s#${TMPWORKINGDIRECTORY}#/tmp/tmp.XXXXXX#g" rootdir/tmp/testsuccess.output
 if grep -iq 'GPG' rootdir/tmp/testsuccess.output; then
 	cat rootdir/tmp/testsuccess.output
 	msgfail
diff --git a/test/libapt/stringview_test.cc b/test/libapt/stringview_test.cc
index 03d82517b..9cfaa3b48 100644
--- a/test/libapt/stringview_test.cc
+++ b/test/libapt/stringview_test.cc
@@ -74,6 +74,14 @@ TEST(StringViewTest,Find)
    EXPECT_EQ(defString.to_string().find('e',3), defString.find('e',3));
    EXPECT_EQ(defString.to_string().find('l',6), defString.find('l',6));
    EXPECT_EQ(defString.to_string().find('l',11), defString.find('l',11));
+
+   EXPECT_EQ(defString.to_string().find("l"), defString.find("l"));
+   EXPECT_EQ(defString.to_string().find("ll"), defString.find("ll"));
+   EXPECT_EQ(defString.to_string().find("lo"), defString.find("lo"));
+   EXPECT_EQ(defString.to_string().find("ll", 1), defString.find("ll", 1));
+   EXPECT_EQ(defString.to_string().find("ll", 6), defString.find("ll", 6));
+   EXPECT_EQ(defString.to_string().find("or"), defString.find("or"));
+   EXPECT_EQ(defString.to_string().find("od"), defString.find("od"));
 }
 
 TEST(StringViewTest,RFind)
diff --git a/test/libapt/strutil_test.cc b/test/libapt/strutil_test.cc
index b7132c35f..469de4403 100644
--- a/test/libapt/strutil_test.cc
+++ b/test/libapt/strutil_test.cc
@@ -1,6 +1,8 @@
 #include <config.h>
 #include <apt-pkg/fileutl.h>
+#include <apt-pkg/string_view.h>
 #include <apt-pkg/strutl.h>
+#include <limits>
 #include <string>
 #include <vector>
 
@@ -21,6 +23,12 @@ TEST(StrUtilTest,DeEscapeString)
    // double slashes
    EXPECT_EQ("foo\\ x", DeEscapeString("foo\\\\ x"));
    EXPECT_EQ("\\foo\\", DeEscapeString("\\\\foo\\\\"));
+
+   // FIXME: the input is bad, the output as well, but we have no indicator for it
+   EXPECT_EQ("aa", DeEscapeString("aa\\x"));
+   EXPECT_EQ("aa0", DeEscapeString("aa\\x0"));
+   EXPECT_EQ("aa", DeEscapeString("aa\\0"));
+   EXPECT_EQ("aaa", DeEscapeString("aa\\0a"));
 }
 TEST(StrUtilTest,StringStrip)
 {
@@ -167,6 +175,10 @@ TEST(StrUtilTest,Base64Encode)
    EXPECT_EQ("ZS4=", Base64Encode("e."));
    EXPECT_EQ("Lg==", Base64Encode("."));
    EXPECT_EQ("", Base64Encode(""));
+   EXPECT_EQ("IA==", Base64Encode("\x20"));
+   EXPECT_EQ("/w==", Base64Encode("\xff"));
+   EXPECT_EQ("/A==", Base64Encode("\xfc"));
+   EXPECT_EQ("//8=", Base64Encode("\xff\xff"));
 }
 static void ReadMessagesTestWithNewLine(char const * const nl, char const * const ab)
 {
@@ -247,6 +259,60 @@ TEST(StrUtilTest,QuoteString)
    EXPECT_EQ("Eltville-Erbach", DeQuoteString(QuoteString("Eltville-Erbach", "")));
 }
 
+static void EXPECT_STRTONUM(APT::StringView const str, bool const success, unsigned long const expected, unsigned const base)
+{
+   SCOPED_TRACE(std::string(str.data(), str.length()));
+   SCOPED_TRACE(base);
+   unsigned long N1 = 1000;
+   unsigned long long N2 = 1000;
+   if (not success)
+   {
+      EXPECT_FALSE(StrToNum(str.data(), N1, str.length(), base));
+      EXPECT_FALSE(StrToNum(str.data(), N2, str.length(), base));
+      return;
+   }
+   EXPECT_TRUE(StrToNum(str.data(), N1, str.length(), base));
+   EXPECT_EQ(expected, N1);
+
+   EXPECT_TRUE(StrToNum(str.data(), N2, str.length(), base));
+   EXPECT_EQ(expected, N2);
+}
+TEST(StrUtilTest,StrToNum)
+{
+   EXPECT_STRTONUM("", true, 0, 10);
+   EXPECT_STRTONUM("  ", true, 0, 10);
+   EXPECT_STRTONUM("0", true, 0, 10);
+   EXPECT_STRTONUM("1", true, 1, 10);
+   EXPECT_STRTONUM(" 1 ", true, 1, 10);
+   EXPECT_STRTONUM("1", true, 1, 8);
+   EXPECT_STRTONUM("10", true, 10, 10);
+   EXPECT_STRTONUM("10", true, 8, 8);
+   EXPECT_STRTONUM("010", true, 8, 8);
+   EXPECT_STRTONUM(" 010 ", true, 8, 8);
+   EXPECT_STRTONUM("-1", false, 0, 10);
+   EXPECT_STRTONUM(" -1 ", false, 0, 10);
+   EXPECT_STRTONUM("11", true, 3, 2);
+
+   unsigned long long bigN = 0;
+   unsigned long smallN = 0;
+   auto bigLimit = std::to_string(std::numeric_limits<unsigned long long>::max());
+   if (std::numeric_limits<unsigned long>::max() < std::numeric_limits<unsigned long long>::max())
+   {
+      EXPECT_TRUE(StrToNum(bigLimit.c_str(), bigN, bigLimit.length(), 10));
+      EXPECT_EQ(std::numeric_limits<unsigned long long>::max(), bigN);
+      EXPECT_FALSE(StrToNum(bigLimit.c_str(), smallN, bigLimit.length(), 10));
+   }
+   bigLimit.append("0");
+   EXPECT_FALSE(StrToNum(bigLimit.c_str(), bigN, bigLimit.length(), 10));
+   EXPECT_FALSE(StrToNum(bigLimit.c_str(), smallN, bigLimit.length(), 10));
+
+   auto const smallLimit = std::to_string(std::numeric_limits<unsigned long>::max());
+   EXPECT_TRUE(StrToNum(smallLimit.c_str(), bigN, smallLimit.length(), 10));
+   EXPECT_EQ(std::numeric_limits<unsigned long>::max(), bigN);
+   EXPECT_TRUE(StrToNum(smallLimit.c_str(), smallN, smallLimit.length(), 10));
+   EXPECT_EQ(std::numeric_limits<unsigned long>::max(), smallN);
+}
+
 TEST(StrUtilTest,RFC1123StrToTime)
 {
    {
@@ -361,3 +427,20 @@ TEST(StrUtilTest, LookupTag)
    EXPECT_EQ("Value4", LookupTag(msg, "Field4", ""));
    EXPECT_EQ("Value5", LookupTag(msg, "Field5", ""));
 }
+
+TEST(StrUtilTest, DisplayLength)
+{
+   EXPECT_EQ(0, APT::String::DisplayLength(""));
+   EXPECT_EQ(1, APT::String::DisplayLength("a"));
+   EXPECT_EQ(3, APT::String::DisplayLength("apt"));
+   EXPECT_EQ(1, APT::String::DisplayLength("@"));
+   EXPECT_EQ(3, APT::String::DisplayLength("き"));
+
+   EXPECT_EQ(1, APT::String::DisplayLength("$"));
+   EXPECT_EQ(2, APT::String::DisplayLength("¢"));
+   EXPECT_EQ(3, APT::String::DisplayLength("ह"));
+   EXPECT_EQ(3, APT::String::DisplayLength("€"));
+   EXPECT_EQ(3, APT::String::DisplayLength("한"));
+   EXPECT_EQ(4, APT::String::DisplayLength("𐍈"));
+   EXPECT_EQ(16, APT::String::DisplayLength("𐍈한€ह¢$"));
+}