add volatile sources support in libapt-pkg

Sources are usually defined in sources.list (and co) and are pretty
stable, but once in a while a frontend might want to add an additional
"source" like a local .deb file to install this package (No support for
'real' sources being added this way as this is a multistep process).

We had a hack in place to allow apt-get and apt to pull this of for a
short while now, but other frontends are either left in the cold by this
and/or the code for it looks dirty with FIXMEs plastering it and has on
top of this also some problems (like including these 'volatile' sources
in the srcpkgcache.bin file).

So the biggest part in this commit is actually the rewrite of the cache
generation as it is now potentially a three step process. The biggest
problem with adding support now through is that this makes a bunch of
previously mostly unusable by externs and therefore hidden classes
public, so a bit of further tuneing on this now public API is in order…

11 files changed, 257 insertions, 315 deletions

diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index 972feba7f..e67233e5f 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -339,18 +339,16 @@ APT_CONST bool debStatusIndex::Exists() const
 }
 									/*}}}*/
 
-// debDebPkgFile - Single .deb file					/*{{{*/
-debDebPkgFileIndex::debDebPkgFileIndex(std::string DebFile)
+// debDebPkgFileIndex - Single .deb file				/*{{{*/
+debDebPkgFileIndex::debDebPkgFileIndex(std::string const &DebFile)
    : pkgIndexFile(true), d(NULL), DebFile(DebFile)
 {
    DebFileFullPath = flAbsPath(DebFile);
 }
-
 std::string debDebPkgFileIndex::ArchiveURI(std::string /*File*/) const
 {
    return "file:" + DebFileFullPath;
 }
-
 bool debDebPkgFileIndex::Exists() const
 {
    return FileExists(DebFile);
@@ -403,6 +401,8 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const
    if (GetContent(content, DebFile) == false)
       return false;
    std::string const contentstr = content.str();
+   if (contentstr.empty())
+      return true;
    DebControl->Write(contentstr.c_str(), contentstr.length());
    // rewind for the listparser
    DebControl->Seek(0);
@@ -431,7 +431,7 @@ pkgCache::PkgFileIterator debDebPkgFileIndex::FindInCache(pkgCache &Cache) const
 
        return File;
    }
-   
+
    return File;
 }
 unsigned long debDebPkgFileIndex::Size() const
@@ -443,12 +443,11 @@ unsigned long debDebPkgFileIndex::Size() const
 }
 									/*}}}*/
 
-// debDscFileIndex stuff
-debDscFileIndex::debDscFileIndex(std::string &DscFile) 
+// debDscFileIndex - a .dsc file					/*{{{*/
+debDscFileIndex::debDscFileIndex(std::string const &DscFile)
    : pkgIndexFile(true), d(NULL), DscFile(DscFile)
 {
 }
-
 bool debDscFileIndex::Exists() const
 {
    return FileExists(DscFile);
@@ -461,8 +460,6 @@ unsigned long debDscFileIndex::Size() const
       return buf.st_size;
    return 0;
 }
-
-// DscFileIndex::CreateSrcParser - Get a parser for the .dsc file	/*{{{*/
 pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const
 {
    if (!FileExists(DscFile))
@@ -471,18 +468,17 @@ pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const
    return new debDscRecordParser(DscFile,this);
 }
 									/*}}}*/
+
 // Index File types for Debian						/*{{{*/
 class APT_HIDDEN debIFTypeSrc : public pkgIndexFile::Type
 {
    public:
-   
    debIFTypeSrc() {Label = "Debian Source Index";};
 };
 class APT_HIDDEN debIFTypePkg : public pkgIndexFile::Type
 {
    public:
-   
-   virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE 
+   virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
    {
       return new debRecordParser(File.FileName(),*File.Cache());
    };
@@ -496,8 +492,7 @@ class APT_HIDDEN debIFTypeTrans : public debIFTypePkg
 class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type
 {
    public:
-   
-   virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE 
+   virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
    {
       return new debRecordParser(File.FileName(),*File.Cache());
    };
@@ -506,7 +501,7 @@ class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type
 class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type
 {
    public:
-   virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE 
+   virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE
    {
       return new debDebFileRecordParser(File.FileName());
    };
diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h
index 71ca22e4d..4a818121a 100644
--- a/apt-pkg/deb/debindexfile.h
+++ b/apt-pkg/deb/debindexfile.h
@@ -27,7 +27,7 @@ class pkgAcquire;
 class pkgCacheGenerator;
 
 
-class APT_HIDDEN debStatusIndex : public pkgIndexFile
+class debStatusIndex : public pkgIndexFile
 {
    void * const d;
    protected:
@@ -51,7 +51,7 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile
    virtual ~debStatusIndex();
 };
 
-class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile
+class debPackagesIndex : public pkgIndexTargetFile
 {
    void * const d;
    public:
@@ -70,7 +70,7 @@ class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile
    virtual ~debPackagesIndex();
 };
 
-class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile
+class debTranslationsIndex : public pkgIndexTargetFile
 {
    void * const d;
    public:
@@ -86,7 +86,7 @@ class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile
    virtual ~debTranslationsIndex();
 };
 
-class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile
+class debSourcesIndex : public pkgIndexTargetFile
 {
    void * const d;
    public:
@@ -107,7 +107,7 @@ class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile
    virtual ~debSourcesIndex();
 };
 
-class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile
+class debDebPkgFileIndex : public pkgIndexFile
 {
  private:
    void * const d;
@@ -141,11 +141,11 @@ class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile
    // Interface for acquire
    virtual std::string ArchiveURI(std::string /*File*/) const APT_OVERRIDE;
 
-   debDebPkgFileIndex(std::string DebFile);
+   debDebPkgFileIndex(std::string const &DebFile);
    virtual ~debDebPkgFileIndex();
 };
 
-class APT_HIDDEN debDscFileIndex : public pkgIndexFile
+class debDscFileIndex : public pkgIndexFile
 {
  private:
    void * const d;
@@ -160,11 +160,11 @@ class APT_HIDDEN debDscFileIndex : public pkgIndexFile
       return DscFile;
    };
 
-   debDscFileIndex(std::string &DscFile);
+   debDscFileIndex(std::string const &DscFile);
    virtual ~debDscFileIndex();
 };
 
-class APT_HIDDEN debDebianSourceDirIndex : public debDscFileIndex
+class debDebianSourceDirIndex : public debDscFileIndex
 {
  public:
    virtual const Type *GetType() const APT_CONST;
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index 1154016a9..b7988d499 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -959,7 +959,7 @@ bool debDebFileParser::UsePackage(pkgCache::PkgIterator &Pkg,
    bool res = debListParser::UsePackage(Pkg, Ver);
    // we use the full file path as a provides so that the file is found
    // by its name
-   if(NewProvidesAllArch(Ver, DebFile, Ver.VerStr(), 0) == false)
+   if(NewProvides(Ver, DebFile, Pkg.Cache()->NativeArch(), Ver.VerStr(), 0) == false)
       return false;
    return res;
 }
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 480317db3..123c91648 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -785,34 +785,5 @@ class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian		/*{{{*/
 };
 									/*}}}*/
 
-debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile)	/*{{{*/
-   : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile)
-{
-   DebIndex = new debDebPkgFileIndex(DebFile);
-   Indexes = new std::vector<pkgIndexFile *>();
-   Indexes->push_back(DebIndex);
-}
-debDebFileMetaIndex::~debDebFileMetaIndex() {}
-									/*}}}*/
-class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type		/*{{{*/
-{
-   public:
-
-   bool CreateItem(std::vector<metaIndex *> &List, std::string const &URI,
-		   std::string const &/*Dist*/, std::string const &/*Section*/,
-		   std::map<std::string, std::string> const &/*Options*/) const APT_OVERRIDE
-   {
-      metaIndex *mi = new debDebFileMetaIndex(URI);
-      List.push_back(mi);
-      return true;
-   }
-
-   debSLTypeDebFile() : Type("deb-file", "Debian local deb file")
-   {
-   }
-};
-									/*}}}*/
-
 APT_HIDDEN debSLTypeDeb _apt_DebType;
 APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType;
-APT_HIDDEN debSLTypeDebFile _apt_DebFileType;
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 8c13237cb..e93959a21 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -17,7 +17,6 @@
 
 class pkgAcquire;
 class pkgIndexFile;
-class debDebPkgFileIndex;
 class IndexTarget;
 class pkgCacheGenerator;
 class OpProgress;
@@ -66,42 +65,4 @@ class APT_HIDDEN debReleaseIndex : public metaIndex
 	 std::vector<std::string> Languages);
 };
 
-class APT_HIDDEN debDebFileMetaIndex : public metaIndex
-{
-private:
-   void * const d;
-   std::string DebFile;
-   debDebPkgFileIndex *DebIndex;
-public:
-   virtual std::string ArchiveURI(std::string const& /*File*/) const APT_OVERRIDE {
-      return DebFile;
-   }
-   virtual bool GetIndexes(pkgAcquire* /*Owner*/, const bool& /*GetAll=false*/) APT_OVERRIDE {
-      return true;
-   }
-   virtual std::vector<IndexTarget> GetIndexTargets() const APT_OVERRIDE {
-      return std::vector<IndexTarget>();
-   }
-   virtual std::vector<pkgIndexFile *> *GetIndexFiles() APT_OVERRIDE {
-      return Indexes;
-   }
-   virtual bool IsTrusted() const APT_OVERRIDE {
-      return true;
-   }
-   virtual bool Load(std::string const &, std::string * const ErrorText) APT_OVERRIDE
-   {
-      LoadedSuccessfully = TRI_NO;
-      if (ErrorText != NULL)
-	 strprintf(*ErrorText, "Unparseable metaindex as it represents the standalone deb file %s", DebFile.c_str());
-      return false;
-   }
-   virtual metaIndex * UnloadedClone() const APT_OVERRIDE
-   {
-      return NULL;
-   }
-   debDebFileMetaIndex(std::string const &DebFile);
-   virtual ~debDebFileMetaIndex();
-
-};
-
 #endif
diff --git a/apt-pkg/edsp/edspsystem.cc b/apt-pkg/edsp/edspsystem.cc
index 4fb34b896..f65fcc0d2 100644
--- a/apt-pkg/edsp/edspsystem.cc
+++ b/apt-pkg/edsp/edspsystem.cc
@@ -91,8 +91,7 @@ signed edspSystem::Score(Configuration const &Cnf)
    return -1000;
 }
 									/*}}}*/
-// System::AddStatusFiles - Register the status files			/*{{{*/
-bool edspSystem::AddStatusFiles(std::vector<pkgIndexFile *> &List)
+bool edspSystem::AddStatusFiles(std::vector<pkgIndexFile *> &List)	/*{{{*/
 {
    if (StatusFile == 0)
    {
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 26a5e60a6..9529f42dc 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -36,6 +36,8 @@
 #include <iostream>
 #include <string>
 #include <vector>
+#include <memory>
+#include <algorithm>
 #include <sys/stat.h>
 #include <unistd.h>
 
@@ -1332,10 +1334,10 @@ map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, cons
 /* This just verifies that each file in the list of index files exists,
    has matching attributes with the cache and the cache does not have
    any extra files. */
-static bool CheckValidity(const string &CacheFile, 
+static bool CheckValidity(const string &CacheFile,
                           pkgSourceList &List,
-                          FileIterator Start, 
-                          FileIterator End,
+                          FileIterator const Start,
+                          FileIterator const End,
                           MMap **OutMap = 0)
 {
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
@@ -1343,7 +1345,7 @@ static bool CheckValidity(const string &CacheFile,
    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
    {
       if (Debug == true)
-	 std::clog << "CacheFile doesn't exist" << std::endl;
+	 std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl;
       return false;
    }
 
@@ -1361,7 +1363,7 @@ static bool CheckValidity(const string &CacheFile,
    if (_error->PendingError() == true || Map->Size() == 0)
    {
       if (Debug == true)
-	 std::clog << "Errors are pending or Map is empty()" << std::endl;
+	 std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl;
       _error->Discard();
       return false;
    }
@@ -1385,10 +1387,9 @@ static bool CheckValidity(const string &CacheFile,
       if (Debug == true)
 	 std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
 
-      std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
-      for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
-	 if ((*j)->HasPackages())
-	    Files.push_back (*j);
+      std::vector <pkgIndexFile *> const * const Indexes = (*i)->GetIndexFiles();
+      std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files),
+	    [](pkgIndexFile const * const I) { return I->HasPackages(); });
    }
    for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
       if (RlsVisited[I] == false)
@@ -1398,8 +1399,7 @@ static bool CheckValidity(const string &CacheFile,
 	 return false;
       }
 
-   for (; Start != End; ++Start)
-      Files.push_back(*Start);
+   std::copy(Start, End, std::back_inserter(Files));
 
    /* Now we check every index file, see if it is in the cache,
       verify the IMS data and check that it is on the disk too.. */
@@ -1482,16 +1482,41 @@ static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator
 }
 									/*}}}*/
 // BuildCache - Merge the list of index files into the cache		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
 static bool BuildCache(pkgCacheGenerator &Gen,
-		       OpProgress *Progress,
+		       OpProgress * const Progress,
 		       map_filesize_t &CurrentSize,map_filesize_t TotalSize,
 		       pkgSourceList const * const List,
-		       FileIterator Start, FileIterator End)
+		       FileIterator const Start, FileIterator const End)
 {
    std::vector<pkgIndexFile *> Files;
    bool const HasFileDeps = Gen.HasFileDeps();
+   bool mergeFailure = false;
+
+   auto const indexFileMerge = [&](pkgIndexFile * const I) {
+      if (HasFileDeps)
+	 Files.push_back(I);
+
+      if (I->HasPackages() == false || mergeFailure)
+	 return;
+
+      if (I->Exists() == false)
+	 return;
+
+      if (I->FindInCache(Gen.GetCache()).end() == false)
+      {
+	 _error->Warning("Duplicate sources.list entry %s",
+	       I->Describe().c_str());
+	 return;
+      }
+
+      map_filesize_t const Size = I->Size();
+      if (Progress != NULL)
+	 Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists"));
+      CurrentSize += Size;
+
+      if (I->Merge(Gen,Progress) == false)
+	 mergeFailure = true;
+   };
 
    if (List !=  NULL)
    {
@@ -1508,63 +1533,20 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 	    return false;
 
 	 std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
-	 for (std::vector<pkgIndexFile *>::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
-	 {
-	    if (HasFileDeps)
-	       Files.push_back(*I);
-
-	    if ((*I)->HasPackages() == false)
-	       continue;
-
-	    if ((*I)->Exists() == false)
-	       continue;
-
-	    if ((*I)->FindInCache(Gen.GetCache()).end() == false)
-	    {
-	       _error->Warning("Duplicate sources.list entry %s",
-		     (*I)->Describe().c_str());
-	       continue;
-	    }
-
-	    map_filesize_t Size = (*I)->Size();
-	    if (Progress != NULL)
-	       Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
-	    CurrentSize += Size;
-
-	    if ((*I)->Merge(Gen,Progress) == false)
-	       return false;
-	 }
+	 if (Indexes != NULL)
+	    std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge);
+	 if (mergeFailure)
+	    return false;
       }
    }
 
-   Gen.SelectReleaseFile("", "");
-   FileIterator I;
-   for (I = Start; I != End; ++I)
+   if (Start != End)
    {
-      if (HasFileDeps)
-	 Files.push_back(*I);
-
-      if ((*I)->HasPackages() == false)
-	 continue;
-      
-      if ((*I)->Exists() == false)
-	 continue;
-
-      if ((*I)->FindInCache(Gen.GetCache()).end() == false)
-      {
-	 _error->Warning("Duplicate sources.list entry %s",
-			 (*I)->Describe().c_str());
-	 continue;
-      }
-      
-      map_filesize_t Size = (*I)->Size();
-      if (Progress != NULL)
-	 Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
-      CurrentSize += Size;
-      
-      if ((*I)->Merge(Gen,Progress) == false)
+      Gen.SelectReleaseFile("", "");
+      std::for_each(Start, End, indexFileMerge);
+      if (mergeFailure)
 	 return false;
-   }   
+   }
 
    if (HasFileDeps == true)
    {
@@ -1582,12 +1564,20 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 	    return false;
       }
    }
-   
+
    return true;
 }
 									/*}}}*/
-// CacheGenerator::CreateDynamicMMap - load an mmap with configuration options	/*{{{*/
-DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) {
+// CacheGenerator::MakeStatusCache - Construct the status cache		/*{{{*/
+// ---------------------------------------------------------------------
+/* This makes sure that the status cache (the cache that has all 
+   index files from the sources list and all local ones) is ready
+   to be mmaped. If OutMap is not zero then a MMap object representing
+   the cache will be stored there. This is pretty much mandetory if you
+   are using AllowMem. AllowMem lets the function be run as non-root
+   where it builds the cache 'fast' into a memory buffer. */
+static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags)
+{
    map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024);
    map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024);
    map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0);
@@ -1599,15 +1589,42 @@ DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long
    else
       return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit);
 }
-									/*}}}*/
-// CacheGenerator::MakeStatusCache - Construct the status cache		/*{{{*/
-// ---------------------------------------------------------------------
-/* This makes sure that the status cache (the cache that has all 
-   index files from the sources list and all local ones) is ready
-   to be mmaped. If OutMap is not zero then a MMap object representing
-   the cache will be stored there. This is pretty much mandetory if you
-   are using AllowMem. AllowMem lets the function be run as non-root
-   where it builds the cache 'fast' into a memory buffer. */
+static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map,
+      std::string const &FileName)
+{
+   FileFd SCacheF(FileName, FileFd::WriteAtomic);
+   if (_error->PendingError() == true)
+      return false;
+
+   fchmod(SCacheF.Fd(),0644);
+
+   // Write out the main data
+   if (SCacheF.Write(Map->Data(),Map->Size()) == false)
+      return _error->Error(_("IO Error saving source cache"));
+   SCacheF.Sync();
+
+   // Write out the proper header
+   Gen->GetCache().HeaderP->Dirty = false;
+   if (SCacheF.Seek(0) == false ||
+	 SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false)
+      return _error->Error(_("IO Error saving source cache"));
+   Gen->GetCache().HeaderP->Dirty = true;
+   SCacheF.Sync();
+   return true;
+}
+static bool loadBackMMapFromFile(std::unique_ptr<pkgCacheGenerator> &Gen,
+      SPtr<DynamicMMap> &Map, OpProgress * const Progress, std::string const &FileName)
+{
+   Map = CreateDynamicMMap(NULL, 0);
+   FileFd CacheF(FileName, FileFd::ReadOnly);
+   map_pointer_t const alloc = Map->RawAllocate(CacheF.Size());
+   if ((alloc == 0 && _error->PendingError())
+	 || CacheF.Read((unsigned char *)Map->Data() + alloc,
+	    CacheF.Size()) == false)
+      return false;
+   Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
+   return true;
+}
 APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 			MMap **OutMap, bool AllowMem)
    { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); }
@@ -1617,18 +1634,6 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 
    std::vector<pkgIndexFile *> Files;
-   /*
-   for (std::vector<metaIndex *>::const_iterator i = List.begin();
-        i != List.end();
-        ++i)
-   {
-      std::vector <pkgIndexFile *> *Indexes = (*i)->GetIndexFiles();
-      for (std::vector<pkgIndexFile *>::const_iterator j = Indexes->begin();
-	   j != Indexes->end();
-	   ++j)
-         Files.push_back (*j);
-   }
-*/
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
@@ -1649,160 +1654,130 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 	 CreateDirectory(dir, flNotFile(SrcCacheFile));
    }
 
-   // Decide if we can write to the cache
-   bool Writeable = false;
-   if (CacheFile.empty() == false)
-      Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
-   else
-      if (SrcCacheFile.empty() == false)
-	 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
-   if (Debug == true)
-      std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
-
-   if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
-      return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
-
    if (Progress != NULL)
       Progress->OverallProgress(0,1,1,_("Reading package lists"));
 
-   // Cache is OK, Fin.
-   if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true)
+   bool pkgcache_fine = false;
+   bool srcpkgcache_fine = false;
+   bool volatile_fine = List.GetVolatileFiles().empty();
+
+   if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true)
    {
-      if (Progress != NULL)
-	 Progress->OverallProgress(1,1,1,_("Reading package lists"));
       if (Debug == true)
-	 std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
-      return true;
+	 std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl;
+      pkgcache_fine = true;
+      srcpkgcache_fine = true;
    }
-   else if (Debug == true)
-	 std::clog << "pkgcache.bin is NOT valid" << std::endl;
-   
-   /* At this point we know we need to reconstruct the package cache,
-      begin. */
-   SPtr<FileFd> CacheF;
-   SPtr<DynamicMMap> Map;
-   if (Writeable == true && CacheFile.empty() == false)
+   if (pkgcache_fine == false)
    {
-      _error->PushToStack();
-      unlink(CacheFile.c_str());
-      CacheF = new FileFd(CacheFile,FileFd::WriteAtomic);
-      fchmod(CacheF->Fd(),0644);
-      Map = CreateDynamicMMap(CacheF, MMap::Public);
-      if (_error->PendingError() == true)
+      if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true)
       {
-	 delete CacheF.UnGuard();
-	 delete Map.UnGuard();
 	 if (Debug == true)
-	    std::clog << "Open filebased MMap FAILED" << std::endl;
-	 Writeable = false;
-	 if (AllowMem == false)
-	 {
-	    _error->MergeWithStack();
-	    return false;
-	 }
-	 _error->RevertToStack();
-      }
-      else
-      {
-	 _error->MergeWithStack();
-	 if (Debug == true)
-	    std::clog << "Open filebased MMap" << std::endl;
+	    std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl;
+	 srcpkgcache_fine = true;
       }
    }
-   if (Writeable == false || CacheFile.empty() == true)
+
+   if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true)
    {
-      // Just build it in memory..
-      Map = CreateDynamicMMap(NULL);
-      if (Debug == true)
-	 std::clog << "Open memory Map (not filebased)" << std::endl;
+      if (Progress != NULL)
+	 Progress->OverallProgress(1,1,1,_("Reading package lists"));
+      return true;
    }
-   
-   // Lets try the source cache.
-   map_filesize_t CurrentSize = 0;
-   map_filesize_t TotalSize = 0;
-   if (CheckValidity(SrcCacheFile, List, Files.end(),
-		     Files.end()) == true)
+
+   bool Writeable = false;
+   if (srcpkgcache_fine == false || pkgcache_fine == false)
    {
+      if (CacheFile.empty() == false)
+	 Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0;
+      else if (SrcCacheFile.empty() == false)
+	 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
+
       if (Debug == true)
-	 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
-      // Preload the map with the source cache
-      FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
-      map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size());
-      if ((alloc == 0 && _error->PendingError())
-		|| SCacheF.Read((unsigned char *)Map->Data() + alloc,
-				SCacheF.Size()) == false)
-	 return false;
+	 std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
+
+      if (Writeable == false && AllowMem == false)
+      {
+	 if (CacheFile.empty() == false)
+	    return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
+	 else if (SrcCacheFile.empty() == false)
+	    return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str());
+	 else
+	    return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!");
+      }
+   }
 
-      TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
+   // At this point we know we need to construct something, so get storage ready
+   SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
+   if (Debug == true)
+      std::clog << "Open memory Map (not filebased)" << std::endl;
 
-      // Build the status cache
-      pkgCacheGenerator Gen(Map.Get(),Progress);
-      if (_error->PendingError() == true)
-	 return false;
-      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
-		     Files.begin(),Files.end()) == false)
+   std::unique_ptr<pkgCacheGenerator> Gen{nullptr};
+   map_filesize_t CurrentSize = 0;
+   std::vector<pkgIndexFile*> VolatileFiles = List.GetVolatileFiles();
+   map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end());
+   if (srcpkgcache_fine == true && pkgcache_fine == false)
+   {
+      if (Debug == true)
+	 std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl;
+      if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false)
 	 return false;
+      srcpkgcache_fine = true;
+      TotalSize += ComputeSize(NULL, Files.begin(), Files.end());
    }
-   else
+   else if (srcpkgcache_fine == false)
    {
       if (Debug == true)
 	 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
-      TotalSize = ComputeSize(&List, Files.begin(),Files.end());
-      
-      // Build the source cache
-      pkgCacheGenerator Gen(Map.Get(),Progress);
-      if (_error->PendingError() == true)
-	 return false;
-      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
-		     Files.end(),Files.end()) == false)
+      Gen.reset(new pkgCacheGenerator(Map.Get(),Progress));
+
+      TotalSize += ComputeSize(&List, Files.begin(),Files.end());
+      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List,
+	       Files.end(),Files.end()) == false)
 	 return false;
-      
-      // Write it back
+
       if (Writeable == true && SrcCacheFile.empty() == false)
-      {
-	 FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic);
-	 if (_error->PendingError() == true)
+	 if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false)
 	    return false;
-	 
-	 fchmod(SCacheF.Fd(),0644);
-	 
-	 // Write out the main data
-	 if (SCacheF.Write(Map->Data(),Map->Size()) == false)
-	    return _error->Error(_("IO Error saving source cache"));
-	 SCacheF.Sync();
-	 
-	 // Write out the proper header
-	 Gen.GetCache().HeaderP->Dirty = false;
-	 if (SCacheF.Seek(0) == false ||
-	     SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false)
-	    return _error->Error(_("IO Error saving source cache"));
-	 Gen.GetCache().HeaderP->Dirty = true;
-	 SCacheF.Sync();
-      }
-      
-      // Build the status cache
-      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
-		     Files.begin(), Files.end()) == false)
+   }
+
+   if (pkgcache_fine == false)
+   {
+      if (Debug == true)
+	 std::clog << "Building status cache in pkgcache.bin now" << std::endl;
+      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
+	       Files.begin(), Files.end()) == false)
 	 return false;
+
+      if (Writeable == true && CacheFile.empty() == false)
+	 if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false)
+	    return false;
    }
+
    if (Debug == true)
-      std::clog << "Caches are ready for shipping" << std::endl;
+      std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl;
 
-   if (_error->PendingError() == true)
-      return false;
-   if (OutMap != 0)
+   if (volatile_fine == false)
    {
-      if (CacheF != 0)
+      if (Gen == nullptr)
       {
-	 delete Map.UnGuard();
-	 *OutMap = new MMap(*CacheF,0);
+	 if (Debug == true)
+	    std::clog << "Populate new MMap with cachefile contents" << std::endl;
+	 if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false)
+	    return false;
       }
-      else
-      {
-	 *OutMap = Map.UnGuard();
-      }      
+
+      Files = List.GetVolatileFiles();
+      if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL,
+	       Files.begin(), Files.end()) == false)
+	 return false;
    }
-   
+
+   if (OutMap != nullptr)
+      *OutMap = Map.UnGuard();
+
+   if (Debug == true)
+      std::clog << "Everything is ready for shipping" << std::endl;
    return true;
 }
 									/*}}}*/
@@ -1817,7 +1792,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
-   SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL);
+   SPtr<DynamicMMap> Map = CreateDynamicMMap(NULL, 0);
    map_filesize_t CurrentSize = 0;
    map_filesize_t TotalSize = 0;
    
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index 0d0fb893f..4b6b91992 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -116,7 +116,6 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    APT_PUBLIC static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress,
 			MMap **OutMap = 0,bool AllowMem = false);
    APT_PUBLIC static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap);
-   APT_PUBLIC static DynamicMMap* CreateDynamicMMap(FileFd *CacheF, unsigned long Flags = 0);
 
    void ReMap(void const * const oldMap, void const * const newMap);
 
diff --git a/apt-pkg/pkgsystem.h b/apt-pkg/pkgsystem.h
index 5be93d059..5b31457e0 100644
--- a/apt-pkg/pkgsystem.h
+++ b/apt-pkg/pkgsystem.h
@@ -52,7 +52,7 @@ class Configuration;
 class pkgIndexFile;
 
 class pkgSystem
-{   
+{
    public:
 
    // Global list of supported systems
@@ -81,7 +81,8 @@ class pkgSystem
    virtual bool ArchiveSupported(const char *Type) = 0;
 
    // Return a list of system index files..
-   virtual bool AddStatusFiles(std::vector<pkgIndexFile *> &List) = 0;   
+   virtual bool AddStatusFiles(std::vector<pkgIndexFile *> &List) = 0;
+
    virtual bool FindIndex(pkgCache::PkgFileIterator File,
 			  pkgIndexFile *&Found) const = 0;
 
diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index eef0ee709..46e51f592 100644
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -274,6 +274,10 @@ pkgSourceList::~pkgSourceList()
 {
    for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
       delete *I;
+   SrcList.clear();
+   for (pkgIndexFile * const File : VolatileFiles)
+      delete File;
+   VolatileFiles.clear();
 }
 									/*}}}*/
 // SourceList::ReadMainList - Read the main source list from etc	/*{{{*/
@@ -339,7 +343,7 @@ bool pkgSourceList::ReadAppend(string const &File)
    else
       return ParseFileOldStyle(File);
 }
-
+									/*}}}*/
 // SourceList::ReadFileOldStyle - Read Traditional style sources.list 	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -454,7 +458,15 @@ bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
          }
       }
    }
-
+   for (vector<pkgIndexFile *>::const_iterator J = VolatileFiles.begin();
+	 J != VolatileFiles.end(); ++J)
+   {
+      if ((*J)->FindInCache(*File.Cache()) == File)
+      {
+	 Found = (*J);
+	 return true;
+      }
+   }
    return false;
 }
 									/*}}}*/
@@ -511,4 +523,14 @@ time_t pkgSourceList::GetLastModifiedTime()
    return mtime_sources;
 }
 									/*}}}*/
-
+std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const	/*{{{*/
+{
+   return VolatileFiles;
+}
+									/*}}}*/
+void pkgSourceList::AddVolatileFile(pkgIndexFile * const File)		/*{{{*/
+{
+   if (File != NULL)
+      VolatileFiles.push_back(File);
+}
+									/*}}}*/
diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h
index d80131438..47a562d18 100644
--- a/apt-pkg/sourcelist.h
+++ b/apt-pkg/sourcelist.h
@@ -50,6 +50,7 @@ class metaIndex;
 class pkgSourceList
 {
    void * const d;
+   std::vector<pkgIndexFile*> VolatileFiles;
    public:
 
    // List of supported source list types
@@ -113,6 +114,24 @@ class pkgSourceList
    // query last-modified time
    time_t GetLastModifiedTime();
 
+   /** \brief add file for parsing, but not to the cache
+    *
+    *  pkgIndexFiles origining from pkgSourcesList are included in
+    *  srcpkgcache, the status files added via #AddStatusFiles are
+    *  included in pkgcache, but these files here are not included in
+    *  any cache to have the possibility of having a file included just
+    *  for a single run like a local .deb/.dsc file.
+    *
+    *  The volatile files do not count as "normal" sourceslist entries,
+    *  can't be iterated over with #begin and #end and can't be
+    *  downloaded, but they can be found via #FindIndex.
+    *
+    *  @param File is an index file; pointer-ownership is transferred
+    */
+   void AddVolatileFile(pkgIndexFile * const File);
+   /** @return list of files registered with #AddVolatileFile */
+   std::vector<pkgIndexFile*> GetVolatileFiles() const;
+
    pkgSourceList();
    virtual ~pkgSourceList();
 };