store Release files data in the Cache

We used to read the Release file for each Packages file and store the
data in the PackageFile struct even through potentially many Packages
(and Translation-*) files could use the same data. The point of the
exercise isn't the duplicated data through. Having the Release files as
first-class citizens in the Cache allows us to properly track their
state as well as allows us to use the information also for files which
aren't in the cache, but where we know to which Release file they
belong (Sources are an example for this).

This modifies the pkgCache structs, especially the PackagesFile struct
which depending on how libapt users access the data in these structs can
mean huge breakage or no visible change. As a single data point:
aptitude seems to be fine with this. Even if there is breakage it is
trivial to fix in a backportable way while avoiding breakage for
everyone would be a huge pain for us.

Note that not all PackageFile structs have a corresponding ReleaseFile.
In particular the dpkg/status file as well as *.deb files have not. As
these have only a Archive property need, the Component property takes
over this duty and the ReleaseFile remains zero. This is also the reason
why it isn't needed nor particularily recommended to change from
PackagesFile to ReleaseFile blindly. Sticking with the earlier is
usually the better option.

7 files changed, 144 insertions, 76 deletions

diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index 7aad65c0e..3a79cbc58 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -128,7 +128,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    if (Dist.empty())
       Dist = "/";
    ::URI Tmp(URI);
-   if (Gen.SelectFile(PackageFile,Tmp.Host,*this) == false)
+   if (Gen.SelectFile(PackageFile, *this, Target.Option(IndexTarget::COMPONENT)) == false)
       return _error->Error("Problem with SelectFile %s",PackageFile.c_str());
 
    // Store the IMS information
@@ -136,31 +136,10 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    pkgCacheGenerator::Dynamic<pkgCache::PkgFileIterator> DynFile(File);
    File->Size = Pkg.FileSize();
    File->mtime = Pkg.ModificationTime();
-   
+
    if (Gen.MergeList(Parser) == false)
       return _error->Error("Problem with MergeList %s",PackageFile.c_str());
 
-   // Check the release file
-   string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease");
-   bool releaseExists = false;
-   if (FileExists(ReleaseFile) == true)
-      releaseExists = true;
-   else
-      ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release");
-
-   if (releaseExists == true || FileExists(ReleaseFile) == true)
-   {
-      FileFd Rel;
-      // Beware: The 'Release' file might be clearsigned in case the
-      // signature for an 'InRelease' file couldn't be checked
-      if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false)
-	 return false;
-
-      if (_error->PendingError() == true)
-	 return false;
-      Parser.LoadReleaseInfo(File, Rel, Target.Option(IndexTarget::COMPONENT));
-   }
-   
    return true;
 }
 									/*}}}*/
@@ -221,17 +200,17 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
      debTranslationsParser TransParser(&Trans);
      if (_error->PendingError() == true)
        return false;
-     
+
      if (Prog != NULL)
 	Prog->SubProgress(0, Target.Description);
-     if (Gen.SelectFile(TranslationFile,string(),*this) == false)
+     if (Gen.SelectFile(TranslationFile, *this, Target.Option(IndexTarget::COMPONENT), pkgCache::Flag::NotSource) == false)
        return _error->Error("Problem with SelectFile %s",TranslationFile.c_str());
 
      // Store the IMS information
      pkgCache::PkgFileIterator TransFile = Gen.GetCurFile();
      TransFile->Size = Trans.FileSize();
      TransFile->mtime = Trans.ModificationTime();
-   
+
      if (Gen.MergeList(TransParser) == false)
        return _error->Error("Problem with MergeList %s",TranslationFile.c_str());
    }
@@ -305,18 +284,17 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 
    if (Prog != NULL)
       Prog->SubProgress(0,File);
-   if (Gen.SelectFile(File,string(),*this,pkgCache::Flag::NotSource) == false)
+   if (Gen.SelectFile(File, *this, "now", pkgCache::Flag::NotSource) == false)
       return _error->Error("Problem with SelectFile %s",File.c_str());
 
    // Store the IMS information
    pkgCache::PkgFileIterator CFile = Gen.GetCurFile();
+   pkgCacheGenerator::Dynamic<pkgCache::PkgFileIterator> DynFile(CFile);
    CFile->Size = Pkg.FileSize();
    CFile->mtime = Pkg.ModificationTime();
-   map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "now");
-   CFile->Archive = storage;
-   
+
    if (Gen.MergeList(Parser) == false)
-      return _error->Error("Problem with MergeList %s",File.c_str());   
+      return _error->Error("Problem with MergeList %s",File.c_str());
    return true;
 }
 									/*}}}*/
@@ -431,7 +409,7 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const
 
    // and give it to the list parser
    debDebFileParser Parser(DebControl, DebFile);
-   if(Gen.SelectFile(DebFile, "local", *this, pkgCache::Flag::LocalSource) == false)
+   if(Gen.SelectFile(DebFile, *this, "now", pkgCache::Flag::LocalSource) == false)
       return _error->Error("Problem with SelectFile %s", DebFile.c_str());
 
    pkgCache::PkgFileIterator File = Gen.GetCurFile();
diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h
index dd3f212b2..6b8c78e5a 100644
--- a/apt-pkg/deb/debindexfile.h
+++ b/apt-pkg/deb/debindexfile.h
@@ -45,7 +45,6 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile
    virtual bool HasPackages() const {return true;};
    virtual unsigned long Size() const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
-   bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog, unsigned long const Flag) const;
    virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const;
 
    debStatusIndex(std::string File);
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index b80b57bc4..c5e77b0ff 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -951,43 +951,6 @@ bool debListParser::Step()
    return false;
 }
 									/*}}}*/
-// ListParser::LoadReleaseInfo - Load the release information		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,
-				    FileFd &File, string component)
-{
-   // apt-secure does no longer download individual (per-section) Release
-   // file. to provide Component pinning we use the section name now
-   map_stringitem_t const storage = StoreString(pkgCacheGenerator::MIXED, component);
-   FileI->Component = storage;
-
-   pkgTagFile TagFile(&File, File.Size());
-   pkgTagSection Section;
-   if (_error->PendingError() == true || TagFile.Step(Section) == false)
-      return false;
-
-   std::string data;
-   #define APT_INRELEASE(TYPE, TAG, STORE) \
-   data = Section.FindS(TAG); \
-   if (data.empty() == false) \
-   { \
-      map_stringitem_t const storage = StoreString(pkgCacheGenerator::TYPE, data); \
-      STORE = storage; \
-   }
-   APT_INRELEASE(MIXED, "Suite", FileI->Archive)
-   APT_INRELEASE(MIXED, "Component", FileI->Component)
-   APT_INRELEASE(VERSIONNUMBER, "Version", FileI->Version)
-   APT_INRELEASE(MIXED, "Origin", FileI->Origin)
-   APT_INRELEASE(MIXED, "Codename", FileI->Codename)
-   APT_INRELEASE(MIXED, "Label", FileI->Label)
-   #undef APT_INRELEASE
-   Section.FindFlag("NotAutomatic", FileI->Flags, pkgCache::Flag::NotAutomatic);
-   Section.FindFlag("ButAutomaticUpgrades", FileI->Flags, pkgCache::Flag::ButAutomaticUpgrades);
-
-   return !_error->PendingError();
-}
-									/*}}}*/
 // ListParser::GetPrio - Convert the priority from a string		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h
index 6279d8399..420d5ff08 100644
--- a/apt-pkg/deb/deblistparser.h
+++ b/apt-pkg/deb/deblistparser.h
@@ -80,9 +80,9 @@ class APT_HIDDEN debListParser : public pkgCacheGenerator::ListParser
    virtual map_filesize_t Size() {return Section.size();};
 
    virtual bool Step();
-   
-   bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File,
-			std::string section);
+
+   bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File,
+			std::string const &section);
 
    APT_PUBLIC static const char *ParseDepends(const char *Start,const char *Stop,
 	 std::string &Package,std::string &Ver,unsigned int &Op);
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 8e4c2be2d..994b95849 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -1,5 +1,6 @@
 #include <config.h>
 
+#include <apt-pkg/error.h>
 #include <apt-pkg/debmetaindex.h>
 #include <apt-pkg/debindexfile.h>
 #include <apt-pkg/strutl.h>
@@ -10,10 +11,12 @@
 #include <apt-pkg/indexrecords.h>
 #include <apt-pkg/sourcelist.h>
 #include <apt-pkg/hashes.h>
-#include <apt-pkg/macros.h>
 #include <apt-pkg/metaindex.h>
+#include <apt-pkg/pkgcachegen.h>
+#include <apt-pkg/tagfile.h>
+#include <apt-pkg/gpgv.h>
+#include <apt-pkg/macros.h>
 
-#include <string.h>
 #include <map>
 #include <string>
 #include <utility>
@@ -21,6 +24,11 @@
 #include <set>
 #include <algorithm>
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <string.h>
+
 using namespace std;
 
 string debReleaseIndex::MetaIndexInfo(const char *Type) const
@@ -37,6 +45,10 @@ string debReleaseIndex::MetaIndexInfo(const char *Type) const
    Info += Type;
    return Info;
 }
+std::string debReleaseIndex::Describe() const
+{
+   return MetaIndexInfo("Release");
+}
 
 string debReleaseIndex::MetaIndexFile(const char *Type) const
 {
@@ -339,6 +351,114 @@ debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section,
 		bool const &IsSrc): Section(Section), IsSrc(IsSrc)
 {}
 
+static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile)
+{
+   ReleaseFile = That->MetaIndexFile("InRelease");
+   bool releaseExists = false;
+   if (FileExists(ReleaseFile) == true)
+      releaseExists = true;
+   else
+   {
+      ReleaseFile = That->MetaIndexFile("Release");
+      if (FileExists(ReleaseFile))
+	 releaseExists = true;
+   }
+   return releaseExists;
+}
+
+bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/
+{
+   std::string ReleaseFile;
+   bool const releaseExists = ReleaseFileName(this, ReleaseFile);
+
+   ::URI Tmp(URI);
+   if (Gen.SelectReleaseFile(ReleaseFile, Tmp.Host) == false)
+      return _error->Error("Problem with SelectReleaseFile %s", ReleaseFile.c_str());
+
+   if (releaseExists == false)
+      return true;
+
+   FileFd Rel;
+   // Beware: The 'Release' file might be clearsigned in case the
+   // signature for an 'InRelease' file couldn't be checked
+   if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false)
+      return false;
+   if (_error->PendingError() == true)
+      return false;
+
+   // Store the IMS information
+   pkgCache::RlsFileIterator File = Gen.GetCurRlsFile();
+   pkgCacheGenerator::Dynamic<pkgCache::RlsFileIterator> DynFile(File);
+   // Rel can't be used as this is potentially a temporary file
+   struct stat Buf;
+   if (stat(ReleaseFile.c_str(), &Buf) != 0)
+      return _error->Errno("fstat", "Unable to stat file %s", ReleaseFile.c_str());
+   File->Size = Buf.st_size;
+   File->mtime = Buf.st_mtime;
+
+   pkgTagFile TagFile(&Rel, Rel.Size());
+   pkgTagSection Section;
+   if (_error->PendingError() == true || TagFile.Step(Section) == false)
+      return false;
+
+   std::string data;
+   #define APT_INRELEASE(TYPE, TAG, STORE) \
+   data = Section.FindS(TAG); \
+   if (data.empty() == false) \
+   { \
+      map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::TYPE, data); \
+      STORE = storage; \
+   }
+   APT_INRELEASE(MIXED, "Suite", File->Archive)
+   APT_INRELEASE(VERSIONNUMBER, "Version", File->Version)
+   APT_INRELEASE(MIXED, "Origin", File->Origin)
+   APT_INRELEASE(MIXED, "Codename", File->Codename)
+   APT_INRELEASE(MIXED, "Label", File->Label)
+   #undef APT_INRELEASE
+   Section.FindFlag("NotAutomatic", File->Flags, pkgCache::Flag::NotAutomatic);
+   Section.FindFlag("ButAutomaticUpgrades", File->Flags, pkgCache::Flag::ButAutomaticUpgrades);
+
+   return !_error->PendingError();
+}
+									/*}}}*/
+// ReleaseIndex::FindInCache - Find this index				/*{{{*/
+pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
+{
+   std::string ReleaseFile;
+   bool const releaseExists = ReleaseFileName(this, ReleaseFile);
+
+   pkgCache::RlsFileIterator File = Cache.RlsFileBegin();
+   for (; File.end() == false; ++File)
+   {
+       if (File->FileName == 0 || ReleaseFile != File.FileName())
+	 continue;
+
+       // empty means the file does not exist by "design"
+       if (releaseExists == false && File->Size == 0)
+	  return File;
+
+      struct stat St;
+      if (stat(File.FileName(),&St) != 0)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "ReleaseIndex::FindInCache - stat failed on " << File.FileName() << std::endl;
+	 return pkgCache::RlsFileIterator(Cache);
+      }
+      if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "ReleaseIndex::FindInCache - size (" << St.st_size << " <> " << File->Size
+			<< ") or mtime (" << St.st_mtime << " <> " << File->mtime
+			<< ") doesn't match for " << File.FileName() << std::endl;
+	 return pkgCache::RlsFileIterator(Cache);
+      }
+      return File;
+   }
+
+   return File;
+}
+									/*}}}*/
+
 class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type
 {
    protected:
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 0b1b08432..7e9942710 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -20,6 +20,8 @@ class pkgAcquire;
 class pkgIndexFile;
 class debDebPkgFileIndex;
 class IndexTarget;
+class pkgCacheGenerator;
+class OpProgress;
 
 class APT_HIDDEN debReleaseIndex : public metaIndex {
    public:
@@ -48,6 +50,10 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const;
    virtual std::vector<IndexTarget> GetIndexTargets() const;
 
+   virtual std::string Describe() const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
+
    std::string MetaIndexInfo(const char *Type) const;
    std::string MetaIndexFile(const char *Types) const;
    std::string MetaIndexURI(const char *Type) const;
diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index a7a66c75d..6ee939edd 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -205,8 +205,10 @@ pkgCache::VerIterator FindNowVersion(const pkgCache::PkgIterator &Pkg)
    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
       for (pkgCache::VerFileIterator Vf = Ver.FileList(); Vf.end() == false; ++Vf)
 	 for (pkgCache::PkgFileIterator F = Vf.File(); F.end() == false; ++F)
-	    if (F->Archive != 0 && strcmp(F.Archive(), "now") == 0)
+	 {
+	    if (F.Archive() != 0 && strcmp(F.Archive(), "now") == 0)
 	       return Ver;
+	 }
    return Ver;
 }
 									/*}}}*/