support compression and by-hash for .diff/Index files

In af81ab9030229b4ce6cbe28f0f0831d4896fda01 by-hash got implemented as a
special compression type for our usual index files like Packages.
Missing in this scheme was the special .diff/Index index file containing
the info about individual patches for this index file. Deriving from the
index file class directly we inherent the compression handling
infrastructure and in this way also by-hash nearly for free.

Closes: #824926

2 files changed, 121 insertions, 114 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index b8786e34b..7abb7b206 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -451,7 +451,11 @@ std::string pkgAcquire::Item::GetFinalFilename() const
 }
 std::string pkgAcqDiffIndex::GetFinalFilename() const
 {
-   return GetFinalFileNameFromURI(GetDiffIndexURI(Target));
+   std::string const FinalFile = GetFinalFileNameFromURI(GetDiffIndexURI(Target));
+   // we don't want recompress, so lets keep whatever we got
+   if (CurrentCompressionExtension == "uncompressed")
+      return FinalFile;
+   return FinalFile + "." + CurrentCompressionExtension;
 }
 std::string pkgAcqIndex::GetFinalFilename() const
 {
@@ -488,7 +492,10 @@ std::string pkgAcqIndex::GetMetaKey() const
 }
 std::string pkgAcqDiffIndex::GetMetaKey() const
 {
-   return GetDiffIndexFileName(Target.MetaKey);
+   auto const metakey = GetDiffIndexFileName(Target.MetaKey);
+   if (CurrentCompressionExtension == "uncompressed")
+      return metakey;
+   return metakey + "." + CurrentCompressionExtension;
 }
 									/*}}}*/
 //pkgAcqTransactionItem::TransactionState and specialisations for child classes	/*{{{*/
@@ -1971,44 +1978,33 @@ pkgAcqBaseIndex::~pkgAcqBaseIndex() {}
 pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner,
                                  pkgAcqMetaClearSig * const TransactionManager,
                                  IndexTarget const &Target)
-   : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL)
+   : pkgAcqIndex(Owner, TransactionManager, Target, true), d(NULL), diffs(NULL)
 {
    // FIXME: Magic number as an upper bound on pdiffs we will reasonably acquire
    ExpectedAdditionalItems = 40;
-
    Debug = _config->FindB("Debug::pkgAcquire::Diffs",false);
 
-   Desc.Owner = this;
-   Desc.Description = GetDiffIndexFileName(Target.Description);
-   Desc.ShortDesc = Target.ShortDesc;
-   Desc.URI = GetDiffIndexURI(Target);
-
-   DestFile = GetPartialFileNameFromURI(Desc.URI);
+   CompressionExtensions.clear();
+   {
+      std::vector<std::string> types = APT::Configuration::getCompressionTypes();
+      if (types.empty() == false)
+      {
+	 std::ostringstream os;
+	 std::copy_if(types.begin(), types.end()-1, std::ostream_iterator<std::string>(os, " "), [&](std::string const type) {
+	       if (type == "uncompressed")
+	          return true;
+	       return TransactionManager->MetaIndexParser->Exists(GetDiffIndexFileName(Target.MetaKey) + '.' + type);
+	 });
+	 os << *types.rbegin();
+	 CompressionExtensions = os.str();
+      }
+   }
+   if (Target.Option(IndexTarget::COMPRESSIONTYPES).find("by-hash") != std::string::npos)
+      CompressionExtensions = "by-hash " + CompressionExtensions;
+   Init(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Target.ShortDesc);
 
    if(Debug)
       std::clog << "pkgAcqDiffIndex: " << Desc.URI << std::endl;
-
-   QueueURI(Desc);
-}
-									/*}}}*/
-// AcqIndex::Custom600Headers - Insert custom request headers		/*{{{*/
-// ---------------------------------------------------------------------
-/* The only header we use is the last-modified header. */
-string pkgAcqDiffIndex::Custom600Headers() const
-{
-   if (TransactionManager->LastMetaIndexParser != NULL)
-      return "\nIndex-File: true";
-
-   string const Final = GetFinalFilename();
-
-   if(Debug)
-      std::clog << "Custom600Header-IMS: " << Final << std::endl;
-
-   struct stat Buf;
-   if (stat(Final.c_str(),&Buf) != 0)
-      return "\nIndex-File: true";
-   
-   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime, false);
 }
 									/*}}}*/
 void pkgAcqDiffIndex::QueueOnIMSHit() const				/*{{{*/
@@ -2039,7 +2035,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
       std::clog << "pkgAcqDiffIndex::ParseIndexDiff() " << IndexDiffFile
 	 << std::endl;
 
-   FileFd Fd(IndexDiffFile,FileFd::ReadOnly);
+   FileFd Fd(IndexDiffFile, FileFd::ReadOnly, FileFd::Extension);
    pkgTagFile TF(&Fd);
    if (Fd.IsOpen() == false || Fd.Failed())
       return false;
@@ -2411,7 +2407,9 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile)	/*{{{*/
 									/*}}}*/
 void pkgAcqDiffIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)/*{{{*/
 {
-   pkgAcqBaseIndex::Failed(Message,Cnf);
+   if (CommonFailed(GetDiffIndexURI(Target), GetDiffIndexFileName(Target.Description), Message, Cnf))
+      return;
+
    Status = StatDone;
    ExpectedAdditionalItems = 0;
 
@@ -2829,10 +2827,12 @@ pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {}
 // AcqIndex::AcqIndex - Constructor					/*{{{*/
 pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner,
                          pkgAcqMetaClearSig * const TransactionManager,
-                         IndexTarget const &Target)
+                         IndexTarget const &Target, bool const Derived)
    : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD),
    CompressionExtensions(Target.Option(IndexTarget::COMPRESSIONTYPES))
 {
+   if (Derived)
+      return;
    Init(Target.URI, Target.Description, Target.ShortDesc);
 
    if(_config->FindB("Debug::Acquire::Transaction", false) == true)
@@ -2938,7 +2938,8 @@ string pkgAcqIndex::Custom600Headers() const
 }
 									/*}}}*/
 // AcqIndex::Failed - getting the indexfile failed			/*{{{*/
-void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
+bool pkgAcqIndex::CommonFailed(std::string const &TargetURI, std::string const TargetDesc,
+      std::string const &Message, pkgAcquire::MethodConfig const * const Cnf)
 {
    pkgAcqBaseIndex::Failed(Message,Cnf);
 
@@ -2950,10 +2951,9 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const *
 	 CompressionExtensions = "by-hash " + CompressionExtensions;
       else
 	 CompressionExtensions = CurrentCompressionExtension + ' ' + CompressionExtensions;
-      Desc.Description = Target.Description;
-      Init(Target.URI, Desc.Description, Desc.ShortDesc);
+      Init(TargetURI, TargetDesc, Desc.ShortDesc);
       Status = StatIdle;
-      return;
+      return true;
    }
 
    // authorisation matches will not be fixed by other compression types
@@ -2961,11 +2961,17 @@ void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const *
    {
       if (CompressionExtensions.empty() == false)
       {
-	 Init(Target.URI, Desc.Description, Desc.ShortDesc);
+	 Init(TargetURI, Desc.Description, Desc.ShortDesc);
 	 Status = StatIdle;
-	 return;
+	 return true;
       }
    }
+   return false;
+}
+void pkgAcqIndex::Failed(string const &Message,pkgAcquire::MethodConfig const * const Cnf)
+{
+   if (CommonFailed(Target.URI, Target.Description, Message, Cnf))
+      return;
 
    if(Target.IsOptional && GetExpectedHashes().empty() && Stage == STAGE_DOWNLOAD)
       Status = StatDone;
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index e6e5ea12b..71a11bcde 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -611,6 +611,79 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem
    virtual ~pkgAcqBaseIndex();
 };
 									/*}}}*/
+/** \brief An acquire item that is responsible for fetching an index	{{{
+ *  file (e.g., Packages or Sources).
+ *
+ *  \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans
+ *
+ *  \todo Why does pkgAcqIndex have protected members?
+ */
+class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
+{
+   void * const d;
+
+   protected:
+
+   /** \brief The stages the method goes through
+    *
+    *  The method first downloads the indexfile, then its decompressed (or
+    *  copied) and verified
+    */
+   enum AllStages {
+      STAGE_DOWNLOAD,
+      STAGE_DECOMPRESS_AND_VERIFY,
+   };
+   AllStages Stage;
+
+   /** \brief Handle what needs to be done when the download is done */
+   void StageDownloadDone(std::string const &Message);
+
+   /** \brief Handle what needs to be done when the decompression/copy is
+    *         done
+    */
+   void StageDecompressDone();
+
+   /** \brief If \b set, this partially downloaded file will be
+    *  removed when the download completes.
+    */
+   std::string EraseFileName;
+
+   /** \brief The compression-related file extensions that are being
+    *  added to the downloaded file one by one if first fails (e.g., "gz bz2").
+    */
+   std::string CompressionExtensions;
+
+   /** \brief The actual compression extension currently used */
+   std::string CurrentCompressionExtension;
+
+   /** \brief Do the changes needed to fetch via AptByHash (if needed) */
+   void InitByHashIfNeeded();
+
+   /** \brief Get the full pathname of the final file for the current URI */
+   virtual std::string GetFinalFilename() const APT_OVERRIDE;
+
+   virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE;
+
+   public:
+   // Specialized action members
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
+   virtual void Done(std::string const &Message, HashStringList const &Hashes,
+		     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
+   virtual std::string Custom600Headers() const APT_OVERRIDE;
+   virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;};
+   virtual std::string GetMetaKey() const APT_OVERRIDE;
+
+   pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager,
+               IndexTarget const &Target, bool const Derived = false) APT_NONNULL(2, 3);
+   virtual ~pkgAcqIndex();
+
+   protected:
+   APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc,
+             std::string const &ShortDesc);
+   APT_HIDDEN bool CommonFailed(std::string const &TargetURI, std::string const TargetDesc,
+             std::string const &Message, pkgAcquire::MethodConfig const * const Cnf);
+};
+									/*}}}*/
 /** \brief An item that is responsible for fetching an index file of	{{{
  *  package list diffs and starting the package list's download.
  *
@@ -620,7 +693,7 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem
  *
  *  \sa pkgAcqIndexDiffs, pkgAcqIndex
  */
-class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
+class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqIndex
 {
    void * const d;
    std::vector<pkgAcqIndexMergeDiffs*> * diffs;
@@ -646,7 +719,6 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex
    virtual void Done(std::string const &Message, HashStringList const &Hashes,
 		     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
    virtual std::string DescURI() const APT_OVERRIDE {return Target.URI + "Index";};
-   virtual std::string Custom600Headers() const APT_OVERRIDE;
    virtual std::string GetMetaKey() const APT_OVERRIDE;
 
    /** \brief Parse the Index file for a set of Packages diffs.
@@ -887,77 +959,6 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex
    virtual ~pkgAcqIndexDiffs();
 };
 									/*}}}*/
-/** \brief An acquire item that is responsible for fetching an index	{{{
- *  file (e.g., Packages or Sources).
- *
- *  \sa pkgAcqDiffIndex, pkgAcqIndexDiffs, pkgAcqIndexTrans
- *
- *  \todo Why does pkgAcqIndex have protected members?
- */
-class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex
-{
-   void * const d;
-
-   protected:
-
-   /** \brief The stages the method goes through
-    *
-    *  The method first downloads the indexfile, then its decompressed (or
-    *  copied) and verified
-    */
-   enum AllStages {
-      STAGE_DOWNLOAD,
-      STAGE_DECOMPRESS_AND_VERIFY,
-   };
-   AllStages Stage;
-
-   /** \brief Handle what needs to be done when the download is done */
-   void StageDownloadDone(std::string const &Message);
-
-   /** \brief Handle what needs to be done when the decompression/copy is
-    *         done 
-    */
-   void StageDecompressDone();
-
-   /** \brief If \b set, this partially downloaded file will be
-    *  removed when the download completes.
-    */
-   std::string EraseFileName;
-
-   /** \brief The compression-related file extensions that are being
-    *  added to the downloaded file one by one if first fails (e.g., "gz bz2").
-    */
-   std::string CompressionExtensions;
-
-   /** \brief The actual compression extension currently used */
-   std::string CurrentCompressionExtension;
-
-   /** \brief Do the changes needed to fetch via AptByHash (if needed) */
-   void InitByHashIfNeeded();
-
-   /** \brief Get the full pathname of the final file for the current URI */
-   virtual std::string GetFinalFilename() const APT_OVERRIDE;
-
-   virtual bool TransactionState(TransactionStates const state) APT_OVERRIDE;
-
-   public:
-   // Specialized action members
-   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
-   virtual void Done(std::string const &Message, HashStringList const &Hashes,
-		     pkgAcquire::MethodConfig const * const Cnf) APT_OVERRIDE;
-   virtual std::string Custom600Headers() const APT_OVERRIDE;
-   virtual std::string DescURI() const APT_OVERRIDE {return Desc.URI;};
-   virtual std::string GetMetaKey() const APT_OVERRIDE;
-
-   pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager,
-               IndexTarget const &Target) APT_NONNULL(2, 3);
-   virtual ~pkgAcqIndex();
-
-   private:
-   APT_HIDDEN void Init(std::string const &URI, std::string const &URIDesc,
-             std::string const &ShortDesc);
-};
-									/*}}}*/
 /** \brief An item that is responsible for fetching a package file.	{{{
  *
  *  If the package file already exists in the cache, nothing will be