Compare SHA256 to check if versions are really the same

If we know both SHA256, and they're different, the packages are. This
approach stores the SHA256 only at runtime, avoiding the overhead of
storing it on-disk, because when we update repositories we update all
of them anyhow.

Note that pkgCacheGenerator is hidden, so we can just modify its
ABI, hooray.

Closes: #931175
LP: #2029268

5 files changed, 59 insertions, 5 deletions

diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h
index eefce2a6a..e6767d7d5 100644
--- a/apt-pkg/deb/deblistparser.h
+++ b/apt-pkg/deb/deblistparser.h
@@ -14,6 +14,9 @@
 #include <apt-pkg/pkgcache.h>
 #include <apt-pkg/pkgcachegen.h>
 #include <apt-pkg/tagfile.h>
+#ifdef APT_COMPILING_APT
+#include <apt-pkg/tagfile-keys.h>
+#endif
 
 #include <string>
 #include <vector>
@@ -90,6 +93,13 @@ class APT_HIDDEN debListParser : public pkgCacheListParser
 
    explicit debListParser(FileFd *File);
    virtual ~debListParser();
+
+#ifdef APT_COMPILING_APT
+   APT::StringView SHA256() const
+   {
+      return Section.Find(pkgTagSection::Key::SHA256);
+   }
+#endif
 };
 
 class APT_HIDDEN debDebFileParser : public debListParser
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 807f3bf6c..9e47ef369 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -12,6 +12,7 @@
 #include <config.h>
 
 #include <apt-pkg/configuration.h>
+#include <apt-pkg/deblistparser.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/indexfile.h>
@@ -376,6 +377,12 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    void const * oldMap = Map.Data();
 
    auto Hash = List.VersionHash();
+   APT::StringView ListSHA256;
+
+   bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
+   auto DebList = dynamic_cast<debListParser *>(&List);
+   if (DebList != nullptr)
+      ListSHA256 = DebList->SHA256();
    if (Ver.end() == false)
    {
       /* We know the list is sorted so we use that fact in the search.
@@ -392,8 +399,18 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
 	 // Versionstrings are equal - is hash also equal?
 	 if (Res == 0)
 	 {
-	    if (List.SameVersion(Hash, Ver) == true)
-	       break;
+	    if (List.SameVersion(Hash, Ver))
+	    {
+               // We do not have SHA256 for both, so we cannot compare them, trust the call from SameVersion()
+               if (ListSHA256.empty() || VersionExtra[Ver->ID].SHA256[0] == 0)
+                  break;
+               // We have SHA256 for both, so they must match.
+               if (ListSHA256 == APT::StringView(VersionExtra[Ver->ID].SHA256, 64))
+                  break;
+	       if (Debug)
+		  std::cerr << "Found differing SHA256 for " << Pkg.Name() << "=" << Version.to_string() << std::endl;
+	    }
+
 	    // sort (volatile) sources above not-sources like the status file
 	    if (CurrentFile == nullptr || (CurrentFile->Flags & pkgCache::Flag::NotSource) == 0)
 	    {
@@ -440,6 +457,8 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
    if (oldMap != Map.Data())
 	 LastVer = static_cast<map_pointer<pkgCache::Version> *>(Map.Data()) + (LastVer - static_cast<map_pointer<pkgCache::Version> const *>(oldMap));
    *LastVer = verindex;
+   if (ListSHA256.size() == 64)
+	 memcpy(VersionExtra[Ver->ID].SHA256, ListSHA256.data(), 64);
 
    if (unlikely(List.NewVersion(Ver) == false))
       return _error->Error(_("Error occurred while processing %s (%s%d)"),
@@ -869,6 +888,10 @@ map_pointer<pkgCache::Version> pkgCacheGenerator::NewVersion(pkgCache::VerIterat
    Ver->Hash = Hash;
    Ver->ID = Cache.HeaderP->VersionCount++;
 
+   // Allocate size for extra store
+   if (VersionExtra.size() <= Ver->ID)
+      VersionExtra.resize(Ver->ID + 1);
+
    // try to find the version string in the group for reuse
    pkgCache::PkgIterator Pkg = Ver.ParentPkg();
    pkgCache::GrpIterator Grp = Pkg.Group();
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index f4781d715..420642684 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -76,6 +76,12 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    std::unordered_set<string_pointer, hash> strSections;
 #endif
 
+   struct VersionExtra
+   {
+      char SHA256[64];
+   };
+   std::vector<VersionExtra> VersionExtra{32 * 1024};
+
    friend class pkgCacheListParser;
    typedef pkgCacheListParser ListParser;
 
diff --git a/test/integration/framework b/test/integration/framework
index 264c228d0..3bf8a3fec 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1022,9 +1022,13 @@ insertpackage() {
 Priority: $PRIORITY
 Section: $SECTION
 Installed-Size: 42
-Size: 42
-SHA256: 0000000000000000000000000000000000000000000000000000000000000000
-Maintainer: Joe Sixpack <joe@example.org>"
+Size: 42"
+                    if echo "$DEPENDENCIES" | grep -q SHA256:; then
+                        :
+                    else
+                        echo "SHA256: 0000000000000000000000000000000000000000000000000000000000000000"
+                    fi
+                    echo "Maintainer: Joe Sixpack <joe@example.org>"
 					test "$arch" = 'none' || echo "Architecture: $arch"
 					echo "Version: $VERSION
 Filename: pool/${DISTSECTION}/${NAME}/${NAME}_${VERSION}_${arch}.deb"
diff --git a/test/integration/test-same-version-but-different b/test/integration/test-same-version-but-different
index df6753a43..595a66fb6 100755
--- a/test/integration/test-same-version-but-different
+++ b/test/integration/test-same-version-but-different
@@ -30,6 +30,9 @@ insertpackage 'testing' 'diff-size' 'all' '1' 'Size: 42'
 insertpackage 'unstable' 'diff-instsize' 'all' '1' 'Installed-Size: 21'
 insertpackage 'testing' 'diff-instsize' 'all' '1' 'Installed-Size: 42'
 
+insertpackage 'unstable' 'diff-sha256' 'all' '1' 'SHA256: 0000000000000000000000000000000000000000000000000000000000000000'
+insertpackage 'testing' 'diff-sha256' 'all' '1' 'SHA256: 0000000000000000000000000000000000000000000000000000000000000001'
+
 setupaptarchive
 
 APTARCHIVE="$(readlink -f ./aptarchive)"
@@ -96,3 +99,11 @@ testsuccessequal "diff-instsize:
         500 file:${APTARCHIVE} testing/main all Packages
      1 500
         500 file:${APTARCHIVE} unstable/main all Packages" apt policy diff-instsize
+testsuccessequal "diff-sha256:
+  Installed: (none)
+  Candidate: 1
+  Version table:
+     1 500
+        500 file:${APTARCHIVE} testing/main all Packages
+     1 500
+        500 file:${APTARCHIVE} unstable/main all Packages" apt policy diff-sha256