From 5576e7f76da73f3f5217f90d816cc19b6c0a5a77 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 1 Aug 2023 13:59:09 +0200 Subject: Compare SHA256 to check if versions are really the same If we know both SHA256, and they're different, the packages are. This approach stores the SHA256 only at runtime, avoiding the overhead of storing it on-disk, because when we update repositories we update all of them anyhow. Note that pkgCacheGenerator is hidden, so we can just modify its ABI, hooray. Closes: #931175 LP: #2029268 --- apt-pkg/deb/deblistparser.h | 10 +++++++++ apt-pkg/pkgcachegen.cc | 27 ++++++++++++++++++++++-- apt-pkg/pkgcachegen.h | 6 ++++++ test/integration/framework | 10 ++++++--- test/integration/test-same-version-but-different | 11 ++++++++++ 5 files changed, 59 insertions(+), 5 deletions(-) diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index eefce2a6a..e6767d7d5 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -14,6 +14,9 @@ #include #include #include +#ifdef APT_COMPILING_APT +#include +#endif #include #include @@ -90,6 +93,13 @@ class APT_HIDDEN debListParser : public pkgCacheListParser explicit debListParser(FileFd *File); virtual ~debListParser(); + +#ifdef APT_COMPILING_APT + APT::StringView SHA256() const + { + return Section.Find(pkgTagSection::Key::SHA256); + } +#endif }; class APT_HIDDEN debDebFileParser : public debListParser diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 807f3bf6c..9e47ef369 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -376,6 +377,12 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator void const * oldMap = Map.Data(); auto Hash = List.VersionHash(); + APT::StringView ListSHA256; + + bool const Debug = _config->FindB("Debug::pkgCacheGen", false); + auto DebList = dynamic_cast(&List); + if (DebList != nullptr) + ListSHA256 = DebList->SHA256(); if (Ver.end() == false) { /* We know the list is sorted so we use that fact in the search. @@ -392,8 +399,18 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator // Versionstrings are equal - is hash also equal? if (Res == 0) { - if (List.SameVersion(Hash, Ver) == true) - break; + if (List.SameVersion(Hash, Ver)) + { + // We do not have SHA256 for both, so we cannot compare them, trust the call from SameVersion() + if (ListSHA256.empty() || VersionExtra[Ver->ID].SHA256[0] == 0) + break; + // We have SHA256 for both, so they must match. + if (ListSHA256 == APT::StringView(VersionExtra[Ver->ID].SHA256, 64)) + break; + if (Debug) + std::cerr << "Found differing SHA256 for " << Pkg.Name() << "=" << Version.to_string() << std::endl; + } + // sort (volatile) sources above not-sources like the status file if (CurrentFile == nullptr || (CurrentFile->Flags & pkgCache::Flag::NotSource) == 0) { @@ -440,6 +457,8 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator if (oldMap != Map.Data()) LastVer = static_cast *>(Map.Data()) + (LastVer - static_cast const *>(oldMap)); *LastVer = verindex; + if (ListSHA256.size() == 64) + memcpy(VersionExtra[Ver->ID].SHA256, ListSHA256.data(), 64); if (unlikely(List.NewVersion(Ver) == false)) return _error->Error(_("Error occurred while processing %s (%s%d)"), @@ -869,6 +888,10 @@ map_pointer pkgCacheGenerator::NewVersion(pkgCache::VerIterat Ver->Hash = Hash; Ver->ID = Cache.HeaderP->VersionCount++; + // Allocate size for extra store + if (VersionExtra.size() <= Ver->ID) + VersionExtra.resize(Ver->ID + 1); + // try to find the version string in the group for reuse pkgCache::PkgIterator Pkg = Ver.ParentPkg(); pkgCache::GrpIterator Grp = Pkg.Group(); diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index f4781d715..420642684 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -76,6 +76,12 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ std::unordered_set strSections; #endif + struct VersionExtra + { + char SHA256[64]; + }; + std::vector VersionExtra{32 * 1024}; + friend class pkgCacheListParser; typedef pkgCacheListParser ListParser; diff --git a/test/integration/framework b/test/integration/framework index 264c228d0..3bf8a3fec 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1022,9 +1022,13 @@ insertpackage() { Priority: $PRIORITY Section: $SECTION Installed-Size: 42 -Size: 42 -SHA256: 0000000000000000000000000000000000000000000000000000000000000000 -Maintainer: Joe Sixpack " +Size: 42" + if echo "$DEPENDENCIES" | grep -q SHA256:; then + : + else + echo "SHA256: 0000000000000000000000000000000000000000000000000000000000000000" + fi + echo "Maintainer: Joe Sixpack " test "$arch" = 'none' || echo "Architecture: $arch" echo "Version: $VERSION Filename: pool/${DISTSECTION}/${NAME}/${NAME}_${VERSION}_${arch}.deb" diff --git a/test/integration/test-same-version-but-different b/test/integration/test-same-version-but-different index df6753a43..595a66fb6 100755 --- a/test/integration/test-same-version-but-different +++ b/test/integration/test-same-version-but-different @@ -30,6 +30,9 @@ insertpackage 'testing' 'diff-size' 'all' '1' 'Size: 42' insertpackage 'unstable' 'diff-instsize' 'all' '1' 'Installed-Size: 21' insertpackage 'testing' 'diff-instsize' 'all' '1' 'Installed-Size: 42' +insertpackage 'unstable' 'diff-sha256' 'all' '1' 'SHA256: 0000000000000000000000000000000000000000000000000000000000000000' +insertpackage 'testing' 'diff-sha256' 'all' '1' 'SHA256: 0000000000000000000000000000000000000000000000000000000000000001' + setupaptarchive APTARCHIVE="$(readlink -f ./aptarchive)" @@ -96,3 +99,11 @@ testsuccessequal "diff-instsize: 500 file:${APTARCHIVE} testing/main all Packages 1 500 500 file:${APTARCHIVE} unstable/main all Packages" apt policy diff-instsize +testsuccessequal "diff-sha256: + Installed: (none) + Candidate: 1 + Version table: + 1 500 + 500 file:${APTARCHIVE} testing/main all Packages + 1 500 + 500 file:${APTARCHIVE} unstable/main all Packages" apt policy diff-sha256 -- cgit v1.2.3-18-g5258