From 4bcb198a67156b2654f3b64173499ab78f0d4d9a Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Sat, 15 Feb 2025 16:37:19 +0100 Subject: cache: Introduce partial SourceVersion support This is the first step that introduces a 1:1 mapping between version and source version. In a future version this can use the fields currently marked unavailable to deduplicate the SourceVersion objects across the group. The policy gains a member for storing pins for sourceversions. Together, in the future we should be able to determine candidates for source versions. --- apt-pkg/cacheiterators.h | 32 ++++++++++++++++++++++++++++-- apt-pkg/deb/deblistparser.cc | 8 ++++---- apt-pkg/edsp/edsplistparser.cc | 2 +- apt-pkg/pkgcache.cc | 4 ++++ apt-pkg/pkgcache.h | 45 ++++++++++++++++++++++++++++++++++-------- apt-pkg/pkgcachegen.cc | 5 +++++ apt-pkg/policy.h | 1 + cmdline/apt-cache.cc | 31 ++++++++++++++++------------- 8 files changed, 99 insertions(+), 29 deletions(-) diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index ceeb4b3a7..443912253 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -188,6 +188,31 @@ class APT_PUBLIC pkgCache::PkgIterator: public Iterator { inline PkgIterator() : Iterator(), HashIndex(0) {} }; /*}}}*/ +// SourceVersion Iterator /*{{{*/ +class APT_PUBLIC pkgCache::SrcVerIterator : public Iterator +{ + public: + inline SourceVersion *OwnerPointer() const + { + return (Owner != 0) ? Owner->SrcVerP : 0; + } +#if 0 + // Iteration + inline SrcVerIterator& operator++() {if (S != Owner->SrcVerP) S = Owner->SrcVerP + S->NextSourceVersion; return *this;} + inline SrcVerIterator operator++(int) { SrcVerIterator const tmp(*this); operator++(); return tmp; } +#endif + inline APT_PURE GrpIterator Group() const { return GrpIterator(*Owner, Owner->GrpP + S->Group); } + inline const char *VerStr() const { return S->VerStr == 0 ? 0 : Owner->StrP + S->VerStr; } + + inline SrcVerIterator(pkgCache &Owner, SourceVersion *Trg = 0) : Iterator(Owner, Trg) + { + if (S == 0) + S = OwnerPointer(); + } + inline SrcVerIterator() : Iterator() {} +}; + /*}}}*/ + // Version Iterator /*{{{*/ class APT_PUBLIC pkgCache::VerIterator : public Iterator { public: @@ -219,12 +244,15 @@ class APT_PUBLIC pkgCache::VerIterator : public Iterator { // Accessors inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;} inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;} + /** \brief source version this version comes from + Always contains the version string, even if it is the same as the binary version */ + SrcVerIterator SourceVersion() const { return SrcVerIterator(*Owner, Owner->SrcVerP + S->SourceVersion); } /** \brief source package name this version comes from Always contains the name, even if it is the same as the binary name */ - inline const char *SourcePkgName() const {return Owner->StrP + S->SourcePkgName;} + inline const char *SourcePkgName() const { return SourceVersion().Group().Name(); } /** \brief source version this version comes from Always contains the version string, even if it is the same as the binary version */ - inline const char *SourceVerStr() const {return Owner->StrP + S->SourceVerStr;} + inline const char *SourceVerStr() const { return SourceVersion().VerStr(); } inline const char *Arch() const { if ((S->MultiArch & pkgCache::Version::All) == pkgCache::Version::All) return "all"; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index e3041f848..dfb07d2a6 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -169,8 +169,8 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) pkgCache::GrpIterator G = Ver.ParentPkg().Group(); // Setup the defaults - Ver->SourcePkgName = G->Name; - Ver->SourceVerStr = Ver->VerStr; + Ver.SourceVersion()->Group = G.MapPointer(); + Ver.SourceVersion()->VerStr = Ver->VerStr; // Parse the name and version str if (Section.Find(pkgTagSection::Key::Source,Start,Stop) == true) @@ -191,7 +191,7 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) { map_stringitem_t const idx = StoreString(pkgCacheGenerator::VERSIONNUMBER, version); G = Ver.ParentPkg().Group(); - Ver->SourceVerStr = idx; + Ver.SourceVersion()->VerStr = idx; } } } @@ -208,7 +208,7 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) } // Link into by source package group. - Ver->SourcePkgName = G->Name; + Ver.SourceVersion()->Group = G.MapPointer(); Ver->NextInSource = G->VersionsInSource; G->VersionsInSource = Ver.MapPointer(); diff --git a/apt-pkg/edsp/edsplistparser.cc b/apt-pkg/edsp/edsplistparser.cc index d3e761be4..f591a6a54 100644 --- a/apt-pkg/edsp/edsplistparser.cc +++ b/apt-pkg/edsp/edsplistparser.cc @@ -52,7 +52,7 @@ bool edspLikeListParser::NewVersion(pkgCache::VerIterator &Ver) if (version != Ver.VerStr()) { map_stringitem_t const idx = StoreString(pkgCacheGenerator::VERSIONNUMBER, version); - Ver->SourceVerStr = idx; + Ver.SourceVersion()->VerStr = idx; } } diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 2beef0e74..a27b1b054 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -67,6 +67,7 @@ pkgCache::Header::Header() APT_HEADER_SET(PackageSz, sizeof(pkgCache::Package)); APT_HEADER_SET(ReleaseFileSz, sizeof(pkgCache::ReleaseFile)); APT_HEADER_SET(PackageFileSz, sizeof(pkgCache::PackageFile)); + APT_HEADER_SET(SourceVersionSz, sizeof(pkgCache::SourceVersion)); APT_HEADER_SET(VersionSz, sizeof(pkgCache::Version) + sizeof(pkgCache::Version::Extra)); APT_HEADER_SET(DescriptionSz, sizeof(pkgCache::Description)); APT_HEADER_SET(DependencySz, sizeof(pkgCache::Dependency)); @@ -79,6 +80,7 @@ pkgCache::Header::Header() GroupCount = 0; PackageCount = 0; VersionCount = 0; + SourceVersionCount = 0; DescriptionCount = 0; DependsCount = 0; DependsDataCount = 0; @@ -112,6 +114,7 @@ bool pkgCache::Header::CheckSizes(Header &Against) const ReleaseFileSz == Against.ReleaseFileSz && PackageFileSz == Against.PackageFileSz && VersionSz == Against.VersionSz && + SourceVersionSz == Against.SourceVersionSz && DescriptionSz == Against.DescriptionSz && DependencySz == Against.DependencySz && DependencyDataSz == Against.DependencyDataSz && @@ -151,6 +154,7 @@ bool pkgCache::ReMap(bool const &Errorchecks) RlsFileP = (ReleaseFile *)Map.Data(); PkgFileP = (PackageFile *)Map.Data(); VerP = (Version *)Map.Data(); + SrcVerP = (SourceVersion *)Map.Data(); DescP = (Description *)Map.Data(); ProvideP = (Provides *)Map.Data(); DepP = (Dependency *)Map.Data(); diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 22eea3f6c..22e13e009 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -130,6 +130,7 @@ class APT_PUBLIC pkgCache /*{{{*/ struct Package; struct ReleaseFile; struct PackageFile; + struct SourceVersion; struct Version; struct Description; struct Provides; @@ -144,6 +145,7 @@ class APT_PUBLIC pkgCache /*{{{*/ class GrpIterator; class PkgIterator; class VerIterator; + class SrcVerIterator; class DescIterator; class DepIterator; class PrvIterator; @@ -224,13 +226,14 @@ class APT_PUBLIC pkgCache /*{{{*/ DescFile *DescFileP; ReleaseFile *RlsFileP; PackageFile *PkgFileP; + SourceVersion *SrcVerP; // reserved for SourceVersion objects Version *VerP; Description *DescP; Provides *ProvideP; Dependency *DepP; DependencyData *DepDataP; char *StrP; - void *reserved[12]; + void *reserved[13]; virtual bool ReMap(bool const &Errorchecks = true); inline bool Sync() {return Map.Sync();} @@ -320,6 +323,7 @@ struct pkgCache::Header map_number_t ReleaseFileSz; map_number_t PackageFileSz; map_number_t VersionSz; + map_number_t SourceVersionSz; map_number_t DescriptionSz; map_number_t DependencySz; map_number_t DependencyDataSz; @@ -335,6 +339,7 @@ struct pkgCache::Header map_id_t GroupCount; map_id_t PackageCount; map_id_t VersionCount; + map_id_t SourceVersionCount; map_id_t DescriptionCount; map_id_t DependsCount; map_id_t DependsDataCount; @@ -371,7 +376,7 @@ struct pkgCache::Header twice the number of pools as there are non-private structure types. The generator stores this information so future additions can make use of any unused pool blocks. */ - DynamicMMap::Pool Pools[2 * 12]; + DynamicMMap::Pool Pools[2 * 13]; /** \brief hash tables providing rapid group/package name lookup @@ -431,6 +436,9 @@ struct pkgCache::Group /** \brief List of binary produces by source package with this name. */ map_pointer VersionsInSource; + /** \brief SourceVersionList */ + map_pointer SourceVersionList; + /** \brief Private pointer */ map_pointer d; }; @@ -612,6 +620,27 @@ struct pkgCache::DescFile map_filesize_t Offset; // File offset }; /*}}}*/ +// SourceVersion structure /*{{{*/ +/** \brief information for a single version of a source package + + The version list is always sorted from highest version to lowest + version by the generator. Equal version numbers are either merged + or handled as separate versions based on the Hash value. */ +struct pkgCache::SourceVersion +{ + /** \brief unique sequel ID */ + map_id_t ID; + /** \brief Group the source package belongs too */ + map_pointer Group; + /** \brief complete version string */ + map_stringitem_t VerStr; + map_pointer VersionList [[gnu::unavailable("not yet available")]]; + map_pointer NextSourceVersion [[gnu::unavailable("not yet available")]]; + + /** \brief Private pointer */ + map_pointer d; +}; + /*}}}*/ // Version structure /*{{{*/ /** \brief information for a single version of a package @@ -626,12 +655,11 @@ struct pkgCache::Version map_stringitem_t VerStr; /** \brief section this version is filled in */ map_stringitem_t Section; - /** \brief source package name this version comes from - Always contains the name, even if it is the same as the binary name */ - map_stringitem_t SourcePkgName; - /** \brief source version this version comes from - Always contains the version string, even if it is the same as the binary version */ - map_stringitem_t SourceVerStr; + + /** \brief the source version object */ + map_pointer SourceVersion; + /** \brief next version in the source package (might be different binary) */ + map_pointer NextInSourceVersion; /** \brief Multi-Arch capabilities of a package version */ enum VerMultiArch { No = 0, /*!< is the default and doesn't trigger special behaviour */ @@ -825,6 +853,7 @@ class pkgCache::Namespace /*{{{*/ typedef pkgCache::GrpIterator GrpIterator; typedef pkgCache::PkgIterator PkgIterator; typedef pkgCache::VerIterator VerIterator; + typedef pkgCache::SrcVerIterator SrcVerIterator; typedef pkgCache::DescIterator DescIterator; typedef pkgCache::DepIterator DepIterator; typedef pkgCache::PrvIterator PrvIterator; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index fec931cf2..6e115e4f4 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -894,6 +894,11 @@ map_pointer pkgCacheGenerator::NewVersion(pkgCache::VerIterat Ver->d = d; if (not Ver.PhasedUpdatePercentage(100)) abort(); + auto SourceVersion = AllocateInMap(); // sequence point so Ver can be moved if needed + Ver->SourceVersion = SourceVersion; + Ver.SourceVersion()->ID = Cache.HeaderP->SourceVersionCount++; + if (not Ver.SourceVersion()) + abort(); //Dynamic DynV(Ver); // caller MergeListVersion already takes care of it Ver->NextVer = Next; diff --git a/apt-pkg/policy.h b/apt-pkg/policy.h index 36c32b644..3f126a7c0 100644 --- a/apt-pkg/policy.h +++ b/apt-pkg/policy.h @@ -58,6 +58,7 @@ class APT_PUBLIC pkgPolicy : public pkgDepCache::Policy explicit PkgPin(std::string const &Pkg) : Pin(), Pkg(Pkg) {}; }; + std::unique_ptr SrcVerPins; std::unique_ptr VerPins; std::unique_ptr PFPriority; std::vector Defaults; diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 67dde67a6..d8e9e7e4b 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -239,6 +239,8 @@ static bool Stats(CommandLine &CmdL) cout << _(" Mixed virtual packages: ") << NVirt << endl; cout << _(" Missing: ") << Missing << endl; + cout << _("Total distinct source versions: ") << Cache->Head().SourceVersionCount << " (" << SizeToStr(Cache->Head().SourceVersionCount * Cache->Head().SourceVersionSz) << ')' << endl; + cout << _("Total distinct versions: ") << Cache->Head().VersionCount << " (" << SizeToStr(Cache->Head().VersionCount*Cache->Head().VersionSz) << ')' << endl; cout << _("Total distinct descriptions: ") << Cache->Head().DescriptionCount << " (" << @@ -266,8 +268,8 @@ static bool Stats(CommandLine &CmdL) stritems.insert(V->VerStr); if (V->Section != 0) stritems.insert(V->Section); - stritems.insert(V->SourcePkgName); - stritems.insert(V->SourceVerStr); + // FIXME: Count the source versions + stritems.insert(V.SourceVersion()->VerStr); for (pkgCache::DepIterator D = V.DependsList(); D.end() == false; ++D) { if (D->Version != 0) @@ -317,18 +319,19 @@ static bool Stats(CommandLine &CmdL) unsigned long Total = 0; #define APT_CACHESIZE(X,Y) (Cache->Head().X * Cache->Head().Y) Total = Slack + Size + - APT_CACHESIZE(GroupCount, GroupSz) + - APT_CACHESIZE(PackageCount, PackageSz) + - APT_CACHESIZE(VersionCount, VersionSz) + - APT_CACHESIZE(DescriptionCount, DescriptionSz) + - APT_CACHESIZE(DependsCount, DependencySz) + - APT_CACHESIZE(DependsDataCount, DependencyDataSz) + - APT_CACHESIZE(ReleaseFileCount, ReleaseFileSz) + - APT_CACHESIZE(PackageFileCount, PackageFileSz) + - APT_CACHESIZE(VerFileCount, VerFileSz) + - APT_CACHESIZE(DescFileCount, DescFileSz) + - APT_CACHESIZE(ProvidesCount, ProvidesSz) + - (2 * Cache->Head().GetHashTableSize() * sizeof(map_id_t)); + APT_CACHESIZE(GroupCount, GroupSz) + + APT_CACHESIZE(PackageCount, PackageSz) + + APT_CACHESIZE(VersionCount, VersionSz) + + APT_CACHESIZE(SourceVersionCount, SourceVersionSz) + + APT_CACHESIZE(DescriptionCount, DescriptionSz) + + APT_CACHESIZE(DependsCount, DependencySz) + + APT_CACHESIZE(DependsDataCount, DependencyDataSz) + + APT_CACHESIZE(ReleaseFileCount, ReleaseFileSz) + + APT_CACHESIZE(PackageFileCount, PackageFileSz) + + APT_CACHESIZE(VerFileCount, VerFileSz) + + APT_CACHESIZE(DescFileCount, DescFileSz) + + APT_CACHESIZE(ProvidesCount, ProvidesSz) + + (2 * Cache->Head().GetHashTableSize() * sizeof(map_id_t)); cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; #undef APT_CACHESIZE -- cgit v1.2.3-70-g09d2