From 773e2c1fe5fd1a315ffaaf79f2f2e4a9e975649a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 9 Dec 2009 19:24:03 +0100 Subject: Refactor the cache iterators by using a common base class This should not change the public interface, but it removes the friend connection between the iterators and pkgcache as it is unused. --- apt-pkg/pkgcache.h | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index e8a3e1064..359f8a590 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -43,6 +43,7 @@ class pkgCache /*{{{*/ struct DescFile; // Iterators + template class Iterator; class PkgIterator; class VerIterator; class DescIterator; @@ -51,14 +52,6 @@ class pkgCache /*{{{*/ class PkgFileIterator; class VerFileIterator; class DescFileIterator; - friend class PkgIterator; - friend class VerIterator; - friend class DescInterator; - friend class DepIterator; - friend class PrvIterator; - friend class PkgFileIterator; - friend class VerFileIterator; - friend class DescFileIterator; class Namespace; -- cgit v1.2.3-70-g09d2 From 5bf15716398fdb767ca6249a0155219b88d7ae60 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 19 Dec 2009 16:25:56 +0100 Subject: Implement the first step toward Multi-Arch by setting up a Group infrastructor for packages. APT is now aware of the fact that a package A in architecture X can't satisfy a dependency on package A in architecture Y - to handle these packages are now identified by name and architecture, so different architectures of the same package are handled internally as completly different packages. This is great for pinning, dependency checking and in many other situations, but sometimes we need to know which archs are available for a given package: Here Groups come to our rescue! --- apt-pkg/cacheiterators.h | 37 ++++++++++++-- apt-pkg/deb/deblistparser.cc | 40 +++++++++++----- apt-pkg/deb/deblistparser.h | 1 + apt-pkg/pkgcache.cc | 112 ++++++++++++++++++++++++++++++++++++------- apt-pkg/pkgcache.h | 30 ++++++++++-- apt-pkg/pkgcachegen.cc | 111 +++++++++++++++++++++++++++++------------- apt-pkg/pkgcachegen.h | 6 ++- 7 files changed, 267 insertions(+), 70 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 1da5c6f02..35d3aa228 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -31,10 +31,8 @@ #define PKGLIB_CACHEITERATORS_H // abstract Iterator template /*{{{*/ /* This template provides the very basic iterator methods we - need to have for doing some walk-over-the-cache magic, */ + need to have for doing some walk-over-the-cache magic */ template class pkgCache::Iterator { - __attribute__ ((deprecated)) void _dummy(); // FIXME: Who on earth uses this method ??? - protected: Str *S; pkgCache *Owner; @@ -75,6 +73,35 @@ template class pkgCache::Iterator { // Constructors - look out for the variable assigning inline Iterator() : S(0), Owner(0) {}; inline Iterator(pkgCache &Owner,Str *T = 0) : S(T), Owner(&Owner) {}; +}; + /*}}}*/ +// Group Iterator /*{{{*/ +/* Packages with the same name are collected in a Group so someone only + interest in package names can iterate easily over the names, so the + different architectures can be treated as of the "same" package + (apt internally treat them as totally different packages) */ +class pkgCache::GrpIterator: public Iterator { + protected: + inline Group* OwnerPointer() const { + return Owner->GrpP; + }; + + public: + void operator ++(int) {if (S != Owner->GrpP) S = Owner->GrpP + S->Next;}; + virtual void operator ++() {operator ++(0);}; + + inline const char *Name() const {return S->Name == 0?0:Owner->StrP + S->Name;}; + inline PkgIterator PackageList() const; + PkgIterator FindPkg(string Arch = "any"); + PkgIterator NextPkg(PkgIterator const &Pkg); + + // Constructors + inline GrpIterator(pkgCache &Owner, Group *Trg) : Iterator(Owner, Trg) { + if (S == 0) + S = OwnerPointer(); + }; + inline GrpIterator() : Iterator() {}; + }; /*}}}*/ // Package Iterator /*{{{*/ @@ -103,6 +130,8 @@ class pkgCache::PkgIterator: public Iterator { inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;}; inline bool Purge() const {return S->CurrentState == pkgCache::State::Purge || (S->CurrentVer == 0 && S->CurrentState == pkgCache::State::NotInstalled);}; + inline const char *Arch() const {return S->Arch == 0?0:Owner->StrP + S->Arch;}; + inline GrpIterator Group() const { return GrpIterator(*Owner, Owner->GrpP + S->Group);}; inline VerIterator VersionList() const; inline VerIterator CurrentVer() const; @@ -339,6 +368,8 @@ class pkgCache::DescFileIterator : public Iterator { }; /*}}}*/ // Inlined Begin functions cant be in the class because of order problems /*{{{*/ +inline pkgCache::PkgIterator pkgCache::GrpIterator::PackageList() const + {return PkgIterator(*Owner,Owner->PkgP + S->FirstPackage);}; inline pkgCache::VerIterator pkgCache::PkgIterator::VersionList() const {return VerIterator(*Owner,Owner->VerP + S->VersionList);}; inline pkgCache::VerIterator pkgCache::PkgIterator::CurrentVer() const diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 25a1df3f9..fe68aec0b 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -53,12 +53,25 @@ unsigned long debListParser::UniqFindTagWrite(const char *Tag) // ListParser::Package - Return the package name /*{{{*/ // --------------------------------------------------------------------- /* This is to return the name of the package this section describes */ -string debListParser::Package() -{ - string Result = Section.FindS("Package"); - if (Result.empty() == true) - _error->Error("Encountered a section with no Package: header"); - return Result; +string debListParser::Package() { + string const Result = Section.FindS("Package"); + if(unlikely(Result.empty() == true)) + _error->Error("Encountered a section with no Package: header"); + return Result; +} + /*}}}*/ +// ListParser::Architecture - Return the package arch /*{{{*/ +// --------------------------------------------------------------------- +/* This will return the Architecture of the package this section describes + Note that architecture "all" packages will get the architecture of the + Packages file parsed here */ +string debListParser::Architecture() { + string const Result = Section.FindS("Architecture"); + if (Result.empty() == true) + return Arch; + if (Result == "all") + return Arch; + return Result; } /*}}}*/ // ListParser::Version - Return the version string /*{{{*/ @@ -78,8 +91,10 @@ bool debListParser::NewVersion(pkgCache::VerIterator Ver) { // Parse the section Ver->Section = UniqFindTagWrite("Section"); - Ver->Arch = UniqFindTagWrite("Architecture"); - + + // Parse the architecture + Ver->Arch = WriteUniqString(Architecture()); + // Archive Size Ver->Size = (unsigned)Section.FindI("Size"); @@ -537,6 +552,7 @@ bool debListParser::ParseDepends(pkgCache::VerIterator Ver, return true; string Package; + string const pkgArch = Ver.Arch(); string Version; unsigned int Op; @@ -546,7 +562,7 @@ bool debListParser::ParseDepends(pkgCache::VerIterator Ver, if (Start == 0) return _error->Error("Problem parsing dependency %s",Tag); - if (NewDepends(Ver,Package,Version,Op,Type) == false) + if (NewDepends(Ver,Package,pkgArch,Version,Op,Type) == false) return false; if (Start == Stop) break; @@ -619,6 +635,7 @@ bool debListParser::Step() if (Section.Find("Architecture",Start,Stop) == false) return true; + //FIXME: Accept different Architectures here if (stringcmp(Arch,Start,Stop) == 0) return true; @@ -641,8 +658,9 @@ bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator FileI, if (Tags.Step(Section) == false) return false; - //mvo: I don't think we need to fill that in (it's unused since apt-0.6) - //FileI->Architecture = WriteUniqString(Arch); + // FIXME: Do we need it now for multi-arch? + // mvo: I don't think we need to fill that in (it's unused since apt-0.6) +// FileI->Architecture = WriteUniqString(Arch); // apt-secure does no longer download individual (per-section) Release // file. to provide Component pinning we use the section name now diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index 1c709229f..5f91e073e 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -46,6 +46,7 @@ class debListParser : public pkgCacheGenerator::ListParser // These all operate against the current section virtual string Package(); + virtual string Architecture(); virtual string Version(); virtual bool NewVersion(pkgCache::VerIterator Ver); virtual string Description(); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 2f8bde6f7..2b5f53f6f 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -79,7 +79,8 @@ pkgCache::Header::Header() StringList = 0; VerSysName = 0; Architecture = 0; - memset(HashTable,0,sizeof(HashTable)); + memset(PkgHashTable,0,sizeof(PkgHashTable)); + memset(GrpHashTable,0,sizeof(GrpHashTable)); memset(Pools,0,sizeof(Pools)); } /*}}}*/ @@ -118,6 +119,7 @@ bool pkgCache::ReMap() { // Apply the typecasts. HeaderP = (Header *)Map.Data(); + GrpP = (Group *)Map.Data(); PkgP = (Package *)Map.Data(); VerFileP = (VerFile *)Map.Data(); DescFileP = (DescFile *)Map.Data(); @@ -165,7 +167,7 @@ unsigned long pkgCache::sHash(const string &Str) const unsigned long Hash = 0; for (string::const_iterator I = Str.begin(); I != Str.end(); I++) Hash = 5*Hash + tolower_ascii(*I); - return Hash % _count(HeaderP->HashTable); + return Hash % _count(HeaderP->PkgHashTable); } unsigned long pkgCache::sHash(const char *Str) const @@ -173,24 +175,40 @@ unsigned long pkgCache::sHash(const char *Str) const unsigned long Hash = 0; for (const char *I = Str; *I != 0; I++) Hash = 5*Hash + tolower_ascii(*I); - return Hash % _count(HeaderP->HashTable); + return Hash % _count(HeaderP->PkgHashTable); } /*}}}*/ // Cache::FindPkg - Locate a package by name /*{{{*/ // --------------------------------------------------------------------- /* Returns 0 on error, pointer to the package otherwise */ -pkgCache::PkgIterator pkgCache::FindPkg(const string &Name) -{ - // Look at the hash bucket - Package *Pkg = PkgP + HeaderP->HashTable[Hash(Name)]; - for (; Pkg != PkgP; Pkg = PkgP + Pkg->NextPackage) - { - if (Pkg->Name != 0 && StrP[Pkg->Name] == Name[0] && - stringcasecmp(Name,StrP + Pkg->Name) == 0) - return PkgIterator(*this,Pkg); - } - return PkgIterator(*this,0); +pkgCache::PkgIterator pkgCache::FindPkg(const string &Name, string Arch) { + /* We make a detour via the GrpIterator here as + on a multi-arch environment a group is easier to + find than a package (less entries in the buckets) */ + pkgCache::GrpIterator Grp = FindGrp(Name); + if (Grp.end() == true) + return PkgIterator(*this,0); + + return Grp.FindPkg(Arch); +} + /*}}}*/ +// Cache::FindGrp - Locate a group by name /*{{{*/ +// --------------------------------------------------------------------- +/* Returns End-Pointer on error, pointer to the group otherwise */ +pkgCache::GrpIterator pkgCache::FindGrp(const string &Name) { + if (unlikely(Name.empty() == true)) + return GrpIterator(*this,0); + + // Look at the hash bucket for the group + Group *Grp = GrpP + HeaderP->GrpHashTable[sHash(Name)]; + for (; Grp != GrpP; Grp = GrpP + Grp->Next) { + if (Grp->Name != 0 && StrP[Grp->Name] == Name[0] && + stringcasecmp(Name, StrP + Grp->Name) == 0) + return GrpIterator(*this, Grp); + } + + return GrpIterator(*this,0); } /*}}}*/ // Cache::CompTypeDeb - Return a string describing the compare type /*{{{*/ @@ -242,6 +260,68 @@ const char *pkgCache::Priority(unsigned char Prio) return 0; } /*}}}*/ +// GrpIterator::FindPkg - Locate a package by arch /*{{{*/ +// --------------------------------------------------------------------- +/* Returns an End-Pointer on error, pointer to the package otherwise */ +pkgCache::PkgIterator pkgCache::GrpIterator::FindPkg(string Arch) { + if (unlikely(IsGood() == false || S->FirstPackage == 0)) + return PkgIterator(*Owner, 0); + + static string const myArch = _config->Find("APT::Architecture"); + /* Most of the time the package for our native architecture is + the one we add at first to the cache, but this would be the + last one we check, so we do it now. */ + if (Arch == "native" || Arch == myArch) { + Arch = myArch; + pkgCache::Package *Pkg = Owner->PkgP + S->LastPackage; + if (stringcasecmp(Arch, Owner->StrP + Pkg->Arch) == 0) + return PkgIterator(*Owner, Pkg); + } + + /* If we accept any package we simply return the "first" + package in this group (the last one added). */ + if (Arch == "any") + return PkgIterator(*Owner, Owner->PkgP + S->FirstPackage); + + /* Iterate over the list to find the matching arch + unfortunately this list includes "package noise" + (= different packages with same calculated hash), + so we need to check the name also */ + for (pkgCache::Package *Pkg = PackageList(); Pkg != Owner->PkgP; + Pkg = Owner->PkgP + Pkg->NextPackage) { + if (S->Name == Pkg->Name && + stringcasecmp(Arch, Owner->StrP + Pkg->Arch) == 0) + return PkgIterator(*Owner, Pkg); + if ((Owner->PkgP + S->LastPackage) == Pkg) + break; + } + + return PkgIterator(*Owner, 0); +} + /*}}}*/ +// GrpIterator::NextPkg - Locate the next package in the group /*{{{*/ +// --------------------------------------------------------------------- +/* Returns an End-Pointer on error, pointer to the package otherwise. + We can't simply ++ to the next as the list of packages includes + "package noise" (= packages with the same hash value but different name) */ +pkgCache::PkgIterator pkgCache::GrpIterator::NextPkg(pkgCache::PkgIterator const &LastPkg) { + if (unlikely(IsGood() == false || S->FirstPackage == 0 || + LastPkg.end() == true)) + return PkgIterator(*Owner, 0); + + // Iterate over the list to find the next package + pkgCache::Package *Pkg = Owner->PkgP + LastPkg.Index(); + Pkg = Owner->PkgP + Pkg->NextPackage; + for (; Pkg != Owner->PkgP; Pkg = Owner->PkgP + Pkg->NextPackage) { + if (S->Name == Pkg->Name) + return PkgIterator(*Owner, Pkg); + if ((Owner->PkgP + S->LastPackage) == Pkg) + break; + } + + return PkgIterator(*Owner, 0); +} + /*}}}*/ // PkgIterator::operator ++ - Postfix incr /*{{{*/ // --------------------------------------------------------------------- /* This will advance to the next logical package in the hash table. */ @@ -252,10 +332,10 @@ void pkgCache::PkgIterator::operator ++(int) S = Owner->PkgP + S->NextPackage; // Follow the hash table - while (S == Owner->PkgP && (HashIndex+1) < (signed)_count(Owner->HeaderP->HashTable)) + while (S == Owner->PkgP && (HashIndex+1) < (signed)_count(Owner->HeaderP->PkgHashTable)) { HashIndex++; - S = Owner->PkgP + Owner->HeaderP->HashTable[HashIndex]; + S = Owner->PkgP + Owner->HeaderP->PkgHashTable[HashIndex]; } }; /*}}}*/ diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 359f8a590..b3d2752d2 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -32,6 +32,7 @@ class pkgCache /*{{{*/ public: // Cache element predeclarations struct Header; + struct Group; struct Package; struct PackageFile; struct Version; @@ -44,6 +45,7 @@ class pkgCache /*{{{*/ // Iterators template class Iterator; + class GrpIterator; class PkgIterator; class VerIterator; class DescIterator; @@ -97,6 +99,7 @@ class pkgCache /*{{{*/ // Pointers to the arrays of items Header *HeaderP; + Group *GrpP; Package *PkgP; VerFile *VerFileP; DescFile *DescFileP; @@ -121,7 +124,9 @@ class pkgCache /*{{{*/ const char *Priority(unsigned char Priority); // Accessors - PkgIterator FindPkg(const string &Name); + GrpIterator FindGrp(const string &Name); + PkgIterator FindPkg(const string &Name, string Arch = "native"); + Header &Head() {return *HeaderP;}; inline PkgIterator PkgBegin(); inline PkgIterator PkgEnd(); @@ -161,6 +166,7 @@ struct pkgCache::Header unsigned short DescFileSz; // Structure counts + unsigned long GroupCount; unsigned long PackageCount; unsigned long VersionCount; unsigned long DescriptionCount; @@ -180,22 +186,36 @@ struct pkgCache::Header /* Allocation pools, there should be one of these for each structure excluding the header */ - DynamicMMap::Pool Pools[8]; + DynamicMMap::Pool Pools[9]; - // Rapid package name lookup - map_ptrloc HashTable[2*1048]; + // Rapid package and group name lookup + // Notice: Increase only both table sizes as the + // hashmethod assume the size of the Pkg one + map_ptrloc PkgHashTable[2*1048]; + map_ptrloc GrpHashTable[2*1048]; bool CheckSizes(Header &Against) const; Header(); }; /*}}}*/ +struct pkgCache::Group { /*{{{*/ + map_ptrloc Name; // Stringtable + + // Linked List + map_ptrloc FirstPackage;// Package + map_ptrloc LastPackage; // Package + map_ptrloc Next; // Group +}; + /*}}}*/ struct pkgCache::Package /*{{{*/ { // Pointers map_ptrloc Name; // Stringtable + map_ptrloc Arch; // StringTable (StringItem) map_ptrloc VersionList; // Version map_ptrloc CurrentVer; // Version map_ptrloc Section; // StringTable (StringItem) + map_ptrloc Group; // Group the Package belongs to // Linked list map_ptrloc NextPackage; // Package @@ -254,7 +274,7 @@ struct pkgCache::Version /*{{{*/ map_ptrloc VerStr; // Stringtable map_ptrloc Section; // StringTable (StringItem) map_ptrloc Arch; // StringTable - + // Lists map_ptrloc FileList; // VerFile map_ptrloc NextVer; // Version diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 68180c702..c37f6f48e 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -108,12 +108,12 @@ bool pkgCacheGenerator::MergeList(ListParser &List, while (List.Step() == true) { // Get a pointer to the package structure - string PackageName = List.Package(); + string const PackageName = List.Package(); if (PackageName.empty() == true) return false; - + pkgCache::PkgIterator Pkg; - if (NewPackage(Pkg,PackageName) == false) + if (NewPackage(Pkg, PackageName, List.Architecture()) == false) return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str()); Counter++; if (Counter % 100 == 0 && Progress != 0) @@ -323,33 +323,71 @@ bool pkgCacheGenerator::MergeFileProvides(ListParser &List) return true; } /*}}}*/ +// CacheGenerator::NewGroup - Add a new group /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a new group structure and adds it to the hash table */ +bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) { + Grp = Cache.FindGrp(Name); + if (Grp.end() == false) + return true; + + // Get a structure + unsigned long const Group = Map.Allocate(sizeof(pkgCache::Group)); + if (unlikely(Group == 0)) + return false; + + Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group); + Grp->Name = Map.WriteString(Name); + if (unlikely(Grp->Name == 0)) + return false; + + // Insert it into the hash table + unsigned long const Hash = Cache.Hash(Name); + Grp->Next = Cache.HeaderP->GrpHashTable[Hash]; + Cache.HeaderP->GrpHashTable[Hash] = Group; + + Cache.HeaderP->GroupCount++; + + return true; +} + /*}}}*/ // CacheGenerator::NewPackage - Add a new package /*{{{*/ // --------------------------------------------------------------------- /* This creates a new package structure and adds it to the hash table */ -bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name) -{ - Pkg = Cache.FindPkg(Name); - if (Pkg.end() == false) - return true; +bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name, + const string &Arch) { + pkgCache::GrpIterator Grp; + if (unlikely(NewGroup(Grp, Name) == false)) + return false; + + Pkg = Grp.FindPkg(Arch); + if (Pkg.end() == false) + return true; // Get a structure - unsigned long Package = Map.Allocate(sizeof(pkgCache::Package)); - if (Package == 0) + unsigned long const Package = Map.Allocate(sizeof(pkgCache::Package)); + if (unlikely(Package == 0)) return false; - Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package); - + // Insert it into the hash table - unsigned long Hash = Cache.Hash(Name); - Pkg->NextPackage = Cache.HeaderP->HashTable[Hash]; - Cache.HeaderP->HashTable[Hash] = Package; - - // Set the name and the ID - Pkg->Name = Map.WriteString(Name); - if (Pkg->Name == 0) + unsigned long const Hash = Cache.Hash(Name); + Pkg->NextPackage = Cache.HeaderP->PkgHashTable[Hash]; + Cache.HeaderP->PkgHashTable[Hash] = Package; + + // remember the packages in the group + Grp->FirstPackage = Package; + if (Grp->LastPackage == 0) + Grp->LastPackage = Package; + + // Set the name, arch and the ID + Pkg->Name = Grp->Name; + Pkg->Group = Grp.Index(); + Pkg->Arch = WriteUniqString(Arch.c_str()); + if (unlikely(Pkg->Arch == 0)) return false; Pkg->ID = Cache.HeaderP->PackageCount++; - + return true; } /*}}}*/ @@ -474,6 +512,7 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, version and to the package that it is pointing to. */ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, const string &PackageName, + const string &Arch, const string &Version, unsigned int Op, unsigned int Type) @@ -481,8 +520,8 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, pkgCache &Cache = Owner->Cache; // Get a structure - unsigned long Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency)); - if (Dependency == 0) + unsigned long const Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency)); + if (unlikely(Dependency == 0)) return false; // Fill it in @@ -492,11 +531,17 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, Dep->CompareOp = Op; Dep->ID = Cache.HeaderP->DependsCount++; - // Locate the target package - pkgCache::PkgIterator Pkg; - if (Owner->NewPackage(Pkg,PackageName) == false) + pkgCache::GrpIterator Grp; + if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) return false; - + + // Locate the target package + pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch); + if (Pkg.end() == true) { + if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) + return false; + } + // Probe the reverse dependency list for a version string that matches if (Version.empty() == false) { @@ -504,7 +549,7 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, if (I->Version != 0 && I.TargetVer() == Version) Dep->Version = I->Version;*/ if (Dep->Version == 0) - if ((Dep->Version = WriteString(Version)) == 0) + if (unlikely((Dep->Version = WriteString(Version)) == 0)) return false; } @@ -524,7 +569,7 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, } // Is it a file dependency? - if (PackageName[0] == '/') + if (unlikely(PackageName[0] == '/')) FoundFileDeps = true; Dep->NextDepends = *OldDepLast; @@ -544,12 +589,12 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, pkgCache &Cache = Owner->Cache; // We do not add self referencing provides - if (Ver.ParentPkg().Name() == PackageName) + if (unlikely(Ver.ParentPkg().Name() == PackageName)) return true; // Get a structure - unsigned long Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides)); - if (Provides == 0) + unsigned long const Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides)); + if (unlikely(Provides == 0)) return false; Cache.HeaderP->ProvidesCount++; @@ -558,12 +603,12 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, Prv->Version = Ver.Index(); Prv->NextPkgProv = Ver->ProvidesList; Ver->ProvidesList = Prv.Index(); - if (Version.empty() == false && (Prv->ProvideVersion = WriteString(Version)) == 0) + if (Version.empty() == false && unlikely((Prv->ProvideVersion = WriteString(Version)) == 0)) return false; // Locate the target package pkgCache::PkgIterator Pkg; - if (Owner->NewPackage(Pkg,PackageName) == false) + if (unlikely(Owner->NewPackage(Pkg,PackageName,string(Ver.Arch())) == false)) return false; // Link it to the package diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 108b34207..4a2419b24 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -51,7 +51,8 @@ class pkgCacheGenerator /*{{{*/ // Flag file dependencies bool FoundFileDeps; - bool NewPackage(pkgCache::PkgIterator &Pkg,const string &PkgName); + bool NewGroup(pkgCache::GrpIterator &Grp,const string &Name); + bool NewPackage(pkgCache::PkgIterator &Pkg,const string &Name, const string &Arch); bool NewFileVer(pkgCache::VerIterator &Ver,ListParser &List); bool NewFileDesc(pkgCache::DescIterator &Desc,ListParser &List); unsigned long NewVersion(pkgCache::VerIterator &Ver,const string &VerStr,unsigned long Next); @@ -96,7 +97,7 @@ class pkgCacheGenerator::ListParser inline unsigned long WriteUniqString(const char *S,unsigned int Size) {return Owner->WriteUniqString(S,Size);}; inline unsigned long WriteString(const string &S) {return Owner->Map.WriteString(S);}; inline unsigned long WriteString(const char *S,unsigned int Size) {return Owner->Map.WriteString(S,Size);}; - bool NewDepends(pkgCache::VerIterator Ver,const string &Package, + bool NewDepends(pkgCache::VerIterator Ver,const string &Package, const string &Arch, const string &Version,unsigned int Op, unsigned int Type); bool NewProvides(pkgCache::VerIterator Ver,const string &Package, @@ -106,6 +107,7 @@ class pkgCacheGenerator::ListParser // These all operate against the current section virtual string Package() = 0; + virtual string Architecture() = 0; virtual string Version() = 0; virtual bool NewVersion(pkgCache::VerIterator Ver) = 0; virtual string Description() = 0; -- cgit v1.2.3-70-g09d2 From 25396fb06350344996a20d05423562f08a4165db Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 27 Dec 2009 19:39:47 +0100 Subject: Split ListParser::NewDepends into two methods to use these new method for creating the dependencies needed for our groups: For now for all groups only one package can be installed at the same time which conflicts with each other packages in the group. The exceptions are architecture all package. Also, the Multi-Arch field is now parsed, but not used for now. --- apt-pkg/cacheiterators.h | 14 ++++- apt-pkg/deb/deblistparser.cc | 24 ++++++++ apt-pkg/pkgcache.cc | 27 +++++++++ apt-pkg/pkgcache.h | 12 +++- apt-pkg/pkgcachegen.cc | 133 ++++++++++++++++++++++++++++++++----------- apt-pkg/pkgcachegen.h | 6 +- 6 files changed, 176 insertions(+), 40 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 35d3aa228..33d2ed6be 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -81,13 +81,21 @@ template class pkgCache::Iterator { different architectures can be treated as of the "same" package (apt internally treat them as totally different packages) */ class pkgCache::GrpIterator: public Iterator { + long HashIndex; + protected: inline Group* OwnerPointer() const { return Owner->GrpP; }; public: - void operator ++(int) {if (S != Owner->GrpP) S = Owner->GrpP + S->Next;}; + // This constructor is the 'begin' constructor, never use it. + inline GrpIterator(pkgCache &Owner) : Iterator(Owner), HashIndex(-1) { + S = OwnerPointer(); + operator ++(0); + }; + + virtual void operator ++(int); virtual void operator ++() {operator ++(0);}; inline const char *Name() const {return S->Name == 0?0:Owner->StrP + S->Name;}; @@ -96,11 +104,11 @@ class pkgCache::GrpIterator: public Iterator { PkgIterator NextPkg(PkgIterator const &Pkg); // Constructors - inline GrpIterator(pkgCache &Owner, Group *Trg) : Iterator(Owner, Trg) { + inline GrpIterator(pkgCache &Owner, Group *Trg) : Iterator(Owner, Trg), HashIndex(0) { if (S == 0) S = OwnerPointer(); }; - inline GrpIterator() : Iterator() {}; + inline GrpIterator() : Iterator(), HashIndex(0) {}; }; /*}}}*/ diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index b57eca813..f683de423 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -104,6 +104,30 @@ bool debListParser::NewVersion(pkgCache::VerIterator Ver) // Parse the architecture Ver->Arch = WriteUniqString(Architecture()); + // Parse multi-arch + if (Section.FindS("Architecture") == "all") + /* Arch all packages can't have a Multi-Arch field, + but we need a special treatment for them nonetheless */ + Ver->MultiArch = pkgCache::Version::All; + else + { + string const MultiArch = Section.FindS("Multi-Arch"); + if (MultiArch.empty() == true) + Ver->MultiArch = pkgCache::Version::None; + else if (MultiArch == "same") + Ver->MultiArch = pkgCache::Version::Same; + else if (MultiArch == "foreign") + Ver->MultiArch = pkgCache::Version::Foreign; + else if (MultiArch == "allowed") + Ver->MultiArch = pkgCache::Version::Allowed; + else + { + _error->Warning("Unknown Multi-Arch type »%s« for package »%s«", + MultiArch.c_str(), Section.FindS("Package").c_str()); + Ver->MultiArch = pkgCache::Version::None; + } + } + // Archive Size Ver->Size = (unsigned)Section.FindI("Size"); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index c945c59bf..1e4c7edb3 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -182,6 +182,16 @@ unsigned long pkgCache::sHash(const char *Str) const // Cache::FindPkg - Locate a package by name /*{{{*/ // --------------------------------------------------------------------- /* Returns 0 on error, pointer to the package otherwise */ +pkgCache::PkgIterator pkgCache::FindPkg(const string &Name) { + size_t const found = Name.find(':'); + if (found == string::npos) + return FindPkg(Name, "native"); + return FindPkg(Name.substr(0, found), Name.substr(found+1, string::npos)); +} + /*}}}*/ +// Cache::FindPkg - Locate a package by name /*{{{*/ +// --------------------------------------------------------------------- +/* Returns 0 on error, pointer to the package otherwise */ pkgCache::PkgIterator pkgCache::FindPkg(const string &Name, string Arch) { /* We make a detour via the GrpIterator here as on a multi-arch environment a group is easier to @@ -322,6 +332,23 @@ pkgCache::PkgIterator pkgCache::GrpIterator::NextPkg(pkgCache::PkgIterator const return PkgIterator(*Owner, 0); } /*}}}*/ +// GrpIterator::operator ++ - Postfix incr /*{{{*/ +// --------------------------------------------------------------------- +/* This will advance to the next logical group in the hash table. */ +void pkgCache::GrpIterator::operator ++(int) +{ + // Follow the current links + if (S != Owner->GrpP) + S = Owner->GrpP + S->Next; + + // Follow the hash table + while (S == Owner->GrpP && (HashIndex+1) < (signed)_count(Owner->HeaderP->GrpHashTable)) + { + HashIndex++; + S = Owner->GrpP + Owner->HeaderP->GrpHashTable[HashIndex]; + } +}; + /*}}}*/ // PkgIterator::operator ++ - Postfix incr /*{{{*/ // --------------------------------------------------------------------- /* This will advance to the next logical package in the hash table. */ diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index b3d2752d2..5f50001d0 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -125,9 +125,12 @@ class pkgCache /*{{{*/ // Accessors GrpIterator FindGrp(const string &Name); - PkgIterator FindPkg(const string &Name, string Arch = "native"); + PkgIterator FindPkg(const string &Name); + PkgIterator FindPkg(const string &Name, string Arch); Header &Head() {return *HeaderP;}; + inline GrpIterator GrpBegin(); + inline GrpIterator GrpEnd(); inline PkgIterator PkgBegin(); inline PkgIterator PkgEnd(); inline PkgFileIterator FileBegin(); @@ -274,7 +277,8 @@ struct pkgCache::Version /*{{{*/ map_ptrloc VerStr; // Stringtable map_ptrloc Section; // StringTable (StringItem) map_ptrloc Arch; // StringTable - + enum {None, All, Foreign, Same, Allowed} MultiArch; + // Lists map_ptrloc FileList; // VerFile map_ptrloc NextVer; // Version @@ -337,6 +341,10 @@ struct pkgCache::StringItem /*{{{*/ /*}}}*/ #include +inline pkgCache::GrpIterator pkgCache::GrpBegin() + {return GrpIterator(*this);}; +inline pkgCache::GrpIterator pkgCache::GrpEnd() + {return GrpIterator(*this,GrpP);}; inline pkgCache::PkgIterator pkgCache::PkgBegin() {return PkgIterator(*this);}; inline pkgCache::PkgIterator pkgCache::PkgEnd() diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index c37f6f48e..2a4a30349 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -38,7 +39,7 @@ typedef vector::iterator FileIterator; // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- -/* We set the diry flag and make sure that is written to the disk */ +/* We set the dirty flag and make sure that is written to the disk */ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : Map(*pMap), Cache(pMap,false), Progress(Prog), FoundFileDeps(0) @@ -506,21 +507,58 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, return Description; } /*}}}*/ -// ListParser::NewDepends - Create a dependency element /*{{{*/ +// CacheGenerator::FinishCache - do various finish operations /*{{{*/ +// --------------------------------------------------------------------- +/* This prepares the Cache for delivery */ +bool pkgCacheGenerator::FinishCache(OpProgress &Progress) { + // FIXME: add progress reporting for this operation + // Do we have different architectures in your groups ? + vector archs = APT::Configuration::getArchitectures(); + if (archs.size() > 1) { + // Create Conflicts in between the group + for (pkgCache::GrpIterator G = GetCache().GrpBegin(); G.end() != true; G++) { + string const PkgName = G.Name(); + for (pkgCache::PkgIterator P = G.PackageList(); P.end() != true; P = G.NextPkg(P)) { + for (pkgCache::VerIterator V = P.VersionList(); V.end() != true; V++) { + // Arch all packages are "co-installable" + if (V->MultiArch == pkgCache::Version::All) + continue; + string const Arch = V.Arch(); + map_ptrloc *OldDepLast = NULL; + for (vector::const_iterator A = archs.begin(); A != archs.end(); ++A) { + if (*A == Arch) + continue; + /* We allow only one installed arch at the time + per group, therefore each group member conflicts + with all other group members */ + pkgCache::PkgIterator D = G.FindPkg(*A); + if (D.end() == true) + continue; + // Conflicts: ${self}:other + NewDepends(D, V, "", + pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts, + OldDepLast); + } + } + } + } + } + return true; +} + /*}}}*/ +// CacheGenerator::NewDepends - Create a dependency element /*{{{*/ // --------------------------------------------------------------------- /* This creates a dependency element in the tree. It is linked to the version and to the package that it is pointing to. */ -bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, - const string &PackageName, - const string &Arch, - const string &Version, - unsigned int Op, - unsigned int Type) +bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver, + string const &Version, + unsigned int const &Op, + unsigned int const &Type, + map_ptrloc *OldDepLast) { - pkgCache &Cache = Owner->Cache; - // Get a structure - unsigned long const Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency)); + unsigned long const Dependency = Map.Allocate(sizeof(pkgCache::Dependency)); if (unlikely(Dependency == 0)) return false; @@ -530,17 +568,6 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, Dep->Type = Type; Dep->CompareOp = Op; Dep->ID = Cache.HeaderP->DependsCount++; - - pkgCache::GrpIterator Grp; - if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) - return false; - - // Locate the target package - pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch); - if (Pkg.end() == true) { - if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) - return false; - } // Probe the reverse dependency list for a version string that matches if (Version.empty() == false) @@ -549,29 +576,23 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, if (I->Version != 0 && I.TargetVer() == Version) Dep->Version = I->Version;*/ if (Dep->Version == 0) - if (unlikely((Dep->Version = WriteString(Version)) == 0)) + if (unlikely((Dep->Version = Map.WriteString(Version)) == 0)) return false; } - + // Link it to the package Dep->Package = Pkg.Index(); Dep->NextRevDepends = Pkg->RevDepends; Pkg->RevDepends = Dep.Index(); - - /* Link it to the version (at the end of the list) - Caching the old end point speeds up generation substantially */ - if (OldDepVer != Ver) + + // Do we know where to link the Dependency to? + if (OldDepLast == NULL) { OldDepLast = &Ver->DependsList; for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++) OldDepLast = &D->NextDepends; - OldDepVer = Ver; } - // Is it a file dependency? - if (unlikely(PackageName[0] == '/')) - FoundFileDeps = true; - Dep->NextDepends = *OldDepLast; *OldDepLast = Dep.Index(); OldDepLast = &Dep->NextDepends; @@ -579,6 +600,41 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, return true; } /*}}}*/ +// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a Group and the Package to link this dependency to if + needed and handles also the caching of the old endpoint */ +bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, + const string &PackageName, + const string &Arch, + const string &Version, + unsigned int Op, + unsigned int Type) +{ + pkgCache::GrpIterator Grp; + if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) + return false; + + // Locate the target package + pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch); + if (Pkg.end() == true) { + if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) + return false; + } + + // Is it a file dependency? + if (unlikely(PackageName[0] == '/')) + FoundFileDeps = true; + + /* Caching the old end point speeds up generation substantially */ + if (OldDepVer != Ver) { + OldDepLast = NULL; + OldDepVer = Ver; + } + + return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast); +} + /*}}}*/ // ListParser::NewProvides - Create a Provides element /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -925,6 +981,9 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, if (BuildCache(Gen,Progress,CurrentSize,TotalSize, Files.begin()+EndOfSource,Files.end()) == false) return false; + + // FIXME: move me to a better place + Gen.FinishCache(Progress); } else { @@ -965,6 +1024,9 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, if (BuildCache(Gen,Progress,CurrentSize,TotalSize, Files.begin()+EndOfSource,Files.end()) == false) return false; + + // FIXME: move me to a better place + Gen.FinishCache(Progress); } if (_error->PendingError() == true) @@ -1010,7 +1072,10 @@ bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap) if (BuildCache(Gen,Progress,CurrentSize,TotalSize, Files.begin()+EndOfSource,Files.end()) == false) return false; - + + // FIXME: move me to a better place + Gen.FinishCache(Progress); + if (_error->PendingError() == true) return false; *OutMap = Map.UnGuard(); diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 4a2419b24..53f09b991 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -55,6 +55,9 @@ class pkgCacheGenerator /*{{{*/ bool NewPackage(pkgCache::PkgIterator &Pkg,const string &Name, const string &Arch); bool NewFileVer(pkgCache::VerIterator &Ver,ListParser &List); bool NewFileDesc(pkgCache::DescIterator &Desc,ListParser &List); + bool NewDepends(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver, + string const &Version, unsigned int const &Op, + unsigned int const &Type, map_ptrloc *OldDepLast); unsigned long NewVersion(pkgCache::VerIterator &Ver,const string &VerStr,unsigned long Next); map_ptrloc NewDescription(pkgCache::DescIterator &Desc,const string &Lang,const MD5SumValue &md5sum,map_ptrloc Next); @@ -73,7 +76,8 @@ class pkgCacheGenerator /*{{{*/ bool HasFileDeps() {return FoundFileDeps;}; bool MergeFileProvides(ListParser &List); - + bool FinishCache(OpProgress &Progress); + pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress); ~pkgCacheGenerator(); }; -- cgit v1.2.3-70-g09d2 From 857e9c13d8d9808fcd1ac8ff3469f6c0b90b7fea Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 7 Feb 2010 12:38:13 +0100 Subject: Drop the Arch information from the Version structure as we can get the information from the parent package now --- apt-pkg/cacheiterators.h | 2 +- apt-pkg/deb/deblistparser.cc | 10 +++++++--- apt-pkg/deb/deblistparser.h | 1 + apt-pkg/pkgcache.cc | 4 ++-- apt-pkg/pkgcache.h | 1 - apt-pkg/pkgcachegen.h | 1 + 6 files changed, 12 insertions(+), 7 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 33d2ed6be..d8e044f88 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -178,7 +178,7 @@ class pkgCache::VerIterator : public Iterator { // Accessors inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;}; inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;}; - inline const char *Arch() const {return S->Arch == 0?0:Owner->StrP + S->Arch;}; + inline const char *Arch() const {return S->ParentPkg == 0?0:Owner->StrP + ParentPkg()->Arch;}; inline PkgIterator ParentPkg() const {return PkgIterator(*Owner,Owner->PkgP + S->ParentPkg);}; inline DescIterator DescriptionList() const; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index f683de423..26841d3d2 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -83,6 +83,13 @@ string debListParser::Architecture() { return Result; } /*}}}*/ +// ListParser::ArchitectureAll /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debListParser::ArchitectureAll() { + return Section.FindS("Architecture") == "all"; +} + /*}}}*/ // ListParser::Version - Return the version string /*{{{*/ // --------------------------------------------------------------------- /* This is to return the string describing the version in debian form, @@ -101,9 +108,6 @@ bool debListParser::NewVersion(pkgCache::VerIterator Ver) // Parse the section Ver->Section = UniqFindTagWrite("Section"); - // Parse the architecture - Ver->Arch = WriteUniqString(Architecture()); - // Parse multi-arch if (Section.FindS("Architecture") == "all") /* Arch all packages can't have a Multi-Arch field, diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index cba4f8e5d..8da051530 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -47,6 +47,7 @@ class debListParser : public pkgCacheGenerator::ListParser // These all operate against the current section virtual string Package(); virtual string Architecture(); + virtual bool ArchitectureAll(); virtual string Version(); virtual bool NewVersion(pkgCache::VerIterator Ver); virtual string Description(); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 1e4c7edb3..7d98869ea 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -399,7 +399,7 @@ const char * pkgCache::PkgIterator::CandVersion() const { //TargetVer is empty, so don't use it. - VerIterator version = pkgPolicy::pkgPolicy(Owner).GetCandidateVer(*this); + VerIterator version = pkgPolicy(Owner).GetCandidateVer(*this); if (version.IsGood()) return version.VerStr(); return 0; @@ -727,7 +727,7 @@ string pkgCache::VerIterator::RelStr() Res += File.Site(); } } - if (S->Arch != 0) + if (S->ParentPkg != 0) Res.append(" [").append(Arch()).append("]"); return Res; } diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 5f50001d0..5edeedfd1 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -276,7 +276,6 @@ struct pkgCache::Version /*{{{*/ { map_ptrloc VerStr; // Stringtable map_ptrloc Section; // StringTable (StringItem) - map_ptrloc Arch; // StringTable enum {None, All, Foreign, Same, Allowed} MultiArch; // Lists diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 53f09b991..ca5d74a9f 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -112,6 +112,7 @@ class pkgCacheGenerator::ListParser // These all operate against the current section virtual string Package() = 0; virtual string Architecture() = 0; + virtual bool ArchitectureAll() = 0; virtual string Version() = 0; virtual bool NewVersion(pkgCache::VerIterator Ver) = 0; virtual string Description() = 0; -- cgit v1.2.3-70-g09d2 From 803ea2a87f81252b2c0d541b8502ed206ce57c84 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 12 Feb 2010 00:04:31 +0100 Subject: Add yet another pseudo package which isn't as pseudo as the others: Arch all packages are now represented by arch depending packages which all depend on a package with the same name and the special arch "all". This packages has NO dependencies, but beside this the same information. It is the only package which has a size, the arch depending ones all have a zero size. While the arch depending pseudo packages are used for dependency resolution the arch "all" package is used for downloading and ordering of the package. --- apt-pkg/algorithms.cc | 52 ++++++++++++++++++++++++++++++++++++++++---- apt-pkg/cacheiterators.h | 1 + apt-pkg/deb/deblistparser.cc | 22 +++++++++++++++++-- apt-pkg/depcache.cc | 4 ++++ apt-pkg/depcache.h | 1 + apt-pkg/orderlist.cc | 4 ++++ apt-pkg/packagemanager.cc | 23 +++++++++++++++----- apt-pkg/pkgcache.cc | 12 ++++++++++ apt-pkg/pkgcache.h | 2 +- apt-pkg/pkgcachegen.cc | 20 +++++++++++++---- cmdline/apt-cache.cc | 2 ++ cmdline/apt-get.cc | 14 +++++++++--- 12 files changed, 137 insertions(+), 20 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/algorithms.cc b/apt-pkg/algorithms.cc index 34da745de..c905cffa9 100644 --- a/apt-pkg/algorithms.cc +++ b/apt-pkg/algorithms.cc @@ -83,13 +83,28 @@ void pkgSimulate::Describe(PkgIterator Pkg,ostream &out,bool Current,bool Candid bool pkgSimulate::Install(PkgIterator iPkg,string /*File*/) { // Adapt the iterator - PkgIterator Pkg = Sim.FindPkg(iPkg.Name()); + PkgIterator Pkg = Sim.FindPkg(iPkg.Name(), iPkg.Arch()); Flags[Pkg->ID] = 1; cout << "Inst "; Describe(Pkg,cout,true,true); Sim.MarkInstall(Pkg,false); - + + if (strcmp(Pkg.Arch(),"all") == 0) + { + pkgCache::GrpIterator G = Pkg.Group(); + pkgCache::GrpIterator iG = iPkg.Group(); + for (pkgCache::PkgIterator P = G.FindPkg("any"); P.end() != true; P = G.NextPkg(P)) + { + if (strcmp(P.Arch(), "all") == 0) + continue; + if (iG.FindPkg(P.Arch())->CurrentVer == 0) + continue; + Flags[P->ID] = 1; + Sim.MarkInstall(P, false); + } + } + // Look for broken conflicts+predepends. for (PkgIterator I = Sim.PkgBegin(); I.end() == false; I++) { @@ -131,9 +146,22 @@ bool pkgSimulate::Install(PkgIterator iPkg,string /*File*/) bool pkgSimulate::Configure(PkgIterator iPkg) { // Adapt the iterator - PkgIterator Pkg = Sim.FindPkg(iPkg.Name()); + PkgIterator Pkg = Sim.FindPkg(iPkg.Name(), iPkg.Arch()); Flags[Pkg->ID] = 2; + + if (strcmp(Pkg.Arch(),"all") == 0) + { + pkgCache::GrpIterator G = Pkg.Group(); + for (pkgCache::PkgIterator P = G.FindPkg("any"); P.end() != true; P = G.NextPkg(P)) + { + if (strcmp(P.Arch(), "all") == 0) + continue; + if (Flags[P->ID] == 1) + Flags[P->ID] = 2; + } + } + // Sim.MarkInstall(Pkg,false); if (Sim[Pkg].InstBroken() == true) { @@ -181,10 +209,26 @@ bool pkgSimulate::Configure(PkgIterator iPkg) bool pkgSimulate::Remove(PkgIterator iPkg,bool Purge) { // Adapt the iterator - PkgIterator Pkg = Sim.FindPkg(iPkg.Name()); + PkgIterator Pkg = Sim.FindPkg(iPkg.Name(), iPkg.Arch()); Flags[Pkg->ID] = 3; Sim.MarkDelete(Pkg); + + if (strcmp(Pkg.Arch(),"all") == 0) + { + pkgCache::GrpIterator G = Pkg.Group(); + pkgCache::GrpIterator iG = iPkg.Group(); + for (pkgCache::PkgIterator P = G.FindPkg("any"); P.end() != true; P = G.NextPkg(P)) + { + if (strcmp(P.Arch(), "all") == 0) + continue; + if (iG.FindPkg(P.Arch())->CurrentVer == 0) + continue; + Flags[P->ID] = 3; + Sim.MarkDelete(P); + } + } + if (Purge == true) cout << "Purg "; else diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index e8cf28496..43cbe1377 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -200,6 +200,7 @@ class pkgCache::VerIterator : public Iterator { string RelStr(); bool Automatic() const; + bool Pseudo() const; VerFileIterator NewestFile() const; inline VerIterator(pkgCache &Owner,Version *Trg = 0) : Iterator(Owner, Trg) { diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 3726a6a04..b3d95164a 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -148,6 +148,24 @@ bool debListParser::NewVersion(pkgCache::VerIterator Ver) Ver->Priority = pkgCache::State::Extra; } + if (Ver->MultiArch == pkgCache::Version::All) + { + /* We maintain a "pseudo" arch=all package for architecture all versions + on which these versions can depend on. This pseudo package is many used + for downloading/installing: The other pseudo-packages will degenerate + to a NOP in the download/install step - this package will ensure that + it is downloaded only one time and installed only one time -- even if + the architecture bound versions coming in and out on regular basis. */ + if (strcmp(Ver.Arch(true),"all") == 0) + return true; + else + { + // our pseudo packages have no size to not confuse the fetcher + Ver->Size = 0; + Ver->InstalledSize = 0; + } + } + if (ParseDepends(Ver,"Depends",pkgCache::Dep::Depends) == false) return false; if (ParseDepends(Ver,"Pre-Depends",pkgCache::Dep::PreDepends) == false) @@ -593,7 +611,7 @@ bool debListParser::ParseDepends(pkgCache::VerIterator Ver, return true; string Package; - string const pkgArch = Ver.Arch(); + string const pkgArch = Ver.Arch(true); string Version; unsigned int Op; @@ -622,7 +640,7 @@ bool debListParser::ParseProvides(pkgCache::VerIterator Ver) { string Package; string Version; - string const Arch = Ver.Arch(); + string const Arch = Ver.Arch(true); unsigned int Op; while (1) diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc index b04181d76..e817adb77 100644 --- a/apt-pkg/depcache.cc +++ b/apt-pkg/depcache.cc @@ -896,6 +896,10 @@ void pkgDepCache::MarkDelete(PkgIterator const &Pkg, bool rPurge, AddStates(Pkg); Update(Pkg); AddSizes(Pkg); + + // if we remove the pseudo package, we also need to remove the "real" + if (Pkg->CurrentVer != 0 && Pkg.CurrentVer().Pseudo() == true) + MarkDelete(Pkg.Group().FindPkg("all"), rPurge, Depth+1, FromUser); } /*}}}*/ // DepCache::IsDeleteOk - check if it is ok to remove this package /*{{{*/ diff --git a/apt-pkg/depcache.h b/apt-pkg/depcache.h index 63cd954ad..ab1021a44 100644 --- a/apt-pkg/depcache.h +++ b/apt-pkg/depcache.h @@ -333,6 +333,7 @@ class pkgDepCache : protected pkgCache::Namespace inline Header &Head() {return *Cache->HeaderP;}; inline PkgIterator PkgBegin() {return Cache->PkgBegin();}; inline PkgIterator FindPkg(string const &Name) {return Cache->FindPkg(Name);}; + inline PkgIterator FindPkg(string const &Name, string const &Arch) {return Cache->FindPkg(Name, Arch);}; inline pkgCache &GetCache() {return *Cache;}; inline pkgVersioningSystem &VS() {return *Cache->VS;}; diff --git a/apt-pkg/orderlist.cc b/apt-pkg/orderlist.cc index 0ee2e2bc8..2e7618b55 100644 --- a/apt-pkg/orderlist.cc +++ b/apt-pkg/orderlist.cc @@ -126,6 +126,10 @@ bool pkgOrderList::IsMissing(PkgIterator Pkg) if (FileList[Pkg->ID].empty() == false) return false; + + if (pkgCache::VerIterator(Cache, Cache[Pkg].CandidateVer).Pseudo() == true) + return false; + return true; } /*}}}*/ diff --git a/apt-pkg/packagemanager.cc b/apt-pkg/packagemanager.cc index 1ab3203a1..08e7fc00f 100644 --- a/apt-pkg/packagemanager.cc +++ b/apt-pkg/packagemanager.cc @@ -80,7 +80,10 @@ bool pkgPackageManager::GetArchives(pkgAcquire *Owner,pkgSourceList *Sources, // Skip already processed packages if (List->IsNow(Pkg) == false) continue; - + + if (pkgCache::VerIterator(Cache, Cache[Pkg].CandidateVer).Pseudo() == true) + continue; + new pkgAcqArchive(Owner,Sources,Recs,Cache[Pkg].InstVerIter(Cache), FileNames[Pkg->ID]); } @@ -277,8 +280,10 @@ bool pkgPackageManager::ConfigureAll() for (pkgOrderList::iterator I = OList.begin(); I != OList.end(); I++) { PkgIterator Pkg(Cache,*I); - - if (ConfigurePkgs == true && Configure(Pkg) == false) + + if (ConfigurePkgs == true && + pkgCache::VerIterator(Cache, Cache[Pkg].CandidateVer).Pseudo() == false && + Configure(Pkg) == false) return false; List->Flag(Pkg,pkgOrderList::Configured,pkgOrderList::States); @@ -310,7 +315,9 @@ bool pkgPackageManager::SmartConfigure(PkgIterator Pkg) { PkgIterator Pkg(Cache,*I); - if (ConfigurePkgs == true && Configure(Pkg) == false) + if (ConfigurePkgs == true && + pkgCache::VerIterator(Cache, Cache[Pkg].CandidateVer).Pseudo() == false && + Configure(Pkg) == false) return false; List->Flag(Pkg,pkgOrderList::Configured,pkgOrderList::States); @@ -457,7 +464,10 @@ bool pkgPackageManager::SmartRemove(PkgIterator Pkg) return true; List->Flag(Pkg,pkgOrderList::Configured,pkgOrderList::States); - return Remove(Pkg,(Cache[Pkg].iFlags & pkgDepCache::Purge) == pkgDepCache::Purge); + + if (pkgCache::VerIterator(Cache, Cache[Pkg].CandidateVer).Pseudo() == false) + return Remove(Pkg,(Cache[Pkg].iFlags & pkgDepCache::Purge) == pkgDepCache::Purge); + return true; } /*}}}*/ // PM::SmartUnPack - Install helper /*{{{*/ @@ -565,7 +575,8 @@ bool pkgPackageManager::SmartUnPack(PkgIterator Pkg) P.end() == false; P++) CheckRConflicts(Pkg,P.ParentPkg().RevDependsList(),P.ProvideVersion()); - if (Install(Pkg,FileNames[Pkg->ID]) == false) + if (pkgCache::VerIterator(Cache, Cache[Pkg].CandidateVer).Pseudo() == false && + Install(Pkg,FileNames[Pkg->ID]) == false) return false; List->Flag(Pkg,pkgOrderList::UnPacked,pkgOrderList::States); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 2d4ee1010..04a2c7234 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -649,6 +649,18 @@ bool pkgCache::VerIterator::Automatic() const return false; } /*}}}*/ +// VerIterator::Pseudo - Check if this version is a pseudo one /*{{{*/ +// --------------------------------------------------------------------- +/* Sometimes you have the need to express dependencies with versions + which doesn't really exist or exist multiply times for "different" + packages. We need these versions for dependency resolution but they + are a problem everytime we need to download/install something. */ +bool pkgCache::VerIterator::Pseudo() const +{ + return (S->MultiArch == pkgCache::Version::All && + strcmp(Arch(true),"all") != 0); +} + /*}}}*/ // VerIterator::NewestFile - Return the newest file version relation /*{{{*/ // --------------------------------------------------------------------- /* This looks at the version numbers associated with all of the sources diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 5edeedfd1..012caac76 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -357,7 +357,7 @@ inline pkgCache::PkgFileIterator pkgCache::FileEnd() class pkgCache::Namespace /*{{{*/ { public: - + typedef pkgCache::GrpIterator GrpIterator; typedef pkgCache::PkgIterator PkgIterator; typedef pkgCache::VerIterator VerIterator; typedef pkgCache::DescIterator DescIterator; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index ebda325f7..c1b546a00 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -115,9 +115,10 @@ bool pkgCacheGenerator::MergeList(ListParser &List, /* As we handle Arch all packages as architecture bounded we add all information to every (simulated) arch package */ std::vector genArch; - if (List.ArchitectureAll() == true) + if (List.ArchitectureAll() == true) { genArch = APT::Configuration::getArchitectures(); - else + genArch.push_back("all"); + } else genArch.push_back(List.Architecture()); for (std::vector::const_iterator arch = genArch.begin(); @@ -531,8 +532,11 @@ bool pkgCacheGenerator::FinishCache(OpProgress &Progress) { for (pkgCache::GrpIterator G = GetCache().GrpBegin(); G.end() != true; G++) { string const PkgName = G.Name(); for (pkgCache::PkgIterator P = G.PackageList(); P.end() != true; P = G.NextPkg(P)) { + if (strcmp(P.Arch(),"all") == 0) + continue; + pkgCache::PkgIterator allPkg; for (pkgCache::VerIterator V = P.VersionList(); V.end() != true; V++) { - string const Arch = V.Arch(); + string const Arch = V.Arch(true); map_ptrloc *OldDepLast = NULL; /* MultiArch handling introduces a lot of implicit Dependencies: - MultiArch: same → Co-Installable if they have the same version @@ -540,6 +544,8 @@ bool pkgCacheGenerator::FinishCache(OpProgress &Progress) { - All others conflict with all other group members */ bool const coInstall = (V->MultiArch == pkgCache::Version::All || V->MultiArch == pkgCache::Version::Same); + if (V->MultiArch == pkgCache::Version::All && allPkg.end() == true) + allPkg = G.FindPkg("all"); for (vector::const_iterator A = archs.begin(); A != archs.end(); ++A) { if (*A == Arch) continue; @@ -561,6 +567,12 @@ bool pkgCacheGenerator::FinishCache(OpProgress &Progress) { NewDepends(D, V, V.VerStr(), pkgCache::Dep::Greater, pkgCache::Dep::DpkgBreaks, OldDepLast); + if (V->MultiArch == pkgCache::Version::All) { + // Depend on ${self}:all which does depend on nothing + NewDepends(allPkg, V, V.VerStr(), + pkgCache::Dep::Equals, pkgCache::Dep::Depends, + OldDepLast); + } } else { // Conflicts: ${self}:other NewDepends(D, V, "", @@ -675,7 +687,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, pkgCache &Cache = Owner->Cache; // We do not add self referencing provides - if (Ver.ParentPkg().Name() == PkgName && PkgArch == Ver.Arch()) + if (Ver.ParentPkg().Name() == PkgName && PkgArch == Ver.Arch(true)) return true; // Get a structure diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index cd806286c..275daa187 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1587,6 +1587,8 @@ bool Policy(CommandLine &CmdL) } for (; Pkg.end() != true; Pkg = Grp.NextPkg(Pkg)) { + if (strcmp(Pkg.Arch(),"all") == 0) + continue; if (myArch == Pkg.Arch()) cout << Pkg.Name() << ":" << endl; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 2597a6acb..93065004c 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -233,7 +233,7 @@ bool ShowList(ostream &out,string Title,string List,string VersionsList) if it is not the main architecture */ string ShowPkg(pkgCache::PkgIterator const Pkg) { string p = Pkg.Name(); - if (_config->Find("APT::Architecture") != Pkg.Arch()) + if (strcmp(Pkg.Arch(),"all") != 0 && _config->Find("APT::Architecture") != Pkg.Arch()) p.append(":").append(Pkg.Arch()); return p; } @@ -385,6 +385,8 @@ void ShowNew(ostream &out,CacheFile &Cache) { pkgCache::PkgIterator I(Cache,Cache.List[J]); if (Cache[I].NewInstall() == true) { + if (Cache[I].CandidateVerIter(Cache).Pseudo() == true) + continue; List += ShowPkg(I) + " "; VersionsList += string(Cache[I].CandVersion) + "\n"; } @@ -407,6 +409,8 @@ void ShowDel(ostream &out,CacheFile &Cache) pkgCache::PkgIterator I(Cache,Cache.List[J]); if (Cache[I].Delete() == true) { + if (Cache[I].CandidateVerIter(Cache).Pseudo() == true) + continue; if ((Cache[I].iFlags & pkgDepCache::Purge) == pkgDepCache::Purge) List += ShowPkg(I) + "* "; else @@ -455,7 +459,9 @@ void ShowUpgraded(ostream &out,CacheFile &Cache) // Not interesting if (Cache[I].Upgrade() == false || Cache[I].NewInstall() == true) continue; - + if (Cache[I].CandidateVerIter(Cache).Pseudo() == true) + continue; + List += ShowPkg(I) + " "; VersionsList += string(Cache[I].CurVersion) + " => " + Cache[I].CandVersion + "\n"; } @@ -476,7 +482,9 @@ bool ShowDowngraded(ostream &out,CacheFile &Cache) // Not interesting if (Cache[I].Downgrade() == false || Cache[I].NewInstall() == true) continue; - + if (Cache[I].CandidateVerIter(Cache).Pseudo() == true) + continue; + List += ShowPkg(I) + " "; VersionsList += string(Cache[I].CurVersion) + " => " + Cache[I].CandVersion + "\n"; } -- cgit v1.2.3-70-g09d2 From 8d4c859d0d20687a8ffcf9e1d60466d061c2e24d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 18 Mar 2010 18:51:14 +0100 Subject: Readd the FindPkg() method implementation used in the singleArch days to use it as a fallback if multiarch is not enabled. The effect is barly noticeable but SingleArch is the realworld scenario. --- apt-pkg/pkgcache.cc | 30 +++++++++++++++++++++++++++++- apt-pkg/pkgcache.h | 8 +++++++- 2 files changed, 36 insertions(+), 2 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 24d9e0329..fe8757ded 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -108,6 +108,7 @@ bool pkgCache::Header::CheckSizes(Header &Against) const /* */ pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map) { + MultiArchEnabled = APT::Configuration::getArchitectures().size() > 1; if (DoMap == true) ReMap(); } @@ -178,11 +179,31 @@ unsigned long pkgCache::sHash(const char *Str) const return Hash % _count(HeaderP->PkgHashTable); } + /*}}}*/ +// Cache::SingleArchFindPkg - Locate a package by name /*{{{*/ +// --------------------------------------------------------------------- +/* Returns 0 on error, pointer to the package otherwise + The multiArch enabled methods will fallback to this one as it is (a bit) + faster for single arch environments and realworld is mostly singlearch… */ +pkgCache::PkgIterator pkgCache::SingleArchFindPkg(const string &Name) +{ + // Look at the hash bucket + Package *Pkg = PkgP + HeaderP->PkgHashTable[Hash(Name)]; + for (; Pkg != PkgP; Pkg = PkgP + Pkg->NextPackage) + { + if (Pkg->Name != 0 && StrP[Pkg->Name] == Name[0] && + stringcasecmp(Name,StrP + Pkg->Name) == 0) + return PkgIterator(*this,Pkg); + } + return PkgIterator(*this,0); +} /*}}}*/ // Cache::FindPkg - Locate a package by name /*{{{*/ // --------------------------------------------------------------------- /* Returns 0 on error, pointer to the package otherwise */ pkgCache::PkgIterator pkgCache::FindPkg(const string &Name) { + if (MultiArchCache() == false) + return SingleArchFindPkg(Name); size_t const found = Name.find(':'); if (found == string::npos) return FindPkg(Name, "native"); @@ -195,7 +216,14 @@ pkgCache::PkgIterator pkgCache::FindPkg(const string &Name) { // Cache::FindPkg - Locate a package by name /*{{{*/ // --------------------------------------------------------------------- /* Returns 0 on error, pointer to the package otherwise */ -pkgCache::PkgIterator pkgCache::FindPkg(const string &Name, string Arch) { +pkgCache::PkgIterator pkgCache::FindPkg(const string &Name, string const &Arch) { + if (MultiArchCache() == false) { + if (Arch == "native" || Arch == "all" || + Arch == _config->Find("APT::Architecture")) + return SingleArchFindPkg(Name); + else + return PkgIterator(*this,0); + } /* We make a detour via the GrpIterator here as on a multi-arch environment a group is easier to find than a package (less entries in the buckets) */ diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 012caac76..577eebad9 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -126,7 +126,7 @@ class pkgCache /*{{{*/ // Accessors GrpIterator FindGrp(const string &Name); PkgIterator FindPkg(const string &Name); - PkgIterator FindPkg(const string &Name, string Arch); + PkgIterator FindPkg(const string &Name, const string &Arch); Header &Head() {return *HeaderP;}; inline GrpIterator GrpBegin(); @@ -136,6 +136,8 @@ class pkgCache /*{{{*/ inline PkgFileIterator FileBegin(); inline PkgFileIterator FileEnd(); + inline bool MultiArchCache() const { return MultiArchEnabled; }; + // Make me a function pkgVersioningSystem *VS; @@ -146,6 +148,10 @@ class pkgCache /*{{{*/ pkgCache(MMap *Map,bool DoMap = true); virtual ~pkgCache() {}; + +private: + bool MultiArchEnabled; + PkgIterator SingleArchFindPkg(const string &Name); }; /*}}}*/ // Header structure /*{{{*/ -- cgit v1.2.3-70-g09d2 From 94449d7cd3eed7637c1ed78863c01ff207faa31e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 4 May 2010 12:30:13 +0200 Subject: * doc/cache.sgml: - drop the file in favor of inplace documentation with doxygen --- apt-pkg/pkgcache.h | 469 ++++++++++++++++++++++++------ debian/changelog | 8 +- doc/cache.sgml | 824 ----------------------------------------------------- 3 files changed, 390 insertions(+), 911 deletions(-) delete mode 100644 doc/cache.sgml (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 577eebad9..a2e63ff03 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -1,20 +1,75 @@ // -*- mode: cpp; mode: fold -*- // Description /*{{{*/ -// $Id: pkgcache.h,v 1.25 2001/07/01 22:28:24 jgg Exp $ -/* ###################################################################### - - Cache - Structure definitions for the cache file - - Please see doc/apt-pkg/cache.sgml for a more detailed description of - this format. Also be sure to keep that file up-to-date!! - +/**\file pkgcache.h + \brief pkgCache - Structure definitions for the cache file + + The goal of the cache file is two fold: + Firstly to speed loading and processing of the package file array and + secondly to reduce memory consumption of the package file array. + + The implementation is aimed at an environment with many primary package + files, for instance someone that has a Package file for their CD-ROM, a + Package file for the latest version of the distribution on the CD-ROM and a + package file for the development version. Always present is the information + contained in the status file which might be considered a separate package + file. + + Please understand, this is designed as a Cache file it is not meant to be + used on any system other than the one it was created for. It is not meant to + be authoritative either, i.e. if a system crash or software failure occurs it + must be perfectly acceptable for the cache file to be in an inconsistent + state. Furthermore at any time the cache file may be erased without losing + any information. + + Also the structures and storage layout is optimized for use by the APT + and may not be suitable for all purposes. However it should be possible + to extend it with associate cache files that contain other information. + + To keep memory use down the cache file only contains often used fields and + fields that are inexpensive to store, the Package file has a full list of + fields. Also the client may assume that all items are perfectly valid and + need not perform checks against their correctness. Removal of information + from the cache is possible, but blanks will be left in the file, and + unused strings will also be present. The recommended implementation is to + simply rebuild the cache each time any of the data files change. It is + possible to add a new package file to the cache without any negative side + effects. + + Note on Pointer access Clients should always use the CacheIterators classes for access to the - cache. They provide a simple STL-like method for traversing the links - of the datastructure. - - See pkgcachegen.h for information about generating cache structures. - - ##################################################################### */ + cache and the data in it. They also provide a simple STL-like method for + traversing the links of the datastructure. + + Every item in every structure is stored as the index to that structure. + What this means is that once the files is mmaped every data access has to + go through a fix up stage to get a real memory pointer. This is done + by taking the index, multiplying it by the type size and then adding + it to the start address of the memory block. This sounds complex, but + in C it is a single array dereference. Because all items are aligned to + their size and indexes are stored as multiples of the size of the structure + the format is immediately portable to all possible architectures - BUT the + generated files are -NOT-. + + This scheme allows code like this to be written: + + void *Map = mmap(...); + Package *PkgList = (Package *)Map; + Header *Head = (Header *)Map; + char *Strings = (char *)Map; + cout << (Strings + PkgList[Head->HashTable[0]]->Name) << endl; + + Notice the lack of casting or multiplication. The net result is to return + the name of the first package in the first hash bucket, without error + checks. + + The generator uses allocation pools to group similarly sized structures in + large blocks to eliminate any alignment overhead. The generator also + assures that no structures overlap and all indexes are unique. Although + at first glance it may seem like there is the potential for two structures + to exist at the same point the generator never allows this to happen. + (See the discussion of free space pools) + + See \ref pkgcachegen.h for more information about generating cache structures. */ /*}}}*/ #ifndef PKGLIB_PKGCACHE_H #define PKGLIB_PKGCACHE_H @@ -66,12 +121,20 @@ class pkgCache /*{{{*/ { enum DepType {Depends=1,PreDepends=2,Suggests=3,Recommends=4, Conflicts=5,Replaces=6,Obsoletes=7,DpkgBreaks=8,Enhances=9}; + /** \brief available compare operators + + The lower 4 bits are used to indicate what operator is being specified and + the upper 4 bits are flags. OR indicates that the next package is + or'd with the current package. */ enum DepCompareOp {Or=0x10,NoOp=0,LessEq=0x1,GreaterEq=0x2,Less=0x3, Greater=0x4,Equals=0x5,NotEquals=0x6}; }; struct State { + /** \brief priority of a package version + + Zero is used for unparsable or absent Priority fields. */ enum VerPriority {Important=1,Required=2,Standard=3,Optional=4,Extra=5}; enum PkgSelectedState {Unknown=0,Install=1,Hold=2,DeInstall=3,Purge=4}; enum PkgInstState {Ok=0,ReInstReq=1,HoldInst=2,HoldReInstReq=3}; @@ -120,7 +183,7 @@ class pkgCache /*{{{*/ inline unsigned long Hash(const string &S) const {return sHash(S);}; inline unsigned long Hash(const char *S) const {return sHash(S);}; - // Usefull transformation things + // Useful transformation things const char *Priority(unsigned char Priority); // Accessors @@ -157,13 +220,29 @@ private: // Header structure /*{{{*/ struct pkgCache::Header { - // Signature information + /** \brief Signature information + + This must contain the hex value 0x98FE76DC which is designed to + verify that the system loading the image has the same byte order + and byte size as the system saving the image */ unsigned long Signature; + /** These contain the version of the cache file */ short MajorVersion; short MinorVersion; + /** \brief indicates if the cache should be erased + + Dirty is true if the cache file was opened for reading, the client + expects to have written things to it and have not fully synced it. + The file should be erased and rebuilt if it is true. */ bool Dirty; - - // Size of structure values + + /** \brief Size of structure values + + All *Sz variables contains the sizeof() that particular structure. + It is used as an extra consistency check on the structure of the file. + + If any of the size values do not exactly match what the client expects + then the client should refuse the load the file. */ unsigned short HeaderSz; unsigned short PackageSz; unsigned short PackageFileSz; @@ -173,8 +252,12 @@ struct pkgCache::Header unsigned short ProvidesSz; unsigned short VerFileSz; unsigned short DescFileSz; - - // Structure counts + + /** \brief Structure counts + + These indicate the number of each structure contained in the cache. + PackageCount is especially useful for generating user state structures. + See Package::Id for more info. */ unsigned long GroupCount; unsigned long PackageCount; unsigned long VersionCount; @@ -184,22 +267,48 @@ struct pkgCache::Header unsigned long VerFileCount; unsigned long DescFileCount; unsigned long ProvidesCount; - - // Offsets - map_ptrloc FileList; // struct PackageFile - map_ptrloc StringList; // struct StringItem - map_ptrloc VerSysName; // StringTable - map_ptrloc Architecture; // StringTable + + /** \brief index of the first PackageFile structure + + The PackageFile structures are singly linked lists that represent + all package files that have been merged into the cache. */ + map_ptrloc FileList; + /** \brief index of the first StringItem structure + + The cache contains a list of all the unique strings (StringItems). + The parser reads this list into memory so it can match strings + against it.*/ + map_ptrloc StringList; + /** \brief String representing the version system used */ + map_ptrloc VerSysName; + /** \brief Architecture(s) the cache was built against */ + map_ptrloc Architecture; + /** \brief The maximum size of a raw entry from the original Package file */ unsigned long MaxVerFileSize; + /** \brief The maximum size of a raw entry from the original Translation file */ unsigned long MaxDescFileSize; - /* Allocation pools, there should be one of these for each structure - excluding the header */ + /** \brief The Pool structures manage the allocation pools that the generator uses + + Start indicates the first byte of the pool, Count is the number of objects + remaining in the pool and ItemSize is the structure size (alignment factor) + of the pool. An ItemSize of 0 indicates the pool is empty. There should be + the same number of pools as there are structure types. The generator + stores this information so future additions can make use of any unused pool + blocks. */ DynamicMMap::Pool Pools[9]; - // Rapid package and group name lookup - // Notice: Increase only both table sizes as the - // hashmethod assume the size of the Pkg one + /** \brief hash tables providing rapid group/package name lookup + + Each group/package name is inserted into the hash table using pkgCache::Hash(const &string) + By iterating over each entry in the hash table it is possible to iterate over + the entire list of packages. Hash Collisions are handled with a singly linked + list of packages based at the hash item. The linked list contains only + packages that match the hashing function. + In the PkgHashTable is it possible that multiple packages have the same name - + these packages are stored as a sequence in the list. + + Beware: The Hashmethod assumes that the hash table sizes are equal */ map_ptrloc PkgHashTable[2*1048]; map_ptrloc GrpHashTable[2*1048]; @@ -207,140 +316,332 @@ struct pkgCache::Header Header(); }; /*}}}*/ -struct pkgCache::Group { /*{{{*/ - map_ptrloc Name; // Stringtable +// Group structure /*{{{*/ +/** \brief groups architecture depending packages together - // Linked List - map_ptrloc FirstPackage;// Package - map_ptrloc LastPackage; // Package - map_ptrloc Next; // Group + On or more packages with the same name form a group, so we have + a simple way to access a package built for different architectures + Group exists in a singly linked list of group records starting at + the hash index of the name in the pkgCache::Header::GrpHashTable */ +struct pkgCache::Group +{ + /** \brief Name of the group */ + map_ptrloc Name; // StringItem + + // Linked List + /** Link to the first package which belongs to the group */ + map_ptrloc FirstPackage; // Package + /** Link to the last package which belongs to the group */ + map_ptrloc LastPackage; // Package + /** Link to the next Group */ + map_ptrloc Next; // Group }; /*}}}*/ -struct pkgCache::Package /*{{{*/ +// Package structure /*{{{*/ +/** \brief contains information for a single unique package + + There can be any number of versions of a given package. + Package exists in a singly linked list of package records starting at + the hash index of the name in the pkgCache::Header::PkgHashTable + + A package can be created for every architecture so package names are + not unique, but it is garanteed that packages with the same name + are sequencel ordered in the list. Packages with the same name can be + accessed with the Group. +*/ +struct pkgCache::Package { - // Pointers - map_ptrloc Name; // Stringtable - map_ptrloc Arch; // StringTable (StringItem) + /** \brief Name of the package */ + map_ptrloc Name; // StringItem + /** \brief Architecture of the package */ + map_ptrloc Arch; // StringItem + /** \brief Base of a singly linked list of versions + + Each structure represents a unique version of the package. + The version structures contain links into PackageFile and the + original text file as well as detailed information about the size + and dependencies of the specific package. In this way multiple + versions of a package can be cleanly handled by the system. + Furthermore, this linked list is guaranteed to be sorted + from Highest version to lowest version with no duplicate entries. */ map_ptrloc VersionList; // Version + /** \brief index to the installed version */ map_ptrloc CurrentVer; // Version - map_ptrloc Section; // StringTable (StringItem) + /** \brief indicates the deduced section + + Should be the index to the string "Unknown" or to the section + of the last parsed item. */ + map_ptrloc Section; // StringItem + /** \brief index of the group this package belongs to */ map_ptrloc Group; // Group the Package belongs to - - // Linked list + + // Linked list + /** \brief Link to the next package in the same bucket */ map_ptrloc NextPackage; // Package + /** \brief List of all dependencies on this package */ map_ptrloc RevDepends; // Dependency + /** \brief List of all "packages" this package provide */ map_ptrloc ProvidesList; // Provides // Install/Remove/Purge etc + /** \brief state that the user wishes the package to be in */ unsigned char SelectedState; // What + /** \brief installation state of the package + + This should be "ok" but in case the installation failed + it will be different. + */ unsigned char InstState; // Flags + /** \brief indicates if the package is installed */ unsigned char CurrentState; // State - + + /** \brief unique sequel ID + + ID is a unique value from 0 to Header->PackageCount assigned by the generator. + This allows clients to create an array of size PackageCount and use it to store + state information for the package map. For instance the status file emitter uses + this to track which packages have been emitted already. */ unsigned int ID; + /** \brief some useful indicators of the package's state */ unsigned long Flags; }; /*}}}*/ -struct pkgCache::PackageFile /*{{{*/ +// Package File structure /*{{{*/ +/** \brief stores information about the files used to generate the cache + + Package files are referenced by Version structures to be able to know + after the generation still from which Packages file includes this Version + as we need this information later on e.g. for pinning. */ +struct pkgCache::PackageFile { - // Names - map_ptrloc FileName; // Stringtable - map_ptrloc Archive; // Stringtable - map_ptrloc Codename; // Stringtable - map_ptrloc Component; // Stringtable - map_ptrloc Version; // Stringtable - map_ptrloc Origin; // Stringtable - map_ptrloc Label; // Stringtable - map_ptrloc Architecture; // Stringtable - map_ptrloc Site; // Stringtable - map_ptrloc IndexType; // Stringtable - unsigned long Size; + /** \brief physical disk file that this PackageFile represents */ + map_ptrloc FileName; // StringItem + /** \brief the release information + + Please see the files document for a description of what the + release information means. */ + map_ptrloc Archive; // StringItem + map_ptrloc Codename; // StringItem + map_ptrloc Component; // StringItem + map_ptrloc Version; // StringItem + map_ptrloc Origin; // StringItem + map_ptrloc Label; // StringItem + map_ptrloc Architecture; // StringItem + /** \brief The site the index file was fetched from */ + map_ptrloc Site; // StringItem + /** \brief indicates what sort of index file this is + + @TODO enumerate at least the possible indexes */ + map_ptrloc IndexType; // StringItem + /** \brief Size of the file + + Used together with the modification time as a + simple check to ensure that the Packages + file has not been altered since Cache generation. */ + unsigned long Size; + /** \brief Modification time for the file */ + time_t mtime; + + /* @TODO document PackageFile::Flags */ unsigned long Flags; - + // Linked list + /** \brief Link to the next PackageFile in the Cache */ map_ptrloc NextFile; // PackageFile + /** \brief unique sequel ID */ unsigned int ID; - time_t mtime; // Modification time for the file }; /*}}}*/ -struct pkgCache::VerFile /*{{{*/ +// VerFile structure /*{{{*/ +/** \brief associates a version with a PackageFile + + This allows a full description of all Versions in all files + (and hence all sources) under consideration. */ +struct pkgCache::VerFile { + /** \brief index of the package file that this version was found in */ map_ptrloc File; // PackageFile + /** \brief next step in the linked list */ map_ptrloc NextFile; // PkgVerFile + /** \brief position in the package file */ map_ptrloc Offset; // File offset + /* @TODO document pkgCache::VerFile::Size */ unsigned long Size; }; /*}}}*/ -struct pkgCache::DescFile /*{{{*/ +// DescFile structure /*{{{*/ +/** \brief associates a description with a Translation file */ +struct pkgCache::DescFile { + /** \brief index of the file that this description was found in */ map_ptrloc File; // PackageFile + /** \brief next step in the linked list */ map_ptrloc NextFile; // PkgVerFile + /** \brief position in the file */ map_ptrloc Offset; // File offset + /* @TODO document pkgCache::DescFile::Size */ unsigned long Size; }; /*}}}*/ -struct pkgCache::Version /*{{{*/ +// Version structure /*{{{*/ +/** \brief information for a single version of a package + + The version list is always sorted from highest version to lowest + version by the generator. Equal version numbers are either merged + or handled as separate versions based on the Hash value. */ +struct pkgCache::Version { - map_ptrloc VerStr; // Stringtable - map_ptrloc Section; // StringTable (StringItem) + /** \brief complete version string */ + map_ptrloc VerStr; // StringItem + /** \brief section this version is filled in */ + map_ptrloc Section; // StringItem + /** \brief stores the MultiArch capabilities of this version + + None is the default and doesn't trigger special behaviour, + Foreign means that this version can fulfill dependencies even + if it is built for another architecture as the requester. + Same indicates that builds for different architectures can + be co-installed on the system and All is the marker for a + version with the Architecture: all. */ enum {None, All, Foreign, Same, Allowed} MultiArch; - // Lists + /** \brief references all the PackageFile's that this version came from + + FileList can be used to determine what distribution(s) the Version + applies to. If FileList is 0 then this is a blank version. + The structure should also have a 0 in all other fields excluding + pkgCache::Version::VerStr and Possibly pkgCache::Version::NextVer. */ map_ptrloc FileList; // VerFile + /** \brief next (lower or equal) version in the linked list */ map_ptrloc NextVer; // Version + /** \brief next description in the linked list */ map_ptrloc DescriptionList; // Description + /** \brief base of the dependency list */ map_ptrloc DependsList; // Dependency + /** \brief links to the owning package + + This allows reverse dependencies to determine the package */ map_ptrloc ParentPkg; // Package + /** \brief list of pkgCache::Provides */ map_ptrloc ProvidesList; // Provides - + + /** \brief archive size for this version + + For Debian this is the size of the .deb file. */ map_ptrloc Size; // These are the .deb size + /** \brief uncompressed size for this version */ map_ptrloc InstalledSize; + /** \brief characteristic value representing this version + + No two packages in existence should have the same VerStr + and Hash with different contents. */ unsigned short Hash; + /** \brief unique sequel ID */ unsigned int ID; + /** \brief parsed priority value */ unsigned char Priority; }; /*}}}*/ -struct pkgCache::Description /*{{{*/ +// Description structure /*{{{*/ +/** \brief datamember of a linked list of available description for a version */ +struct pkgCache::Description { - // Language Code store the description translation language code. If - // the value has a 0 lenght then this is readed using the Package - // file else the Translation-CODE are used. - map_ptrloc language_code; // StringTable - map_ptrloc md5sum; // StringTable + /** \brief Language code of this description (translation) + + If the value has a 0 length then this is read using the Package + file else the Translation-CODE file is used. */ + map_ptrloc language_code; // StringItem + /** \brief MD5sum of the original description + + Used to map Translations of a description to a version + and to check that the Translation is up-to-date. */ + map_ptrloc md5sum; // StringItem - // Linked list + /* @TODO document pkgCache::Description::FileList */ map_ptrloc FileList; // DescFile + /** \brief next translation for this description */ map_ptrloc NextDesc; // Description + /** \brief the text is a description of this package */ map_ptrloc ParentPkg; // Package + /** \brief unique sequel ID */ unsigned int ID; }; /*}}}*/ -struct pkgCache::Dependency /*{{{*/ +// Dependency structure /*{{{*/ +/** \brief information for a single dependency record + + The records are split up like this to ease processing by the client. + The base of the linked list is pkgCache::Version::DependsList. + All forms of dependencies are recorded here including Depends, + Recommends, Suggests, Enhances, Conflicts, Replaces and Breaks. */ +struct pkgCache::Dependency { - map_ptrloc Version; // Stringtable + /** \brief string of the version the dependency is applied against */ + map_ptrloc Version; // StringItem + /** \brief index of the package this depends applies to + + The generator will - if the package does not already exist - + create a blank (no version records) package. */ map_ptrloc Package; // Package + /** \brief next dependency of this version */ map_ptrloc NextDepends; // Dependency + /** \brief next reverse dependency of this package */ map_ptrloc NextRevDepends; // Dependency + /** \brief version of the package which has the reverse depends */ map_ptrloc ParentVer; // Version - - // Specific types of depends - map_ptrloc ID; + + /** \brief unique sequel ID */ + map_ptrloc ID; + /** \brief Dependency type - Depends, Recommends, Conflicts, etc */ unsigned char Type; + /** \brief comparison operator specified on the depends line + + If the high bit is set then it is a logical OR with the previous record. */ unsigned char CompareOp; }; /*}}}*/ -struct pkgCache::Provides /*{{{*/ +// Provides structure /*{{{*/ +/** \brief handles virtual packages + + When a Provides: line is encountered a new provides record is added + associating the package with a virtual package name. + The provides structures are linked off the package structures. + This simplifies the analysis of dependencies and other aspects A provides + refers to a specific version of a specific package, not all versions need to + provide that provides.*/ +struct pkgCache::Provides { - map_ptrloc ParentPkg; // Pacakge + /** \brief index of the package providing this */ + map_ptrloc ParentPkg; // Package + /** \brief index of the version this provide line applies to */ map_ptrloc Version; // Version - map_ptrloc ProvideVersion; // Stringtable + /** \brief version in the provides line (if any) + + This version allows dependencies to depend on specific versions of a + Provides, as well as allowing Provides to override existing packages. + This is experimental. Note that Debian doesn't allow versioned provides */ + map_ptrloc ProvideVersion; // StringItem + /** \brief next provides (based of package) */ map_ptrloc NextProvides; // Provides + /** \brief next provides (based of version) */ map_ptrloc NextPkgProv; // Provides }; /*}}}*/ -struct pkgCache::StringItem /*{{{*/ +// StringItem structure /*{{{*/ +/** \brief used for generating single instances of strings + + Some things like Section Name are are useful to have as unique tags. + It is part of a linked list based at pkgCache::Header::StringList + + All strings are simply inlined any place in the file that is natural + for the writer. The client should make no assumptions about the positioning + of strings. All StringItems should be null-terminated. */ +struct pkgCache::StringItem { - map_ptrloc String; // Stringtable + /** \brief string this refers to */ + map_ptrloc String; // StringItem + /** \brief Next link in the chain */ map_ptrloc NextItem; // StringItem }; /*}}}*/ diff --git a/debian/changelog b/debian/changelog index 2f262a3a9..76fc2550b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -38,6 +38,11 @@ apt (0.7.26~exp4) experimental; urgency=low - be less strict and accept [option=value] as well * apt-pkg/contrib/configuration.cc: - error out if #clear directive has no argument + * doc/files.sgml: + - sync documentation with status quo, regarding files/directories in + use, extended_states and uri schemes. + * doc/cache.sgml: + - drop the file in favor of inplace documentation with doxygen [ Jari Aalto ] * cmdline/apt-get.cc: @@ -114,9 +119,6 @@ apt (0.7.26~exp3) experimental; urgency=low Thanks to Chris Leick and Georg Koppen! (Closes: #574962) * apt-pkg/contrib/strutl.cc: - convert all toupper calls to tolower_ascii for a little speedup - * doc/files.sgml: - - sync documentation with status quo, regarding files/directories in - use, extended_states and uri schemes. [ Jean-Baptiste Lallement ] * apt-pkg/contrib/strutl.cc: diff --git a/doc/cache.sgml b/doc/cache.sgml deleted file mode 100644 index aea5a45c3..000000000 --- a/doc/cache.sgml +++ /dev/null @@ -1,824 +0,0 @@ - - - -APT Cache File Format - -Jason Gunthorpe jgg@debian.org -$Id: cache.sgml,v 1.11 2003/02/12 15:05:44 doogie Exp $ - - -This document describes the complete implementation and format of the APT -Cache file. The APT Cache file is a way for APT to parse and store a -large number of package files for display in the UI. It's primary design -goal is to make display of a single package in the tree very fast by -pre-linking important things like dependencies and provides. - -The specification doubles as documentation for one of the in-memory -structures used by the package library and the APT GUI. - - - - -Copyright © Jason Gunthorpe, 1997-1998. -

-APT and this document are free software; you can redistribute them and/or -modify them under the terms of the GNU General Public License as published -by the Free Software Foundation; either version 2 of the License, or (at your -option) any later version. - -

-For more details, on Debian GNU/Linux systems, see the file -/usr/share/common-licenses/GPL for the full license. - - - - -Introduction - - -Purpose - -

-This document describes the implementation of an architecture -dependent binary cache file. The goal of this cache file is two fold, -firstly to speed loading and processing of the package file array and -secondly to reduce memory consumption of the package file array. - -

-The implementation is aimed at an environment with many primary package -files, for instance someone that has a Package file for their CD-ROM, a -Package file for the latest version of the distribution on the CD-ROM and a -package file for the development version. Always present is the information -contained in the status file which might be considered a separate package -file. - -

-Please understand, this is designed as a -CACHE FILE- it is not meant to be -used on any system other than the one it was created for. It is not meant to -be authoritative either, i.e. if a system crash or software failure occurs it -must be perfectly acceptable for the cache file to be in an inconsistent -state. Furthermore at any time the cache file may be erased without losing -any information. - -

-Also the structures and storage layout is optimized for use by the APT -GUI and may not be suitable for all purposes. However it should be possible -to extend it with associate cache files that contain other information. - -

-To keep memory use down the cache file only contains often used fields and -fields that are inexpensive to store, the Package file has a full list of -fields. Also the client may assume that all items are perfectly valid and -need not perform checks against their correctness. Removal of information -from the cache is possible, but blanks will be left in the file, and -unused strings will also be present. The recommended implementation is to -simply rebuild the cache each time any of the data files change. It is -possible to add a new package file to the cache without any negative side -effects. - -Note on Pointer access -

-Every item in every structure is stored as the index to that structure. -What this means is that once the files is mmaped every data access has to -go through a fixup stage to get a real memory pointer. This is done -by taking the index, multiplying it by the type size and then adding -it to the start address of the memory block. This sounds complex, but -in C it is a single array dereference. Because all items are aligned to -their size and indexes are stored as multiples of the size of the structure -the format is immediately portable to all possible architectures - BUT the -generated files are -NOT-. - -

-This scheme allows code like this to be written: - - void *Map = mmap(...); - Package *PkgList = (Package *)Map; - Header *Head = (Header *)Map; - char *Strings = (char *)Map; - cout << (Strings + PkgList[Head->HashTable[0]]->Name) << endl; - -

-Notice the lack of casting or multiplication. The net result is to return -the name of the first package in the first hash bucket, without error -checks. - -

-The generator uses allocation pools to group similarly sized structures in -large blocks to eliminate any alignment overhead. The generator also -assures that no structures overlap and all indexes are unique. Although -at first glance it may seem like there is the potential for two structures -to exist at the same point the generator never allows this to happen. -(See the discussion of free space pools) - - -Structures - - -Header -

-This is the first item in the file. - - struct Header - { - // Signature information - unsigned long Signature; - short MajorVersion; - short MinorVersion; - bool Dirty; - - // Size of structure values - unsigned short HeaderSz; - unsigned short PackageSz; - unsigned short PackageFileSz; - unsigned short VersionSz; - unsigned short DependencySz; - unsigned short ProvidesSz; - unsigned short VerFileSz; - - // Structure counts - unsigned long PackageCount; - unsigned long VersionCount; - unsigned long DependsCount; - unsigned long PackageFileCount; - - // Offsets - unsigned long FileList; // PackageFile - unsigned long StringList; // StringItem - unsigned long VerSysName; // StringTable - unsigned long Architecture; // StringTable - unsigned long MaxVerFileSize; - - // Allocation pools - struct - { - unsigned long ItemSize; - unsigned long Start; - unsigned long Count; - } Pools[7]; - - // Package name lookup - unsigned long HashTable[2*1024]; // Package - }; - - -Signature -This must contain the hex value 0x98FE76DC which is designed to verify -that the system loading the image has the same byte order and byte size as -the system saving the image - -MajorVersion -MinorVersion -These contain the version of the cache file, currently 0.2. - -Dirty -Dirty is true if the cache file was opened for reading, the client expects -to have written things to it and have not fully synced it. The file should -be erased and rebuilt if it is true. - -HeaderSz -PackageSz -PackageFileSz -VersionSz -DependencySz -VerFileSz -ProvidesSz -*Sz contains the sizeof() that particular structure. It is used as an -extra consistency check on the structure of the file. - -If any of the size values do not exactly match what the client expects then -the client should refuse the load the file. - -PackageCount -VersionCount -DependsCount -PackageFileCount -These indicate the number of each structure contained in the cache. -PackageCount is especially useful for generating user state structures. -See Package::Id for more info. - -VerSysName -String representing the version system used for this cache - -Architecture -Architecture the cache was built against. - -MaxVerFileSize -The maximum size of a raw entry from the original Package file -(i.e. VerFile::Size) is stored here. - -FileList -This contains the index of the first PackageFile structure. The PackageFile -structures are singly linked lists that represent all package files that -have been merged into the cache. - -StringList -This contains a list of all the unique strings (string item type strings) in -the cache. The parser reads this list into memory so it can match strings -against it. - -Pools -The Pool structures manage the allocation pools that the generator uses. -Start indicates the first byte of the pool, Count is the number of objects -remaining in the pool and ItemSize is the structure size (alignment factor) -of the pool. An ItemSize of 0 indicates the pool is empty. There should be -the same number of pools as there are structure types. The generator -stores this information so future additions can make use of any unused pool -blocks. - -HashTable -HashTable is a hash table that provides indexing for all of the packages. -Each package name is inserted into the hash table using the following has -function: - - unsigned long Hash(string Str) - { - unsigned long Hash = 0; - for (const char *I = Str.begin(); I != Str.end(); I++) - Hash += *I * ((Str.end() - I + 1)); - return Hash % _count(Head.HashTable); - } - -

-By iterating over each entry in the hash table it is possible to iterate over -the entire list of packages. Hash Collisions are handled with a singly linked -list of packages based at the hash item. The linked list contains only -packages that match the hashing function. - - - - - -Package -

-This contains information for a single unique package. There can be any -number of versions of a given package. Package exists in a singly -linked list of package records starting at the hash index of the name in -the Header->HashTable. - - struct Pacakge - { - // Pointers - unsigned long Name; // Stringtable - unsigned long VersionList; // Version - unsigned long CurrentVer; // Version - unsigned long Section; // StringTable (StringItem) - - // Linked lists - unsigned long NextPackage; // Package - unsigned long RevDepends; // Dependency - unsigned long ProvidesList; // Provides - - // Install/Remove/Purge etc - unsigned char SelectedState; // What - unsigned char InstState; // Flags - unsigned char CurrentState; // State - - // Unique ID for this pkg - unsigned short ID; - unsigned long Flags; - }; - - - -Name -Name of the package. - -VersionList -Base of a singly linked list of version structures. Each structure -represents a unique version of the package. The version structures -contain links into PackageFile and the original text file as well as -detailed information about the size and dependencies of the specific -package. In this way multiple versions of a package can be cleanly handled -by the system. Furthermore, this linked list is guaranteed to be sorted -from Highest version to lowest version with no duplicate entries. - -CurrentVer -CurrentVer is an index to the installed version, either can be -0. - -Section -This indicates the deduced section. It should be "Unknown" or the section -of the last parsed item. - -NextPackage -Next link in this hash item. This linked list is based at Header.HashTable -and contains only packages with the same hash value. - -RevDepends -Reverse Depends is a linked list of all dependencies linked to this package. - -ProvidesList -This is a linked list of all provides for this package name. - -SelectedState -InstState -CurrentState -These correspond to the 3 items in the Status field found in the status -file. See the section on defines for the possible values. -

-SelectedState is the state that the user wishes the package to be -in. -

-InstState is the installation state of the package. This normally -should be OK, but if the installation had an accident it may be otherwise. -

-CurrentState indicates if the package is installed, partially installed or -not installed. - -ID -ID is a value from 0 to Header->PackageCount. It is a unique value assigned -by the generator. This allows clients to create an array of size PackageCount -and use it to store state information for the package map. For instance the -status file emitter uses this to track which packages have been emitted -already. - -Flags -Flags are some useful indicators of the package's state. - - - - - - -PackageFile -

-This contains information for a single package file. Package files are -referenced by Version structures. This is a singly linked list based from -Header.FileList - - struct PackageFile - { - // Names - unsigned long FileName; // Stringtable - unsigned long Archive; // Stringtable - unsigned long Component; // Stringtable - unsigned long Version; // Stringtable - unsigned long Origin; // Stringtable - unsigned long Label; // Stringtable - unsigned long Architecture; // Stringtable - unsigned long Site; // Stringtable - unsigned long IndexType; // Stringtable - unsigned long Size; - - // Linked list - unsigned long NextFile; // PackageFile - unsigned short ID; - unsigned long Flags; - time_t mtime; // Modification time - }; - - - -FileName -Refers the the physical disk file that this PacakgeFile represents. - -Archive -Component -Version -Origin -Label -Architecture -NotAutomatic -This is the release information. Please see the files document for a -description of what the release information means. - -Site -The site the index file was fetched from. - -IndexType -A string indicating what sort of index file this is. - -Size -Size is provided as a simple check to ensure that the package file has not -been altered. - -ID -See Package::ID. - -Flags -Provides some flags for the PackageFile, see the section on defines. - -mtime -Modification time for the file at time of cache generation. - - - - - - -Version -

-This contains the information for a single version of a package. This is a -single linked list based from Package.Versionlist. - -

-The version list is always sorted from highest version to lowest version by -the generator. Also there may not be any duplicate entries in the list (same -VerStr). - - - struct Version - { - unsigned long VerStr; // Stringtable - unsigned long Section; // StringTable (StringItem) - unsigned long Arch; // StringTable - - // Lists - unsigned long FileList; // VerFile - unsigned long NextVer; // Version - unsigned long DependsList; // Dependency - unsigned long ParentPkg; // Package - unsigned long ProvidesList; // Provides - - unsigned long Size; - unsigned long InstalledSize; - unsigned long Hash; - unsigned short ID; - unsigned char Priority; - }; - - - -VerStr -This is the complete version string. - -FileList -References the all the PackageFile's that this version came out of. FileList -can be used to determine what distribution(s) the Version applies to. If -FileList is 0 then this is a blank version. The structure should also have -a 0 in all other fields excluding VerStr and Possibly NextVer. - -Section -This string indicates which section it is part of. The string should be -contained in the StringItem list. - -Arch -Architecture the package was compiled for. - -NextVer -Next step in the linked list. - -DependsList -This is the base of the dependency list. - -ParentPkg -This links the version to the owning package, allowing reverse dependencies -to determine the package. - -ProvidesList -Head of the linked list of Provides::NextPkgProv, forward provides. - -Size -InstalledSize -The archive size for this version. For Debian this is the size of the .deb -file. Installed size is the uncompressed size for this version - -Hash -This is a characteristic value representing this package. No two packages -in existence should have the same VerStr and Hash with different contents. - -ID -See Package::ID. - -Priority -This is the parsed priority value of the package. - - - - - -Dependency -

-Dependency contains the information for a single dependency record. The records -are split up like this to ease processing by the client. The base of list -linked list is Version.DependsList. All forms of dependencies are recorded -here including Conflicts, Breaks, Suggests and Recommends. - -

-Multiple depends on the same package must be grouped together in -the Dependency lists. Clients should assume this is always true. - - - struct Dependency - { - unsigned long Version; // Stringtable - unsigned long Package; // Package - unsigned long NextDepends; // Dependency - unsigned long NextRevDepends; // Reverse dependency linking - unsigned long ParentVer; // Upwards parent version link - - // Specific types of depends - unsigned char Type; - unsigned char CompareOp; - unsigned short ID; - }; - - -Version -The string form of the version that the dependency is applied against. - -Package -The index of the package file this depends applies to. If the package file -does not already exist when the dependency is inserted a blank one (no -version records) should be created. - -NextDepends -Linked list based off a Version structure of all the dependencies in that -version. - -NextRevDepends -Reverse dependency linking, based off a Package structure. This linked list -is a list of all packages that have a depends line for a given package. - -ParentVer -Parent version linking, allows the reverse dependency list to link -back to the version and package that the dependency are for. - -Type -Describes weather it is depends, predepends, recommends, suggests, etc. - -CompareOp -Describes the comparison operator specified on the depends line. If the high -bit is set then it is a logical or with the previous record. - -ID -See Package::ID. - - - - - - -Provides -

-Provides handles virtual packages. When a Provides: line is encountered -a new provides record is added associating the package with a virtual -package name. The provides structures are linked off the package structures. -This simplifies the analysis of dependencies and other aspects A provides -refers to a specific version of a specific package, not all versions need to -provide that provides. - -

-There is a linked list of provided package names started from each -version that provides packages. This is the forwards provides mechanism. - - struct Provides - { - unsigned long ParentPkg; // Package - unsigned long Version; // Version - unsigned long ProvideVersion; // Stringtable - unsigned long NextProvides; // Provides - unsigned long NextPkgProv; // Provides - }; - - -ParentPkg -The index of the package that head of this linked list is in. ParentPkg->Name -is the name of the provides. - -Version -The index of the version this provide line applies to. - -ProvideVersion -Each provides can specify a version in the provides line. This version allows -dependencies to depend on specific versions of a Provides, as well as allowing -Provides to override existing packages. This is experimental. - -NextProvides -Next link in the singly linked list of provides (based off package) - -NextPkgProv -Next link in the singly linked list of provides for 'Version'. - - - - - - -VerFile -

-VerFile associates a version with a PackageFile, this allows a full -description of all Versions in all files (and hence all sources) under -consideration. - - - struct pkgCache::VerFile - { - unsigned long File; // PackageFile - unsigned long NextFile; // PkgVerFile - unsigned long Offset; - unsigned short Size; - } - - -File -The index of the package file that this version was found in. - -NextFile -The next step in the linked list. - -Offset -Size -These describe the exact position in the package file for the section from -this version. - - - - - -StringItem -

-StringItem is used for generating single instances of strings. Some things -like Section Name are are useful to have as unique tags. It is part of -a linked list based at Header::StringList. - - struct StringItem - { - unsigned long String; // Stringtable - unsigned long NextItem; // StringItem - }; - - -String -The string this refers to. - -NextItem -Next link in the chain. - - - - -StringTable -

-All strings are simply inlined any place in the file that is natural for the -writer. The client should make no assumptions about the positioning of -strings. All stringtable values point to a byte offset from the start of the -file that a null terminated string will begin. - - - -Defines -

-Several structures use variables to indicate things. Here is a list of all -of them. - -Definitions for Dependency::Type -

- -#define pkgDEP_Depends 1 -#define pkgDEP_PreDepends 2 -#define pkgDEP_Suggests 3 -#define pkgDEP_Recommends 4 -#define pkgDEP_Conflicts 5 -#define pkgDEP_Replaces 6 -#define pkgDEP_Breaks 8 - - - -Definitions for Dependency::CompareOp -

- -#define pkgOP_OR 0x10 -#define pkgOP_LESSEQ 0x1 -#define pkgOP_GREATEREQ 0x2 -#define pkgOP_LESS 0x3 -#define pkgOP_GREATER 0x4 -#define pkgOP_EQUALS 0x5 - -The lower 4 bits are used to indicate what operator is being specified and -the upper 4 bits are flags. pkgOP_OR indicates that the next package is -or'd with the current package. - - -Definitions for Package::SelectedState -

- -#define pkgSTATE_Unkown 0 -#define pkgSTATE_Install 1 -#define pkgSTATE_Hold 2 -#define pkgSTATE_DeInstall 3 -#define pkgSTATE_Purge 4 - - - -Definitions for Package::InstState -

- -#define pkgSTATE_Ok 0 -#define pkgSTATE_ReInstReq 1 -#define pkgSTATE_Hold 2 -#define pkgSTATE_HoldReInstReq 3 - - - -Definitions for Package::CurrentState -

- -#define pkgSTATE_NotInstalled 0 -#define pkgSTATE_UnPacked 1 -#define pkgSTATE_HalfConfigured 2 -#define pkgSTATE_UnInstalled 3 -#define pkgSTATE_HalfInstalled 4 -#define pkgSTATE_ConfigFiles 5 -#define pkgSTATE_Installed 6 -#define pkgSTATE_TriggersAwaited 7 -#define pkgSTATE_TriggersPending 8 - - - -Definitions for Package::Flags -

- -#define pkgFLAG_Auto (1 << 0) -#define pkgFLAG_New (1 << 1) -#define pkgFLAG_Obsolete (1 << 2) -#define pkgFLAG_Essential (1 << 3) -#define pkgFLAG_ImmediateConf (1 << 4) - - - -Definitions for Version::Priority -

-Zero is used for unparsable or absent Priority fields. - -#define pkgPRIO_Important 1 -#define pkgPRIO_Required 2 -#define pkgPRIO_Standard 3 -#define pkgPRIO_Optional 4 -#define pkgPRIO_Extra 5 - - - -Definitions for PackageFile::Flags -

- -#define pkgFLAG_NotSource (1 << 0) -#define pkgFLAG_NotAutomatic (1 << 1) - - - - - -Notes on the Generator - - -

-The pkgCache::MergePackageFile function is currently the only generator of -the cache file. It implements a conversion from the normal textual package -file into the cache file. - -

-The generator assumes any package declaration with a -Status: line is a 'Status of the package' type of package declaration. -A Package with a Target-Version field should also really have a status field. -The processing of a Target-Version field can create a place-holder Version -structure that is empty to refer to the specified version (See Version -for info on what a empty Version looks like). The Target-Version syntax -allows the specification of a specific version and a target distribution. - -

-Different section names on different versions is supported, but I -do not expect to use it. To simplify the GUI it will merely use the section -in the Package structure. This should be okay as I hope sections do not change -much. - -

-The generator goes through a number of post processing steps after producing -a disk file. It sorts all of the version lists to be in descending order -and then generates the reverse dependency lists for all of the packages. -ID numbers and count values are also generated in the post processing step. - -

-It is possible to extend many of the structures in the cache with extra data. -This is done by using the ID member. ID will be a unique number from 0 to -Header->??Count. For example - -struct MyPkgData; -MyPkgData *Data = new MyPkgData[Header->PackageCount]; -Data[Package->ID]->Item = 0; - -This provides a one way reference between package structures and user data. To -get a two way reference would require a member inside the MyPkgData structure. - -

-The generators use of free space pools tend to make the package file quite -large, and quite full of blank space. This could be fixed with sparse files. - - - -Future Directions - - -

-Some good directions to take the cache file is into a cache directory that -contains many associated caches that cache other important bits of -information. (/var/cache/apt, FHS2) - -

-Caching of the info/*.list is an excellent place to start, by generating all -the list files into a tree structure and reverse linking them to the package -structures in the main cache file major speed gains in dpkg might be achieved. - - - - -- cgit v1.2.3-70-g09d2 From 52c41485092d6da77d2848a955609f22da50372c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 4 May 2010 12:43:08 +0200 Subject: * apt-pkg/pkgcache.h: - enhance the Groups ABI by providing a ID as the other structs does - check also the size of the Group struct then checking for the others --- apt-pkg/pkgcache.cc | 3 +++ apt-pkg/pkgcache.h | 10 +++++++--- apt-pkg/pkgcachegen.cc | 3 +-- debian/changelog | 3 +++ 4 files changed, 14 insertions(+), 5 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 1bbd74bd9..ba3c5cbf8 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -55,6 +55,7 @@ pkgCache::Header::Header() Dirty = false; HeaderSz = sizeof(pkgCache::Header); + GroupSz = sizeof(pkgCache::Group); PackageSz = sizeof(pkgCache::Package); PackageFileSz = sizeof(pkgCache::PackageFile); VersionSz = sizeof(pkgCache::Version); @@ -64,6 +65,7 @@ pkgCache::Header::Header() VerFileSz = sizeof(pkgCache::VerFile); DescFileSz = sizeof(pkgCache::DescFile); + GroupCount = 0; PackageCount = 0; VersionCount = 0; DescriptionCount = 0; @@ -90,6 +92,7 @@ pkgCache::Header::Header() bool pkgCache::Header::CheckSizes(Header &Against) const { if (HeaderSz == Against.HeaderSz && + GroupSz == Against.GroupSz && PackageSz == Against.PackageSz && PackageFileSz == Against.PackageFileSz && VersionSz == Against.VersionSz && diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index a2e63ff03..643f240b0 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -244,6 +244,7 @@ struct pkgCache::Header If any of the size values do not exactly match what the client expects then the client should refuse the load the file. */ unsigned short HeaderSz; + unsigned short GroupSz; unsigned short PackageSz; unsigned short PackageFileSz; unsigned short VersionSz; @@ -329,12 +330,15 @@ struct pkgCache::Group map_ptrloc Name; // StringItem // Linked List - /** Link to the first package which belongs to the group */ + /** \brief Link to the first package which belongs to the group */ map_ptrloc FirstPackage; // Package - /** Link to the last package which belongs to the group */ + /** \brief Link to the last package which belongs to the group */ map_ptrloc LastPackage; // Package - /** Link to the next Group */ + /** \brief Link to the next Group */ map_ptrloc Next; // Group + /** \brief unique sequel ID */ + unsigned int ID; + }; /*}}}*/ // Package structure /*{{{*/ diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 114c9d5ed..d96d3370f 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -356,8 +356,7 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) Grp->Next = Cache.HeaderP->GrpHashTable[Hash]; Cache.HeaderP->GrpHashTable[Hash] = Group; - Cache.HeaderP->GroupCount++; - + Grp->ID = Cache.HeaderP->GroupCount++; return true; } /*}}}*/ diff --git a/debian/changelog b/debian/changelog index 76fc2550b..5885f9246 100644 --- a/debian/changelog +++ b/debian/changelog @@ -43,6 +43,9 @@ apt (0.7.26~exp4) experimental; urgency=low use, extended_states and uri schemes. * doc/cache.sgml: - drop the file in favor of inplace documentation with doxygen + * apt-pkg/pkgcache.h: + - enhance the Groups ABI by providing a ID as the other structs does + - check also the size of the Group struct then checking for the others [ Jari Aalto ] * cmdline/apt-get.cc: -- cgit v1.2.3-70-g09d2 From e2c66de5c5e63d8400efb0522c31fbe1ec225f93 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 3 Jun 2010 09:22:00 +0200 Subject: * apt-pkg/pkgcache.h: - switch {,Install-}Size to unsigned long long * apt-pkg/depcache.cc: - deal with long long, not with int to remove 2GB Limit (LP: #250909) --- apt-pkg/acquire-item.h | 4 ++-- apt-pkg/deb/deblistparser.cc | 5 ++--- apt-pkg/depcache.cc | 14 +++++++------- apt-pkg/pkgcache.h | 4 ++-- apt-pkg/tagfile.cc | 24 ++++++++++++++++++++++++ apt-pkg/tagfile.h | 1 + debian/changelog | 10 ++++++++++ 7 files changed, 48 insertions(+), 14 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index b338b2a41..36fc53b92 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -112,10 +112,10 @@ class pkgAcquire::Item : public WeakPointable string ErrorText; /** \brief The size of the object to fetch. */ - unsigned long FileSize; + unsigned long long FileSize; /** \brief How much of the object was already fetched. */ - unsigned long PartialSize; + unsigned long long PartialSize; /** \brief If not \b NULL, contains the name of a subprocess that * is operating on this object (for instance, "gzip" or "gpgv"). diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 0551a5f7c..83c5b8d2e 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -133,10 +133,9 @@ bool debListParser::NewVersion(pkgCache::VerIterator Ver) } // Archive Size - Ver->Size = (unsigned)Section.FindI("Size"); - + Ver->Size = Section.FindULL("Size"); // Unpacked Size (in K) - Ver->InstalledSize = (unsigned)Section.FindI("Installed-Size"); + Ver->InstalledSize = Section.FindULL("Installed-Size"); Ver->InstalledSize *= 1024; // Priority diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc index 411ae5f62..6e0eeab5b 100644 --- a/apt-pkg/depcache.cc +++ b/apt-pkg/depcache.cc @@ -422,8 +422,8 @@ void pkgDepCache::AddSizes(const PkgIterator &Pkg,signed long Mult) // Compute the size data if (P.NewInstall() == true) { - iUsrSize += (signed)(Mult*P.InstVerIter(*this)->InstalledSize); - iDownloadSize += (signed)(Mult*P.InstVerIter(*this)->Size); + iUsrSize += (signed long long)(Mult*P.InstVerIter(*this)->InstalledSize); + iDownloadSize += (signed long long)(Mult*P.InstVerIter(*this)->Size); return; } @@ -432,9 +432,9 @@ void pkgDepCache::AddSizes(const PkgIterator &Pkg,signed long Mult) (P.InstallVer != (Version *)Pkg.CurrentVer() || (P.iFlags & ReInstall) == ReInstall) && P.InstallVer != 0) { - iUsrSize += (signed)(Mult*((signed)P.InstVerIter(*this)->InstalledSize - - (signed)Pkg.CurrentVer()->InstalledSize)); - iDownloadSize += (signed)(Mult*P.InstVerIter(*this)->Size); + iUsrSize += (signed long long)(Mult*((signed long long)P.InstVerIter(*this)->InstalledSize - + (signed long long)Pkg.CurrentVer()->InstalledSize)); + iDownloadSize += (signed long long)(Mult*P.InstVerIter(*this)->Size); return; } @@ -442,14 +442,14 @@ void pkgDepCache::AddSizes(const PkgIterator &Pkg,signed long Mult) if (Pkg.State() == pkgCache::PkgIterator::NeedsUnpack && P.Delete() == false) { - iDownloadSize += (signed)(Mult*P.InstVerIter(*this)->Size); + iDownloadSize += (signed long long)(Mult*P.InstVerIter(*this)->Size); return; } // Removing if (Pkg->CurrentVer != 0 && P.InstallVer == 0) { - iUsrSize -= (signed)(Mult*Pkg.CurrentVer()->InstalledSize); + iUsrSize -= (signed long long)(Mult*Pkg.CurrentVer()->InstalledSize); return; } } diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 643f240b0..426bb9f13 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -532,9 +532,9 @@ struct pkgCache::Version /** \brief archive size for this version For Debian this is the size of the .deb file. */ - map_ptrloc Size; // These are the .deb size + unsigned long long Size; // These are the .deb size /** \brief uncompressed size for this version */ - map_ptrloc InstalledSize; + unsigned long long InstalledSize; /** \brief characteristic value representing this version No two packages in existence should have the same VerStr diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 0d4999ee7..1394d7e24 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -365,6 +365,30 @@ signed int pkgTagSection::FindI(const char *Tag,signed long Default) const return Result; } /*}}}*/ +// TagSection::FindULL - Find an unsigned long long integer /*{{{*/ +// --------------------------------------------------------------------- +/* */ +unsigned long long pkgTagSection::FindULL(const char *Tag, unsigned long long const &Default) const +{ + const char *Start; + const char *Stop; + if (Find(Tag,Start,Stop) == false) + return Default; + + // Copy it into a temp buffer so we can use strtoull + char S[100]; + if ((unsigned)(Stop - Start) >= sizeof(S)) + return Default; + strncpy(S,Start,Stop-Start); + S[Stop - Start] = 0; + + char *End; + unsigned long long Result = strtoull(S,&End,10); + if (S == End) + return Default; + return Result; +} + /*}}}*/ // TagSection::FindFlag - Locate a yes/no type flag /*{{{*/ // --------------------------------------------------------------------- /* The bits marked in Flag are masked on/off in Flags */ diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index f63a51d07..6891c1d81 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -57,6 +57,7 @@ class pkgTagSection bool Find(const char *Tag,unsigned &Pos) const; string FindS(const char *Tag) const; signed int FindI(const char *Tag,signed long Default = 0) const ; + unsigned long long FindULL(const char *Tag, unsigned long long const &Default = 0) const; bool FindFlag(const char *Tag,unsigned long &Flags, unsigned long Flag) const; bool Scan(const char *Start,unsigned long MaxLength); diff --git a/debian/changelog b/debian/changelog index 3771ca415..e153027b3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +apt (0.7.26~exp6) UNRELEASED; urgency=low + + [ David Kalnischkies ] + * apt-pkg/pkgcache.h: + - switch {,Install-}Size to unsigned long long + * apt-pkg/depcache.cc: + - deal with long long, not with int to remove 2GB Limit (LP: #250909) + + -- David Kalnischkies Thu, 03 Jun 2010 09:19:01 +0200 + apt (0.7.26~exp5) experimental; urgency=low [ David Kalnischkies ] -- cgit v1.2.3-70-g09d2 From a9fe592842bfa17d91f4904d7fb0e3af3adebb17 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 8 Jul 2010 15:28:53 +0200 Subject: * apt-pkg/pkgcachegen.{cc,h}: - make the used MMap moveable (and therefore dynamic resizeable) by applying (some) mad pointer magic (Closes: #195018) --- apt-pkg/cacheiterators.h | 6 ++ apt-pkg/contrib/mmap.cc | 27 ++++-- apt-pkg/deb/debindexfile.cc | 1 + apt-pkg/pkgcache.cc | 5 +- apt-pkg/pkgcache.h | 2 +- apt-pkg/pkgcachegen.cc | 229 ++++++++++++++++++++++++++++++++------------ apt-pkg/pkgcachegen.h | 30 ++++-- debian/changelog | 3 + 8 files changed, 226 insertions(+), 77 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index dfe5707e1..eb8dee5e3 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -75,6 +75,12 @@ template class pkgCache::Iterator : inline bool IsGood() const { return S && Owner && ! end();}; inline unsigned long Index() const {return S - OwnerPointer();}; + void ReOwn(pkgCache &newOwner, void const * const oldMap, void const * const newMap) { + if (S == 0) + return; + S += (Str*)(newMap) - (Str*)(oldMap); + } + // Constructors - look out for the variable assigning inline Iterator() : S(0), Owner(0) {}; inline Iterator(pkgCache &Owner,Str *T = 0) : S(T), Owner(&Owner) {}; diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index d71066243..aa184b130 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -225,22 +225,22 @@ DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace, // disable Moveable if we don't grow if (Grow == 0) - Flags &= ~Moveable; + this->Flags &= ~Moveable; #ifndef __linux__ // kfreebsd doesn't have mremap, so we use the fallback - if ((Flags & Moveable) == Moveable) - Flags |= Fallback; + if ((this->Flags & Moveable) == Moveable) + this->Flags |= Fallback; #endif #ifdef _POSIX_MAPPED_FILES - if ((Flags & Fallback) != Fallback) { + if ((this->Flags & Fallback) != Fallback) { // Set the permissions. int Prot = PROT_READ; int Map = MAP_PRIVATE | MAP_ANONYMOUS; - if ((Flags & ReadOnly) != ReadOnly) + if ((this->Flags & ReadOnly) != ReadOnly) Prot |= PROT_WRITE; - if ((Flags & Public) == Public) + if ((this->Flags & Public) == Public) Map = MAP_SHARED | MAP_ANONYMOUS; // use anonymous mmap() to get the memory @@ -314,7 +314,7 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) // Look for a matching pool entry Pool *I; Pool *Empty = 0; - for (I = Pools; I != Pools + PoolCount; I++) + for (I = Pools; I != Pools + PoolCount; ++I) { if (I->ItemSize == 0) Empty = I; @@ -342,7 +342,11 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) { const unsigned long size = 20*1024; I->Count = size/ItemSize; + Pool* oldPools = Pools; Result = RawAllocate(size,ItemSize); + if (Pools != oldPools) + I += Pools - oldPools; + // Does the allocation failed ? if (Result == 0 && _error->PendingError()) return 0; @@ -365,7 +369,7 @@ unsigned long DynamicMMap::WriteString(const char *String, if (Len == (unsigned long)-1) Len = strlen(String); - unsigned long Result = RawAllocate(Len+1,0); + unsigned long const Result = RawAllocate(Len+1,0); if (Result == 0 && _error->PendingError()) return 0; @@ -395,16 +399,20 @@ bool DynamicMMap::Grow() { return _error->Error(_("Unable to increase the size of the MMap as the " "limit of %lu bytes is already reached."), Limit); - unsigned long const newSize = WorkSpace + 1024*1024; + unsigned long const newSize = WorkSpace + GrowFactor; if(Fd != 0) { Fd->Seek(newSize - 1); char C = 0; Fd->Write(&C,sizeof(C)); } + + unsigned long const poolOffset = Pools - ((Pool*) Base); + if ((Flags & Fallback) != Fallback) { #if defined(_POSIX_MAPPED_FILES) && defined(__linux__) #ifdef MREMAP_MAYMOVE + if ((Flags & Moveable) == Moveable) Base = mremap(Base, WorkSpace, newSize, MREMAP_MAYMOVE); else @@ -425,6 +433,7 @@ bool DynamicMMap::Grow() { return false; } + Pools =(Pool*) Base + poolOffset; WorkSpace = newSize; return true; } diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 6d9e99497..5e6db3f38 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -277,6 +277,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator File = Gen.GetCurFile(); + pkgCacheGenerator::Dynamic DynFile(File); struct stat St; if (fstat(Pkg.Fd(),&St) != 0) return _error->Errno("fstat","Failed to stat"); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 68aa83d0c..8af8ef7de 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -119,7 +119,7 @@ pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map) // Cache::ReMap - Reopen the cache file /*{{{*/ // --------------------------------------------------------------------- /* If the file is already closed then this will open it open it. */ -bool pkgCache::ReMap() +bool pkgCache::ReMap(bool const &Errorchecks) { // Apply the typecasts. HeaderP = (Header *)Map.Data(); @@ -135,6 +135,9 @@ bool pkgCache::ReMap() StringItemP = (StringItem *)Map.Data(); StrP = (char *)Map.Data(); + if (Errorchecks == false) + return true; + if (Map.Size() == 0 || HeaderP == 0) return _error->Error(_("Empty package cache")); diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 426bb9f13..799521784 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -174,7 +174,7 @@ class pkgCache /*{{{*/ StringItem *StringItemP; char *StrP; - virtual bool ReMap(); + virtual bool ReMap(bool const &Errorchecks = true); inline bool Sync() {return Map.Sync();}; inline MMap &GetMap() {return Map;}; inline void *DataEnd() {return ((unsigned char *)Map.Data()) + Map.Size();}; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 62e457734..18bad6727 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -36,6 +36,7 @@ #include /*}}}*/ typedef vector::iterator FileIterator; +template std::set pkgCacheGenerator::Dynamic::toReMap; // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- @@ -61,8 +62,12 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : // Starting header *Cache.HeaderP = pkgCache::Header(); - Cache.HeaderP->VerSysName = WriteStringInMap(_system->VS->Label); - Cache.HeaderP->Architecture = WriteStringInMap(_config->Find("APT::Architecture")); + map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label); + Cache.HeaderP->VerSysName = idxVerSysName; + map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture")); + Cache.HeaderP->Architecture = idxArchitecture; + if (unlikely(idxVerSysName == 0 || idxArchitecture == 0)) + return; Cache.ReMap(); } else @@ -96,19 +101,65 @@ pkgCacheGenerator::~pkgCacheGenerator() Map.Sync(0,sizeof(pkgCache::Header)); } /*}}}*/ +void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/ + if (oldMap == newMap) + return; + + Cache.ReMap(false); + + CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap; + + for (size_t i = 0; i < _count(UniqHash); ++i) + if (UniqHash[i] != 0) + UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap; + + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); +} /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ -unsigned long pkgCacheGenerator::WriteStringInMap(const char *String, +map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String, const unsigned long &Len) { - return Map.WriteString(String, Len); + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.WriteString(String, Len); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ -unsigned long pkgCacheGenerator::WriteStringInMap(const char *String) { - return Map.WriteString(String); +map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) { + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.WriteString(String); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ -unsigned long pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ - return Map.Allocate(size); +map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.Allocate(size); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ // CacheGenerator::MergeList - Merge the package list /*{{{*/ @@ -142,6 +193,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List, { // Get a pointer to the package structure pkgCache::PkgIterator Pkg; + Dynamic DynPkg(Pkg); if (NewPackage(Pkg, PackageName, *arch) == false) return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str()); Counter++; @@ -158,6 +210,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List, // (this has the bonus that we get MMap error when we run out // of MMap space) pkgCache::VerIterator Ver(Cache); + Dynamic DynVer(Ver); if (List.UsePackage(Pkg, Ver) == false) return _error->Error(_("Error occurred while processing %s (UsePackage1)"), PackageName.c_str()); @@ -165,11 +218,11 @@ bool pkgCacheGenerator::MergeList(ListParser &List, // Find the right version to write the description MD5SumValue CurMd5 = List.Description_md5(); Ver = Pkg.VersionList(); - map_ptrloc *LastVer = &Pkg->VersionList; - for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) + for (; Ver.end() == false; ++Ver) { pkgCache::DescIterator Desc = Ver.DescriptionList(); + Dynamic DynDesc(Desc); map_ptrloc *LastDesc = &Ver->DescriptionList; bool duplicate=false; @@ -189,7 +242,11 @@ bool pkgCacheGenerator::MergeList(ListParser &List, if (MD5SumValue(Desc.md5()) == CurMd5) { // Add new description - *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc); + void const * const oldMap = Map.Data(); + map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc); + if (oldMap != Map.Data()) + LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastDesc = descindex; Desc->ParentPkg = Pkg.Index(); if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) @@ -203,7 +260,9 @@ bool pkgCacheGenerator::MergeList(ListParser &List, } pkgCache::VerIterator Ver = Pkg.VersionList(); + Dynamic DynVer(Ver); map_ptrloc *LastVer = &Pkg->VersionList; + void const * oldMap = Map.Data(); int Res = 1; unsigned long const Hash = List.VersionHash(); for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) @@ -242,21 +301,28 @@ bool pkgCacheGenerator::MergeList(ListParser &List, } // Add a new version - *LastVer = NewVersion(Ver,Version,*LastVer); + map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer); + if (verindex == 0 && _error->PendingError()) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 1); + + if (oldMap != Map.Data()) + LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastVer = verindex; Ver->ParentPkg = Pkg.Index(); Ver->Hash = Hash; - if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false) - return _error->Error(_("Error occurred while processing %s (NewVersion1)"), - PackageName.c_str()); + if (List.NewVersion(Ver) == false) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 2); if (List.UsePackage(Pkg,Ver) == false) return _error->Error(_("Error occurred while processing %s (UsePackage3)"), PackageName.c_str()); if (NewFileVer(Ver,List) == false) - return _error->Error(_("Error occurred while processing %s (NewVersion2)"), - PackageName.c_str()); + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 3); // Read only a single record and return if (OutVer != 0) @@ -269,13 +335,18 @@ bool pkgCacheGenerator::MergeList(ListParser &List, /* Record the Description data. Description data always exist in Packages and Translation-* files. */ pkgCache::DescIterator Desc = Ver.DescriptionList(); + Dynamic DynDesc(Desc); map_ptrloc *LastDesc = &Ver->DescriptionList; - + // Skip to the end of description set for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++); // Add new description - *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc); + oldMap = Map.Data(); + map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc); + if (oldMap != Map.Data()) + LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastDesc = descindex; Desc->ParentPkg = Pkg.Index(); if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) @@ -322,6 +393,7 @@ bool pkgCacheGenerator::MergeFileProvides(ListParser &List) continue; pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName); + Dynamic DynPkg(Pkg); if (Pkg.end() == true) return _error->Error(_("Error occurred while processing %s (FindPkg)"), PackageName.c_str()); @@ -331,6 +403,7 @@ bool pkgCacheGenerator::MergeFileProvides(ListParser &List) unsigned long Hash = List.VersionHash(); pkgCache::VerIterator Ver = Pkg.VersionList(); + Dynamic DynVer(Ver); for (; Ver.end() == false; Ver++) { if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr()) @@ -358,14 +431,15 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) return true; // Get a structure - unsigned long const Group = AllocateInMap(sizeof(pkgCache::Group)); + map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group)); if (unlikely(Group == 0)) return false; Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group); - Grp->Name = WriteStringInMap(Name); - if (unlikely(Grp->Name == 0)) + map_ptrloc const idxName = WriteStringInMap(Name); + if (unlikely(idxName == 0)) return false; + Grp->Name = idxName; // Insert it into the hash table unsigned long const Hash = Cache.Hash(Name); @@ -382,6 +456,7 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name, const string &Arch) { pkgCache::GrpIterator Grp; + Dynamic DynGrp(Grp); if (unlikely(NewGroup(Grp, Name) == false)) return false; @@ -390,7 +465,7 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name return true; // Get a structure - unsigned long const Package = AllocateInMap(sizeof(pkgCache::Package)); + map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package)); if (unlikely(Package == 0)) return false; Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package); @@ -416,9 +491,10 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name // Set the name, arch and the ID Pkg->Name = Grp->Name; Pkg->Group = Grp.Index(); - Pkg->Arch = WriteUniqString(Arch.c_str()); - if (unlikely(Pkg->Arch == 0)) + map_ptrloc const idxArch = WriteUniqString(Arch.c_str()); + if (unlikely(idxArch == 0)) return false; + Pkg->Arch = idxArch; Pkg->ID = Cache.HeaderP->PackageCount++; return true; @@ -434,7 +510,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, return true; // Get a structure - unsigned long VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); + map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); if (VerFile == 0) return 0; @@ -465,7 +541,7 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, unsigned long Next) { // Get a structure - unsigned long Version = AllocateInMap(sizeof(pkgCache::Version)); + map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version)); if (Version == 0) return 0; @@ -473,9 +549,10 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version); Ver->NextVer = Next; Ver->ID = Cache.HeaderP->VersionCount++; - Ver->VerStr = WriteStringInMap(VerStr); - if (Ver->VerStr == 0) + map_ptrloc const idxVerStr = WriteStringInMap(VerStr); + if (unlikely(idxVerStr == 0)) return 0; + Ver->VerStr = idxVerStr; return Version; } @@ -490,7 +567,7 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, return true; // Get a structure - unsigned long DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); + map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); if (DescFile == 0) return false; @@ -523,7 +600,7 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, map_ptrloc Next) { // Get a structure - map_ptrloc Description = AllocateInMap(sizeof(pkgCache::Description)); + map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description)); if (Description == 0) return 0; @@ -531,10 +608,12 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description); Desc->NextDesc = Next; Desc->ID = Cache.HeaderP->DescriptionCount++; - Desc->language_code = WriteStringInMap(Lang); - Desc->md5sum = WriteStringInMap(md5sum.Value()); - if (Desc->language_code == 0 || Desc->md5sum == 0) + map_ptrloc const idxlanguage_code = WriteStringInMap(Lang); + map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value()); + if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0)) return 0; + Desc->language_code = idxlanguage_code; + Desc->md5sum = idxmd5sum; return Description; } @@ -550,15 +629,22 @@ bool pkgCacheGenerator::FinishCache(OpProgress *Progress) if (archs.size() > 1) { // Create Conflicts in between the group - for (pkgCache::GrpIterator G = GetCache().GrpBegin(); G.end() != true; G++) + pkgCache::GrpIterator G = GetCache().GrpBegin(); + Dynamic DynG(G); + for (; G.end() != true; G++) { string const PkgName = G.Name(); - for (pkgCache::PkgIterator P = G.PackageList(); P.end() != true; P = G.NextPkg(P)) + pkgCache::PkgIterator P = G.PackageList(); + Dynamic DynP(P); + for (; P.end() != true; P = G.NextPkg(P)) { if (strcmp(P.Arch(),"all") == 0) continue; pkgCache::PkgIterator allPkg; - for (pkgCache::VerIterator V = P.VersionList(); V.end() != true; V++) + Dynamic DynallPkg(allPkg); + pkgCache::VerIterator V = P.VersionList(); + Dynamic DynV(V); + for (; V.end() != true; V++) { string const Arch = V.Arch(true); map_ptrloc *OldDepLast = NULL; @@ -578,6 +664,7 @@ bool pkgCacheGenerator::FinishCache(OpProgress *Progress) per group, therefore each group member conflicts with all other group members */ pkgCache::PkgIterator D = G.FindPkg(*A); + Dynamic DynD(D); if (D.end() == true) continue; if (coInstall == true) @@ -622,13 +709,15 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, unsigned int const &Type, map_ptrloc *OldDepLast) { + void const * const oldMap = Map.Data(); // Get a structure - unsigned long const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); + map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); if (unlikely(Dependency == 0)) return false; // Fill it in pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency); + Dynamic DynDep(Dep); Dep->ParentVer = Ver.Index(); Dep->Type = Type; Dep->CompareOp = Op; @@ -640,9 +729,12 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, /* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++) if (I->Version != 0 && I.TargetVer() == Version) Dep->Version = I->Version;*/ - if (Dep->Version == 0) - if (unlikely((Dep->Version = WriteStringInMap(Version)) == 0)) + if (Dep->Version == 0) { + map_ptrloc const index = WriteStringInMap(Version); + if (unlikely(index == 0)) return false; + Dep->Version = index; + } } // Link it to the package @@ -656,7 +748,8 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, OldDepLast = &Ver->DependsList; for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++) OldDepLast = &D->NextDepends; - } + } else if (oldMap != Map.Data()) + OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; Dep->NextDepends = *OldDepLast; *OldDepLast = Dep.Index(); @@ -677,11 +770,13 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver, unsigned int Type) { pkgCache::GrpIterator Grp; + Dynamic DynGrp(Grp); if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) return false; // Locate the target package pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch); + Dynamic DynPkg(Pkg); if (Pkg.end() == true) { if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) return false; @@ -715,13 +810,14 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, return true; // Get a structure - unsigned long const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); + map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); if (unlikely(Provides == 0)) return false; Cache.HeaderP->ProvidesCount++; // Fill it in pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP); + Dynamic DynPrv(Prv); Prv->Version = Ver.Index(); Prv->NextPkgProv = Ver->ProvidesList; Ver->ProvidesList = Prv.Index(); @@ -730,6 +826,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, // Locate the target package pkgCache::PkgIterator Pkg; + Dynamic DynPkg(Pkg); if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false)) return false; @@ -750,24 +847,29 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, unsigned long Flags) { // Get some space for the structure - CurrentFile = Cache.PkgFileP + AllocateInMap(sizeof(*CurrentFile)); - if (CurrentFile == Cache.PkgFileP) + map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile)); + if (unlikely(idxFile == 0)) return false; - + CurrentFile = Cache.PkgFileP + idxFile; + // Fill it in - CurrentFile->FileName = WriteStringInMap(File); - CurrentFile->Site = WriteUniqString(Site); + map_ptrloc const idxFileName = WriteStringInMap(File); + map_ptrloc const idxSite = WriteUniqString(Site); + if (unlikely(idxFileName == 0 || idxSite == 0)) + return false; + CurrentFile->FileName = idxFileName; + CurrentFile->Site = idxSite; CurrentFile->NextFile = Cache.HeaderP->FileList; CurrentFile->Flags = Flags; CurrentFile->ID = Cache.HeaderP->PackageFileCount; - CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label); + map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label); + if (unlikely(idxIndexType == 0)) + return false; + CurrentFile->IndexType = idxIndexType; PkgFileName = File; Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP; Cache.HeaderP->PackageFileCount++; - if (CurrentFile->FileName == 0) - return false; - if (Progress != 0) Progress->SubProgress(Index.Size()); return true; @@ -807,18 +909,25 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, } // Get a structure - unsigned long Item = AllocateInMap(sizeof(pkgCache::StringItem)); + void const * const oldMap = Map.Data(); + map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem)); if (Item == 0) return 0; + map_ptrloc const idxString = WriteStringInMap(S,Size); + if (unlikely(idxString == 0)) + return 0; + if (oldMap != Map.Data()) { + Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap; + } + *Last = Item; + // Fill in the structure pkgCache::StringItem *ItemP = Cache.StringItemP + Item; ItemP->NextItem = I - Cache.StringItemP; - *Last = Item; - ItemP->String = WriteStringInMap(S,Size); - if (ItemP->String == 0) - return 0; - + ItemP->String = idxString; + Bucket = ItemP; return ItemP->String; } @@ -1072,7 +1181,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress unlink(CacheFile.c_str()); CacheF = new FileFd(CacheFile,FileFd::WriteEmpty); fchmod(CacheF->Fd(),0644); - Map = new DynamicMMap(*CacheF,MMap::Public,MapSize); + Map = new DynamicMMap(*CacheF,MMap::Public | MMap::Moveable, MapSize); if (_error->PendingError() == true) return false; if (Debug == true) @@ -1081,7 +1190,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress else { // Just build it in memory.. - Map = new DynamicMMap(0,MapSize); + Map = new DynamicMMap(MMap::Moveable, MapSize); if (Debug == true) std::clog << "Open memory Map (not filebased)" << std::endl; } @@ -1194,7 +1303,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O if (_system->AddStatusFiles(Files) == false) return false; - SPtr Map = new DynamicMMap(0,MapSize); + SPtr Map = new DynamicMMap(MMap::Moveable, MapSize); unsigned long CurrentSize = 0; unsigned long TotalSize = 0; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 8f7739165..3bee1f958 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -23,6 +23,8 @@ #include #include +#include + class pkgSourceList; class OpProgress; class MMap; @@ -33,18 +35,32 @@ class pkgCacheGenerator /*{{{*/ private: pkgCache::StringItem *UniqHash[26]; - unsigned long WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; - unsigned long WriteStringInMap(const char *String); - unsigned long WriteStringInMap(const char *String, const unsigned long &Len); - unsigned long AllocateInMap(const unsigned long &size); + map_ptrloc WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; + map_ptrloc WriteStringInMap(const char *String); + map_ptrloc WriteStringInMap(const char *String, const unsigned long &Len); + map_ptrloc AllocateInMap(const unsigned long &size); public: class ListParser; friend class ListParser; - + + template class Dynamic { + Iter *I; + + public: + static std::set toReMap; + Dynamic(Iter &It) : I(&It) { + toReMap.insert(I); + } + + ~Dynamic() { + toReMap.erase(I); + } + }; + protected: - + DynamicMMap ⤅ pkgCache Cache; OpProgress *Progress; @@ -86,6 +102,8 @@ class pkgCacheGenerator /*{{{*/ MMap **OutMap = 0,bool AllowMem = false); static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); + void ReMap(void const * const oldMap, void const * const newMap); + pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress); ~pkgCacheGenerator(); }; diff --git a/debian/changelog b/debian/changelog index 878d5238a..dab703c6b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -37,6 +37,9 @@ apt (0.7.26~exp8) UNRELEASED; urgency=low * apt-pkg/policy.h: - add another round of const& madness as the previous round accidentally NOT overrides the virtual GetCandidateVer() method (Closes: #587725) + * apt-pkg/pkgcachegen.{cc,h}: + - make the used MMap moveable (and therefore dynamic resizeable) by + applying (some) mad pointer magic (Closes: #195018) [ Julian Andres Klode ] * methods/ftp.h: -- cgit v1.2.3-70-g09d2