From 5bf15716398fdb767ca6249a0155219b88d7ae60 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 19 Dec 2009 16:25:56 +0100 Subject: Implement the first step toward Multi-Arch by setting up a Group infrastructor for packages. APT is now aware of the fact that a package A in architecture X can't satisfy a dependency on package A in architecture Y - to handle these packages are now identified by name and architecture, so different architectures of the same package are handled internally as completly different packages. This is great for pinning, dependency checking and in many other situations, but sometimes we need to know which archs are available for a given package: Here Groups come to our rescue! --- apt-pkg/pkgcachegen.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 108b34207..4a2419b24 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -51,7 +51,8 @@ class pkgCacheGenerator /*{{{*/ // Flag file dependencies bool FoundFileDeps; - bool NewPackage(pkgCache::PkgIterator &Pkg,const string &PkgName); + bool NewGroup(pkgCache::GrpIterator &Grp,const string &Name); + bool NewPackage(pkgCache::PkgIterator &Pkg,const string &Name, const string &Arch); bool NewFileVer(pkgCache::VerIterator &Ver,ListParser &List); bool NewFileDesc(pkgCache::DescIterator &Desc,ListParser &List); unsigned long NewVersion(pkgCache::VerIterator &Ver,const string &VerStr,unsigned long Next); @@ -96,7 +97,7 @@ class pkgCacheGenerator::ListParser inline unsigned long WriteUniqString(const char *S,unsigned int Size) {return Owner->WriteUniqString(S,Size);}; inline unsigned long WriteString(const string &S) {return Owner->Map.WriteString(S);}; inline unsigned long WriteString(const char *S,unsigned int Size) {return Owner->Map.WriteString(S,Size);}; - bool NewDepends(pkgCache::VerIterator Ver,const string &Package, + bool NewDepends(pkgCache::VerIterator Ver,const string &Package, const string &Arch, const string &Version,unsigned int Op, unsigned int Type); bool NewProvides(pkgCache::VerIterator Ver,const string &Package, @@ -106,6 +107,7 @@ class pkgCacheGenerator::ListParser // These all operate against the current section virtual string Package() = 0; + virtual string Architecture() = 0; virtual string Version() = 0; virtual bool NewVersion(pkgCache::VerIterator Ver) = 0; virtual string Description() = 0; -- cgit v1.2.3-70-g09d2 From 25396fb06350344996a20d05423562f08a4165db Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 27 Dec 2009 19:39:47 +0100 Subject: Split ListParser::NewDepends into two methods to use these new method for creating the dependencies needed for our groups: For now for all groups only one package can be installed at the same time which conflicts with each other packages in the group. The exceptions are architecture all package. Also, the Multi-Arch field is now parsed, but not used for now. --- apt-pkg/cacheiterators.h | 14 ++++- apt-pkg/deb/deblistparser.cc | 24 ++++++++ apt-pkg/pkgcache.cc | 27 +++++++++ apt-pkg/pkgcache.h | 12 +++- apt-pkg/pkgcachegen.cc | 133 ++++++++++++++++++++++++++++++++----------- apt-pkg/pkgcachegen.h | 6 +- 6 files changed, 176 insertions(+), 40 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 35d3aa228..33d2ed6be 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -81,13 +81,21 @@ template class pkgCache::Iterator { different architectures can be treated as of the "same" package (apt internally treat them as totally different packages) */ class pkgCache::GrpIterator: public Iterator { + long HashIndex; + protected: inline Group* OwnerPointer() const { return Owner->GrpP; }; public: - void operator ++(int) {if (S != Owner->GrpP) S = Owner->GrpP + S->Next;}; + // This constructor is the 'begin' constructor, never use it. + inline GrpIterator(pkgCache &Owner) : Iterator(Owner), HashIndex(-1) { + S = OwnerPointer(); + operator ++(0); + }; + + virtual void operator ++(int); virtual void operator ++() {operator ++(0);}; inline const char *Name() const {return S->Name == 0?0:Owner->StrP + S->Name;}; @@ -96,11 +104,11 @@ class pkgCache::GrpIterator: public Iterator { PkgIterator NextPkg(PkgIterator const &Pkg); // Constructors - inline GrpIterator(pkgCache &Owner, Group *Trg) : Iterator(Owner, Trg) { + inline GrpIterator(pkgCache &Owner, Group *Trg) : Iterator(Owner, Trg), HashIndex(0) { if (S == 0) S = OwnerPointer(); }; - inline GrpIterator() : Iterator() {}; + inline GrpIterator() : Iterator(), HashIndex(0) {}; }; /*}}}*/ diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index b57eca813..f683de423 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -104,6 +104,30 @@ bool debListParser::NewVersion(pkgCache::VerIterator Ver) // Parse the architecture Ver->Arch = WriteUniqString(Architecture()); + // Parse multi-arch + if (Section.FindS("Architecture") == "all") + /* Arch all packages can't have a Multi-Arch field, + but we need a special treatment for them nonetheless */ + Ver->MultiArch = pkgCache::Version::All; + else + { + string const MultiArch = Section.FindS("Multi-Arch"); + if (MultiArch.empty() == true) + Ver->MultiArch = pkgCache::Version::None; + else if (MultiArch == "same") + Ver->MultiArch = pkgCache::Version::Same; + else if (MultiArch == "foreign") + Ver->MultiArch = pkgCache::Version::Foreign; + else if (MultiArch == "allowed") + Ver->MultiArch = pkgCache::Version::Allowed; + else + { + _error->Warning("Unknown Multi-Arch type »%s« for package »%s«", + MultiArch.c_str(), Section.FindS("Package").c_str()); + Ver->MultiArch = pkgCache::Version::None; + } + } + // Archive Size Ver->Size = (unsigned)Section.FindI("Size"); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index c945c59bf..1e4c7edb3 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -182,6 +182,16 @@ unsigned long pkgCache::sHash(const char *Str) const // Cache::FindPkg - Locate a package by name /*{{{*/ // --------------------------------------------------------------------- /* Returns 0 on error, pointer to the package otherwise */ +pkgCache::PkgIterator pkgCache::FindPkg(const string &Name) { + size_t const found = Name.find(':'); + if (found == string::npos) + return FindPkg(Name, "native"); + return FindPkg(Name.substr(0, found), Name.substr(found+1, string::npos)); +} + /*}}}*/ +// Cache::FindPkg - Locate a package by name /*{{{*/ +// --------------------------------------------------------------------- +/* Returns 0 on error, pointer to the package otherwise */ pkgCache::PkgIterator pkgCache::FindPkg(const string &Name, string Arch) { /* We make a detour via the GrpIterator here as on a multi-arch environment a group is easier to @@ -322,6 +332,23 @@ pkgCache::PkgIterator pkgCache::GrpIterator::NextPkg(pkgCache::PkgIterator const return PkgIterator(*Owner, 0); } /*}}}*/ +// GrpIterator::operator ++ - Postfix incr /*{{{*/ +// --------------------------------------------------------------------- +/* This will advance to the next logical group in the hash table. */ +void pkgCache::GrpIterator::operator ++(int) +{ + // Follow the current links + if (S != Owner->GrpP) + S = Owner->GrpP + S->Next; + + // Follow the hash table + while (S == Owner->GrpP && (HashIndex+1) < (signed)_count(Owner->HeaderP->GrpHashTable)) + { + HashIndex++; + S = Owner->GrpP + Owner->HeaderP->GrpHashTable[HashIndex]; + } +}; + /*}}}*/ // PkgIterator::operator ++ - Postfix incr /*{{{*/ // --------------------------------------------------------------------- /* This will advance to the next logical package in the hash table. */ diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index b3d2752d2..5f50001d0 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -125,9 +125,12 @@ class pkgCache /*{{{*/ // Accessors GrpIterator FindGrp(const string &Name); - PkgIterator FindPkg(const string &Name, string Arch = "native"); + PkgIterator FindPkg(const string &Name); + PkgIterator FindPkg(const string &Name, string Arch); Header &Head() {return *HeaderP;}; + inline GrpIterator GrpBegin(); + inline GrpIterator GrpEnd(); inline PkgIterator PkgBegin(); inline PkgIterator PkgEnd(); inline PkgFileIterator FileBegin(); @@ -274,7 +277,8 @@ struct pkgCache::Version /*{{{*/ map_ptrloc VerStr; // Stringtable map_ptrloc Section; // StringTable (StringItem) map_ptrloc Arch; // StringTable - + enum {None, All, Foreign, Same, Allowed} MultiArch; + // Lists map_ptrloc FileList; // VerFile map_ptrloc NextVer; // Version @@ -337,6 +341,10 @@ struct pkgCache::StringItem /*{{{*/ /*}}}*/ #include +inline pkgCache::GrpIterator pkgCache::GrpBegin() + {return GrpIterator(*this);}; +inline pkgCache::GrpIterator pkgCache::GrpEnd() + {return GrpIterator(*this,GrpP);}; inline pkgCache::PkgIterator pkgCache::PkgBegin() {return PkgIterator(*this);}; inline pkgCache::PkgIterator pkgCache::PkgEnd() diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index c37f6f48e..2a4a30349 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -38,7 +39,7 @@ typedef vector::iterator FileIterator; // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- -/* We set the diry flag and make sure that is written to the disk */ +/* We set the dirty flag and make sure that is written to the disk */ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : Map(*pMap), Cache(pMap,false), Progress(Prog), FoundFileDeps(0) @@ -506,21 +507,58 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, return Description; } /*}}}*/ -// ListParser::NewDepends - Create a dependency element /*{{{*/ +// CacheGenerator::FinishCache - do various finish operations /*{{{*/ +// --------------------------------------------------------------------- +/* This prepares the Cache for delivery */ +bool pkgCacheGenerator::FinishCache(OpProgress &Progress) { + // FIXME: add progress reporting for this operation + // Do we have different architectures in your groups ? + vector archs = APT::Configuration::getArchitectures(); + if (archs.size() > 1) { + // Create Conflicts in between the group + for (pkgCache::GrpIterator G = GetCache().GrpBegin(); G.end() != true; G++) { + string const PkgName = G.Name(); + for (pkgCache::PkgIterator P = G.PackageList(); P.end() != true; P = G.NextPkg(P)) { + for (pkgCache::VerIterator V = P.VersionList(); V.end() != true; V++) { + // Arch all packages are "co-installable" + if (V->MultiArch == pkgCache::Version::All) + continue; + string const Arch = V.Arch(); + map_ptrloc *OldDepLast = NULL; + for (vector::const_iterator A = archs.begin(); A != archs.end(); ++A) { + if (*A == Arch) + continue; + /* We allow only one installed arch at the time + per group, therefore each group member conflicts + with all other group members */ + pkgCache::PkgIterator D = G.FindPkg(*A); + if (D.end() == true) + continue; + // Conflicts: ${self}:other + NewDepends(D, V, "", + pkgCache::Dep::NoOp, pkgCache::Dep::Conflicts, + OldDepLast); + } + } + } + } + } + return true; +} + /*}}}*/ +// CacheGenerator::NewDepends - Create a dependency element /*{{{*/ // --------------------------------------------------------------------- /* This creates a dependency element in the tree. It is linked to the version and to the package that it is pointing to. */ -bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, - const string &PackageName, - const string &Arch, - const string &Version, - unsigned int Op, - unsigned int Type) +bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver, + string const &Version, + unsigned int const &Op, + unsigned int const &Type, + map_ptrloc *OldDepLast) { - pkgCache &Cache = Owner->Cache; - // Get a structure - unsigned long const Dependency = Owner->Map.Allocate(sizeof(pkgCache::Dependency)); + unsigned long const Dependency = Map.Allocate(sizeof(pkgCache::Dependency)); if (unlikely(Dependency == 0)) return false; @@ -530,17 +568,6 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, Dep->Type = Type; Dep->CompareOp = Op; Dep->ID = Cache.HeaderP->DependsCount++; - - pkgCache::GrpIterator Grp; - if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) - return false; - - // Locate the target package - pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch); - if (Pkg.end() == true) { - if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) - return false; - } // Probe the reverse dependency list for a version string that matches if (Version.empty() == false) @@ -549,29 +576,23 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, if (I->Version != 0 && I.TargetVer() == Version) Dep->Version = I->Version;*/ if (Dep->Version == 0) - if (unlikely((Dep->Version = WriteString(Version)) == 0)) + if (unlikely((Dep->Version = Map.WriteString(Version)) == 0)) return false; } - + // Link it to the package Dep->Package = Pkg.Index(); Dep->NextRevDepends = Pkg->RevDepends; Pkg->RevDepends = Dep.Index(); - - /* Link it to the version (at the end of the list) - Caching the old end point speeds up generation substantially */ - if (OldDepVer != Ver) + + // Do we know where to link the Dependency to? + if (OldDepLast == NULL) { OldDepLast = &Ver->DependsList; for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++) OldDepLast = &D->NextDepends; - OldDepVer = Ver; } - // Is it a file dependency? - if (unlikely(PackageName[0] == '/')) - FoundFileDeps = true; - Dep->NextDepends = *OldDepLast; *OldDepLast = Dep.Index(); OldDepLast = &Dep->NextDepends; @@ -579,6 +600,41 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, return true; } /*}}}*/ +// ListParser::NewDepends - Create the environment for a new dependency /*{{{*/ +// --------------------------------------------------------------------- +/* This creates a Group and the Package to link this dependency to if + needed and handles also the caching of the old endpoint */ +bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, + const string &PackageName, + const string &Arch, + const string &Version, + unsigned int Op, + unsigned int Type) +{ + pkgCache::GrpIterator Grp; + if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) + return false; + + // Locate the target package + pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch); + if (Pkg.end() == true) { + if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) + return false; + } + + // Is it a file dependency? + if (unlikely(PackageName[0] == '/')) + FoundFileDeps = true; + + /* Caching the old end point speeds up generation substantially */ + if (OldDepVer != Ver) { + OldDepLast = NULL; + OldDepVer = Ver; + } + + return Owner->NewDepends(Pkg, Ver, Version, Op, Type, OldDepLast); +} + /*}}}*/ // ListParser::NewProvides - Create a Provides element /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -925,6 +981,9 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, if (BuildCache(Gen,Progress,CurrentSize,TotalSize, Files.begin()+EndOfSource,Files.end()) == false) return false; + + // FIXME: move me to a better place + Gen.FinishCache(Progress); } else { @@ -965,6 +1024,9 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, if (BuildCache(Gen,Progress,CurrentSize,TotalSize, Files.begin()+EndOfSource,Files.end()) == false) return false; + + // FIXME: move me to a better place + Gen.FinishCache(Progress); } if (_error->PendingError() == true) @@ -1010,7 +1072,10 @@ bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap) if (BuildCache(Gen,Progress,CurrentSize,TotalSize, Files.begin()+EndOfSource,Files.end()) == false) return false; - + + // FIXME: move me to a better place + Gen.FinishCache(Progress); + if (_error->PendingError() == true) return false; *OutMap = Map.UnGuard(); diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 4a2419b24..53f09b991 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -55,6 +55,9 @@ class pkgCacheGenerator /*{{{*/ bool NewPackage(pkgCache::PkgIterator &Pkg,const string &Name, const string &Arch); bool NewFileVer(pkgCache::VerIterator &Ver,ListParser &List); bool NewFileDesc(pkgCache::DescIterator &Desc,ListParser &List); + bool NewDepends(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver, + string const &Version, unsigned int const &Op, + unsigned int const &Type, map_ptrloc *OldDepLast); unsigned long NewVersion(pkgCache::VerIterator &Ver,const string &VerStr,unsigned long Next); map_ptrloc NewDescription(pkgCache::DescIterator &Desc,const string &Lang,const MD5SumValue &md5sum,map_ptrloc Next); @@ -73,7 +76,8 @@ class pkgCacheGenerator /*{{{*/ bool HasFileDeps() {return FoundFileDeps;}; bool MergeFileProvides(ListParser &List); - + bool FinishCache(OpProgress &Progress); + pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress); ~pkgCacheGenerator(); }; -- cgit v1.2.3-70-g09d2 From 857e9c13d8d9808fcd1ac8ff3469f6c0b90b7fea Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 7 Feb 2010 12:38:13 +0100 Subject: Drop the Arch information from the Version structure as we can get the information from the parent package now --- apt-pkg/cacheiterators.h | 2 +- apt-pkg/deb/deblistparser.cc | 10 +++++++--- apt-pkg/deb/deblistparser.h | 1 + apt-pkg/pkgcache.cc | 4 ++-- apt-pkg/pkgcache.h | 1 - apt-pkg/pkgcachegen.h | 1 + 6 files changed, 12 insertions(+), 7 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 33d2ed6be..d8e044f88 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -178,7 +178,7 @@ class pkgCache::VerIterator : public Iterator { // Accessors inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;}; inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;}; - inline const char *Arch() const {return S->Arch == 0?0:Owner->StrP + S->Arch;}; + inline const char *Arch() const {return S->ParentPkg == 0?0:Owner->StrP + ParentPkg()->Arch;}; inline PkgIterator ParentPkg() const {return PkgIterator(*Owner,Owner->PkgP + S->ParentPkg);}; inline DescIterator DescriptionList() const; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index f683de423..26841d3d2 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -83,6 +83,13 @@ string debListParser::Architecture() { return Result; } /*}}}*/ +// ListParser::ArchitectureAll /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool debListParser::ArchitectureAll() { + return Section.FindS("Architecture") == "all"; +} + /*}}}*/ // ListParser::Version - Return the version string /*{{{*/ // --------------------------------------------------------------------- /* This is to return the string describing the version in debian form, @@ -101,9 +108,6 @@ bool debListParser::NewVersion(pkgCache::VerIterator Ver) // Parse the section Ver->Section = UniqFindTagWrite("Section"); - // Parse the architecture - Ver->Arch = WriteUniqString(Architecture()); - // Parse multi-arch if (Section.FindS("Architecture") == "all") /* Arch all packages can't have a Multi-Arch field, diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index cba4f8e5d..8da051530 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -47,6 +47,7 @@ class debListParser : public pkgCacheGenerator::ListParser // These all operate against the current section virtual string Package(); virtual string Architecture(); + virtual bool ArchitectureAll(); virtual string Version(); virtual bool NewVersion(pkgCache::VerIterator Ver); virtual string Description(); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 1e4c7edb3..7d98869ea 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -399,7 +399,7 @@ const char * pkgCache::PkgIterator::CandVersion() const { //TargetVer is empty, so don't use it. - VerIterator version = pkgPolicy::pkgPolicy(Owner).GetCandidateVer(*this); + VerIterator version = pkgPolicy(Owner).GetCandidateVer(*this); if (version.IsGood()) return version.VerStr(); return 0; @@ -727,7 +727,7 @@ string pkgCache::VerIterator::RelStr() Res += File.Site(); } } - if (S->Arch != 0) + if (S->ParentPkg != 0) Res.append(" [").append(Arch()).append("]"); return Res; } diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 5f50001d0..5edeedfd1 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -276,7 +276,6 @@ struct pkgCache::Version /*{{{*/ { map_ptrloc VerStr; // Stringtable map_ptrloc Section; // StringTable (StringItem) - map_ptrloc Arch; // StringTable enum {None, All, Foreign, Same, Allowed} MultiArch; // Lists diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 53f09b991..ca5d74a9f 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -112,6 +112,7 @@ class pkgCacheGenerator::ListParser // These all operate against the current section virtual string Package() = 0; virtual string Architecture() = 0; + virtual bool ArchitectureAll() = 0; virtual string Version() = 0; virtual bool NewVersion(pkgCache::VerIterator Ver) = 0; virtual string Description() = 0; -- cgit v1.2.3-70-g09d2 From 67e0766f0eab85ce1aeacee75fc6b200f36996c9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 10 Feb 2010 22:45:58 +0100 Subject: Foreign Versions add an implicit Provides to the other packages in the group --- apt-pkg/deb/deblistparser.cc | 55 +++++++++++++++++++++++++++++--------------- apt-pkg/pkgcachegen.cc | 7 +++--- apt-pkg/pkgcachegen.h | 4 ++-- 3 files changed, 42 insertions(+), 24 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 26841d3d2..b6082cdd5 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -618,29 +618,46 @@ bool debListParser::ParseProvides(pkgCache::VerIterator Ver) { const char *Start; const char *Stop; - if (Section.Find("Provides",Start,Stop) == false) - return true; - - string Package; - string Version; - unsigned int Op; - - while (1) + if (Section.Find("Provides",Start,Stop) == true) { - Start = ParseDepends(Start,Stop,Package,Version,Op); - if (Start == 0) - return _error->Error("Problem parsing Provides line"); - if (Op != pkgCache::Dep::NoOp) { - _error->Warning("Ignoring Provides line with DepCompareOp for package %s", Package.c_str()); - } else { - if (NewProvides(Ver,Package,Version) == false) - return false; + string Package; + string Version; + string const Arch = Ver.Arch(); + unsigned int Op; + + while (1) + { + Start = ParseDepends(Start,Stop,Package,Version,Op); + if (Start == 0) + return _error->Error("Problem parsing Provides line"); + if (Op != pkgCache::Dep::NoOp) { + _error->Warning("Ignoring Provides line with DepCompareOp for package %s", Package.c_str()); + } else { + if (NewProvides(Ver, Package, Arch, Version) == false) + return false; + } + + if (Start == Stop) + break; } + } - if (Start == Stop) - break; + if (Ver->MultiArch != pkgCache::Version::Foreign) + return true; + + std::vector const archs = APT::Configuration::getArchitectures(); + if (archs.size() <= 1) + return true; + + string const Package = Ver.ParentPkg().Name(); + string const Version = Ver.VerStr(); + for (std::vector::const_iterator a = archs.begin(); + a != archs.end(); ++a) + { + if (NewProvides(Ver, Package, *a, Version) == false) + return false; } - + return true; } /*}}}*/ diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 037c0cb63..ebda325f7 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -668,13 +668,14 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, // --------------------------------------------------------------------- /* */ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, - const string &PackageName, + const string &PkgName, + const string &PkgArch, const string &Version) { pkgCache &Cache = Owner->Cache; // We do not add self referencing provides - if (unlikely(Ver.ParentPkg().Name() == PackageName)) + if (Ver.ParentPkg().Name() == PkgName && PkgArch == Ver.Arch()) return true; // Get a structure @@ -693,7 +694,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, // Locate the target package pkgCache::PkgIterator Pkg; - if (unlikely(Owner->NewPackage(Pkg,PackageName,string(Ver.Arch())) == false)) + if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false)) return false; // Link it to the package diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index ca5d74a9f..46d0cd893 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -104,8 +104,8 @@ class pkgCacheGenerator::ListParser bool NewDepends(pkgCache::VerIterator Ver,const string &Package, const string &Arch, const string &Version,unsigned int Op, unsigned int Type); - bool NewProvides(pkgCache::VerIterator Ver,const string &Package, - const string &Version); + bool NewProvides(pkgCache::VerIterator Ver,const string &PkgName, + const string &PkgArch, const string &Version); public: -- cgit v1.2.3-70-g09d2 From 2e5f4e45f593535e2c88181ff7a9e2d32a5e60f9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 4 Jun 2010 14:43:03 +0200 Subject: * apt-pkg/cachefile.{cc,h}: - split Open() into submethods to be able to build only parts - make the OpProgress optional in the Cache buildprocess --- apt-pkg/cachefile.cc | 50 ++++++++++++++++++++++++++++++++------------- apt-pkg/cachefile.h | 15 ++++++++------ apt-pkg/deb/debindexfile.cc | 18 ++++++++-------- apt-pkg/deb/debindexfile.h | 6 +++--- apt-pkg/indexfile.h | 8 ++++++-- apt-pkg/pkgcachegen.cc | 45 +++++++++++++++++++++++++--------------- apt-pkg/pkgcachegen.h | 10 +++++++-- debian/changelog | 3 +++ 8 files changed, 103 insertions(+), 52 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc index 790312dc8..b0f8bc424 100644 --- a/apt-pkg/cachefile.cc +++ b/apt-pkg/cachefile.cc @@ -46,7 +46,7 @@ pkgCacheFile::~pkgCacheFile() // CacheFile::BuildCaches - Open and build the cache files /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgCacheFile::BuildCaches(OpProgress &Progress,bool WithLock) +bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock) { const bool ErrorWasEmpty = _error->empty(); if (WithLock == true) @@ -65,8 +65,9 @@ bool pkgCacheFile::BuildCaches(OpProgress &Progress,bool WithLock) return _error->Error(_("The list of sources could not be read.")); // Read the caches - bool Res = pkgMakeStatusCache(List,Progress,&Map,!WithLock); - Progress.Done(); + bool Res = pkgCacheGenerator::MakeStatusCache(List,Progress,&Map,!WithLock); + if (Progress != NULL) + Progress->Done(); if (Res == false) return _error->Error(_("The package lists or status file could not be parsed or opened.")); @@ -80,29 +81,50 @@ bool pkgCacheFile::BuildCaches(OpProgress &Progress,bool WithLock) return true; } /*}}}*/ -// CacheFile::Open - Open the cache files, creating if necessary /*{{{*/ +// CacheFile::BuildPolicy - Open and build all relevant preferences /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgCacheFile::Open(OpProgress &Progress,bool WithLock) +bool pkgCacheFile::BuildPolicy(OpProgress *Progress) { - if (BuildCaches(Progress,WithLock) == false) - return false; - - // The policy engine Policy = new pkgPolicy(Cache); if (_error->PendingError() == true) return false; if (ReadPinFile(*Policy) == false || ReadPinDir(*Policy) == false) return false; - - // Create the dependency cache + + return true; +} + /*}}}*/ +// CacheFile::BuildDepCache - Open and build the dependency cache /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheFile::BuildDepCache(OpProgress *Progress) +{ DCache = new pkgDepCache(Cache,Policy); if (_error->PendingError() == true) return false; - - DCache->Init(&Progress); - Progress.Done(); + + DCache->Init(Progress); + return true; +} + /*}}}*/ +// CacheFile::Open - Open the cache files, creating if necessary /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool pkgCacheFile::Open(OpProgress *Progress, bool WithLock) +{ + if (BuildCaches(Progress,WithLock) == false) + return false; + + if (BuildPolicy(Progress) == false) + return false; + + if (BuildDepCache(Progress) == false) + return false; + + if (Progress != NULL) + Progress->Done(); if (_error->PendingError() == true) return false; diff --git a/apt-pkg/cachefile.h b/apt-pkg/cachefile.h index 3b057951c..b75a7dbe4 100644 --- a/apt-pkg/cachefile.h +++ b/apt-pkg/cachefile.h @@ -30,11 +30,10 @@ class pkgCacheFile MMap *Map; pkgCache *Cache; pkgDepCache *DCache; - + pkgPolicy *Policy; + public: - pkgPolicy *Policy; - // We look pretty much exactly like a pointer to a dep cache inline operator pkgCache &() {return *Cache;}; inline operator pkgCache *() {return Cache;}; @@ -45,12 +44,16 @@ class pkgCacheFile inline pkgDepCache::StateCache &operator [](pkgCache::PkgIterator const &I) {return (*DCache)[I];}; inline unsigned char &operator [](pkgCache::DepIterator const &I) {return (*DCache)[I];}; - bool BuildCaches(OpProgress &Progress,bool WithLock = true); - bool Open(OpProgress &Progress,bool WithLock = true); + bool BuildCaches(OpProgress *Progress = NULL,bool WithLock = true); + __deprecated bool BuildCaches(OpProgress &Progress,bool const &WithLock = true) { return BuildCaches(&Progress, WithLock); }; + bool BuildPolicy(OpProgress *Progress = NULL); + bool BuildDepCache(OpProgress *Progress = NULL); + bool Open(OpProgress *Progress = NULL, bool WithLock = true); + __deprecated bool Open(OpProgress &Progress,bool const &WithLock = true) { return Open(&Progress, WithLock); }; void Close(); pkgCacheFile(); - ~pkgCacheFile(); + virtual ~pkgCacheFile(); }; #endif diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index b89429d86..6d9e99497 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -262,15 +262,15 @@ unsigned long debPackagesIndex::Size() const // PackagesIndex::Merge - Load the index file into a cache /*{{{*/ // --------------------------------------------------------------------- /* */ -bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const +bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const { string PackageFile = IndexFile("Packages"); FileFd Pkg(PackageFile,FileFd::ReadOnly); debListParser Parser(&Pkg, Architecture); if (_error->PendingError() == true) return _error->Error("Problem opening %s",PackageFile.c_str()); - - Prog.SubProgress(0,Info("Packages")); + if (Prog != NULL) + Prog->SubProgress(0,Info("Packages")); ::URI Tmp(URI); if (Gen.SelectFile(PackageFile,Tmp.Host,*this) == false) return _error->Error("Problem with SelectFile %s",PackageFile.c_str()); @@ -445,7 +445,7 @@ unsigned long debTranslationsIndex::Size() const // TranslationsIndex::Merge - Load the index file into a cache /*{{{*/ // --------------------------------------------------------------------- /* */ -bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const +bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const { // Check the translation file, if in use string TranslationFile = IndexFile(Language); @@ -456,7 +456,8 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const if (_error->PendingError() == true) return false; - Prog.SubProgress(0, Info(TranslationFile.c_str())); + if (Prog != NULL) + Prog->SubProgress(0, Info(TranslationFile.c_str())); if (Gen.SelectFile(TranslationFile,string(),*this) == false) return _error->Error("Problem with SelectFile %s",TranslationFile.c_str()); @@ -529,7 +530,7 @@ unsigned long debStatusIndex::Size() const // StatusIndex::Merge - Load the index file into a cache /*{{{*/ // --------------------------------------------------------------------- /* */ -bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const +bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const { FileFd Pkg(File,FileFd::ReadOnly); if (_error->PendingError() == true) @@ -537,8 +538,9 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const debListParser Parser(&Pkg); if (_error->PendingError() == true) return false; - - Prog.SubProgress(0,File); + + if (Prog != NULL) + Prog->SubProgress(0,File); if (Gen.SelectFile(File,string(),*this,pkgCache::Flag::NotSource) == false) return _error->Error("Problem with SelectFile %s",File.c_str()); diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index 766e8b214..b5085992d 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -35,7 +35,7 @@ class debStatusIndex : public pkgIndexFile virtual bool Exists() const; virtual bool HasPackages() const {return true;}; virtual unsigned long Size() const; - virtual bool Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const; + virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; debStatusIndex(string File); @@ -67,7 +67,7 @@ class debPackagesIndex : public pkgIndexFile virtual bool Exists() const; virtual bool HasPackages() const {return true;}; virtual unsigned long Size() const; - virtual bool Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const; + virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; debPackagesIndex(string const &URI, string const &Dist, string const &Section, @@ -99,7 +99,7 @@ class debTranslationsIndex : public pkgIndexFile virtual bool Exists() const; virtual bool HasPackages() const; virtual unsigned long Size() const; - virtual bool Merge(pkgCacheGenerator &Gen,OpProgress &Prog) const; + virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; debTranslationsIndex(string URI,string Dist,string Section, char const * const Language); diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h index 3cc501629..2b5ae6342 100644 --- a/apt-pkg/indexfile.h +++ b/apt-pkg/indexfile.h @@ -74,8 +74,12 @@ class pkgIndexFile virtual bool Exists() const = 0; virtual bool HasPackages() const = 0; virtual unsigned long Size() const = 0; - virtual bool Merge(pkgCacheGenerator &/*Gen*/,OpProgress &/*Prog*/) const {return false;}; - virtual bool MergeFileProvides(pkgCacheGenerator &/*Gen*/,OpProgress &/*Prog*/) const {return true;}; + virtual bool Merge(pkgCacheGenerator &/*Gen*/,OpProgress* /*Prog*/) const { return false; }; + __deprecated virtual bool Merge(pkgCacheGenerator &Gen, OpProgress &Prog) const + { return Merge(Gen, &Prog); }; + virtual bool MergeFileProvides(pkgCacheGenerator &/*Gen*/,OpProgress* /*Prog*/) const {return true;}; + __deprecated virtual bool MergeFileProvides(pkgCacheGenerator &Gen, OpProgress &Prog) const + {return MergeFileProvides(Gen, &Prog);}; virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; static bool TranslationsAvailable(); diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index d96d3370f..5649cd6f8 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -526,7 +526,7 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, // CacheGenerator::FinishCache - do various finish operations /*{{{*/ // --------------------------------------------------------------------- /* This prepares the Cache for delivery */ -bool pkgCacheGenerator::FinishCache(OpProgress &Progress) +bool pkgCacheGenerator::FinishCache(OpProgress *Progress) { // FIXME: add progress reporting for this operation // Do we have different architectures in your groups ? @@ -923,7 +923,7 @@ static unsigned long ComputeSize(FileIterator Start,FileIterator End) // --------------------------------------------------------------------- /* */ static bool BuildCache(pkgCacheGenerator &Gen, - OpProgress &Progress, + OpProgress *Progress, unsigned long &CurrentSize,unsigned long TotalSize, FileIterator Start, FileIterator End) { @@ -944,7 +944,8 @@ static bool BuildCache(pkgCacheGenerator &Gen, } unsigned long Size = (*I)->Size(); - Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); + if (Progress != NULL) + Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); CurrentSize += Size; if ((*I)->Merge(Gen,Progress) == false) @@ -953,13 +954,15 @@ static bool BuildCache(pkgCacheGenerator &Gen, if (Gen.HasFileDeps() == true) { - Progress.Done(); + if (Progress != NULL) + Progress->Done(); TotalSize = ComputeSize(Start, End); CurrentSize = 0; for (I = Start; I != End; I++) { unsigned long Size = (*I)->Size(); - Progress.OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides")); + if (Progress != NULL) + Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides")); CurrentSize += Size; if ((*I)->MergeFileProvides(Gen,Progress) == false) return false; @@ -969,7 +972,7 @@ static bool BuildCache(pkgCacheGenerator &Gen, return true; } /*}}}*/ -// MakeStatusCache - Construct the status cache /*{{{*/ +// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ // --------------------------------------------------------------------- /* This makes sure that the status cache (the cache that has all index files from the sources list and all local ones) is ready @@ -977,7 +980,10 @@ static bool BuildCache(pkgCacheGenerator &Gen, the cache will be stored there. This is pretty much mandetory if you are using AllowMem. AllowMem lets the function be run as non-root where it builds the cache 'fast' into a memory buffer. */ -bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, +__deprecated bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, + MMap **OutMap, bool AllowMem) + { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); } +bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress, MMap **OutMap,bool AllowMem) { bool const Debug = _config->FindB("Debug::pkgCacheGen", false); @@ -1028,13 +1034,15 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, if (Writeable == false && AllowMem == false && CacheFile.empty() == false) return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str()); - - Progress.OverallProgress(0,1,1,_("Reading package lists")); - + + if (Progress != NULL) + Progress->OverallProgress(0,1,1,_("Reading package lists")); + // Cache is OK, Fin. if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true) { - Progress.OverallProgress(1,1,1,_("Reading package lists")); + if (Progress != NULL) + Progress->OverallProgress(1,1,1,_("Reading package lists")); if (Debug == true) std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl; return true; @@ -1084,7 +1092,7 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end()); // Build the status cache - pkgCacheGenerator Gen(Map.Get(),&Progress); + pkgCacheGenerator Gen(Map.Get(),Progress); if (_error->PendingError() == true) return false; if (BuildCache(Gen,Progress,CurrentSize,TotalSize, @@ -1101,7 +1109,7 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, TotalSize = ComputeSize(Files.begin(),Files.end()); // Build the source cache - pkgCacheGenerator Gen(Map.Get(),&Progress); + pkgCacheGenerator Gen(Map.Get(),Progress); if (_error->PendingError() == true) return false; if (BuildCache(Gen,Progress,CurrentSize,TotalSize, @@ -1160,10 +1168,12 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, return true; } /*}}}*/ -// MakeOnlyStatusCache - Build a cache with just the status files /*{{{*/ +// CacheGenerator::MakeOnlyStatusCache - Build only a status files cache/*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap) +__deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap) + { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); } +bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap) { unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024); vector Files; @@ -1178,8 +1188,9 @@ bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap) TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end()); // Build the status cache - Progress.OverallProgress(0,1,1,_("Reading package lists")); - pkgCacheGenerator Gen(Map.Get(),&Progress); + if (Progress != NULL) + Progress->OverallProgress(0,1,1,_("Reading package lists")); + pkgCacheGenerator Gen(Map.Get(),Progress); if (_error->PendingError() == true) return false; if (BuildCache(Gen,Progress,CurrentSize,TotalSize, diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 46d0cd893..ff0941e0c 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -76,7 +76,11 @@ class pkgCacheGenerator /*{{{*/ bool HasFileDeps() {return FoundFileDeps;}; bool MergeFileProvides(ListParser &List); - bool FinishCache(OpProgress &Progress); + bool FinishCache(OpProgress *Progress); + + static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress, + MMap **OutMap = 0,bool AllowMem = false); + static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress); ~pkgCacheGenerator(); @@ -134,10 +138,12 @@ class pkgCacheGenerator::ListParser virtual ~ListParser() {}; }; /*}}}*/ + bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, MMap **OutMap = 0,bool AllowMem = false); bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap); + #ifdef APT_COMPATIBILITY #if APT_COMPATIBILITY != 986 #warning "Using APT_COMPATIBILITY" @@ -145,7 +151,7 @@ bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutMap); MMap *pkgMakeStatusCacheMem(pkgSourceList &List,OpProgress &Progress) { MMap *Map = 0; - if (pkgMakeStatusCache(List,Progress,&Map,true) == false) + if (pkgCacheGenerator::MakeStatusCache(List,&Progress,&Map,true) == false) return 0; return Map; } diff --git a/debian/changelog b/debian/changelog index 994be2bbc..097916d1e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -32,6 +32,9 @@ apt (0.7.26~exp6) UNRELEASED; urgency=low - get the best matching arch package from a group with FindPreferredPkg * cmdline/apt-cache.cc: - make the search multiarch compatible by using GrpIterator instead + * apt-pkg/cachefile.{cc,h}: + - split Open() into submethods to be able to build only parts + - make the OpProgress optional in the Cache buildprocess -- David Kalnischkies Mon, 31 May 2010 22:36:35 +0200 -- cgit v1.2.3-70-g09d2 From 7e58ab0c0db9e5f27ae91251bf692bf79a046534 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 6 Jul 2010 10:21:45 +0200 Subject: wrap the mmap actions in the CacheGenerator in their own methods to be able to react on condition changes later then we can move mmap --- apt-pkg/pkgcachegen.cc | 53 ++++++++++++++++++++++++++++++++------------------ apt-pkg/pkgcachegen.h | 12 ++++++++---- 2 files changed, 42 insertions(+), 23 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 05c01494b..6a9da4a92 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -61,8 +61,8 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : // Starting header *Cache.HeaderP = pkgCache::Header(); - Cache.HeaderP->VerSysName = Map.WriteString(_system->VS->Label); - Cache.HeaderP->Architecture = Map.WriteString(_config->Find("APT::Architecture")); + Cache.HeaderP->VerSysName = WriteStringInMap(_system->VS->Label); + Cache.HeaderP->Architecture = WriteStringInMap(_config->Find("APT::Architecture")); Cache.ReMap(); } else @@ -96,6 +96,21 @@ pkgCacheGenerator::~pkgCacheGenerator() Map.Sync(0,sizeof(pkgCache::Header)); } /*}}}*/ +// CacheGenerator::WriteStringInMap /*{{{*/ +unsigned long pkgCacheGenerator::WriteStringInMap(const char *String, + const unsigned long &Len) { + return Map.WriteString(String, Len); +} + /*}}}*/ +// CacheGenerator::WriteStringInMap /*{{{*/ +unsigned long pkgCacheGenerator::WriteStringInMap(const char *String) { + return Map.WriteString(String); +} + /*}}}*/ +unsigned long pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ + return Map.Allocate(size); +} + /*}}}*/ // CacheGenerator::MergeList - Merge the package list /*{{{*/ // --------------------------------------------------------------------- /* This provides the generation of the entries in the cache. Each loop @@ -342,12 +357,12 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) return true; // Get a structure - unsigned long const Group = Map.Allocate(sizeof(pkgCache::Group)); + unsigned long const Group = AllocateInMap(sizeof(pkgCache::Group)); if (unlikely(Group == 0)) return false; Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group); - Grp->Name = Map.WriteString(Name); + Grp->Name = WriteStringInMap(Name); if (unlikely(Grp->Name == 0)) return false; @@ -374,7 +389,7 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name return true; // Get a structure - unsigned long const Package = Map.Allocate(sizeof(pkgCache::Package)); + unsigned long const Package = AllocateInMap(sizeof(pkgCache::Package)); if (unlikely(Package == 0)) return false; Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package); @@ -418,7 +433,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, return true; // Get a structure - unsigned long VerFile = Map.Allocate(sizeof(pkgCache::VerFile)); + unsigned long VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); if (VerFile == 0) return 0; @@ -449,7 +464,7 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, unsigned long Next) { // Get a structure - unsigned long Version = Map.Allocate(sizeof(pkgCache::Version)); + unsigned long Version = AllocateInMap(sizeof(pkgCache::Version)); if (Version == 0) return 0; @@ -457,7 +472,7 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version); Ver->NextVer = Next; Ver->ID = Cache.HeaderP->VersionCount++; - Ver->VerStr = Map.WriteString(VerStr); + Ver->VerStr = WriteStringInMap(VerStr); if (Ver->VerStr == 0) return 0; @@ -474,7 +489,7 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, return true; // Get a structure - unsigned long DescFile = Map.Allocate(sizeof(pkgCache::DescFile)); + unsigned long DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); if (DescFile == 0) return false; @@ -507,7 +522,7 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, map_ptrloc Next) { // Get a structure - map_ptrloc Description = Map.Allocate(sizeof(pkgCache::Description)); + map_ptrloc Description = AllocateInMap(sizeof(pkgCache::Description)); if (Description == 0) return 0; @@ -515,8 +530,8 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description); Desc->NextDesc = Next; Desc->ID = Cache.HeaderP->DescriptionCount++; - Desc->language_code = Map.WriteString(Lang); - Desc->md5sum = Map.WriteString(md5sum.Value()); + Desc->language_code = WriteStringInMap(Lang); + Desc->md5sum = WriteStringInMap(md5sum.Value()); if (Desc->language_code == 0 || Desc->md5sum == 0) return 0; @@ -607,7 +622,7 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, map_ptrloc *OldDepLast) { // Get a structure - unsigned long const Dependency = Map.Allocate(sizeof(pkgCache::Dependency)); + unsigned long const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); if (unlikely(Dependency == 0)) return false; @@ -625,7 +640,7 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, if (I->Version != 0 && I.TargetVer() == Version) Dep->Version = I->Version;*/ if (Dep->Version == 0) - if (unlikely((Dep->Version = Map.WriteString(Version)) == 0)) + if (unlikely((Dep->Version = WriteStringInMap(Version)) == 0)) return false; } @@ -699,7 +714,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, return true; // Get a structure - unsigned long const Provides = Owner->Map.Allocate(sizeof(pkgCache::Provides)); + unsigned long const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); if (unlikely(Provides == 0)) return false; Cache.HeaderP->ProvidesCount++; @@ -734,12 +749,12 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, unsigned long Flags) { // Get some space for the structure - CurrentFile = Cache.PkgFileP + Map.Allocate(sizeof(*CurrentFile)); + CurrentFile = Cache.PkgFileP + AllocateInMap(sizeof(*CurrentFile)); if (CurrentFile == Cache.PkgFileP) return false; // Fill it in - CurrentFile->FileName = Map.WriteString(File); + CurrentFile->FileName = WriteStringInMap(File); CurrentFile->Site = WriteUniqString(Site); CurrentFile->NextFile = Cache.HeaderP->FileList; CurrentFile->Flags = Flags; @@ -791,7 +806,7 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, } // Get a structure - unsigned long Item = Map.Allocate(sizeof(pkgCache::StringItem)); + unsigned long Item = AllocateInMap(sizeof(pkgCache::StringItem)); if (Item == 0) return 0; @@ -799,7 +814,7 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, pkgCache::StringItem *ItemP = Cache.StringItemP + Item; ItemP->NextItem = I - Cache.StringItemP; *Last = Item; - ItemP->String = Map.WriteString(S,Size); + ItemP->String = WriteStringInMap(S,Size); if (ItemP->String == 0) return 0; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index ff0941e0c..a88c49451 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -31,9 +31,13 @@ class pkgIndexFile; class pkgCacheGenerator /*{{{*/ { private: - + pkgCache::StringItem *UniqHash[26]; - + unsigned long WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; + unsigned long WriteStringInMap(const char *String); + unsigned long WriteStringInMap(const char *String, const unsigned long &Len); + unsigned long AllocateInMap(const unsigned long &size); + public: class ListParser; @@ -103,8 +107,8 @@ class pkgCacheGenerator::ListParser inline unsigned long WriteUniqString(string S) {return Owner->WriteUniqString(S);}; inline unsigned long WriteUniqString(const char *S,unsigned int Size) {return Owner->WriteUniqString(S,Size);}; - inline unsigned long WriteString(const string &S) {return Owner->Map.WriteString(S);}; - inline unsigned long WriteString(const char *S,unsigned int Size) {return Owner->Map.WriteString(S,Size);}; + inline unsigned long WriteString(const string &S) {return Owner->WriteStringInMap(S);}; + inline unsigned long WriteString(const char *S,unsigned int Size) {return Owner->WriteStringInMap(S,Size);}; bool NewDepends(pkgCache::VerIterator Ver,const string &Package, const string &Arch, const string &Version,unsigned int Op, unsigned int Type); -- cgit v1.2.3-70-g09d2 From 32b9a14cb4c6bdcddfe84c4451c225ced1a34bb7 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 8 Jul 2010 11:10:46 +0200 Subject: use references instead of copies in the Cache generation methods --- apt-pkg/deb/deblistparser.cc | 16 ++++++++-------- apt-pkg/deb/deblistparser.h | 14 +++++++------- apt-pkg/pkgcachegen.cc | 9 +++++---- apt-pkg/pkgcachegen.h | 12 ++++++------ 4 files changed, 26 insertions(+), 25 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 24df57a5c..2cfeb23e9 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -106,7 +106,7 @@ string debListParser::Version() // ListParser::NewVersion - Fill in the version structure /*{{{*/ // --------------------------------------------------------------------- /* */ -bool debListParser::NewVersion(pkgCache::VerIterator Ver) +bool debListParser::NewVersion(pkgCache::VerIterator &Ver) { // Parse the section Ver->Section = UniqFindTagWrite("Section"); @@ -251,8 +251,8 @@ MD5SumValue debListParser::Description_md5() // --------------------------------------------------------------------- /* This is called to update the package with any new information that might be found in the section */ -bool debListParser::UsePackage(pkgCache::PkgIterator Pkg, - pkgCache::VerIterator Ver) +bool debListParser::UsePackage(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) { if (Pkg->Section == 0) Pkg->Section = UniqFindTagWrite("Section"); @@ -332,8 +332,8 @@ unsigned short debListParser::VersionHash() Some of the above are obsolete (I think?) flag = hold-* and status = post-inst-failed, removal-failed at least. */ -bool debListParser::ParseStatus(pkgCache::PkgIterator Pkg, - pkgCache::VerIterator Ver) +bool debListParser::ParseStatus(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) { const char *Start; const char *Stop; @@ -634,7 +634,7 @@ const char *debListParser::ParseDepends(const char *Start,const char *Stop, // --------------------------------------------------------------------- /* This is the higher level depends parser. It takes a tag and generates a complete depends tree for the given version. */ -bool debListParser::ParseDepends(pkgCache::VerIterator Ver, +bool debListParser::ParseDepends(pkgCache::VerIterator &Ver, const char *Tag,unsigned int Type) { const char *Start; @@ -674,7 +674,7 @@ bool debListParser::ParseDepends(pkgCache::VerIterator Ver, // ListParser::ParseProvides - Parse the provides list /*{{{*/ // --------------------------------------------------------------------- /* */ -bool debListParser::ParseProvides(pkgCache::VerIterator Ver) +bool debListParser::ParseProvides(pkgCache::VerIterator &Ver) { const char *Start; const char *Stop; @@ -779,7 +779,7 @@ bool debListParser::Step() // ListParser::LoadReleaseInfo - Load the release information /*{{{*/ // --------------------------------------------------------------------- /* */ -bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator FileI, +bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, FileFd &File, string component) { pkgTagFile Tags(&File, File.Size() + 256); // XXX diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index 6c81d9fa0..4bc1bd93c 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -36,10 +36,10 @@ class debListParser : public pkgCacheGenerator::ListParser bool MultiArchEnabled; unsigned long UniqFindTagWrite(const char *Tag); - bool ParseStatus(pkgCache::PkgIterator Pkg,pkgCache::VerIterator Ver); - bool ParseDepends(pkgCache::VerIterator Ver,const char *Tag, + bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver); + bool ParseDepends(pkgCache::VerIterator &Ver,const char *Tag, unsigned int Type); - bool ParseProvides(pkgCache::VerIterator Ver); + bool ParseProvides(pkgCache::VerIterator &Ver); static bool GrabWord(string Word,WordList *List,unsigned char &Out); public: @@ -51,19 +51,19 @@ class debListParser : public pkgCacheGenerator::ListParser virtual string Architecture(); virtual bool ArchitectureAll(); virtual string Version(); - virtual bool NewVersion(pkgCache::VerIterator Ver); + virtual bool NewVersion(pkgCache::VerIterator &Ver); virtual string Description(); virtual string DescriptionLanguage(); virtual MD5SumValue Description_md5(); virtual unsigned short VersionHash(); - virtual bool UsePackage(pkgCache::PkgIterator Pkg, - pkgCache::VerIterator Ver); + virtual bool UsePackage(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver); virtual unsigned long Offset() {return iOffset;}; virtual unsigned long Size() {return Section.size();}; virtual bool Step(); - bool LoadReleaseInfo(pkgCache::PkgFileIterator FileI,FileFd &File, + bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File, string section); static const char *ParseDepends(const char *Start,const char *Stop, diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 6a9da4a92..62e457734 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -157,13 +157,14 @@ bool pkgCacheGenerator::MergeList(ListParser &List, // we first process the package, then the descriptions // (this has the bonus that we get MMap error when we run out // of MMap space) - if (List.UsePackage(Pkg,pkgCache::VerIterator(Cache)) == false) + pkgCache::VerIterator Ver(Cache); + if (List.UsePackage(Pkg, Ver) == false) return _error->Error(_("Error occurred while processing %s (UsePackage1)"), PackageName.c_str()); // Find the right version to write the description MD5SumValue CurMd5 = List.Description_md5(); - pkgCache::VerIterator Ver = Pkg.VersionList(); + Ver = Pkg.VersionList(); map_ptrloc *LastVer = &Pkg->VersionList; for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) @@ -668,7 +669,7 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, // --------------------------------------------------------------------- /* This creates a Group and the Package to link this dependency to if needed and handles also the caching of the old endpoint */ -bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, +bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver, const string &PackageName, const string &Arch, const string &Version, @@ -702,7 +703,7 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator Ver, // ListParser::NewProvides - Create a Provides element /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator Ver, +bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, const string &PkgName, const string &PkgArch, const string &Version) diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index a88c49451..8f7739165 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -109,10 +109,10 @@ class pkgCacheGenerator::ListParser inline unsigned long WriteUniqString(const char *S,unsigned int Size) {return Owner->WriteUniqString(S,Size);}; inline unsigned long WriteString(const string &S) {return Owner->WriteStringInMap(S);}; inline unsigned long WriteString(const char *S,unsigned int Size) {return Owner->WriteStringInMap(S,Size);}; - bool NewDepends(pkgCache::VerIterator Ver,const string &Package, const string &Arch, + bool NewDepends(pkgCache::VerIterator &Ver,const string &Package, const string &Arch, const string &Version,unsigned int Op, unsigned int Type); - bool NewProvides(pkgCache::VerIterator Ver,const string &PkgName, + bool NewProvides(pkgCache::VerIterator &Ver,const string &PkgName, const string &PkgArch, const string &Version); public: @@ -122,13 +122,13 @@ class pkgCacheGenerator::ListParser virtual string Architecture() = 0; virtual bool ArchitectureAll() = 0; virtual string Version() = 0; - virtual bool NewVersion(pkgCache::VerIterator Ver) = 0; + virtual bool NewVersion(pkgCache::VerIterator &Ver) = 0; virtual string Description() = 0; virtual string DescriptionLanguage() = 0; virtual MD5SumValue Description_md5() = 0; virtual unsigned short VersionHash() = 0; - virtual bool UsePackage(pkgCache::PkgIterator Pkg, - pkgCache::VerIterator Ver) = 0; + virtual bool UsePackage(pkgCache::PkgIterator &Pkg, + pkgCache::VerIterator &Ver) = 0; virtual unsigned long Offset() = 0; virtual unsigned long Size() = 0; @@ -136,7 +136,7 @@ class pkgCacheGenerator::ListParser inline bool HasFileDeps() {return FoundFileDeps;}; virtual bool CollectFileProvides(pkgCache &Cache, - pkgCache::VerIterator Ver) {return true;}; + pkgCache::VerIterator &Ver) {return true;}; ListParser() : FoundFileDeps(false) {}; virtual ~ListParser() {}; -- cgit v1.2.3-70-g09d2 From a9fe592842bfa17d91f4904d7fb0e3af3adebb17 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 8 Jul 2010 15:28:53 +0200 Subject: * apt-pkg/pkgcachegen.{cc,h}: - make the used MMap moveable (and therefore dynamic resizeable) by applying (some) mad pointer magic (Closes: #195018) --- apt-pkg/cacheiterators.h | 6 ++ apt-pkg/contrib/mmap.cc | 27 ++++-- apt-pkg/deb/debindexfile.cc | 1 + apt-pkg/pkgcache.cc | 5 +- apt-pkg/pkgcache.h | 2 +- apt-pkg/pkgcachegen.cc | 229 ++++++++++++++++++++++++++++++++------------ apt-pkg/pkgcachegen.h | 30 ++++-- debian/changelog | 3 + 8 files changed, 226 insertions(+), 77 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index dfe5707e1..eb8dee5e3 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -75,6 +75,12 @@ template class pkgCache::Iterator : inline bool IsGood() const { return S && Owner && ! end();}; inline unsigned long Index() const {return S - OwnerPointer();}; + void ReOwn(pkgCache &newOwner, void const * const oldMap, void const * const newMap) { + if (S == 0) + return; + S += (Str*)(newMap) - (Str*)(oldMap); + } + // Constructors - look out for the variable assigning inline Iterator() : S(0), Owner(0) {}; inline Iterator(pkgCache &Owner,Str *T = 0) : S(T), Owner(&Owner) {}; diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index d71066243..aa184b130 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -225,22 +225,22 @@ DynamicMMap::DynamicMMap(unsigned long Flags,unsigned long const &WorkSpace, // disable Moveable if we don't grow if (Grow == 0) - Flags &= ~Moveable; + this->Flags &= ~Moveable; #ifndef __linux__ // kfreebsd doesn't have mremap, so we use the fallback - if ((Flags & Moveable) == Moveable) - Flags |= Fallback; + if ((this->Flags & Moveable) == Moveable) + this->Flags |= Fallback; #endif #ifdef _POSIX_MAPPED_FILES - if ((Flags & Fallback) != Fallback) { + if ((this->Flags & Fallback) != Fallback) { // Set the permissions. int Prot = PROT_READ; int Map = MAP_PRIVATE | MAP_ANONYMOUS; - if ((Flags & ReadOnly) != ReadOnly) + if ((this->Flags & ReadOnly) != ReadOnly) Prot |= PROT_WRITE; - if ((Flags & Public) == Public) + if ((this->Flags & Public) == Public) Map = MAP_SHARED | MAP_ANONYMOUS; // use anonymous mmap() to get the memory @@ -314,7 +314,7 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) // Look for a matching pool entry Pool *I; Pool *Empty = 0; - for (I = Pools; I != Pools + PoolCount; I++) + for (I = Pools; I != Pools + PoolCount; ++I) { if (I->ItemSize == 0) Empty = I; @@ -342,7 +342,11 @@ unsigned long DynamicMMap::Allocate(unsigned long ItemSize) { const unsigned long size = 20*1024; I->Count = size/ItemSize; + Pool* oldPools = Pools; Result = RawAllocate(size,ItemSize); + if (Pools != oldPools) + I += Pools - oldPools; + // Does the allocation failed ? if (Result == 0 && _error->PendingError()) return 0; @@ -365,7 +369,7 @@ unsigned long DynamicMMap::WriteString(const char *String, if (Len == (unsigned long)-1) Len = strlen(String); - unsigned long Result = RawAllocate(Len+1,0); + unsigned long const Result = RawAllocate(Len+1,0); if (Result == 0 && _error->PendingError()) return 0; @@ -395,16 +399,20 @@ bool DynamicMMap::Grow() { return _error->Error(_("Unable to increase the size of the MMap as the " "limit of %lu bytes is already reached."), Limit); - unsigned long const newSize = WorkSpace + 1024*1024; + unsigned long const newSize = WorkSpace + GrowFactor; if(Fd != 0) { Fd->Seek(newSize - 1); char C = 0; Fd->Write(&C,sizeof(C)); } + + unsigned long const poolOffset = Pools - ((Pool*) Base); + if ((Flags & Fallback) != Fallback) { #if defined(_POSIX_MAPPED_FILES) && defined(__linux__) #ifdef MREMAP_MAYMOVE + if ((Flags & Moveable) == Moveable) Base = mremap(Base, WorkSpace, newSize, MREMAP_MAYMOVE); else @@ -425,6 +433,7 @@ bool DynamicMMap::Grow() { return false; } + Pools =(Pool*) Base + poolOffset; WorkSpace = newSize; return true; } diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 6d9e99497..5e6db3f38 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -277,6 +277,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator File = Gen.GetCurFile(); + pkgCacheGenerator::Dynamic DynFile(File); struct stat St; if (fstat(Pkg.Fd(),&St) != 0) return _error->Errno("fstat","Failed to stat"); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 68aa83d0c..8af8ef7de 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -119,7 +119,7 @@ pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map) // Cache::ReMap - Reopen the cache file /*{{{*/ // --------------------------------------------------------------------- /* If the file is already closed then this will open it open it. */ -bool pkgCache::ReMap() +bool pkgCache::ReMap(bool const &Errorchecks) { // Apply the typecasts. HeaderP = (Header *)Map.Data(); @@ -135,6 +135,9 @@ bool pkgCache::ReMap() StringItemP = (StringItem *)Map.Data(); StrP = (char *)Map.Data(); + if (Errorchecks == false) + return true; + if (Map.Size() == 0 || HeaderP == 0) return _error->Error(_("Empty package cache")); diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 426bb9f13..799521784 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -174,7 +174,7 @@ class pkgCache /*{{{*/ StringItem *StringItemP; char *StrP; - virtual bool ReMap(); + virtual bool ReMap(bool const &Errorchecks = true); inline bool Sync() {return Map.Sync();}; inline MMap &GetMap() {return Map;}; inline void *DataEnd() {return ((unsigned char *)Map.Data()) + Map.Size();}; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 62e457734..18bad6727 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -36,6 +36,7 @@ #include /*}}}*/ typedef vector::iterator FileIterator; +template std::set pkgCacheGenerator::Dynamic::toReMap; // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- @@ -61,8 +62,12 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : // Starting header *Cache.HeaderP = pkgCache::Header(); - Cache.HeaderP->VerSysName = WriteStringInMap(_system->VS->Label); - Cache.HeaderP->Architecture = WriteStringInMap(_config->Find("APT::Architecture")); + map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label); + Cache.HeaderP->VerSysName = idxVerSysName; + map_ptrloc const idxArchitecture = WriteStringInMap(_config->Find("APT::Architecture")); + Cache.HeaderP->Architecture = idxArchitecture; + if (unlikely(idxVerSysName == 0 || idxArchitecture == 0)) + return; Cache.ReMap(); } else @@ -96,19 +101,65 @@ pkgCacheGenerator::~pkgCacheGenerator() Map.Sync(0,sizeof(pkgCache::Header)); } /*}}}*/ +void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newMap) {/*{{{*/ + if (oldMap == newMap) + return; + + Cache.ReMap(false); + + CurrentFile += (pkgCache::PackageFile*) newMap - (pkgCache::PackageFile*) oldMap; + + for (size_t i = 0; i < _count(UniqHash); ++i) + if (UniqHash[i] != 0) + UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap; + + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); + for (std::set::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReOwn(Cache, oldMap, newMap); +} /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ -unsigned long pkgCacheGenerator::WriteStringInMap(const char *String, +map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String, const unsigned long &Len) { - return Map.WriteString(String, Len); + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.WriteString(String, Len); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ -unsigned long pkgCacheGenerator::WriteStringInMap(const char *String) { - return Map.WriteString(String); +map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) { + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.WriteString(String); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ -unsigned long pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ - return Map.Allocate(size); +map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ + void const * const oldMap = Map.Data(); + map_ptrloc const index = Map.Allocate(size); + if (index != 0) + ReMap(oldMap, Map.Data()); + return index; } /*}}}*/ // CacheGenerator::MergeList - Merge the package list /*{{{*/ @@ -142,6 +193,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List, { // Get a pointer to the package structure pkgCache::PkgIterator Pkg; + Dynamic DynPkg(Pkg); if (NewPackage(Pkg, PackageName, *arch) == false) return _error->Error(_("Error occurred while processing %s (NewPackage)"),PackageName.c_str()); Counter++; @@ -158,6 +210,7 @@ bool pkgCacheGenerator::MergeList(ListParser &List, // (this has the bonus that we get MMap error when we run out // of MMap space) pkgCache::VerIterator Ver(Cache); + Dynamic DynVer(Ver); if (List.UsePackage(Pkg, Ver) == false) return _error->Error(_("Error occurred while processing %s (UsePackage1)"), PackageName.c_str()); @@ -165,11 +218,11 @@ bool pkgCacheGenerator::MergeList(ListParser &List, // Find the right version to write the description MD5SumValue CurMd5 = List.Description_md5(); Ver = Pkg.VersionList(); - map_ptrloc *LastVer = &Pkg->VersionList; - for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) + for (; Ver.end() == false; ++Ver) { pkgCache::DescIterator Desc = Ver.DescriptionList(); + Dynamic DynDesc(Desc); map_ptrloc *LastDesc = &Ver->DescriptionList; bool duplicate=false; @@ -189,7 +242,11 @@ bool pkgCacheGenerator::MergeList(ListParser &List, if (MD5SumValue(Desc.md5()) == CurMd5) { // Add new description - *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc); + void const * const oldMap = Map.Data(); + map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), CurMd5, *LastDesc); + if (oldMap != Map.Data()) + LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastDesc = descindex; Desc->ParentPkg = Pkg.Index(); if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) @@ -203,7 +260,9 @@ bool pkgCacheGenerator::MergeList(ListParser &List, } pkgCache::VerIterator Ver = Pkg.VersionList(); + Dynamic DynVer(Ver); map_ptrloc *LastVer = &Pkg->VersionList; + void const * oldMap = Map.Data(); int Res = 1; unsigned long const Hash = List.VersionHash(); for (; Ver.end() == false; LastVer = &Ver->NextVer, Ver++) @@ -242,21 +301,28 @@ bool pkgCacheGenerator::MergeList(ListParser &List, } // Add a new version - *LastVer = NewVersion(Ver,Version,*LastVer); + map_ptrloc const verindex = NewVersion(Ver,Version,*LastVer); + if (verindex == 0 && _error->PendingError()) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 1); + + if (oldMap != Map.Data()) + LastVer += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastVer = verindex; Ver->ParentPkg = Pkg.Index(); Ver->Hash = Hash; - if ((*LastVer == 0 && _error->PendingError()) || List.NewVersion(Ver) == false) - return _error->Error(_("Error occurred while processing %s (NewVersion1)"), - PackageName.c_str()); + if (List.NewVersion(Ver) == false) + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 2); if (List.UsePackage(Pkg,Ver) == false) return _error->Error(_("Error occurred while processing %s (UsePackage3)"), PackageName.c_str()); if (NewFileVer(Ver,List) == false) - return _error->Error(_("Error occurred while processing %s (NewVersion2)"), - PackageName.c_str()); + return _error->Error(_("Error occurred while processing %s (NewVersion%d)"), + PackageName.c_str(), 3); // Read only a single record and return if (OutVer != 0) @@ -269,13 +335,18 @@ bool pkgCacheGenerator::MergeList(ListParser &List, /* Record the Description data. Description data always exist in Packages and Translation-* files. */ pkgCache::DescIterator Desc = Ver.DescriptionList(); + Dynamic DynDesc(Desc); map_ptrloc *LastDesc = &Ver->DescriptionList; - + // Skip to the end of description set for (; Desc.end() == false; LastDesc = &Desc->NextDesc, Desc++); // Add new description - *LastDesc = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc); + oldMap = Map.Data(); + map_ptrloc const descindex = NewDescription(Desc, List.DescriptionLanguage(), List.Description_md5(), *LastDesc); + if (oldMap != Map.Data()) + LastDesc += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + *LastDesc = descindex; Desc->ParentPkg = Pkg.Index(); if ((*LastDesc == 0 && _error->PendingError()) || NewFileDesc(Desc,List) == false) @@ -322,6 +393,7 @@ bool pkgCacheGenerator::MergeFileProvides(ListParser &List) continue; pkgCache::PkgIterator Pkg = Cache.FindPkg(PackageName); + Dynamic DynPkg(Pkg); if (Pkg.end() == true) return _error->Error(_("Error occurred while processing %s (FindPkg)"), PackageName.c_str()); @@ -331,6 +403,7 @@ bool pkgCacheGenerator::MergeFileProvides(ListParser &List) unsigned long Hash = List.VersionHash(); pkgCache::VerIterator Ver = Pkg.VersionList(); + Dynamic DynVer(Ver); for (; Ver.end() == false; Ver++) { if (Ver->Hash == Hash && Version.c_str() == Ver.VerStr()) @@ -358,14 +431,15 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) return true; // Get a structure - unsigned long const Group = AllocateInMap(sizeof(pkgCache::Group)); + map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group)); if (unlikely(Group == 0)) return false; Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group); - Grp->Name = WriteStringInMap(Name); - if (unlikely(Grp->Name == 0)) + map_ptrloc const idxName = WriteStringInMap(Name); + if (unlikely(idxName == 0)) return false; + Grp->Name = idxName; // Insert it into the hash table unsigned long const Hash = Cache.Hash(Name); @@ -382,6 +456,7 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name, const string &Arch) { pkgCache::GrpIterator Grp; + Dynamic DynGrp(Grp); if (unlikely(NewGroup(Grp, Name) == false)) return false; @@ -390,7 +465,7 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name return true; // Get a structure - unsigned long const Package = AllocateInMap(sizeof(pkgCache::Package)); + map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package)); if (unlikely(Package == 0)) return false; Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package); @@ -416,9 +491,10 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name // Set the name, arch and the ID Pkg->Name = Grp->Name; Pkg->Group = Grp.Index(); - Pkg->Arch = WriteUniqString(Arch.c_str()); - if (unlikely(Pkg->Arch == 0)) + map_ptrloc const idxArch = WriteUniqString(Arch.c_str()); + if (unlikely(idxArch == 0)) return false; + Pkg->Arch = idxArch; Pkg->ID = Cache.HeaderP->PackageCount++; return true; @@ -434,7 +510,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, return true; // Get a structure - unsigned long VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); + map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); if (VerFile == 0) return 0; @@ -465,7 +541,7 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, unsigned long Next) { // Get a structure - unsigned long Version = AllocateInMap(sizeof(pkgCache::Version)); + map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version)); if (Version == 0) return 0; @@ -473,9 +549,10 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, Ver = pkgCache::VerIterator(Cache,Cache.VerP + Version); Ver->NextVer = Next; Ver->ID = Cache.HeaderP->VersionCount++; - Ver->VerStr = WriteStringInMap(VerStr); - if (Ver->VerStr == 0) + map_ptrloc const idxVerStr = WriteStringInMap(VerStr); + if (unlikely(idxVerStr == 0)) return 0; + Ver->VerStr = idxVerStr; return Version; } @@ -490,7 +567,7 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, return true; // Get a structure - unsigned long DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); + map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); if (DescFile == 0) return false; @@ -523,7 +600,7 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, map_ptrloc Next) { // Get a structure - map_ptrloc Description = AllocateInMap(sizeof(pkgCache::Description)); + map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description)); if (Description == 0) return 0; @@ -531,10 +608,12 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description); Desc->NextDesc = Next; Desc->ID = Cache.HeaderP->DescriptionCount++; - Desc->language_code = WriteStringInMap(Lang); - Desc->md5sum = WriteStringInMap(md5sum.Value()); - if (Desc->language_code == 0 || Desc->md5sum == 0) + map_ptrloc const idxlanguage_code = WriteStringInMap(Lang); + map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value()); + if (unlikely(idxlanguage_code == 0 || idxmd5sum == 0)) return 0; + Desc->language_code = idxlanguage_code; + Desc->md5sum = idxmd5sum; return Description; } @@ -550,15 +629,22 @@ bool pkgCacheGenerator::FinishCache(OpProgress *Progress) if (archs.size() > 1) { // Create Conflicts in between the group - for (pkgCache::GrpIterator G = GetCache().GrpBegin(); G.end() != true; G++) + pkgCache::GrpIterator G = GetCache().GrpBegin(); + Dynamic DynG(G); + for (; G.end() != true; G++) { string const PkgName = G.Name(); - for (pkgCache::PkgIterator P = G.PackageList(); P.end() != true; P = G.NextPkg(P)) + pkgCache::PkgIterator P = G.PackageList(); + Dynamic DynP(P); + for (; P.end() != true; P = G.NextPkg(P)) { if (strcmp(P.Arch(),"all") == 0) continue; pkgCache::PkgIterator allPkg; - for (pkgCache::VerIterator V = P.VersionList(); V.end() != true; V++) + Dynamic DynallPkg(allPkg); + pkgCache::VerIterator V = P.VersionList(); + Dynamic DynV(V); + for (; V.end() != true; V++) { string const Arch = V.Arch(true); map_ptrloc *OldDepLast = NULL; @@ -578,6 +664,7 @@ bool pkgCacheGenerator::FinishCache(OpProgress *Progress) per group, therefore each group member conflicts with all other group members */ pkgCache::PkgIterator D = G.FindPkg(*A); + Dynamic DynD(D); if (D.end() == true) continue; if (coInstall == true) @@ -622,13 +709,15 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, unsigned int const &Type, map_ptrloc *OldDepLast) { + void const * const oldMap = Map.Data(); // Get a structure - unsigned long const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); + map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); if (unlikely(Dependency == 0)) return false; // Fill it in pkgCache::DepIterator Dep(Cache,Cache.DepP + Dependency); + Dynamic DynDep(Dep); Dep->ParentVer = Ver.Index(); Dep->Type = Type; Dep->CompareOp = Op; @@ -640,9 +729,12 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, /* for (pkgCache::DepIterator I = Pkg.RevDependsList(); I.end() == false; I++) if (I->Version != 0 && I.TargetVer() == Version) Dep->Version = I->Version;*/ - if (Dep->Version == 0) - if (unlikely((Dep->Version = WriteStringInMap(Version)) == 0)) + if (Dep->Version == 0) { + map_ptrloc const index = WriteStringInMap(Version); + if (unlikely(index == 0)) return false; + Dep->Version = index; + } } // Link it to the package @@ -656,7 +748,8 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, OldDepLast = &Ver->DependsList; for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; D++) OldDepLast = &D->NextDepends; - } + } else if (oldMap != Map.Data()) + OldDepLast += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; Dep->NextDepends = *OldDepLast; *OldDepLast = Dep.Index(); @@ -677,11 +770,13 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver, unsigned int Type) { pkgCache::GrpIterator Grp; + Dynamic DynGrp(Grp); if (unlikely(Owner->NewGroup(Grp, PackageName) == false)) return false; // Locate the target package pkgCache::PkgIterator Pkg = Grp.FindPkg(Arch); + Dynamic DynPkg(Pkg); if (Pkg.end() == true) { if (unlikely(Owner->NewPackage(Pkg, PackageName, Arch) == false)) return false; @@ -715,13 +810,14 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, return true; // Get a structure - unsigned long const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); + map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); if (unlikely(Provides == 0)) return false; Cache.HeaderP->ProvidesCount++; // Fill it in pkgCache::PrvIterator Prv(Cache,Cache.ProvideP + Provides,Cache.PkgP); + Dynamic DynPrv(Prv); Prv->Version = Ver.Index(); Prv->NextPkgProv = Ver->ProvidesList; Ver->ProvidesList = Prv.Index(); @@ -730,6 +826,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, // Locate the target package pkgCache::PkgIterator Pkg; + Dynamic DynPkg(Pkg); if (unlikely(Owner->NewPackage(Pkg,PkgName, PkgArch) == false)) return false; @@ -750,24 +847,29 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, unsigned long Flags) { // Get some space for the structure - CurrentFile = Cache.PkgFileP + AllocateInMap(sizeof(*CurrentFile)); - if (CurrentFile == Cache.PkgFileP) + map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile)); + if (unlikely(idxFile == 0)) return false; - + CurrentFile = Cache.PkgFileP + idxFile; + // Fill it in - CurrentFile->FileName = WriteStringInMap(File); - CurrentFile->Site = WriteUniqString(Site); + map_ptrloc const idxFileName = WriteStringInMap(File); + map_ptrloc const idxSite = WriteUniqString(Site); + if (unlikely(idxFileName == 0 || idxSite == 0)) + return false; + CurrentFile->FileName = idxFileName; + CurrentFile->Site = idxSite; CurrentFile->NextFile = Cache.HeaderP->FileList; CurrentFile->Flags = Flags; CurrentFile->ID = Cache.HeaderP->PackageFileCount; - CurrentFile->IndexType = WriteUniqString(Index.GetType()->Label); + map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label); + if (unlikely(idxIndexType == 0)) + return false; + CurrentFile->IndexType = idxIndexType; PkgFileName = File; Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP; Cache.HeaderP->PackageFileCount++; - if (CurrentFile->FileName == 0) - return false; - if (Progress != 0) Progress->SubProgress(Index.Size()); return true; @@ -807,18 +909,25 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, } // Get a structure - unsigned long Item = AllocateInMap(sizeof(pkgCache::StringItem)); + void const * const oldMap = Map.Data(); + map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem)); if (Item == 0) return 0; + map_ptrloc const idxString = WriteStringInMap(S,Size); + if (unlikely(idxString == 0)) + return 0; + if (oldMap != Map.Data()) { + Last += (map_ptrloc*) Map.Data() - (map_ptrloc*) oldMap; + I += (pkgCache::StringItem*) Map.Data() - (pkgCache::StringItem*) oldMap; + } + *Last = Item; + // Fill in the structure pkgCache::StringItem *ItemP = Cache.StringItemP + Item; ItemP->NextItem = I - Cache.StringItemP; - *Last = Item; - ItemP->String = WriteStringInMap(S,Size); - if (ItemP->String == 0) - return 0; - + ItemP->String = idxString; + Bucket = ItemP; return ItemP->String; } @@ -1072,7 +1181,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress unlink(CacheFile.c_str()); CacheF = new FileFd(CacheFile,FileFd::WriteEmpty); fchmod(CacheF->Fd(),0644); - Map = new DynamicMMap(*CacheF,MMap::Public,MapSize); + Map = new DynamicMMap(*CacheF,MMap::Public | MMap::Moveable, MapSize); if (_error->PendingError() == true) return false; if (Debug == true) @@ -1081,7 +1190,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress else { // Just build it in memory.. - Map = new DynamicMMap(0,MapSize); + Map = new DynamicMMap(MMap::Moveable, MapSize); if (Debug == true) std::clog << "Open memory Map (not filebased)" << std::endl; } @@ -1194,7 +1303,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O if (_system->AddStatusFiles(Files) == false) return false; - SPtr Map = new DynamicMMap(0,MapSize); + SPtr Map = new DynamicMMap(MMap::Moveable, MapSize); unsigned long CurrentSize = 0; unsigned long TotalSize = 0; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 8f7739165..3bee1f958 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -23,6 +23,8 @@ #include #include +#include + class pkgSourceList; class OpProgress; class MMap; @@ -33,18 +35,32 @@ class pkgCacheGenerator /*{{{*/ private: pkgCache::StringItem *UniqHash[26]; - unsigned long WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; - unsigned long WriteStringInMap(const char *String); - unsigned long WriteStringInMap(const char *String, const unsigned long &Len); - unsigned long AllocateInMap(const unsigned long &size); + map_ptrloc WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; + map_ptrloc WriteStringInMap(const char *String); + map_ptrloc WriteStringInMap(const char *String, const unsigned long &Len); + map_ptrloc AllocateInMap(const unsigned long &size); public: class ListParser; friend class ListParser; - + + template class Dynamic { + Iter *I; + + public: + static std::set toReMap; + Dynamic(Iter &It) : I(&It) { + toReMap.insert(I); + } + + ~Dynamic() { + toReMap.erase(I); + } + }; + protected: - + DynamicMMap ⤅ pkgCache Cache; OpProgress *Progress; @@ -86,6 +102,8 @@ class pkgCacheGenerator /*{{{*/ MMap **OutMap = 0,bool AllowMem = false); static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); + void ReMap(void const * const oldMap, void const * const newMap); + pkgCacheGenerator(DynamicMMap *Map,OpProgress *Progress); ~pkgCacheGenerator(); }; diff --git a/debian/changelog b/debian/changelog index 878d5238a..dab703c6b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -37,6 +37,9 @@ apt (0.7.26~exp8) UNRELEASED; urgency=low * apt-pkg/policy.h: - add another round of const& madness as the previous round accidentally NOT overrides the virtual GetCandidateVer() method (Closes: #587725) + * apt-pkg/pkgcachegen.{cc,h}: + - make the used MMap moveable (and therefore dynamic resizeable) by + applying (some) mad pointer magic (Closes: #195018) [ Julian Andres Klode ] * methods/ftp.h: -- cgit v1.2.3-70-g09d2 From dcdf1ef18b37c243fc707869149f7761d964915c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 9 Jul 2010 17:00:28 +0200 Subject: * doc/apt.conf.5.xml: - add and document APT::Cache-{Start,Grow,Limit} options for mmap control --- apt-pkg/contrib/mmap.cc | 2 ++ apt-pkg/pkgcachegen.cc | 22 ++++++++++++++++------ apt-pkg/pkgcachegen.h | 1 + debian/changelog | 4 +++- doc/apt.conf.5.xml | 17 ++++++++++++++--- doc/examples/configure-index | 4 +++- 6 files changed, 39 insertions(+), 11 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index aa184b130..69fb61fca 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -398,6 +398,8 @@ bool DynamicMMap::Grow() { if (Limit != 0 && WorkSpace >= Limit) return _error->Error(_("Unable to increase the size of the MMap as the " "limit of %lu bytes is already reached."), Limit); + if (GrowFactor <= 0) + return _error->Error(_("Unable to increase size of the MMap as automatic growing is disabled by user.")); unsigned long const newSize = WorkSpace + GrowFactor; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 18bad6727..7ca8fbfda 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -1094,6 +1094,18 @@ static bool BuildCache(pkgCacheGenerator &Gen, return true; } /*}}}*/ +DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) { + unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024); + unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024); + unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0); + Flags |= MMap::Moveable; + if (_config->FindB("APT::Cache-Fallback", false) == true) + Flags |= MMap::Fallback; + if (CacheF != NULL) + return new DynamicMMap(*CacheF, Flags, MapStart, MapGrow, MapLimit); + else + return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit); +} // CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ // --------------------------------------------------------------------- /* This makes sure that the status cache (the cache that has all @@ -1109,7 +1121,6 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress MMap **OutMap,bool AllowMem) { bool const Debug = _config->FindB("Debug::pkgCacheGen", false); - unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024); vector Files; for (vector::const_iterator i = List.begin(); @@ -1181,7 +1192,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress unlink(CacheFile.c_str()); CacheF = new FileFd(CacheFile,FileFd::WriteEmpty); fchmod(CacheF->Fd(),0644); - Map = new DynamicMMap(*CacheF,MMap::Public | MMap::Moveable, MapSize); + Map = CreateDynamicMMap(CacheF, MMap::Public); if (_error->PendingError() == true) return false; if (Debug == true) @@ -1190,7 +1201,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress else { // Just build it in memory.. - Map = new DynamicMMap(MMap::Moveable, MapSize); + Map = CreateDynamicMMap(NULL); if (Debug == true) std::clog << "Open memory Map (not filebased)" << std::endl; } @@ -1297,13 +1308,12 @@ __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutM { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); } bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap) { - unsigned long MapSize = _config->FindI("APT::Cache-Limit",20*1024*1024); vector Files; unsigned long EndOfSource = Files.size(); if (_system->AddStatusFiles(Files) == false) return false; - - SPtr Map = new DynamicMMap(MMap::Moveable, MapSize); + + SPtr Map = CreateDynamicMMap(NULL); unsigned long CurrentSize = 0; unsigned long TotalSize = 0; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 3bee1f958..20dd28030 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -101,6 +101,7 @@ class pkgCacheGenerator /*{{{*/ static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress, MMap **OutMap = 0,bool AllowMem = false); static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); + static DynamicMMap* CreateDynamicMMap(FileFd *CacheF, unsigned long Flags = 0); void ReMap(void const * const oldMap, void const * const newMap); diff --git a/debian/changelog b/debian/changelog index dab703c6b..f819908ad 100644 --- a/debian/changelog +++ b/debian/changelog @@ -40,6 +40,8 @@ apt (0.7.26~exp8) UNRELEASED; urgency=low * apt-pkg/pkgcachegen.{cc,h}: - make the used MMap moveable (and therefore dynamic resizeable) by applying (some) mad pointer magic (Closes: #195018) + * doc/apt.conf.5.xml: + - add and document APT::Cache-{Start,Grow,Limit} options for mmap control [ Julian Andres Klode ] * methods/ftp.h: @@ -55,7 +57,7 @@ apt (0.7.26~exp8) UNRELEASED; urgency=low * debian/control: - Set Standards-Version to 3.9.0 - -- David Kalnischkies Mon, 05 Jul 2010 12:05:30 +0200 + -- David Kalnischkies Fri, 09 Jul 2010 16:55:53 +0200 apt (0.7.26~exp7) experimental; urgency=low diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index 39e2c8e6b..7b7290794 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -199,9 +199,20 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; anything that those packages depend on. - Cache-Limit - APT uses a fixed size memory mapped cache file to store the 'available' - information. This sets the size of that cache (in bytes). + Cache-Start, Cache-Grow and Cache-Limit + APT uses since version 0.7.26 a resizable memory mapped cache file to store the 'available' + information. Cache-Start acts as a hint to which size the Cache will grow + and is therefore the amount of memory APT will request at startup. The default value is + 20971520 bytes (~20 MB). Note that these amount of space need to be available for APT + otherwise it will likely fail ungracefully, so for memory restricted devices these value should + be lowered while on systems with a lot of configured sources this might be increased. + Cache-Grow defines in byte with the default of 1048576 (~1 MB) how much + the Cache size will be increased in the event the space defined by Cache-Start + is not enough. These value will be applied again and again until either the cache is big + enough to store all information or the size of the cache reaches the Cache-Limit. + The default of Cache-Limit is 0 which stands for no limit. + If Cache-Grow is set to 0 the automatic grow of the cache is disabled. + Build-Essential diff --git a/doc/examples/configure-index b/doc/examples/configure-index index fdec32c2c..26fb53fec 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -95,7 +95,9 @@ APT Clean-Installed "true"; Immediate-Configure "true"; // DO NOT turn this off, see the man page Force-LoopBreak "false"; // DO NOT turn this on, see the man page - Cache-Limit "4194304"; + Cache-Start "20971520"; + Cache-Grow "1048576"; + Cache-Limit "0"; Default-Release ""; // consider Recommends, Suggests as important dependencies that should -- cgit v1.2.3-70-g09d2 From 7635093c1c015e385a9e72bdd8a089fd9d48ab57 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 9 Jul 2010 19:51:19 +0200 Subject: switch from std::set to std::vector as it is way more simple, a bit faster and still provides everything we need for the Cache generator --- apt-pkg/pkgcachegen.cc | 16 ++++++++-------- apt-pkg/pkgcachegen.h | 12 +++++------- 2 files changed, 13 insertions(+), 15 deletions(-) (limited to 'apt-pkg/pkgcachegen.h') diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 7ca8fbfda..1175d5129 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -36,7 +36,7 @@ #include /*}}}*/ typedef vector::iterator FileIterator; -template std::set pkgCacheGenerator::Dynamic::toReMap; +template std::vector pkgCacheGenerator::Dynamic::toReMap(6); // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- @@ -113,25 +113,25 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM if (UniqHash[i] != 0) UniqHash[i] += (pkgCache::StringItem*) newMap - (pkgCache::StringItem*) oldMap; - for (std::set::const_iterator i = Dynamic::toReMap.begin(); + for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) (*i)->ReOwn(Cache, oldMap, newMap); - for (std::set::const_iterator i = Dynamic::toReMap.begin(); + for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) (*i)->ReOwn(Cache, oldMap, newMap); - for (std::set::const_iterator i = Dynamic::toReMap.begin(); + for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) (*i)->ReOwn(Cache, oldMap, newMap); - for (std::set::const_iterator i = Dynamic::toReMap.begin(); + for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) (*i)->ReOwn(Cache, oldMap, newMap); - for (std::set::const_iterator i = Dynamic::toReMap.begin(); + for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) (*i)->ReOwn(Cache, oldMap, newMap); - for (std::set::const_iterator i = Dynamic::toReMap.begin(); + for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) (*i)->ReOwn(Cache, oldMap, newMap); - for (std::set::const_iterator i = Dynamic::toReMap.begin(); + for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) (*i)->ReOwn(Cache, oldMap, newMap); } /*}}}*/ diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 20dd28030..ff198833a 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -23,7 +23,7 @@ #include #include -#include +#include class pkgSourceList; class OpProgress; @@ -46,16 +46,14 @@ class pkgCacheGenerator /*{{{*/ friend class ListParser; template class Dynamic { - Iter *I; - public: - static std::set toReMap; - Dynamic(Iter &It) : I(&It) { - toReMap.insert(I); + static std::vector toReMap; + Dynamic(Iter &I) { + toReMap.push_back(&I); } ~Dynamic() { - toReMap.erase(I); + toReMap.pop_back(); } }; -- cgit v1.2.3-70-g09d2