From 8d041b4f4f353079268039dcbfd8b5e575196b66 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 8 Jun 2015 15:22:01 +0200 Subject: do not request files if we expect an IMS hit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we have a file on disk and the hashes are the same in the new Release file and the old one we have on disk we know that if we ask the server for the file, we will at best get an IMS hit – at worse the server doesn't support this and sends us the (unchanged) file and we have to run all our checks on it again for nothing. So, we can save ourselves (and the servers) some unneeded requests if we figure this out on our own. --- test/integration/framework | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'test/integration/framework') diff --git a/test/integration/framework b/test/integration/framework index b253deb91..56c4a1216 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -835,7 +835,9 @@ buildaptarchivefromincoming() { buildaptarchivefromfiles() { msginfo "Build APT archive for ${CCMD}$(basename $0)${CINFO} based on prebuild files…" - find aptarchive -name 'Packages' -o -name 'Sources' -o -name 'Translation-*' | while read line; do + local DIR='aptarchive' + if [ -d "${DIR}/dists" ]; then DIR="${DIR}/dists"; fi + find "$DIR" -name 'Packages' -o -name 'Sources' -o -name 'Translation-*' | while read line; do msgninfo "\t${line} file… " compressfile "$line" "$1" msgdone "info" -- cgit v1.2.3-70-g09d2 From 59148d9630bbbd53c6aa10da0fcdbd579797502a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 10 Jun 2015 19:22:41 +0200 Subject: abstract the code to iterate over all targets a bit We have two places in the code which need to iterate over targets and do certain things with it. The first one is actually creating these targets for download and the second instance pepares certain targets for reading. Git-Dch: Ignore --- apt-pkg/deb/debmetaindex.cc | 197 +++++++++++---------- test/integration/framework | 7 +- test/integration/test-apt-acquire-additional-files | 33 ++++ 3 files changed, 142 insertions(+), 95 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 8fef05ab0..c35f3b0a4 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -90,10 +90,11 @@ debReleaseIndex::~debReleaseIndex() { delete *S; } -vector * debReleaseIndex::ComputeIndexTargets() const +template +void foreachTarget(std::string const URI, std::string const Dist, + std::map > const &ArchEntries, + CallC Call) { - vector * IndexTargets = new vector ; - bool const flatArchive = (Dist[Dist.length() - 1] == '/'); std::string baseURI = URI; if (flatArchive) @@ -108,7 +109,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const std::vector lang = APT::Configuration::getLanguages(true); if (lang.empty()) lang.push_back("none"); - map >::const_iterator const src = ArchEntries.find("source"); + map >::const_iterator const src = ArchEntries.find("source"); if (src != ArchEntries.end()) { std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true); @@ -123,8 +124,8 @@ vector * debReleaseIndex::ComputeIndexTargets() const if (URI.empty()) continue; - vector const SectionEntries = src->second; - for (vector::const_iterator I = SectionEntries.begin(); + vector const SectionEntries = src->second; + for (vector::const_iterator I = SectionEntries.begin(); I != SectionEntries.end(); ++I) { for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) @@ -139,27 +140,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const { "$(LANGUAGE)", &(*l) }, { NULL, NULL } }; - std::string const name = SubstVar(URI, subst); - IndexTarget * Target; - if (IsOptional == true) - { - Target = new OptionalIndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - else - { - Target = new IndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - IndexTargets->push_back(Target); + Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst); if (URI.find("$(LANGUAGE)") == std::string::npos) break; @@ -171,10 +152,6 @@ vector * debReleaseIndex::ComputeIndexTargets() const } } - // Only source release - if (IndexTargets->empty() == false && ArchEntries.size() == 1) - return IndexTargets; - std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true); for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) { @@ -187,13 +164,13 @@ vector * debReleaseIndex::ComputeIndexTargets() const if (URI.empty()) continue; - for (map >::const_iterator a = ArchEntries.begin(); + for (map >::const_iterator a = ArchEntries.begin(); a != ArchEntries.end(); ++a) { if (a->first == "source") continue; - for (vector ::const_iterator I = a->second.begin(); + for (vector ::const_iterator I = a->second.begin(); I != a->second.end(); ++I) { for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) @@ -209,27 +186,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const { "$(ARCHITECTURE)", &(a->first) }, { NULL, NULL } }; - std::string const name = SubstVar(URI, subst); - IndexTarget * Target; - if (IsOptional == true) - { - Target = new OptionalIndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - else - { - Target = new IndexTarget( - name, - SubstVar(ShortDesc, subst), - SubstVar(LongDesc, subst), - baseURI + name - ); - } - IndexTargets->push_back(Target); + Call(baseURI, *T, URI, ShortDesc, LongDesc, IsOptional, subst); if (URI.find("$(LANGUAGE)") == std::string::npos) break; @@ -243,9 +200,51 @@ vector * debReleaseIndex::ComputeIndexTargets() const break; } } +} + + +struct ComputeIndexTargetsClass +{ + vector * const IndexTargets; + + void operator()(std::string const &baseURI, std::string const &/*TargetName*/, + std::string const &URI, std::string const &ShortDesc, std::string const &LongDesc, + bool const IsOptional, struct SubstVar const * const subst) + { + std::string const name = SubstVar(URI, subst); + IndexTarget * Target; + if (IsOptional == true) + { + Target = new OptionalIndexTarget( + name, + SubstVar(ShortDesc, subst), + SubstVar(LongDesc, subst), + baseURI + name + ); + } + else + { + Target = new IndexTarget( + name, + SubstVar(ShortDesc, subst), + SubstVar(LongDesc, subst), + baseURI + name + ); + } + IndexTargets->push_back(Target); + } - return IndexTargets; + ComputeIndexTargetsClass() : IndexTargets(new vector ) {} +}; + +vector * debReleaseIndex::ComputeIndexTargets() const +{ + ComputeIndexTargetsClass comp; + foreachTarget(URI, Dist, ArchEntries, comp); + return comp.IndexTargets; } + + /*}}}*/ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const { @@ -303,45 +302,55 @@ bool debReleaseIndex::IsTrusted() const return FileExists(VerifiedSigFile); } -vector *debReleaseIndex::GetIndexFiles() { - if (Indexes != NULL) - return Indexes; - - Indexes = new vector ; - map >::const_iterator const src = ArchEntries.find("source"); - if (src != ArchEntries.end()) { - vector const SectionEntries = src->second; - for (vector::const_iterator I = SectionEntries.begin(); - I != SectionEntries.end(); ++I) - Indexes->push_back(new debSourcesIndex (URI, Dist, (*I)->Section, IsTrusted())); - } - - // Only source release - if (Indexes->empty() == false && ArchEntries.size() == 1) - return Indexes; - - std::vector const lang = APT::Configuration::getLanguages(true); - map > sections; - for (map >::const_iterator a = ArchEntries.begin(); - a != ArchEntries.end(); ++a) { - if (a->first == "source") - continue; - for (vector::const_iterator I = a->second.begin(); - I != a->second.end(); ++I) { - Indexes->push_back(new debPackagesIndex (URI, Dist, (*I)->Section, IsTrusted(), a->first)); - sections[(*I)->Section].insert(lang.begin(), lang.end()); - } - } - - for (map >::const_iterator s = sections.begin(); - s != sections.end(); ++s) - for (set::const_iterator l = s->second.begin(); - l != s->second.end(); ++l) { - if (*l == "none") continue; - Indexes->push_back(new debTranslationsIndex(URI,Dist,s->first,(*l).c_str())); - } - - return Indexes; +struct GetIndexFilesClass +{ + vector * const Indexes; + std::string const URI; + std::string const Release; + bool const IsTrusted; + + void operator()(std::string const &/*baseURI*/, std::string const &TargetName, + std::string const &/*URI*/, std::string const &/*ShortDesc*/, std::string const &/*LongDesc*/, + bool const /*IsOptional*/, struct SubstVar const * const subst) + { + if (TargetName == "Packages") + { + Indexes->push_back(new debPackagesIndex( + URI, + Release, + SubstVar("$(COMPONENT)", subst), + IsTrusted, + SubstVar("$(ARCHITECTURE)", subst) + )); + } + else if (TargetName == "Sources") + Indexes->push_back(new debSourcesIndex( + URI, + Release, + SubstVar("$(COMPONENT)", subst), + IsTrusted + )); + else if (TargetName == "Translations") + Indexes->push_back(new debTranslationsIndex( + URI, + Release, + SubstVar("$(COMPONENT)", subst), + SubstVar("$(LANGUAGE)", subst) + )); + } + + GetIndexFilesClass(std::string const &URI, std::string const &Release, bool const IsTrusted) : + Indexes(new vector ), URI(URI), Release(Release), IsTrusted(IsTrusted) {} +}; + +std::vector *debReleaseIndex::GetIndexFiles() +{ + if (Indexes != NULL) + return Indexes; + + GetIndexFilesClass comp(URI, Dist, IsTrusted()); + foreachTarget(URI, Dist, ArchEntries, comp); + return Indexes = comp.Indexes; } void debReleaseIndex::PushSectionEntry(vector const &Archs, const debSectionEntry *Entry) { diff --git a/test/integration/framework b/test/integration/framework index 56c4a1216..110758e82 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1531,6 +1531,11 @@ aptautotest() { } aptautotest_aptget_update() { + local TESTCALL="$1" + while [ -n "$2" ]; do + if [ "$2" = '--print-uris' ]; then return; fi # simulation mode + shift + done if ! test -d "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"; then return; fi testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755" testfilestats "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:755" @@ -1538,7 +1543,7 @@ aptautotest_aptget_update() { for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -type f ! -name 'lock'); do testfilestats "$file" '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" done - if [ "$1" = 'testsuccess' ]; then + if [ "$TESTCALL" = 'testsuccess' ]; then # failure cases can retain partial files and such testempty find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists/partial" -mindepth 1 ! \( -name 'lock' -o -name '*.FAILED' \) fi diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files index 150a50980..0e59233e9 100755 --- a/test/integration/test-apt-acquire-additional-files +++ b/test/integration/test-apt-acquire-additional-files @@ -6,6 +6,9 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture 'amd64' + +# note that in --print-uri we talk about .bz2 because that is the default. +# This doesn't mean it is actually attempt to download it. configcompression '.' 'gz' buildsimplenativepackage 'foo' 'amd64' '1' 'unstable' @@ -13,6 +16,11 @@ buildsimplenativepackage 'foo' 'amd64' '1' 'unstable' setupaptarchive --no-update changetowebserver +testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 +'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 +'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris + testsuccessequal "Get:1 http://localhost:8080 unstable InRelease [$(stat -c%s aptarchive/dists/unstable/InRelease) B] Get:2 http://localhost:8080 unstable/main Sources [$(stat -c%s aptarchive/dists/unstable/main/source/Sources.gz) B] Get:3 http://localhost:8080 unstable/main amd64 Packages [$(stat -c%s aptarchive/dists/unstable/main/binary-amd64/Packages.gz) B] @@ -29,6 +37,12 @@ APT::Acquire::Targets::deb::Contents { }; EOF +testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 +'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 +'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 +'http://localhost:8080/dists/unstable/main/Contents-amd64.bz2' localhost:8080_dists_unstable_main_Contents-amd64 0 " aptget update --print-uris + testsuccessequal "Hit http://localhost:8080 unstable InRelease Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B] Reading package lists..." aptget update @@ -46,9 +60,28 @@ APT::Acquire::Targets::deb::Contents { }; EOF +# the last line is utter bogus of course, but how should apt know… +testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 +'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 +'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 +'http://localhost:8080/dists/unstable/main/Contents-amd64.gz.bz2' localhost:8080_dists_unstable_main_Contents-amd64.gz 0 " aptget update --print-uris + testsuccessequal "Hit http://localhost:8080 unstable InRelease Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarchive/dists/unstable/main/Contents-amd64.gz) B] Reading package lists..." aptget update testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*' testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz' + +rm -f rootdir/etc/apt/apt.conf.d/content-target.conf + +testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 +'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 +'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 +'http://localhost:8080/dists/unstable/main/i18n/Translation-en.bz2' localhost:8080_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris + +testsuccessequal "Hit http://localhost:8080 unstable InRelease +Reading package lists..." aptget update + +testempty find rootdir/var/lib/apt/lists -name '*Contents*' -- cgit v1.2.3-70-g09d2 From b07aeb1a6e24825e534167a737043441e871de9f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 12 Jun 2015 02:08:53 +0200 Subject: store Release files data in the Cache We used to read the Release file for each Packages file and store the data in the PackageFile struct even through potentially many Packages (and Translation-*) files could use the same data. The point of the exercise isn't the duplicated data through. Having the Release files as first-class citizens in the Cache allows us to properly track their state as well as allows us to use the information also for files which aren't in the cache, but where we know to which Release file they belong (Sources are an example for this). This modifies the pkgCache structs, especially the PackagesFile struct which depending on how libapt users access the data in these structs can mean huge breakage or no visible change. As a single data point: aptitude seems to be fine with this. Even if there is breakage it is trivial to fix in a backportable way while avoiding breakage for everyone would be a huge pain for us. Note that not all PackageFile structs have a corresponding ReleaseFile. In particular the dpkg/status file as well as *.deb files have not. As these have only a Archive property need, the Component property takes over this duty and the ReleaseFile remains zero. This is also the reason why it isn't needed nor particularily recommended to change from PackagesFile to ReleaseFile blindly. Sticking with the earlier is usually the better option. --- apt-pkg/acquire-item.cc | 6 +- apt-pkg/cacheiterators.h | 46 ++++++- apt-pkg/clean.cc | 2 +- apt-pkg/deb/debindexfile.cc | 42 ++----- apt-pkg/deb/debindexfile.h | 1 - apt-pkg/deb/deblistparser.cc | 37 ------ apt-pkg/deb/deblistparser.h | 6 +- apt-pkg/deb/debmetaindex.cc | 124 ++++++++++++++++++- apt-pkg/deb/debmetaindex.h | 6 + apt-pkg/deb/dpkgpm.cc | 4 +- apt-pkg/depcache.cc | 6 +- apt-pkg/edsp.cc | 2 +- apt-pkg/edsp/edspindexfile.cc | 5 +- apt-pkg/edsp/edsplistparser.cc | 4 +- apt-pkg/edsp/edsplistparser.h | 4 +- apt-pkg/metaindex.cc | 17 +++ apt-pkg/metaindex.h | 6 + apt-pkg/pkgcache.cc | 77 ++++++++---- apt-pkg/pkgcache.h | 75 ++++++++++-- apt-pkg/pkgcachegen.cc | 226 +++++++++++++++++++++++++++-------- apt-pkg/pkgcachegen.h | 10 +- apt-pkg/policy.cc | 15 ++- apt-pkg/versionmatch.cc | 45 +++---- apt-pkg/versionmatch.h | 4 +- cmdline/apt-cache.cc | 11 +- test/integration/framework | 9 +- test/integration/test-policy-pinning | 2 +- 27 files changed, 573 insertions(+), 219 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d4191f00e..d6e9ccbe0 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -2568,7 +2568,7 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour // Skip not source sources, they do not have file fields. for (; Vf.end() == false; ++Vf) { - if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0) + if (Vf.File().Flagged(pkgCache::Flag::NotSource)) continue; break; } @@ -2636,14 +2636,14 @@ bool pkgAcqArchive::QueueNext() { pkgCache::PkgFileIterator const PkgF = Vf.File(); // Ignore not source sources - if ((PkgF->Flags & pkgCache::Flag::NotSource) != 0) + if (PkgF.Flagged(pkgCache::Flag::NotSource)) continue; // Try to cross match against the source list pkgIndexFile *Index; if (Sources->FindIndex(PkgF, Index) == false) continue; - LocalSource = (PkgF->Flags & pkgCache::Flag::LocalSource) == pkgCache::Flag::LocalSource; + LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource); // only try to get a trusted package from another source if that source // is also trusted diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index fe798799c..301da6fc4 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -367,27 +367,61 @@ class pkgCache::PrvIterator : public Iterator { } }; /*}}}*/ -// Package file /*{{{*/ -class pkgCache::PkgFileIterator : public Iterator { +// Release file /*{{{*/ +class pkgCache::RlsFileIterator : public Iterator { protected: - inline PackageFile* OwnerPointer() const { - return (Owner != 0) ? Owner->PkgFileP : 0; + inline ReleaseFile* OwnerPointer() const { + return (Owner != 0) ? Owner->RlsFileP : 0; } public: // Iteration - void operator ++(int) {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile;} + void operator ++(int) {if (S != Owner->RlsFileP) S = Owner->RlsFileP + S->NextFile;} inline void operator ++() {operator ++(0);} // Accessors inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;} inline const char *Archive() const {return S->Archive == 0?0:Owner->StrP + S->Archive;} - inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;} inline const char *Version() const {return S->Version == 0?0:Owner->StrP + S->Version;} inline const char *Origin() const {return S->Origin == 0?0:Owner->StrP + S->Origin;} inline const char *Codename() const {return S->Codename ==0?0:Owner->StrP + S->Codename;} inline const char *Label() const {return S->Label == 0?0:Owner->StrP + S->Label;} inline const char *Site() const {return S->Site == 0?0:Owner->StrP + S->Site;} + inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return (S->Flags & flag) == flag; } + + bool IsOk(); + std::string RelStr(); + + // Constructors + inline RlsFileIterator() : Iterator() {} + inline RlsFileIterator(pkgCache &Owner) : Iterator(Owner, Owner.RlsFileP) {} + inline RlsFileIterator(pkgCache &Owner,ReleaseFile *Trg) : Iterator(Owner, Trg) {} +}; + /*}}}*/ +// Package file /*{{{*/ +class pkgCache::PkgFileIterator : public Iterator { + protected: + inline PackageFile* OwnerPointer() const { + return (Owner != 0) ? Owner->PkgFileP : 0; + } + + public: + // Iteration + void operator ++(int) {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile;} + inline void operator ++() {operator ++(0);} + + // Accessors + inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;} + inline pkgCache::RlsFileIterator ReleaseFile() const {return RlsFileIterator(*Owner, Owner->RlsFileP + S->Release);} + inline const char *Archive() const {return S->Release == 0 ? Component() : ReleaseFile().Archive();} + inline const char *Version() const {return S->Release == 0 ? NULL : ReleaseFile().Version();} + inline const char *Origin() const {return S->Release == 0 ? NULL : ReleaseFile().Origin();} + inline const char *Codename() const {return S->Release == 0 ? NULL : ReleaseFile().Codename();} + inline const char *Label() const {return S->Release == 0 ? NULL : ReleaseFile().Label();} + inline const char *Site() const {return S->Release == 0 ? NULL : ReleaseFile().Site();} + inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return S->Release== 0 ? false : ReleaseFile().Flagged(flag);} + inline bool Flagged(pkgCache::Flag::PkgFFlags const flag) const {return (S->Flags & flag) == flag;} + inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;} inline const char *Architecture() const {return S->Architecture == 0?0:Owner->StrP + S->Architecture;} inline const char *IndexType() const {return S->IndexType == 0?0:Owner->StrP + S->IndexType;} diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc index 6edce5b6d..0fca60ba9 100644 --- a/apt-pkg/clean.cc +++ b/apt-pkg/clean.cc @@ -106,7 +106,7 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache) J.end() == false; ++J) { if (CleanInstalled == true && - (J.File()->Flags & pkgCache::Flag::NotSource) != 0) + J.File().Flagged(pkgCache::Flag::NotSource)) continue; IsFetchable = true; break; diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 7aad65c0e..3a79cbc58 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -128,7 +128,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const if (Dist.empty()) Dist = "/"; ::URI Tmp(URI); - if (Gen.SelectFile(PackageFile,Tmp.Host,*this) == false) + if (Gen.SelectFile(PackageFile, *this, Target.Option(IndexTarget::COMPONENT)) == false) return _error->Error("Problem with SelectFile %s",PackageFile.c_str()); // Store the IMS information @@ -136,31 +136,10 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const pkgCacheGenerator::Dynamic DynFile(File); File->Size = Pkg.FileSize(); File->mtime = Pkg.ModificationTime(); - + if (Gen.MergeList(Parser) == false) return _error->Error("Problem with MergeList %s",PackageFile.c_str()); - // Check the release file - string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease"); - bool releaseExists = false; - if (FileExists(ReleaseFile) == true) - releaseExists = true; - else - ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release"); - - if (releaseExists == true || FileExists(ReleaseFile) == true) - { - FileFd Rel; - // Beware: The 'Release' file might be clearsigned in case the - // signature for an 'InRelease' file couldn't be checked - if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false) - return false; - - if (_error->PendingError() == true) - return false; - Parser.LoadReleaseInfo(File, Rel, Target.Option(IndexTarget::COMPONENT)); - } - return true; } /*}}}*/ @@ -221,17 +200,17 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const debTranslationsParser TransParser(&Trans); if (_error->PendingError() == true) return false; - + if (Prog != NULL) Prog->SubProgress(0, Target.Description); - if (Gen.SelectFile(TranslationFile,string(),*this) == false) + if (Gen.SelectFile(TranslationFile, *this, Target.Option(IndexTarget::COMPONENT), pkgCache::Flag::NotSource) == false) return _error->Error("Problem with SelectFile %s",TranslationFile.c_str()); // Store the IMS information pkgCache::PkgFileIterator TransFile = Gen.GetCurFile(); TransFile->Size = Trans.FileSize(); TransFile->mtime = Trans.ModificationTime(); - + if (Gen.MergeList(TransParser) == false) return _error->Error("Problem with MergeList %s",TranslationFile.c_str()); } @@ -305,18 +284,17 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const if (Prog != NULL) Prog->SubProgress(0,File); - if (Gen.SelectFile(File,string(),*this,pkgCache::Flag::NotSource) == false) + if (Gen.SelectFile(File, *this, "now", pkgCache::Flag::NotSource) == false) return _error->Error("Problem with SelectFile %s",File.c_str()); // Store the IMS information pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); + pkgCacheGenerator::Dynamic DynFile(CFile); CFile->Size = Pkg.FileSize(); CFile->mtime = Pkg.ModificationTime(); - map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "now"); - CFile->Archive = storage; - + if (Gen.MergeList(Parser) == false) - return _error->Error("Problem with MergeList %s",File.c_str()); + return _error->Error("Problem with MergeList %s",File.c_str()); return true; } /*}}}*/ @@ -431,7 +409,7 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const // and give it to the list parser debDebFileParser Parser(DebControl, DebFile); - if(Gen.SelectFile(DebFile, "local", *this, pkgCache::Flag::LocalSource) == false) + if(Gen.SelectFile(DebFile, *this, "now", pkgCache::Flag::LocalSource) == false) return _error->Error("Problem with SelectFile %s", DebFile.c_str()); pkgCache::PkgFileIterator File = Gen.GetCurFile(); diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index dd3f212b2..6b8c78e5a 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -45,7 +45,6 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile virtual bool HasPackages() const {return true;}; virtual unsigned long Size() const; virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; - bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog, unsigned long const Flag) const; virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; debStatusIndex(std::string File); diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index b80b57bc4..c5e77b0ff 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -951,43 +951,6 @@ bool debListParser::Step() return false; } /*}}}*/ -// ListParser::LoadReleaseInfo - Load the release information /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, - FileFd &File, string component) -{ - // apt-secure does no longer download individual (per-section) Release - // file. to provide Component pinning we use the section name now - map_stringitem_t const storage = StoreString(pkgCacheGenerator::MIXED, component); - FileI->Component = storage; - - pkgTagFile TagFile(&File, File.Size()); - pkgTagSection Section; - if (_error->PendingError() == true || TagFile.Step(Section) == false) - return false; - - std::string data; - #define APT_INRELEASE(TYPE, TAG, STORE) \ - data = Section.FindS(TAG); \ - if (data.empty() == false) \ - { \ - map_stringitem_t const storage = StoreString(pkgCacheGenerator::TYPE, data); \ - STORE = storage; \ - } - APT_INRELEASE(MIXED, "Suite", FileI->Archive) - APT_INRELEASE(MIXED, "Component", FileI->Component) - APT_INRELEASE(VERSIONNUMBER, "Version", FileI->Version) - APT_INRELEASE(MIXED, "Origin", FileI->Origin) - APT_INRELEASE(MIXED, "Codename", FileI->Codename) - APT_INRELEASE(MIXED, "Label", FileI->Label) - #undef APT_INRELEASE - Section.FindFlag("NotAutomatic", FileI->Flags, pkgCache::Flag::NotAutomatic); - Section.FindFlag("ButAutomaticUpgrades", FileI->Flags, pkgCache::Flag::ButAutomaticUpgrades); - - return !_error->PendingError(); -} - /*}}}*/ // ListParser::GetPrio - Convert the priority from a string /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index 6279d8399..420d5ff08 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -80,9 +80,9 @@ class APT_HIDDEN debListParser : public pkgCacheGenerator::ListParser virtual map_filesize_t Size() {return Section.size();}; virtual bool Step(); - - bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File, - std::string section); + + bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File, + std::string const §ion); APT_PUBLIC static const char *ParseDepends(const char *Start,const char *Stop, std::string &Package,std::string &Ver,unsigned int &Op); diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 8e4c2be2d..994b95849 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -10,10 +11,12 @@ #include #include #include -#include #include +#include +#include +#include +#include -#include #include #include #include @@ -21,6 +24,11 @@ #include #include +#include +#include +#include +#include + using namespace std; string debReleaseIndex::MetaIndexInfo(const char *Type) const @@ -37,6 +45,10 @@ string debReleaseIndex::MetaIndexInfo(const char *Type) const Info += Type; return Info; } +std::string debReleaseIndex::Describe() const +{ + return MetaIndexInfo("Release"); +} string debReleaseIndex::MetaIndexFile(const char *Type) const { @@ -339,6 +351,114 @@ debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section, bool const &IsSrc): Section(Section), IsSrc(IsSrc) {} +static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile) +{ + ReleaseFile = That->MetaIndexFile("InRelease"); + bool releaseExists = false; + if (FileExists(ReleaseFile) == true) + releaseExists = true; + else + { + ReleaseFile = That->MetaIndexFile("Release"); + if (FileExists(ReleaseFile)) + releaseExists = true; + } + return releaseExists; +} + +bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/ +{ + std::string ReleaseFile; + bool const releaseExists = ReleaseFileName(this, ReleaseFile); + + ::URI Tmp(URI); + if (Gen.SelectReleaseFile(ReleaseFile, Tmp.Host) == false) + return _error->Error("Problem with SelectReleaseFile %s", ReleaseFile.c_str()); + + if (releaseExists == false) + return true; + + FileFd Rel; + // Beware: The 'Release' file might be clearsigned in case the + // signature for an 'InRelease' file couldn't be checked + if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false) + return false; + if (_error->PendingError() == true) + return false; + + // Store the IMS information + pkgCache::RlsFileIterator File = Gen.GetCurRlsFile(); + pkgCacheGenerator::Dynamic DynFile(File); + // Rel can't be used as this is potentially a temporary file + struct stat Buf; + if (stat(ReleaseFile.c_str(), &Buf) != 0) + return _error->Errno("fstat", "Unable to stat file %s", ReleaseFile.c_str()); + File->Size = Buf.st_size; + File->mtime = Buf.st_mtime; + + pkgTagFile TagFile(&Rel, Rel.Size()); + pkgTagSection Section; + if (_error->PendingError() == true || TagFile.Step(Section) == false) + return false; + + std::string data; + #define APT_INRELEASE(TYPE, TAG, STORE) \ + data = Section.FindS(TAG); \ + if (data.empty() == false) \ + { \ + map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::TYPE, data); \ + STORE = storage; \ + } + APT_INRELEASE(MIXED, "Suite", File->Archive) + APT_INRELEASE(VERSIONNUMBER, "Version", File->Version) + APT_INRELEASE(MIXED, "Origin", File->Origin) + APT_INRELEASE(MIXED, "Codename", File->Codename) + APT_INRELEASE(MIXED, "Label", File->Label) + #undef APT_INRELEASE + Section.FindFlag("NotAutomatic", File->Flags, pkgCache::Flag::NotAutomatic); + Section.FindFlag("ButAutomaticUpgrades", File->Flags, pkgCache::Flag::ButAutomaticUpgrades); + + return !_error->PendingError(); +} + /*}}}*/ +// ReleaseIndex::FindInCache - Find this index /*{{{*/ +pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const +{ + std::string ReleaseFile; + bool const releaseExists = ReleaseFileName(this, ReleaseFile); + + pkgCache::RlsFileIterator File = Cache.RlsFileBegin(); + for (; File.end() == false; ++File) + { + if (File->FileName == 0 || ReleaseFile != File.FileName()) + continue; + + // empty means the file does not exist by "design" + if (releaseExists == false && File->Size == 0) + return File; + + struct stat St; + if (stat(File.FileName(),&St) != 0) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "ReleaseIndex::FindInCache - stat failed on " << File.FileName() << std::endl; + return pkgCache::RlsFileIterator(Cache); + } + if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "ReleaseIndex::FindInCache - size (" << St.st_size << " <> " << File->Size + << ") or mtime (" << St.st_mtime << " <> " << File->mtime + << ") doesn't match for " << File.FileName() << std::endl; + return pkgCache::RlsFileIterator(Cache); + } + return File; + } + + return File; +} + /*}}}*/ + class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type { protected: diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 0b1b08432..7e9942710 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -20,6 +20,8 @@ class pkgAcquire; class pkgIndexFile; class debDebPkgFileIndex; class IndexTarget; +class pkgCacheGenerator; +class OpProgress; class APT_HIDDEN debReleaseIndex : public metaIndex { public: @@ -48,6 +50,10 @@ class APT_HIDDEN debReleaseIndex : public metaIndex { virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const; virtual std::vector GetIndexTargets() const; + virtual std::string Describe() const; + virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const; + virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; + std::string MetaIndexInfo(const char *Type) const; std::string MetaIndexFile(const char *Types) const; std::string MetaIndexURI(const char *Type) const; diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index a7a66c75d..6ee939edd 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -205,8 +205,10 @@ pkgCache::VerIterator FindNowVersion(const pkgCache::PkgIterator &Pkg) for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver) for (pkgCache::VerFileIterator Vf = Ver.FileList(); Vf.end() == false; ++Vf) for (pkgCache::PkgFileIterator F = Vf.File(); F.end() == false; ++F) - if (F->Archive != 0 && strcmp(F.Archive(), "now") == 0) + { + if (F.Archive() != 0 && strcmp(F.Archive(), "now") == 0) return Ver; + } return Ver; } /*}}}*/ diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc index b73c336db..36e1ac9ec 100644 --- a/apt-pkg/depcache.cc +++ b/apt-pkg/depcache.cc @@ -1685,13 +1685,13 @@ pkgCache::VerIterator pkgDepCache::Policy::GetCandidateVer(PkgIterator const &Pk for (VerFileIterator J = I.FileList(); J.end() == false; ++J) { - if ((J.File()->Flags & Flag::NotSource) != 0) + if (J.File().Flagged(Flag::NotSource)) continue; /* Stash the highest version of a not-automatic source, we use it if there is nothing better */ - if ((J.File()->Flags & Flag::NotAutomatic) != 0 || - (J.File()->Flags & Flag::ButAutomaticUpgrades) != 0) + if (J.File().Flagged(Flag::NotAutomatic) || + J.File().Flagged(Flag::ButAutomaticUpgrades)) { if (Last.end() == true) Last = I; diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc index 3c6a7e30f..41cc2cdfe 100644 --- a/apt-pkg/edsp.cc +++ b/apt-pkg/edsp.cc @@ -129,7 +129,7 @@ void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgI signed short const p = Cache.GetPolicy().GetPriority(File); if (Pin < p) Pin = p; - if ((File->Flags & pkgCache::Flag::NotSource) != pkgCache::Flag::NotSource) { + if (File.Flagged(pkgCache::Flag::NotSource) == false) { string Release = File.RelStr(); if (!Release.empty()) Releases.insert(Release); diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc index d00536362..43dd44a79 100644 --- a/apt-pkg/edsp/edspindexfile.cc +++ b/apt-pkg/edsp/edspindexfile.cc @@ -49,15 +49,14 @@ bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const if (Prog != NULL) Prog->SubProgress(0,File); - if (Gen.SelectFile(File,std::string(),*this) == false) + if (Gen.SelectFile(File, *this, "edsp") == false) return _error->Error("Problem with SelectFile %s",File.c_str()); // Store the IMS information pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); + pkgCacheGenerator::Dynamic DynFile(CFile); CFile->Size = Pkg.FileSize(); CFile->mtime = Pkg.ModificationTime(); - map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "edsp::scenario"); - CFile->Archive = storage; if (Gen.MergeList(Parser) == false) return _error->Error("Problem with MergeList %s",File.c_str()); diff --git a/apt-pkg/edsp/edsplistparser.cc b/apt-pkg/edsp/edsplistparser.cc index 212dc7840..d62abe709 100644 --- a/apt-pkg/edsp/edsplistparser.cc +++ b/apt-pkg/edsp/edsplistparser.cc @@ -86,8 +86,8 @@ bool edspListParser::ParseStatus(pkgCache::PkgIterator &Pkg, } /*}}}*/ // ListParser::LoadReleaseInfo - Load the release information /*{{{*/ -APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::PkgFileIterator & /*FileI*/, - FileFd & /*File*/, std::string /*component*/) +APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::RlsFileIterator & /*FileI*/, + FileFd & /*File*/, std::string const &/*component*/) { return true; } diff --git a/apt-pkg/edsp/edsplistparser.h b/apt-pkg/edsp/edsplistparser.h index 86cd77606..abe2ef139 100644 --- a/apt-pkg/edsp/edsplistparser.h +++ b/apt-pkg/edsp/edsplistparser.h @@ -34,8 +34,8 @@ class APT_HIDDEN edspListParser : public debListParser virtual MD5SumValue Description_md5(); virtual unsigned short VersionHash(); - bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File, - std::string section); + bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File, + std::string const §ion); edspListParser(FileFd *File, std::string const &Arch = ""); diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc index 31a8ec009..35ab6c53b 100644 --- a/apt-pkg/metaindex.cc +++ b/apt-pkg/metaindex.cc @@ -1,4 +1,5 @@ // Include Files /*{{{*/ +#include #include #include @@ -22,6 +23,22 @@ std::string metaIndex::LocalFileName() const } #endif +std::string metaIndex::Describe() const +{ + return "Release"; +} + +pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache) const +{ + return pkgCache::RlsFileIterator(Cache); +} + +bool metaIndex::Merge(pkgCacheGenerator &Gen,OpProgress *) const +{ + return Gen.SelectReleaseFile("", ""); +} + + metaIndex::metaIndex(std::string const &URI, std::string const &Dist, char const * const Type) : Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(false) diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index 4e5de8be0..20c879a89 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -23,6 +23,8 @@ using std::string; class pkgAcquire; class IndexTarget; +class pkgCacheGenerator; +class OpProgress; class metaIndex { @@ -51,6 +53,10 @@ class metaIndex virtual std::vector *GetIndexFiles() = 0; virtual bool IsTrusted() const = 0; + virtual std::string Describe() const; + virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const; + virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; + metaIndex(std::string const &URI, std::string const &Dist, char const * const Type); virtual ~metaIndex(); diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 864ae0f60..9fe382108 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -61,6 +61,7 @@ pkgCache::Header::Header() HeaderSz = sizeof(pkgCache::Header); GroupSz = sizeof(pkgCache::Group); PackageSz = sizeof(pkgCache::Package); + ReleaseFileSz = sizeof(pkgCache::ReleaseFile); PackageFileSz = sizeof(pkgCache::PackageFile); VersionSz = sizeof(pkgCache::Version); DescriptionSz = sizeof(pkgCache::Description); @@ -74,6 +75,7 @@ pkgCache::Header::Header() VersionCount = 0; DescriptionCount = 0; DependsCount = 0; + ReleaseFileCount = 0; PackageFileCount = 0; VerFileCount = 0; DescFileCount = 0; @@ -82,6 +84,7 @@ pkgCache::Header::Header() MaxDescFileSize = 0; FileList = 0; + RlsFileList = 0; #if APT_PKG_ABI < 413 APT_IGNORE_DEPRECATED(StringList = 0;) #endif @@ -102,6 +105,7 @@ bool pkgCache::Header::CheckSizes(Header &Against) const if (HeaderSz == Against.HeaderSz && GroupSz == Against.GroupSz && PackageSz == Against.PackageSz && + ReleaseFileSz == Against.ReleaseFileSz && PackageFileSz == Against.PackageFileSz && VersionSz == Against.VersionSz && DescriptionSz == Against.DescriptionSz && @@ -140,6 +144,7 @@ bool pkgCache::ReMap(bool const &Errorchecks) PkgP = (Package *)Map.Data(); VerFileP = (VerFile *)Map.Data(); DescFileP = (DescFile *)Map.Data(); + RlsFileP = (ReleaseFile *)Map.Data(); PkgFileP = (PackageFile *)Map.Data(); VerP = (Version *)Map.Data(); DescP = (Description *)Map.Data(); @@ -814,7 +819,7 @@ APT_PURE bool pkgCache::VerIterator::Downloadable() const { VerFileIterator Files = FileList(); for (; Files.end() == false; ++Files) - if ((Files.File()->Flags & pkgCache::Flag::NotSource) != pkgCache::Flag::NotSource) + if (Files.File().Flagged(pkgCache::Flag::NotSource) == false) return true; return false; } @@ -828,7 +833,7 @@ APT_PURE bool pkgCache::VerIterator::Automatic() const VerFileIterator Files = FileList(); for (; Files.end() == false; ++Files) // Do not check ButAutomaticUpgrades here as it is kind of automatic… - if ((Files.File()->Flags & pkgCache::Flag::NotAutomatic) != pkgCache::Flag::NotAutomatic) + if (Files.File().Flagged(pkgCache::Flag::NotAutomatic) == false) return true; return false; } @@ -861,27 +866,27 @@ string pkgCache::VerIterator::RelStr() const for (pkgCache::VerFileIterator I = this->FileList(); I.end() == false; ++I) { // Do not print 'not source' entries' - pkgCache::PkgFileIterator File = I.File(); - if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource) + pkgCache::PkgFileIterator const File = I.File(); + if (File.Flagged(pkgCache::Flag::NotSource)) continue; // See if we have already printed this out.. bool Seen = false; for (pkgCache::VerFileIterator J = this->FileList(); I != J; ++J) { - pkgCache::PkgFileIterator File2 = J.File(); - if (File2->Label == 0 || File->Label == 0) + pkgCache::PkgFileIterator const File2 = J.File(); + if (File2.Label() == 0 || File.Label() == 0) continue; if (strcmp(File.Label(),File2.Label()) != 0) continue; - if (File2->Version == File->Version) + if (File2.Version() == File.Version()) { Seen = true; break; } - if (File2->Version == 0 || File->Version == 0) + if (File2.Version() == 0 || File.Version() == 0) break; if (strcmp(File.Version(),File2.Version()) == 0) Seen = true; @@ -895,12 +900,12 @@ string pkgCache::VerIterator::RelStr() const else First = false; - if (File->Label != 0) + if (File.Label() != 0) Res = Res + File.Label() + ':'; - if (File->Archive != 0) + if (File.Archive() != 0) { - if (File->Version == 0) + if (File.Version() == 0) Res += File.Archive(); else Res = Res + File.Version() + '/' + File.Archive(); @@ -908,7 +913,7 @@ string pkgCache::VerIterator::RelStr() const else { // No release file, print the host name that this came from - if (File->Site == 0 || File.Site()[0] == 0) + if (File.Site() == 0 || File.Site()[0] == 0) Res += "localhost"; else Res += File.Site(); @@ -931,12 +936,12 @@ const char * pkgCache::VerIterator::MultiArchType() const return "none"; } /*}}}*/ -// PkgFileIterator::IsOk - Checks if the cache is in sync with the file /*{{{*/ +// RlsFileIterator::IsOk - Checks if the cache is in sync with the file /*{{{*/ // --------------------------------------------------------------------- /* This stats the file and compares its stats with the ones that were - stored during generation. Date checks should probably also be + stored during generation. Date checks should probably also be included here. */ -bool pkgCache::PkgFileIterator::IsOk() +bool pkgCache::RlsFileIterator::IsOk() { struct stat Buf; if (stat(FileName(),&Buf) != 0) @@ -948,10 +953,8 @@ bool pkgCache::PkgFileIterator::IsOk() return true; } /*}}}*/ -// PkgFileIterator::RelStr - Return the release string /*{{{*/ -// --------------------------------------------------------------------- -/* */ -string pkgCache::PkgFileIterator::RelStr() +// RlsFileIterator::RelStr - Return the release string /*{{{*/ +string pkgCache::RlsFileIterator::RelStr() { string Res; if (Version() != 0) @@ -964,8 +967,40 @@ string pkgCache::PkgFileIterator::RelStr() Res = Res + (Res.empty() == true?"n=":",n=") + Codename(); if (Label() != 0) Res = Res + (Res.empty() == true?"l=":",l=") + Label(); - if (Component() != 0) - Res = Res + (Res.empty() == true?"c=":",c=") + Component(); + return Res; +} + /*}}}*/ +// PkgFileIterator::IsOk - Checks if the cache is in sync with the file /*{{{*/ +// --------------------------------------------------------------------- +/* This stats the file and compares its stats with the ones that were + stored during generation. Date checks should probably also be + included here. */ +bool pkgCache::PkgFileIterator::IsOk() +{ + struct stat Buf; + if (stat(FileName(),&Buf) != 0) + return false; + + if (Buf.st_size != (signed)S->Size || Buf.st_mtime != S->mtime) + return false; + + return true; +} + /*}}}*/ +string pkgCache::PkgFileIterator::RelStr() /*{{{*/ +{ + std::string Res; + if (ReleaseFile() == 0) + { + if (Component() != 0) + Res = Res + (Res.empty() == true?"a=":",a=") + Component(); + } + else + { + Res = ReleaseFile().RelStr(); + if (Component() != 0) + Res = Res + (Res.empty() == true?"c=":",c=") + Component(); + } if (Architecture() != 0) Res = Res + (Res.empty() == true?"b=":",b=") + Architecture(); return Res; diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index b4d56611a..696b3b94d 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -133,6 +133,7 @@ class pkgCache /*{{{*/ struct Header; struct Group; struct Package; + struct ReleaseFile; struct PackageFile; struct Version; struct Description; @@ -150,6 +151,7 @@ class pkgCache /*{{{*/ class DescIterator; class DepIterator; class PrvIterator; + class RlsFileIterator; class PkgFileIterator; class VerFileIterator; class DescFileIterator; @@ -192,9 +194,11 @@ class pkgCache /*{{{*/ enum PkgFlags {Auto=(1<<0),Essential=(1<<3),Important=(1<<4)}; enum PkgFFlags { NotSource=(1<<0), /*!< packages can't be fetched from here, e.g. dpkg/status file */ - NotAutomatic=(1<<1), /*!< archive has a default pin of 1 */ - ButAutomaticUpgrades=(1<<2), /*!< (together with the previous) archive has a default pin of 100 */ - LocalSource=(1<<3), /*!< local sources can't and will not be verified by hashes */ + LocalSource=(1<<1), /*!< local sources can't and will not be verified by hashes */ + }; + enum ReleaseFileFlags { + NotAutomatic=(1<<0), /*!< archive has a default pin of 1 */ + ButAutomaticUpgrades=(1<<1), /*!< (together with the previous) archive has a default pin of 100 */ }; }; @@ -215,6 +219,7 @@ class pkgCache /*{{{*/ Package *PkgP; VerFile *VerFileP; DescFile *DescFileP; + ReleaseFile *RlsFileP; PackageFile *PkgFileP; Version *VerP; Description *DescP; @@ -247,6 +252,8 @@ class pkgCache /*{{{*/ inline PkgIterator PkgEnd(); inline PkgFileIterator FileBegin(); inline PkgFileIterator FileEnd(); + inline RlsFileIterator RlsFileBegin(); + inline RlsFileIterator RlsFileEnd(); inline bool MultiArchCache() const { return MultiArchEnabled; } inline char const * NativeArch(); @@ -296,6 +303,7 @@ struct pkgCache::Header unsigned short HeaderSz; unsigned short GroupSz; unsigned short PackageSz; + unsigned short ReleaseFileSz; unsigned short PackageFileSz; unsigned short VersionSz; unsigned short DescriptionSz; @@ -314,6 +322,7 @@ struct pkgCache::Header map_id_t VersionCount; map_id_t DescriptionCount; map_id_t DependsCount; + map_fileid_t ReleaseFileCount; map_fileid_t PackageFileCount; map_fileid_t VerFileCount; map_fileid_t DescFileCount; @@ -324,6 +333,9 @@ struct pkgCache::Header The PackageFile structures are singly linked lists that represent all package files that have been merged into the cache. */ map_pointer_t FileList; + /** \brief index of the first ReleaseFile structure */ + map_pointer_t RlsFileList; + #if APT_PKG_ABI < 413 APT_DEPRECATED map_pointer_t StringList; #endif @@ -482,15 +494,14 @@ struct pkgCache::Package unsigned long Flags; }; /*}}}*/ -// Package File structure /*{{{*/ -/** \brief stores information about the files used to generate the cache +// Release File structure /*{{{*/ +/** \brief stores information about the release files used to generate the cache - Package files are referenced by Version structures to be able to know - after the generation still from which Packages file includes this Version - as we need this information later on e.g. for pinning. */ -struct pkgCache::PackageFile + PackageFiles reference ReleaseFiles as we need to keep record of which + version belongs to which release e.g. for pinning. */ +struct pkgCache::ReleaseFile { - /** \brief physical disk file that this PackageFile represents */ + /** \brief physical disk file that this ReleaseFile represents */ map_stringitem_t FileName; /** \brief the release information @@ -498,13 +509,47 @@ struct pkgCache::PackageFile release information means. */ map_stringitem_t Archive; map_stringitem_t Codename; - map_stringitem_t Component; map_stringitem_t Version; map_stringitem_t Origin; map_stringitem_t Label; - map_stringitem_t Architecture; /** \brief The site the index file was fetched from */ map_stringitem_t Site; + + /** \brief Size of the file + + Used together with the modification time as a + simple check to ensure that the Packages + file has not been altered since Cache generation. */ + map_filesize_t Size; + /** \brief Modification time for the file */ + time_t mtime; + + /** @TODO document PackageFile::Flags */ + unsigned long Flags; + + // Linked list + /** \brief Link to the next ReleaseFile in the Cache */ + map_pointer_t NextFile; + /** \brief unique sequel ID */ + should_be_map_fileid_t ID; +}; + /*}}}*/ +// Package File structure /*{{{*/ +/** \brief stores information about the files used to generate the cache + + Package files are referenced by Version structures to be able to know + after the generation still from which Packages file includes this Version + as we need this information later on e.g. for pinning. */ +struct pkgCache::PackageFile +{ + /** \brief physical disk file that this PackageFile represents */ + map_stringitem_t FileName; + /** \brief the release information */ + map_pointer_t Release; + + map_stringitem_t Component; + map_stringitem_t Architecture; + /** \brief indicates what sort of index file this is @TODO enumerate at least the possible indexes */ @@ -744,6 +789,11 @@ inline pkgCache::PkgFileIterator pkgCache::FileBegin() {return PkgFileIterator(*this,PkgFileP + HeaderP->FileList);} inline pkgCache::PkgFileIterator pkgCache::FileEnd() {return PkgFileIterator(*this,PkgFileP);} +inline pkgCache::RlsFileIterator pkgCache::RlsFileBegin() + {return RlsFileIterator(*this,RlsFileP + HeaderP->RlsFileList);} +inline pkgCache::RlsFileIterator pkgCache::RlsFileEnd() + {return RlsFileIterator(*this,RlsFileP);} + // Oh I wish for Real Name Space Support class pkgCache::Namespace /*{{{*/ @@ -755,6 +805,7 @@ class pkgCache::Namespace /*{{{*/ typedef pkgCache::DescIterator DescIterator; typedef pkgCache::DepIterator DepIterator; typedef pkgCache::PrvIterator PrvIterator; + typedef pkgCache::RlsFileIterator RlsFileIterator; typedef pkgCache::PkgFileIterator PkgFileIterator; typedef pkgCache::VerFileIterator VerFileIterator; typedef pkgCache::Version Version; diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index ba454f057..2d0e3960d 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -54,10 +54,8 @@ using std::string; /* We set the dirty flag and make sure that is written to the disk */ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : Map(*pMap), Cache(pMap,false), Progress(Prog), - FoundFileDeps(0) + CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0) { - CurrentFile = 0; - if (_error->PendingError() == true) return; @@ -145,6 +143,7 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM Cache.ReMap(false); CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap; + CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap; for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) @@ -167,6 +166,9 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM for (std::vector::const_iterator i = Dynamic::toReMap.begin(); i != Dynamic::toReMap.end(); ++i) (*i)->ReMap(oldMap, newMap); + for (std::vector::const_iterator i = Dynamic::toReMap.begin(); + i != Dynamic::toReMap.end(); ++i) + (*i)->ReMap(oldMap, newMap); } /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String, @@ -1072,13 +1074,47 @@ bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{* return Hash == Ver->Hash; } /*}}}*/ +// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to /*{{{*/ +bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site, + unsigned long Flags) +{ + if (File.empty() && Site.empty()) + { + CurrentRlsFile = NULL; + return true; + } + + // Get some space for the structure + map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile)); + if (unlikely(idxFile == 0)) + return false; + CurrentRlsFile = Cache.RlsFileP + idxFile; + + // Fill it in + map_stringitem_t const idxFileName = WriteStringInMap(File); + map_stringitem_t const idxSite = StoreString(MIXED, Site); + if (unlikely(idxFileName == 0 || idxSite == 0)) + return false; + CurrentRlsFile->FileName = idxFileName; + CurrentRlsFile->Site = idxSite; + CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList; + CurrentRlsFile->Flags = Flags; + CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount; + RlsFileName = File; + Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP; + Cache.HeaderP->ReleaseFileCount++; + + return true; +} + /*}}}*/ // CacheGenerator::SelectFile - Select the current file being parsed /*{{{*/ // --------------------------------------------------------------------- /* This is used to select which file is to be associated with all newly added versions. The caller is responsible for setting the IMS fields. */ -bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, - const pkgIndexFile &Index, - unsigned long Flags) +bool pkgCacheGenerator::SelectFile(std::string const &File, + pkgIndexFile const &Index, + std::string const &Component, + unsigned long const Flags) { // Get some space for the structure map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile)); @@ -1088,18 +1124,24 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, // Fill it in map_stringitem_t const idxFileName = WriteStringInMap(File); - map_stringitem_t const idxSite = StoreString(MIXED, Site); - if (unlikely(idxFileName == 0 || idxSite == 0)) + if (unlikely(idxFileName == 0)) return false; CurrentFile->FileName = idxFileName; - CurrentFile->Site = idxSite; CurrentFile->NextFile = Cache.HeaderP->FileList; - CurrentFile->Flags = Flags; CurrentFile->ID = Cache.HeaderP->PackageFileCount; map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label); if (unlikely(idxIndexType == 0)) return false; CurrentFile->IndexType = idxIndexType; + map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component); + if (unlikely(component == 0)) + return false; + CurrentFile->Component = component; + CurrentFile->Flags = Flags; + if (CurrentRlsFile != NULL) + CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP; + else + CurrentFile->Release = 0; PkgFileName = File; Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP; Cache.HeaderP->PackageFileCount++; @@ -1174,35 +1216,59 @@ static bool CheckValidity(const string &CacheFile, _error->Discard(); return false; } - + + SPtrArray RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount]; + memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount); + std::vector Files; + for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i) + { + if (Debug == true) + std::clog << "Checking RlsFile " << (*i)->Describe() << ": "; + pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache); + if (RlsFile.end() == true) + { + if (Debug == true) + std::clog << "FindInCache returned end-Pointer" << std::endl; + return false; + } + + RlsVisited[RlsFile->ID] = true; + if (Debug == true) + std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl; + + std::vector *Indexes = (*i)->GetIndexFiles(); + for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j) + if ((*j)->HasPackages()) + Files.push_back (*j); + } + for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I) + if (RlsVisited[I] == false) + { + if (Debug == true) + std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl; + return false; + } + + for (; Start != End; ++Start) + Files.push_back(*Start); + /* Now we check every index file, see if it is in the cache, verify the IMS data and check that it is on the disk too.. */ SPtrArray Visited = new bool[Cache.HeaderP->PackageFileCount]; memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount); - for (; Start != End; ++Start) + for (std::vector::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile) { if (Debug == true) - std::clog << "Checking PkgFile " << (*Start)->Describe() << ": "; - if ((*Start)->HasPackages() == false) - { - if (Debug == true) - std::clog << "Has NO packages" << std::endl; - continue; - } - - if ((*Start)->Exists() == false) + std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": "; + if ((*PkgFile)->Exists() == false) { -#if 0 // mvo: we no longer give a message here (Default Sources spec) - _error->WarningE("stat",_("Couldn't stat source package list %s"), - (*Start)->Describe().c_str()); -#endif if (Debug == true) std::clog << "file doesn't exist" << std::endl; continue; } // FindInCache is also expected to do an IMS check. - pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache); + pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache); if (File.end() == true) { if (Debug == true) @@ -1214,15 +1280,15 @@ static bool CheckValidity(const string &CacheFile, if (Debug == true) std::clog << "with ID " << File->ID << " is valid" << std::endl; } - + for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++) if (Visited[I] == false) { if (Debug == true) - std::clog << "File with ID" << I << " wasn't visited" << std::endl; + std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl; return false; } - + if (_error->PendingError() == true) { if (Debug == true) @@ -1243,9 +1309,20 @@ static bool CheckValidity(const string &CacheFile, // --------------------------------------------------------------------- /* Size is kind of an abstract notion that is only used for the progress meter */ -static map_filesize_t ComputeSize(FileIterator Start,FileIterator End) +static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End) { map_filesize_t TotalSize = 0; + if (List != NULL) + { + for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i) + { + std::vector *Indexes = (*i)->GetIndexFiles(); + for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j) + if ((*j)->HasPackages() == true) + TotalSize += (*j)->Size(); + } + } + for (; Start < End; ++Start) { if ((*Start)->HasPackages() == false) @@ -1261,11 +1338,63 @@ static map_filesize_t ComputeSize(FileIterator Start,FileIterator End) static bool BuildCache(pkgCacheGenerator &Gen, OpProgress *Progress, map_filesize_t &CurrentSize,map_filesize_t TotalSize, + pkgSourceList const * const List, FileIterator Start, FileIterator End) { + std::vector Files; + bool const HasFileDeps = Gen.HasFileDeps(); + + if (List != NULL) + { + for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i) + { + if ((*i)->FindInCache(Gen.GetCache()).end() == false) + { + _error->Warning("Duplicate sources.list entry %s", + (*i)->Describe().c_str()); + continue; + } + + if ((*i)->Merge(Gen, Progress) == false) + return false; + + std::vector *Indexes = (*i)->GetIndexFiles(); + for (std::vector::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I) + { + if (HasFileDeps) + Files.push_back(*I); + + if ((*I)->HasPackages() == false) + continue; + + if ((*I)->Exists() == false) + continue; + + if ((*I)->FindInCache(Gen.GetCache()).end() == false) + { + _error->Warning("Duplicate sources.list entry %s", + (*I)->Describe().c_str()); + continue; + } + + map_filesize_t Size = (*I)->Size(); + if (Progress != NULL) + Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); + CurrentSize += Size; + + if ((*I)->Merge(Gen,Progress) == false) + return false; + } + } + } + + Gen.SelectReleaseFile("", ""); FileIterator I; for (I = Start; I != End; ++I) { + if (HasFileDeps) + Files.push_back(*I); + if ((*I)->HasPackages() == false) continue; @@ -1288,13 +1417,13 @@ static bool BuildCache(pkgCacheGenerator &Gen, return false; } - if (Gen.HasFileDeps() == true) + if (HasFileDeps == true) { if (Progress != NULL) Progress->Done(); - TotalSize = ComputeSize(Start, End); + TotalSize = ComputeSize(List, Start, End); CurrentSize = 0; - for (I = Start; I != End; ++I) + for (std::vector::const_iterator I = Files.begin(); I != Files.end(); ++I) { map_filesize_t Size = (*I)->Size(); if (Progress != NULL) @@ -1339,6 +1468,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress bool const Debug = _config->FindB("Debug::pkgCacheGen", false); std::vector Files; + /* for (std::vector::const_iterator i = List.begin(); i != List.end(); ++i) @@ -1349,8 +1479,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress ++j) Files.push_back (*j); } - - map_filesize_t const EndOfSource = Files.size(); +*/ if (_system->AddStatusFiles(Files) == false) return false; @@ -1442,8 +1571,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress // Lets try the source cache. map_filesize_t CurrentSize = 0; map_filesize_t TotalSize = 0; - if (CheckValidity(SrcCacheFile, List, Files.begin(), - Files.begin()+EndOfSource) == true) + if (CheckValidity(SrcCacheFile, List, Files.end(), + Files.end()) == true) { if (Debug == true) std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl; @@ -1455,28 +1584,28 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress SCacheF.Size()) == false) return false; - TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end()); + TotalSize = ComputeSize(NULL, Files.begin(), Files.end()); // Build the status cache pkgCacheGenerator Gen(Map.Get(),Progress); if (_error->PendingError() == true) return false; - if (BuildCache(Gen,Progress,CurrentSize,TotalSize, - Files.begin()+EndOfSource,Files.end()) == false) + if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(),Files.end()) == false) return false; } else { if (Debug == true) std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl; - TotalSize = ComputeSize(Files.begin(),Files.end()); + TotalSize = ComputeSize(&List, Files.begin(),Files.end()); // Build the source cache pkgCacheGenerator Gen(Map.Get(),Progress); if (_error->PendingError() == true) return false; - if (BuildCache(Gen,Progress,CurrentSize,TotalSize, - Files.begin(),Files.begin()+EndOfSource) == false) + if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List, + Files.end(),Files.end()) == false) return false; // Write it back @@ -1503,8 +1632,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress } // Build the status cache - if (BuildCache(Gen,Progress,CurrentSize,TotalSize, - Files.begin()+EndOfSource,Files.end()) == false) + if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) return false; } if (Debug == true) @@ -1536,7 +1665,6 @@ APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **Ou bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap) { std::vector Files; - map_filesize_t EndOfSource = Files.size(); if (_system->AddStatusFiles(Files) == false) return false; @@ -1544,7 +1672,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O map_filesize_t CurrentSize = 0; map_filesize_t TotalSize = 0; - TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end()); + TotalSize = ComputeSize(NULL, Files.begin(), Files.end()); // Build the status cache if (Progress != NULL) @@ -1552,8 +1680,8 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O pkgCacheGenerator Gen(Map.Get(),Progress); if (_error->PendingError() == true) return false; - if (BuildCache(Gen,Progress,CurrentSize,TotalSize, - Files.begin()+EndOfSource,Files.end()) == false) + if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL, + Files.begin(), Files.end()) == false) return false; if (_error->PendingError() == true) diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index c4ace713d..2795d09d6 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -69,7 +69,9 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ DynamicMMap ⤅ pkgCache Cache; OpProgress *Progress; - + + std::string RlsFileName; + pkgCache::ReleaseFile *CurrentRlsFile; std::string PkgFileName; pkgCache::PackageFile *CurrentFile; @@ -100,12 +102,14 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ inline map_stringitem_t StoreString(enum StringType const type, const std::string &S) {return StoreString(type, S.c_str(),S.length());}; void DropProgress() {Progress = 0;}; - bool SelectFile(const std::string &File,const std::string &Site,pkgIndexFile const &Index, - unsigned long Flags = 0); + bool SelectFile(const std::string &File,pkgIndexFile const &Index, std::string const &Component, unsigned long Flags = 0); + bool SelectReleaseFile(const std::string &File, const std::string &Site, unsigned long Flags = 0); bool MergeList(ListParser &List,pkgCache::VerIterator *Ver = 0); inline pkgCache &GetCache() {return Cache;}; inline pkgCache::PkgFileIterator GetCurFile() {return pkgCache::PkgFileIterator(Cache,CurrentFile);}; + inline pkgCache::RlsFileIterator GetCurRlsFile() + {return pkgCache::RlsFileIterator(Cache,CurrentRlsFile);}; bool HasFileDeps() {return FoundFileDeps;}; bool MergeFileProvides(ListParser &List); diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc index 00693ce54..bd40ad2d9 100644 --- a/apt-pkg/policy.cc +++ b/apt-pkg/policy.cc @@ -63,9 +63,9 @@ pkgPolicy::pkgPolicy(pkgCache *Owner) : Pins(0), PFPriority(0), Cache(Owner) pkgVersionMatch vm("", pkgVersionMatch::None); for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F) { - if ((F->Archive != 0 && vm.ExpressionMatches(DefRel, F.Archive()) == true) || - (F->Codename != 0 && vm.ExpressionMatches(DefRel, F.Codename()) == true) || - (F->Version != 0 && vm.ExpressionMatches(DefRel, F.Version()) == true) || + if (vm.ExpressionMatches(DefRel, F.Archive()) || + vm.ExpressionMatches(DefRel, F.Codename()) || + vm.ExpressionMatches(DefRel, F.Version()) || (DefRel.length() > 2 && DefRel[1] == '=')) found = true; } @@ -86,11 +86,11 @@ bool pkgPolicy::InitDefaults() for (pkgCache::PkgFileIterator I = Cache->FileBegin(); I != Cache->FileEnd(); ++I) { PFPriority[I->ID] = 500; - if ((I->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource) + if (I.Flagged(pkgCache::Flag::NotSource)) PFPriority[I->ID] = 100; - else if ((I->Flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades) + else if (I.Flagged(pkgCache::Flag::ButAutomaticUpgrades)) PFPriority[I->ID] = 100; - else if ((I->Flags & pkgCache::Flag::NotAutomatic) == pkgCache::Flag::NotAutomatic) + else if (I.Flagged(pkgCache::Flag::NotAutomatic)) PFPriority[I->ID] = 1; } @@ -170,8 +170,7 @@ pkgCache::VerIterator pkgPolicy::GetCandidateVer(pkgCache::PkgIterator const &Pk then it is not a candidate for installation, ever. This weeds out bogus entries that may be due to config-file states, or other. */ - if ((VF.File()->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource && - instVer == false) + if (VF.File().Flagged(pkgCache::Flag::NotSource) && instVer == false) continue; signed Prio = PFPriority[VF.File()->ID]; diff --git a/apt-pkg/versionmatch.cc b/apt-pkg/versionmatch.cc index 284098bdf..86c1b7d4a 100644 --- a/apt-pkg/versionmatch.cc +++ b/apt-pkg/versionmatch.cc @@ -137,7 +137,10 @@ pkgVersionMatch::pkgVersionMatch(string Data,MatchType Type) : Type(Type) // --------------------------------------------------------------------- /* */ bool pkgVersionMatch::MatchVer(const char *A,string B,bool Prefix) -{ +{ + if (A == NULL) + return false; + const char *Ab = A; const char *Ae = Ab + strlen(A); @@ -178,13 +181,16 @@ pkgCache::VerIterator pkgVersionMatch::Find(pkgCache::PkgIterator Pkg) // This will be Ended by now. return Ver; } + /*}}}*/ #ifndef FNM_CASEFOLD #define FNM_CASEFOLD 0 #endif -bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string) +bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string)/*{{{*/ { + if (pattern == NULL || string == NULL) + return false; if (pattern[0] == '/') { size_t length = strlen(pattern); if (pattern[length - 1] == '/') { @@ -230,38 +236,30 @@ bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File) return false; if (RelVerStr.empty() == false) - if (File->Version == 0 || - (MatchVer(File.Version(),RelVerStr,RelVerPrefixMatch) == false && - ExpressionMatches(RelVerStr, File.Version()) == false)) + if (MatchVer(File.Version(),RelVerStr,RelVerPrefixMatch) == false && + ExpressionMatches(RelVerStr, File.Version()) == false) return false; if (RelOrigin.empty() == false) - if (File->Origin == 0 || !ExpressionMatches(RelOrigin,File.Origin())) + if (!ExpressionMatches(RelOrigin,File.Origin())) return false; if (RelArchive.empty() == false) - if (File->Archive == 0 || - !ExpressionMatches(RelArchive,File.Archive())) + if (!ExpressionMatches(RelArchive,File.Archive())) return false; if (RelCodename.empty() == false) - if (File->Codename == 0 || - !ExpressionMatches(RelCodename,File.Codename())) + if (!ExpressionMatches(RelCodename,File.Codename())) return false; if (RelRelease.empty() == false) - if ((File->Archive == 0 || - !ExpressionMatches(RelRelease,File.Archive())) && - (File->Codename == 0 || - !ExpressionMatches(RelRelease,File.Codename()))) + if (!ExpressionMatches(RelRelease,File.Archive()) && + !ExpressionMatches(RelRelease,File.Codename())) return false; if (RelLabel.empty() == false) - if (File->Label == 0 || - !ExpressionMatches(RelLabel,File.Label())) + if (!ExpressionMatches(RelLabel,File.Label())) return false; if (RelComponent.empty() == false) - if (File->Component == 0 || - !ExpressionMatches(RelComponent,File.Component())) + if (!ExpressionMatches(RelComponent,File.Component())) return false; if (RelArchitecture.empty() == false) - if (File->Architecture == 0 || - !ExpressionMatches(RelArchitecture,File.Architecture())) + if (!ExpressionMatches(RelArchitecture,File.Architecture())) return false; return true; } @@ -269,11 +267,14 @@ bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File) if (Type == Origin) { if (OrSite.empty() == false) { - if (File->Site == 0) + if (File.Site() == NULL) return false; } else // so we are talking about file:// or status file - if (strcmp(File.Site(),"") == 0 && File->Archive != 0 && strcmp(File.Archive(),"now") == 0) // skip the status file + { + pkgCache::RlsFileIterator const RlsFile = File.ReleaseFile(); + if (strcmp(File.Site(),"") == 0 && RlsFile->Archive != 0 && strcmp(RlsFile.Archive(),"now") == 0) // skip the status file return false; + } return (ExpressionMatches(OrSite, File.Site())); /* both strings match */ } diff --git a/apt-pkg/versionmatch.h b/apt-pkg/versionmatch.h index 4c8f704c8..bb950b9c7 100644 --- a/apt-pkg/versionmatch.h +++ b/apt-pkg/versionmatch.h @@ -70,8 +70,8 @@ class pkgVersionMatch enum MatchType {None = 0,Version,Release,Origin} Type; bool MatchVer(const char *A,std::string B,bool Prefix) APT_PURE; - bool ExpressionMatches(const char *pattern, const char *string); - bool ExpressionMatches(const std::string& pattern, const char *string); + static bool ExpressionMatches(const char *pattern, const char *string); + static bool ExpressionMatches(const std::string& pattern, const char *string); bool FileMatch(pkgCache::PkgFileIterator File); pkgCache::VerIterator Find(pkgCache::PkgIterator Pkg); diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 690b03bcc..3b8ebc96d 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -413,17 +413,21 @@ static bool Stats(CommandLine &) stritems.insert(Prv->ProvideVersion); } } - for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F) + for (pkgCache::RlsFileIterator F = Cache->RlsFileBegin(); F != Cache->RlsFileEnd(); ++F) { stritems.insert(F->FileName); stritems.insert(F->Archive); stritems.insert(F->Codename); - stritems.insert(F->Component); stritems.insert(F->Version); stritems.insert(F->Origin); stritems.insert(F->Label); - stritems.insert(F->Architecture); stritems.insert(F->Site); + } + for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F) + { + stritems.insert(F->FileName); + stritems.insert(F->Architecture); + stritems.insert(F->Component); stritems.insert(F->IndexType); } unsigned long Size = 0; @@ -446,6 +450,7 @@ static bool Stats(CommandLine &) APT_CACHESIZE(VersionCount, VersionSz) + APT_CACHESIZE(DescriptionCount, DescriptionSz) + APT_CACHESIZE(DependsCount, DependencySz) + + APT_CACHESIZE(ReleaseFileCount, ReleaseFileSz) + APT_CACHESIZE(PackageFileCount, PackageFileSz) + APT_CACHESIZE(VerFileCount, VerFileSz) + APT_CACHESIZE(DescFileCount, DescFileSz) + diff --git a/test/integration/framework b/test/integration/framework index 110758e82..7b03c09ef 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -137,7 +137,14 @@ dpkgcheckbuilddeps() { command dpkg-checkbuilddeps --admindir=${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg "$@" } gdb() { - local CMD="$1" + local CMD + case "$1" in + aptget) CMD="apt-get";; + aptcache) CMD="apt-cache";; + aptmark) CMD="apt-mark";; + apthelper) CMD="apt-helper";; + *) CMD="$1";; + esac shift runapt command gdb --quiet -ex run "${BUILDDIRECTORY}/$CMD" --args "${BUILDDIRECTORY}/$CMD" "$@" } diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning index 9cd54264b..9f7f457ae 100755 --- a/test/integration/test-policy-pinning +++ b/test/integration/test-policy-pinning @@ -22,7 +22,7 @@ testequalpolicy() { release a=now $(echo "$AP" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE} Packages release c= -Pinned packages:" aptcache policy $* +Pinned packages:" aptcache policy "$@" } testglobalpolicy() { -- cgit v1.2.3-70-g09d2 From d56e2917f27a722b54685de13aeb1bb7592fc61b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 13 Jun 2015 11:13:45 +0200 Subject: provide a public interface for acquiring changelogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provided is a specialized acquire item which given a version can figure out the correct URI to try by itself and if not provides an error message alongside with static methods to get just the URI it would try to download if it should just be displayed or similar such. The URI is constructed as follows: Release files can provide an URI template in the "Changelogs" field, otherwise we lookup a configuration item based on the "Label" or "Origin" of the Release file to get a (hopefully known) default value for now. This template should contain the string CHANGEPATH which is replaced with the information about the version we want the changelog for (e.g. main/a/apt/apt_1.1). This middleway was choosen as this path part was consistent over the three known implementations (+1 defunct), while the rest of the URI varies widely between them. The benefit of this construct is that it is now easy to get changelogs for Debian packages on Ubuntu and vice versa – even at the moment where the Changelogs field is present nowhere. Strictly better than what apt-get had before as it would even fail to get changelogs from security… Now it will notice that security identifies as Origin: Debian and pick this setting (assuming again that no Changelogs field exists). If on the other hand security would ship its changelogs in a different location we could set it via the Label option overruling Origin. Closes: 687147, 739854, 784027, 787190 --- apt-pkg/acquire-item.cc | 211 +++++++++++++++++++++ apt-pkg/acquire-item.h | 114 +++++++++++ apt-pkg/init.cc | 4 + cmdline/apt-get.cc | 204 ++++---------------- doc/apt-get.8.xml | 16 +- doc/apt.conf.5.xml | 27 +++ doc/examples/configure-index | 11 +- test/integration/framework | 24 ++- test/integration/test-apt-get-changelog | 104 +++++++--- .../test-bug-722207-print-uris-even-if-very-quiet | 3 +- test/integration/test-bug-738785-switch-protocol | 7 +- vendor/ubuntu/apt.conf-01-vendor-ubuntu | 6 - 12 files changed, 507 insertions(+), 224 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d6e9ccbe0..4bf4e62f8 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -2836,6 +2837,216 @@ std::string pkgAcqArchive::ShortDesc() const /*{{{*/ } /*}}}*/ +// AcqChangelog::pkgAcqChangelog - Constructors /*{{{*/ +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver, + std::string const &DestDir, std::string const &DestFilename) : + pkgAcquire::Item(Owner), d(NULL), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr()) +{ + Desc.URI = URI(Ver); + Init(DestDir, DestFilename); +} +// some parameters are char* here as they come likely from char* interfaces – which can also return NULL +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile, + char const * const Component, char const * const SrcName, char const * const SrcVersion, + const string &DestDir, const string &DestFilename) : + pkgAcquire::Item(Owner), d(NULL), SrcName(SrcName), SrcVersion(SrcVersion) +{ + Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion); + Init(DestDir, DestFilename); +} +pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, + std::string const &URI, char const * const SrcName, char const * const SrcVersion, + const string &DestDir, const string &DestFilename) : + pkgAcquire::Item(Owner), d(NULL), SrcName(SrcName), SrcVersion(SrcVersion) +{ + Desc.URI = URI; + Init(DestDir, DestFilename); +} +void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename) +{ + if (Desc.URI.empty()) + { + Status = StatError; + // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1 + strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str()); + // Let the error message print something sensible rather than "Failed to fetch /" + if (DestFilename.empty()) + DestFile = SrcName + ".changelog"; + else + DestFile = DestFilename; + Desc.URI = "changelog:/" + DestFile; + return; + } + + if (DestDir.empty()) + { + std::string const systemTemp = GetTempDir(); + char tmpname[100]; + snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str()); + if (NULL == mkdtemp(tmpname)) + { + _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str()); + Status = StatError; + return; + } + DestFile = TemporaryDirectory = tmpname; + } + else + DestFile = DestDir; + + if (DestFilename.empty()) + DestFile = flCombine(DestFile, SrcName + ".changelog"); + else + DestFile = flCombine(DestFile, DestFilename); + + Desc.ShortDesc = "Changelog"; + strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str()); + Desc.Owner = this; + QueueURI(Desc); + + if (Status == StatDone) // this happens if we queue the same changelog two times + { + Complete = true; + for (pkgAcquire::UriIterator I = Owner->UriBegin(); I != Owner->UriEnd(); ++I) + if (I->URI == Desc.URI) + if (DestFile != I->Owner->DestFile) + if (symlink(I->Owner->DestFile.c_str(), DestFile.c_str()) != 0) + { + ; // ignore error, there isn't anthing we could do to handle the edgecase of an edgecase + } + } +} + /*}}}*/ +std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver) /*{{{*/ +{ + char const * const SrcName = Ver.SourcePkgName(); + char const * const SrcVersion = Ver.SourceVerStr(); + pkgCache::PkgFileIterator PkgFile; + // find the first source for this version which promises a changelog + for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF) + { + pkgCache::PkgFileIterator const PF = VF.File(); + if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0) + continue; + PkgFile = PF; + pkgCache::RlsFileIterator const RF = PF.ReleaseFile(); + std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion); + if (uri.empty()) + continue; + return uri; + } + return ""; +} +std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls) +{ + if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0)) + return ""; + std::string const serverConfig = "Acquire::Changelogs::URI"; + std::string server; +#define APT_EMPTY_SERVER \ + if (server.empty() == false) \ + { \ + if (server != "no") \ + return server; \ + return ""; \ + } +#define APT_CHECK_SERVER(X, Y) \ + if (Rls->X != 0) \ + { \ + std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \ + server = _config->Find(specialServerConfig); \ + APT_EMPTY_SERVER \ + } + // this way e.g. Debian-Security can fallback to Debian + APT_CHECK_SERVER(Label, "Override::") + APT_CHECK_SERVER(Origin, "Override::") + + if (RealFileExists(Rls.FileName())) + { + _error->PushToStack(); + FileFd rf; + /* This can be costly. A caller wanting to get millions of URIs might + want to do this on its own once and use Override settings. + We don't do this here as Origin/Label are not as unique as they + should be so this could produce request order-dependent anomalies */ + if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true) + { + pkgTagFile TagFile(&rf, rf.Size()); + pkgTagSection Section; + if (TagFile.Step(Section) == true) + server = Section.FindS("Changelogs"); + } + _error->RevertToStack(); + APT_EMPTY_SERVER + } + + APT_CHECK_SERVER(Label, "") + APT_CHECK_SERVER(Origin, "") +#undef APT_CHECK_SERVER +#undef APT_EMPTY_SERVER + return ""; +} +std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, + char const * const SrcVersion) +{ + return URI(URITemplate(Rls), Component, SrcName, SrcVersion); +} +std::string pkgAcqChangelog::URI(std::string const &Template, + char const * const Component, char const * const SrcName, + char const * const SrcVersion) +{ + if (Template.find("CHANGEPATH") == std::string::npos) + return ""; + + // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/1.1 or contrib/liba/libapt/2.0 + std::string Src = SrcName; + std::string path = APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1); + path.append("/").append(Src).append("/"); + path.append(Src).append("_").append(StripEpoch(SrcVersion)); + // we omit component for releases without one (= flat-style repositories) + if (Component != NULL && strlen(Component) != 0) + path = std::string(Component) + "/" + path; + + return SubstVar(Template, "CHANGEPATH", path); +} + /*}}}*/ +// AcqChangelog::Failed - Failure handler /*{{{*/ +void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Failed(Message,Cnf); + + std::string errText; + // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1 + strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str()); + + // Error is probably something techy like 404 Not Found + if (ErrorText.empty()) + ErrorText = errText; + else + ErrorText = errText + " (" + ErrorText + ")"; + return; +} + /*}}}*/ +// AcqChangelog::Done - Item downloaded OK /*{{{*/ +void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf) +{ + Item::Done(Message,CalcHashes,Cnf); + + Complete = true; +} + /*}}}*/ +pkgAcqChangelog::~pkgAcqChangelog() /*{{{*/ +{ + if (TemporaryDirectory.empty() == false) + { + unlink(DestFile.c_str()); + rmdir(TemporaryDirectory.c_str()); + } +} + /*}}}*/ + // AcqFile::pkgAcqFile - Constructor /*{{{*/ pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes, unsigned long long const Size,string const &Dsc,string const &ShortDesc, diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index b6d569737..606fd4173 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -1031,6 +1031,120 @@ class pkgAcqArchive : public pkgAcquire::Item std::string &StoreFilename); }; /*}}}*/ +/** \brief Retrieve the changelog for the given version {{{ + * + * Downloads the changelog to a temporary file it will also remove again + * while it is deconstructed or downloads it to a named location. + */ +class pkgAcqChangelog : public pkgAcquire::Item +{ + void *d; + std::string TemporaryDirectory; + std::string const SrcName; + std::string const SrcVersion; + + public: + // we will never have hashes for changelogs. + // If you need verified ones, download the deb and extract the changelog. + virtual HashStringList GetExpectedHashes() const { return HashStringList(); } + virtual bool HashesRequired() const { return false; } + + // Specialized action members + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf); + virtual void Done(std::string const &Message, HashStringList const &CalcHashes, + pkgAcquire::MethodConfig const * const Cnf); + virtual std::string DescURI() const {return Desc.URI;}; + + /** returns the URI to the changelog of this version + * + * @param Ver is the version to get the changelog for + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(pkgCache::VerIterator const &Ver); + + /** returns the URI to the changelog of this version + * + * \param Rls is the Release file the package comes from + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, + char const * const SrcVersion); + + /** returns the URI to the changelog of this version + * + * \param Template URI where CHANGEPATH has to be filled in + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * @return the URI which will be used to acquire the changelog + */ + static std::string URI(std::string const &Template, + char const * const Component, char const * const SrcName, + char const * const SrcVersion); + + /** returns the URI template for this release file + * + * \param Rls is a Release file + * @return the URI template to use for this release file + */ + static std::string URITemplate(pkgCache::RlsFileIterator const &Rls); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param Ver is the version to get the changelog for + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver, + std::string const &DestDir="", std::string const &DestFilename=""); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param Rls is the Release file the package comes from + * \param Component in which the package resides, can be empty + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &Rls, + char const * const Component, char const * const SrcName, char const * const SrcVersion, + std::string const &DestDir="", std::string const &DestFilename=""); + + /** \brief Create a new pkgAcqChangelog object. + * + * \param Owner The pkgAcquire object with which this object is + * associated. + * \param URI is to be used to get the changelog + * \param SrcName is the source package name + * \param SrcVersion is the source package version + * \param DestDir The directory the file should be downloaded into. + * Will be an autocreated (and cleaned up) temporary directory if not set. + * \param DestFilename The filename the file should have in #DestDir + * Defaults to sourcepackagename.changelog if not set. + */ + pkgAcqChangelog(pkgAcquire * const Owner, std::string const &URI, + char const * const SrcName, char const * const SrcVersion, + std::string const &DestDir="", std::string const &DestFilename=""); + + virtual ~pkgAcqChangelog(); + +private: + APT_HIDDEN void Init(std::string const &DestDir, std::string const &DestFilename); +}; + /*}}}*/ /** \brief Retrieve an arbitrary file to the current directory. {{{ * * The file is retrieved even if it is accessed via a URL type that diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index e2239a906..96966b249 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -119,6 +119,10 @@ bool pkgInitConfig(Configuration &Cnf) Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatDescription", "$(SITE) $(RELEASE) Sources"); Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Optional", false); + Cnf.CndSet("Acquire::Changelogs::URI::Origin::Debian", "http://metadata.ftp-master.debian.org/changelogs/CHANGEPATH_changelog"); + Cnf.CndSet("Acquire::Changelogs::URI::Origin::Ubuntu", "http://changelogs.ubuntu.com/changelogs/pool/CHANGEPATH/changelog"); + Cnf.CndSet("Acquire::Changelogs::URI::Origin::Ultimedia", "http://packages.ultimediaos.com/changelogs/pool/CHANGEPATH/changelog.txt"); + bool Res = true; // Read an alternate config file diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index a4cd3c377..184b51d23 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1410,196 +1410,70 @@ static bool DoBuildDep(CommandLine &CmdL) return true; } /*}}}*/ -// GetChangelogPath - return a path pointing to a changelog file or dir /*{{{*/ -// --------------------------------------------------------------------- -/* This returns a "path" string for the changelog url construction. - * Please note that its not complete, it either needs a "/changelog" - * appended (for the packages.debian.org/changelogs site) or a - * ".changelog" (for third party sites that store the changelog in the - * pool/ next to the deb itself) - * Example return: "pool/main/a/apt/apt_0.8.8ubuntu3" - */ -static string GetChangelogPath(CacheFile &Cache, - pkgCache::VerIterator Ver) -{ - pkgRecords Recs(Cache); - pkgRecords::Parser &rec=Recs.Lookup(Ver.FileList()); - string path = flNotFile(rec.FileName()); -#if APT_PKG_ABI >= 413 - path.append(Ver.SourcePkgName()); - path.append("_"); - path.append(StripEpoch(Ver.SourceVerStr())); -#else - string srcpkg = rec.SourcePkg().empty() ? Ver.ParentPkg().Name() : rec.SourcePkg(); - string ver = Ver.VerStr(); - // if there is a source version it always wins - if (rec.SourceVer() != "") - ver = rec.SourceVer(); - path += srcpkg + "_" + StripEpoch(ver); -#endif - return path; -} - /*}}}*/ -// GuessThirdPartyChangelogUri - return url /*{{{*/ -// --------------------------------------------------------------------- -/* Contruct a changelog file path for third party sites that do not use - * packages.debian.org/changelogs - * This simply uses the ArchiveURI() of the source pkg and looks for - * a .changelog file there, Example for "mediabuntu": - * apt-get changelog mplayer-doc: - * http://packages.medibuntu.org/pool/non-free/m/mplayer/mplayer_1.0~rc4~try1.dsfg1-1ubuntu1+medibuntu1.changelog - */ -static bool GuessThirdPartyChangelogUri(CacheFile &Cache, - pkgCache::VerIterator Ver, - string &out_uri) -{ - // get the binary deb server path - pkgCache::VerFileIterator Vf = Ver.FileList(); - if (Vf.end() == true) - return false; - pkgCache::PkgFileIterator F = Vf.File(); - pkgIndexFile *index; - pkgSourceList *SrcList = Cache.GetSourceList(); - if(SrcList->FindIndex(F, index) == false) - return false; - - // get archive uri for the binary deb - string path_without_dot_changelog = GetChangelogPath(Cache, Ver); - out_uri = index->ArchiveURI(path_without_dot_changelog + ".changelog"); - - // now strip away the filename and add srcpkg_srcver.changelog - return true; -} - /*}}}*/ -// DownloadChangelog - Download the changelog /*{{{*/ -// --------------------------------------------------------------------- -static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, - pkgCache::VerIterator Ver, string targetfile) -/* Download a changelog file for the given package version to - * targetfile. This will first try the server from Apt::Changelogs::Server - * (http://packages.debian.org/changelogs by default) and if that gives - * a 404 tries to get it from the archive directly (see - * GuessThirdPartyChangelogUri for details how) - */ -{ - // make the server root configurable - string const server = _config->Find("Apt::Changelogs::Server", - "http://packages.debian.org/changelogs"); - string const path = GetChangelogPath(CacheFile, Ver); - string changelog_uri; - if (APT::String::Endswith(server, "/") == true) - strprintf(changelog_uri, "%s%s/changelog", server.c_str(), path.c_str()); - else - strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str()); - if (_config->FindB("APT::Get::Print-URIs", false) == true) - { - std::cout << '\'' << changelog_uri << '\'' << std::endl; - return true; - } - pkgCache::PkgIterator const Pkg = Ver.ParentPkg(); - - string descr; - strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), changelog_uri.c_str()); - // queue it - pkgAcquire::Item const * itm = new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); - - // Disable drop-privs if "_apt" can not write to the target dir - CheckDropPrivsMustBeDisabled(Fetcher); - - // try downloading it, if that fails, try third-party-changelogs location - // FIXME: Fetcher.Run() is "Continue" even if I get a 404?!? - Fetcher.Run(); - if (itm->Status != pkgAcquire::Item::StatDone) - { - string third_party_uri; - if (GuessThirdPartyChangelogUri(CacheFile, Ver, third_party_uri)) - { - strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), third_party_uri.c_str()); - itm = new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); - Fetcher.Run(); - } - } - - if (itm->Status == pkgAcquire::Item::StatDone) - return true; - - // error - return _error->Error("changelog download failed"); -} - /*}}}*/ // DoChangelog - Get changelog from the command line /*{{{*/ -// --------------------------------------------------------------------- static bool DoChangelog(CommandLine &CmdL) { CacheFile Cache; if (Cache.ReadOnlyOpen() == false) return false; - + APT::CacheSetHelper helper; APT::VersionList verset = APT::VersionList::FromCommandLine(Cache, CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper); if (verset.empty() == true) return false; pkgAcquire Fetcher; + AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0)); + Fetcher.SetLog(&Stat); - if (_config->FindB("APT::Get::Print-URIs", false) == true) + bool const downOnly = _config->FindB("APT::Get::Download-Only", false); + bool const printOnly = _config->FindB("APT::Get::Print-URIs", false); + + for (APT::VersionList::const_iterator Ver = verset.begin(); + Ver != verset.end(); + ++Ver) { - bool Success = true; - for (APT::VersionList::const_iterator Ver = verset.begin(); - Ver != verset.end(); ++Ver) - Success &= DownloadChangelog(Cache, Fetcher, Ver, ""); - return Success; + if (printOnly) + new pkgAcqChangelog(&Fetcher, Ver, "/dev/null"); + else if (downOnly) + new pkgAcqChangelog(&Fetcher, Ver, "."); + else + new pkgAcqChangelog(&Fetcher, Ver); } - AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0)); - Fetcher.SetLog(&Stat); + if (printOnly == false) + { + // Disable drop-privs if "_apt" can not write to the target dir + CheckDropPrivsMustBeDisabled(Fetcher); + if (_error->PendingError() == true) + return false; - bool const downOnly = _config->FindB("APT::Get::Download-Only", false); + bool Failed = false; + if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true) + return false; + } - char tmpname[100]; - const char* tmpdir = NULL; - if (downOnly == false) + if (downOnly == false || printOnly == true) { - std::string systemTemp = GetTempDir(); - snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", - systemTemp.c_str()); - tmpdir = mkdtemp(tmpname); - if (tmpdir == NULL) - return _error->Errno("mkdtemp", "mkdtemp failed"); - - std::string const SandboxUser = _config->Find("APT::Sandbox::User"); - if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it + bool Failed = false; + for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin(); I != Fetcher.ItemsEnd(); ++I) { - struct passwd const * const pw = getpwnam(SandboxUser.c_str()); - struct group const * const gr = getgrnam("root"); - if (pw != NULL && gr != NULL) + if (printOnly) { - // chown the tmp dir directory we use to the sandbox user - if(chown(tmpdir, pw->pw_uid, gr->gr_gid) != 0) - _error->WarningE("DoChangelog", "chown to %s:%s of directory %s failed", SandboxUser.c_str(), "root", tmpdir); + if ((*I)->ErrorText.empty() == false) + { + Failed = true; + _error->Error("%s", (*I)->ErrorText.c_str()); + } + else + cout << '\'' << (*I)->DescURI() << "' " << flNotDir((*I)->DestFile) << std::endl; } + else + DisplayFileInPager((*I)->DestFile); } + return Failed == false; } - for (APT::VersionList::const_iterator Ver = verset.begin(); - Ver != verset.end(); - ++Ver) - { - string changelogfile; - if (downOnly == false) - changelogfile.append(tmpname).append("/changelog"); - else - changelogfile.append(Ver.ParentPkg().Name()).append(".changelog"); - if (DownloadChangelog(Cache, Fetcher, Ver, changelogfile) && downOnly == false) - { - DisplayFileInPager(changelogfile); - // cleanup temp file - unlink(changelogfile.c_str()); - } - } - // clenaup tmp dir - if (tmpdir != NULL) - rmdir(tmpdir); return true; } /*}}}*/ diff --git a/doc/apt-get.8.xml b/doc/apt-get.8.xml index da077afa7..5b6788ed4 100644 --- a/doc/apt-get.8.xml +++ b/doc/apt-get.8.xml @@ -230,16 +230,12 @@ - changelog downloads a package changelog and displays - it through sensible-pager. The server name and base - directory is defined in the APT::Changelogs::Server - variable (e.g. packages.debian.org/changelogs for - Debian or changelogs.ubuntu.com/changelogs for - Ubuntu). - By default it displays the changelog for the version that is - installed. However, you can specify the same options as for - the command. - + changelog tries to download the + changelog of a package and displays it through + sensible-pager. By default it + displays the changelog for the version that is installed. + However, you can specify the same options as for the + command. diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index efe986ea8..7d5f7e9b3 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -618,6 +618,33 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; + scope + + Acquiring changelogs can only be done if an URI is known from where to get them. + Preferable the Release file indicates this in a 'Changelogs' field. If this isn't + available the Label/Origin field of the Release file is used to check if a + Acquire::Changelogs::URI::Label::LABEL or + Acquire::Changelogs::URI::Origin::ORIGIN option + exists and if so this value is taken. The value in the Release file can be overridden + with Acquire::Changelogs::URI::Override::Label::LABEL + or Acquire::Changelogs::URI::Override::Origin::ORIGIN. + + The value should be a normal URI to a text file, expect that package specific data is + replaced with the placeholder CHANGEPATH. The + value for it is: 1. if the package is from a component (e.g. main) + this is the first part otherwise it is omitted, 2. the first letter of source package name, + expect if the source package name starts with 'lib' in which case it will + be the first four letters. 3. The complete source package name. 4. the complete name again and + 5. the source version. + The first (if present), second, third and fourth part are separated by a slash ('/') + and between the fourth and fifth part is an underscore ('_'). + + The special value 'no' is available for this option indicating that + this source can't be used to acquire changelog files from. Another source will be tried + if available in this case. + + + diff --git a/doc/examples/configure-index b/doc/examples/configure-index index ef1ae056d..1339335fa 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -117,14 +117,6 @@ APT // does a ExecFork) Keep-Fds {}; - Changelogs - { - // server the provides the changelogs, the code will assume - // the changlogs are in the pool/ under a srcpkg_ver directory - // with the name "changelog" - Server "http://packages.debian.org/changelogs"; - }: - // control parameters for cron jobs by /etc/cron.daily/apt Periodic { @@ -305,6 +297,9 @@ Acquire "none"; "fr"; }; + + // Location of the changelogs with the placeholder CHANGEPATH (e.g. "main/a/apt/apt_1.1") + Changelogs::URI::Origin::Debian "http://metadata.ftp-master.debian.org/changelogs/CHANGEPATH_changelog"; }; // Directory layout diff --git a/test/integration/framework b/test/integration/framework index 7b03c09ef..1b99929e2 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -520,6 +520,12 @@ Package: $NAME" > debian/control buildsimplenativepackage() { local NAME="$1" + local NM + if [ "$(echo "$NAME" | cut -c 1-3)" = 'lib' ]; then + NM="$(echo "$NAME" | cut -c 1-4)" + else + NM="$(echo "$NAME" | cut -c 1)" + fi local ARCH="$2" local VERSION="$3" local RELEASE="${4:-unstable}" @@ -600,15 +606,15 @@ Package: $NAME" >> ${BUILDDIR}/debian/control (cd ${BUILDDIR}; dpkg-gencontrol -DArchitecture=$arch) (cd ${BUILDDIR}/debian/tmp; md5sum $(find usr/ -type f) > DEBIAN/md5sums) local LOG="${BUILDDIR}/../${NAME}_${VERSION}_${arch}.dpkg-deb.log" - # ensure the right permissions as dpkg-deb ensists + # ensure the right permissions as dpkg-deb insists chmod 755 ${BUILDDIR}/debian/tmp/DEBIAN testsuccess --nomsg dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/.. echo "pool/${NAME}_${VERSION}_${arch}.deb" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.pkglist done - mkdir -p ${BUILDDIR}/../${NAME}_${VERSION} - cp ${BUILDDIR}/debian/changelog ${BUILDDIR}/../${NAME}_${VERSION}/ - cp ${BUILDDIR}/debian/changelog ${BUILDDIR}/../${NAME}_${VERSION}.changelog + local CHANGEPATH="${BUILDDIR}/../${DISTSECTION}/${NM}/${NAME}/${NAME}_${VERSION}" + mkdir -p $CHANGEPATH + cp ${BUILDDIR}/debian/changelog $CHANGEPATH rm -rf "${BUILDDIR}" msgdone "info" } @@ -876,6 +882,7 @@ getcodenamefromsuite() { } getreleaseversionfromsuite() { true; } getlabelfromsuite() { true; } +getoriginfromsuite() { true; } generatereleasefiles() { # $1 is the Date header and $2 is the ValidUntil header to be set @@ -887,16 +894,21 @@ generatereleasefiles() { local CODENAME="$(getcodenamefromsuite $SUITE)" local VERSION="$(getreleaseversionfromsuite $SUITE)" local LABEL="$(getlabelfromsuite $SUITE)" + local ORIGIN="$(getoriginfromsuite $SUITE)" if [ -n "$VERSION" ]; then VERSION="-o APT::FTPArchive::Release::Version=${VERSION}" fi if [ -n "$LABEL" ]; then LABEL="-o APT::FTPArchive::Release::Label=${LABEL}" fi + if [ -n "$ORIGIN" ]; then + ORIGIN="-o APT::FTPArchive::Release::Origin=${ORIGIN}" + fi aptftparchive -qq release $dir \ -o APT::FTPArchive::Release::Suite="${SUITE}" \ -o APT::FTPArchive::Release::Codename="${CODENAME}" \ ${LABEL} \ + ${ORIGIN} \ ${VERSION} \ | sed -e '/0 Release$/ d' > $dir/Release # remove the self reference if [ "$SUITE" = "experimental" -o "$SUITE" = "experimental2" ]; then @@ -1450,9 +1462,9 @@ testfilestats() { msgpass else echo >&2 - ls -ld >&2 "$1" + ls -ld >&2 "$1" || true echo -n >&2 "stat(1) reports for $2: " - stat --format "$2" "$1" + stat --format "$2" "$1" || true msgfail fi } diff --git a/test/integration/test-apt-get-changelog b/test/integration/test-apt-get-changelog index 7e81c71b6..5fa8543b9 100755 --- a/test/integration/test-apt-get-changelog +++ b/test/integration/test-apt-get-changelog @@ -5,44 +5,98 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'native' -buildsimplenativepackage 'apt' 'all' '1.0' 'stable' +buildsimplenativepackage 'foo' 'all' '1.0' 'stable' +buildsimplenativepackage 'libbar' 'all' '1.0' 'stable' + +getlabelfromsuite() { echo 'Testcases'; } +getoriginfromsuite() { echo 'Debian'; } setupaptarchive --no-update changetowebserver testsuccess aptget update -# simulate normal user with non-existent root-owned directories -rm -rf rootdir/var/cache/apt/archives/ -mkdir rootdir/var/cache/apt/archives/ -addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/cache/apt/archives || true;" -chmod -R -w rootdir/var/cache/apt/archives +testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/f/foo/foo_1.0_changelog' foo.changelog +'http://metadata.ftp-master.debian.org/changelogs/main/libb/libbar/libbar_1.0_changelog' libbar.changelog" aptget changelog foo libbar --print-uris + +releasechanger() { + # modifying the Release files in lists… bad stuff. Good that this is only a test… + sed -i "s#^${1}: .*#${1}: ${2}#" $(find rootdir/var/lib/apt/lists -name '*Release') + rm -f rootdir/var/cache/apt/*.bin +} +releasechanger 'Origin' 'Ubuntu' +testsuccessequal "'http://changelogs.ubuntu.com/changelogs/pool/main/f/foo/foo_1.0/changelog' foo.changelog +'http://changelogs.ubuntu.com/changelogs/pool/main/libb/libbar/libbar_1.0/changelog' libbar.changelog" aptget changelog foo libbar --print-uris + +releasechanger 'Label' 'Debian' +testsuccessequal "'http://changelogs.ubuntu.com/changelogs/pool/main/f/foo/foo_1.0/changelog' foo.changelog +'http://changelogs.ubuntu.com/changelogs/pool/main/libb/libbar/libbar_1.0/changelog' libbar.changelog" aptget changelog foo libbar --print-uris + +testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0.changelog' foo.changelog +'http://localhost:8080/main/libb/libbar/libbar_1.0.changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Label::Debian='http://localhost:8080/CHANGEPATH.changelog' + +sed -i '/^Origin: / a\ +Changelogs: http://example.org/CHANGEPATH-changelog' $(find rootdir/var/lib/apt/lists -name '*Release') +rm -f rootdir/var/cache/apt/*.bin -echo 'Apt::Changelogs::Server "http://localhost:8080/";' > rootdir/etc/apt/apt.conf.d/changelog.conf +testsuccessequal "'http://example.org/main/f/foo/foo_1.0-changelog' foo.changelog +'http://example.org/main/libb/libbar/libbar_1.0-changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Label::Debian='http://localhost:8080/CHANGEPATH.changelog' -testsuccessequal "'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt --print-uris +testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0.changelog' foo.changelog +'http://localhost:8080/main/libb/libbar/libbar_1.0.changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Override::Label::Debian='http://localhost:8080/CHANGEPATH.changelog' -testsuccessequal "'http://localhost:8080/pool/apt_1.0/changelog' -'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt apt --print-uris +releasechanger 'Changelogs' 'no' +testequal 'E: Failed to fetch changelog:/foo.changelog Changelog unavailable for foo=1.0 +' aptget changelog foo -qq -d + +sed -i '/^Changelogs: / d' $(find rootdir/var/lib/apt/lists -name '*Release') +releasechanger 'Label' 'Testcases' + +echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/CHANGEPATH/change.txt";' > rootdir/etc/apt/apt.conf.d/changelog.conf +testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0/change.txt' foo.changelog +'http://localhost:8080/main/libb/libbar/libbar_1.0/change.txt' libbar.changelog" aptget changelog foo libbar --print-uris + +echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/pool/CHANGEPATH/changelog";' > rootdir/etc/apt/apt.conf.d/changelog.conf +testsuccessequal "'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog" aptget changelog foo --print-uris cd downloaded -testsuccess aptget changelog apt -qq -testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/apt_1.0/changelog)" +testsuccess aptget changelog foo -qq +testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)" + +testsuccess aptget changelog foo libbar -qq +testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog) +$(cat ../aptarchive/pool/main/libb/libbar/libbar_1.0/changelog)" + +testsuccess aptget changelog foo -d +testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" +testfileequal 'foo.changelog' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)" +rm -f foo.changelog -testsuccess aptget changelog apt -d -testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0/changelog)" -testfilestats 'apt.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" -rm -f apt.changelog ../aptarchive/pool/apt_1.0/changelog +testsuccess aptget changelog libbar foo -d +testfilestats 'libbar.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" +testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" +testfileequal 'libbar.changelog' "$(cat ../aptarchive/pool/main/libb/libbar/libbar_1.0/changelog)" +testfileequal 'foo.changelog' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)" +rm -f libbar.changelog foo.changelog -testequal "$(cat ../aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \ - -qq -o APT::Changelogs::Server='http://not-on-the-main-server:8080/' +# as such bogus, but can happen with multiple binaries from the same source +testsuccessequal "'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog +'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog" aptget changelog foo foo --print-uris +testsuccess aptget changelog foo foo -qq +testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog) +$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)" +testsuccess aptget changelog foo foo -d +testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" +testfileequal 'foo.changelog' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)" +rm -f foo.changelog -testsuccess aptget changelog apt -d -testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0.changelog)" -testfilestats 'apt.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644" -rm -f apt.changelog ../aptarchive/pool/apt_1.0.changelog +# no CHANGEPATH in the URI +testequal 'E: Failed to fetch changelog:/foo.changelog Changelog unavailable for foo=1.0 +' aptget changelog foo -qq -d -o Acquire::Changelogs::URI::Label::Testcases='http://localhost:8080/change.txt' +testfailure test -e foo.changelog -testequal 'E: changelog download failed' aptget changelog apt -qq -d -o APT::Changelogs::Server='http://not-on-the-main-server:8080/' -testfailure test -e apt.changelog +testequal 'E: Failed to fetch http://localhost:8080/does/not/exist/main/f/foo/foo_1.0/change.txt Changelog unavailable for foo=1.0 (404 Not Found) +' aptget changelog foo -qq -d -o Acquire::Changelogs::URI::Label::Testcases='http://localhost:8080/does/not/exist/CHANGEPATH/change.txt' +testfailure test -e foo.changelog diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet index 2cad929cc..e51d72ccd 100755 --- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet +++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet @@ -12,6 +12,7 @@ insertpackage 'unstable' 'apt' 'all' '2' insertsource 'unstable' 'apt' 'all' '2' insertsource 'unstable' 'apt2' 'all' '1' +getoriginfromsuite() { echo 'Debian'; } setupaptarchive APTARCHIVE=$(readlink -f ./aptarchive) @@ -22,7 +23,7 @@ testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.d testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris -testsuccessequal "'http://packages.debian.org/changelogs/pool/main/apt/apt_2/changelog'" aptget changelog apt -qq --print-uris +testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol index f6336ffe3..539080200 100755 --- a/test/integration/test-bug-738785-switch-protocol +++ b/test/integration/test-bug-738785-switch-protocol @@ -10,6 +10,7 @@ configarchitecture "i386" buildsimplenativepackage 'apt' 'all' '1.0' 'stable' # setup http redirecting to https +getlabelfromsuite() { echo 'Testcases'; } setupaptarchive --no-update changetowebserver -o 'aptwebserver::redirect::replace::/redirectme/=https://localhost:4433/' \ -o 'aptwebserver::redirect::replace::/downgrademe/=http://localhost:8080/' \ @@ -20,10 +21,10 @@ sed -i -e 's#:4433/#:8080/redirectme#' -e 's# https:# http:#' rootdir/etc/apt/so testsuccess aptget update -o Debug::Acquire::http=1 -o Debug::Acquire::https=1 -o Debug::pkgAcquire::Worker=1 msgtest 'Test that the webserver does not answer' 'http requests' -downloadfile 'http://localhost:8080/pool/apt_1.0/changelog' changelog >/dev/null 2>&1 && msgfail || msgpass +downloadfile 'http://localhost:8080/pool/main/a/apt/apt_1.0/changelog' changelog >/dev/null 2>&1 && msgfail || msgpass -echo 'Apt::Changelogs::Server "http://localhost:8080/redirectme";' > rootdir/etc/apt/apt.conf.d/changelog.conf -testsuccessequal "'http://localhost:8080/redirectme/pool/apt_1.0/changelog'" aptget changelog apt --print-uris +echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/redirectme/pool/CHANGEPATH/changelog";' > rootdir/etc/apt/apt.conf.d/changelog.conf +testsuccessequal "'http://localhost:8080/redirectme/pool/main/a/apt/apt_1.0/changelog' apt.changelog" aptget changelog apt --print-uris cd downloaded testsuccess aptget changelog apt -d diff --git a/vendor/ubuntu/apt.conf-01-vendor-ubuntu b/vendor/ubuntu/apt.conf-01-vendor-ubuntu index c4092ff44..e69de29bb 100644 --- a/vendor/ubuntu/apt.conf-01-vendor-ubuntu +++ b/vendor/ubuntu/apt.conf-01-vendor-ubuntu @@ -1,6 +0,0 @@ -// Server information for apt-changelog -APT { - Changelogs { - Server "http://changelogs.ubuntu.com/changelogs"; - }; -}; -- cgit v1.2.3-70-g09d2 From 9d2a8a7388cf3b0bbbe92f6b0b30a533e1167f40 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 15 Jun 2015 23:06:56 +0200 Subject: condense parallel requests with the same hashes to one It shouldn't be too common, but sometimes people have multiple mirrors in the sources or otherwise repositories with the same content. Now that we gracefully can handle multiple requests to the same URI, we can also fold multiple requests with the same expected hashes into one. Note that this isn't trying to find oppertunities for merging, but just merges if it happens to encounter the oppertunity for it. This is most obvious in the new testcase actually as it needs to delay the action to give the acquire system enough time to figure out that they can be merged. --- apt-pkg/acquire.cc | 5 +- apt-private/acqprogress.cc | 6 +- test/integration/framework | 8 ++- test/integration/skip-aptwebserver | 25 ------- .../test-acquire-same-file-multiple-times | 4 +- .../test-acquire-same-repository-multiple-times | 81 ++++++++++++++++++++++ test/integration/test-apt-get-source | 42 ++++++----- test/integration/test-apt-get-source-arch | 23 +++--- test/integration/test-apt-get-source-multisources | 12 ++-- .../integration/test-apt-get-update-unauth-warning | 37 +++++++++- test/integration/test-apt-progress-fd | 4 -- test/integration/test-apt-update-ims | 18 +++++ .../test-bug-722207-print-uris-even-if-very-quiet | 12 ++-- test/integration/test-bug-738785-switch-protocol | 22 +++--- 14 files changed, 198 insertions(+), 101 deletions(-) delete mode 100755 test/integration/skip-aptwebserver create mode 100755 test/integration/test-acquire-same-repository-multiple-times (limited to 'test/integration/framework') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 14c8863dc..34afab181 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -680,9 +680,12 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) { QItem **I = &Items; // move to the end of the queue and check for duplicates here + HashStringList const hsl = Item.Owner->GetExpectedHashes(); for (; *I != 0; I = &(*I)->Next) - if (Item.URI == (*I)->URI) + if (Item.URI == (*I)->URI || hsl == (*I)->Owner->GetExpectedHashes()) { + if (_config->FindB("Debug::pkgAcquire::Worker",false) == true) + std::cerr << " @ Queue: Action combined for " << Item.URI << " and " << (*I)->URI << std::endl; (*I)->Owners.push_back(Item.Owner); Item.Owner->Status = (*I)->Owner->Status; return false; diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc index dc92e3b2a..f6c3d1204 100644 --- a/apt-private/acqprogress.cc +++ b/apt-private/acqprogress.cc @@ -116,14 +116,10 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm) if (Quiet > 1) return; - // Ignore certain kinds of transient failures (bad code) - if (Itm.Owner->Status == pkgAcquire::Item::StatIdle) - return; - AssignItemID(Itm); clearLastLine(); - if (Itm.Owner->Status == pkgAcquire::Item::StatDone) + if (Itm.Owner->Status == pkgAcquire::Item::StatDone || Itm.Owner->Status == pkgAcquire::Item::StatIdle) { // TRANSLATOR: Very short word to be displayed for files in 'apt-get update' // which failed to download, but the error is ignored (compare "Err:") diff --git a/test/integration/framework b/test/integration/framework index 1b99929e2..d8f7567d9 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -245,7 +245,7 @@ setupenvironment() { echo "Dir \"${TMPWORKINGDIRECTORY}/rootdir\";" > aptconfig.conf echo "Dir::state::status \"${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg/status\";" >> aptconfig.conf echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf - echo "Dir::Bin::Methods \"${METHODSDIR}\";" >> aptconfig.conf + echo "Dir::Bin::Methods \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/methods\";" >> aptconfig.conf # store apt-key were we can access it, even if we run it as a different user # destroys coverage reporting though, so just do it for root for now if [ "$(id -u)" = '0' ]; then @@ -786,6 +786,8 @@ insertsource() { local SPATH="aptarchive/dists/${RELEASE}/main/source" mkdir -p $SPATH local FILE="${SPATH}/Sources" + local DSCFILE="${NAME}_${VERSION}.dsc" + local TARFILE="${NAME}_${VERSION}.tar.gz" echo "Package: $NAME Binary: $NAME Version: $VERSION @@ -793,8 +795,8 @@ Maintainer: Joe Sixpack Architecture: $ARCH" >> $FILE test -z "$DEPENDENCIES" || echo "$DEPENDENCIES" >> $FILE echo "Files: - d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.dsc - d41d8cd98f00b204e9800998ecf8427e 0 ${NAME}_${VERSION}.tar.gz + $(echo -n "$DSCFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$DSCFILE" | wc -c) $DSCFILE + $(echo -n "$TARFILE" | md5sum | cut -d' ' -f 1) $(echo -n "$TARFILE" | wc -c) $TARFILE " >> $FILE } diff --git a/test/integration/skip-aptwebserver b/test/integration/skip-aptwebserver deleted file mode 100755 index 0622941ce..000000000 --- a/test/integration/skip-aptwebserver +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -set -e - -TESTDIR=$(readlink -f $(dirname $0)) -. $TESTDIR/framework - -setupenvironment -configarchitecture 'amd64' - -buildsimplenativepackage 'apt' 'all' '1.0' 'stable' - -setupaptarchive -changetowebserver - -rm -rf rootdir/var/lib/apt/lists -aptget update -qq -testequal 'Hit http://localhost stable InRelease -Hit http://localhost stable/main Sources -Hit http://localhost stable/main amd64 Packages -Hit http://localhost stable/main Translation-en -Reading package lists...' aptget update - -mv rootdir/var/lib/apt/lists/localhost* rootdir/var/lib/apt/lists/partial -aptget update - diff --git a/test/integration/test-acquire-same-file-multiple-times b/test/integration/test-acquire-same-file-multiple-times index d329a39cb..526765521 100755 --- a/test/integration/test-acquire-same-file-multiple-times +++ b/test/integration/test-acquire-same-file-multiple-times @@ -45,7 +45,7 @@ changetowebserver -o aptwebserver::redirect::replace::/foo2=/foo httpdown() { msgtest 'Downloading the same URI to different files' 'twice over http' - testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 + testsuccess --nomsg apthelper download-file http://localhost:8080/foo ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 cp rootdir/tmp/testsuccess.output download.log testsuccess cmp $TESTDIR/framework ./downloaded/foo1 testsuccess cmp ./downloaded/foo1 ./downloaded/foo2 @@ -57,7 +57,7 @@ testrun 'httpdown' httpredirectdown() { msgtest 'Redirect leads' 'first URI to the second URI' - testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 + testsuccess --nomsg apthelper download-file http://localhost:8080/foo2 ./downloaded/foo1 '' http://localhost:8080/foo ./downloaded/foo2 '' -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 cp rootdir/tmp/testsuccess.output download.log testsuccess cmp $TESTDIR/framework ./downloaded/foo1 testsuccess cmp ./downloaded/foo1 ./downloaded/foo2 diff --git a/test/integration/test-acquire-same-repository-multiple-times b/test/integration/test-acquire-same-repository-multiple-times new file mode 100755 index 000000000..bfeaf88db --- /dev/null +++ b/test/integration/test-acquire-same-repository-multiple-times @@ -0,0 +1,81 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'amd64' + +TESTFILE="$TESTDIR/framework" +cp $TESTFILE aptarchive/foo +APTARCHIVE="$(readlink -f ./aptarchive)" + +getcodenamefromsuite() { echo "jessie"; } +buildsimplenativepackage 'foo' 'all' '1.0' 'stable' +setupaptarchive --no-update +ln -s "${APTARCHIVE}/dists/stable" "${APTARCHIVE}/dists/jessie" +for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do + sed 's#stable#jessie#g' $FILE > $(echo "$FILE" | sed 's#stable#jessie#g') +done + +# install a slowed down file: otherwise its to fast to reproduce combining +NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods" +OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)" +rm $NEWMETHODS +mkdir $NEWMETHODS +for METH in $(find $OLDMETHODS ! -type d); do + ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS +done +rm $NEWMETHODS/file +cat >$NEWMETHODS/file < file.lst + testequal "$(find $(readlink -f ./rootdir/var/lib/apt/lists) -name '*_dists_*' \( ! -name '*InRelease' \) -type f | sort)" cat file.lst + testsuccess aptcache policy + testequal "foo: + Installed: (none) + Candidate: 1.0 + Version table: + 1.0 0 + 500 $1:$2 jessie/main amd64 Packages + 500 $1:$2 stable/main amd64 Packages" aptcache policy foo + testfailure aptcache show foo/unstable + testsuccess aptcache show foo/stable + testsuccess aptcache show foo/jessie +} + +tworepos 'file' "$APTARCHIVE" 'no partial' +testequal '12' grep -c '200%20URI%20Start' ./download.log +testequal '12' grep -c '201%20URI%20Done' ./download.log +testequal '6' grep -c '^ @ Queue: Action combined' ./download.log +tworepos 'file' "$APTARCHIVE" 'hit' +testequal '6' grep -c '200%20URI%20Start' ./download.log +testequal '6' grep -c '201%20URI%20Done' ./download.log +testequal '0' grep -c '^ @ Queue: Action combined' ./download.log +rm -rf rootdir/var/lib/apt/lists + +changetowebserver + +tworepos 'http' '//localhost:8080' 'no partial' +testequal '10' grep -c '200%20URI%20Start' ./download.log +testequal '10' grep -c '201%20URI%20Done' ./download.log +testequal '6' grep -c '^ @ Queue: Action combined' ./download.log +tworepos 'http' '//localhost:8080' 'hit' +testequal '2' grep -c '200%20URI%20Start' ./download.log +testequal '4' grep -c '201%20URI%20Done' ./download.log +testequal '0' grep -c '^ @ Queue: Action combined' ./download.log +rm -rf rootdir/var/lib/apt/lists diff --git a/test/integration/test-apt-get-source b/test/integration/test-apt-get-source index 9db24370f..22f01b997 100755 --- a/test/integration/test-apt-get-source +++ b/test/integration/test-apt-get-source @@ -34,43 +34,40 @@ APTARCHIVE=$(readlink -f ./aptarchive) # normal operation gets highest version number HEADER="Reading package lists... Building dependency tree..." +DOWNLOAD1="Need to get 0 B/25 B of source archives. +'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef +'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184" +DOWNLOAD2="Need to get 0 B/25 B of source archives. +'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55 +'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f" testsuccessequal "$HEADER -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo +$DOWNLOAD2" aptget source -q --print-uris foo # select by release: suite testsuccessequal "$HEADER Selected version '1.0' (stable) for foo -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/stable +$DOWNLOAD1" aptget source -q --print-uris foo/stable testsuccessequal "$HEADER Selected version '2.0' (unstable) for foo -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/unstable +$DOWNLOAD2" aptget source -q --print-uris foo/unstable # select by release: codename testsuccessequal "$HEADER Selected version '2.0' (sid) for foo -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/sid +$DOWNLOAD2" aptget source -q --print-uris foo/sid # select by version testsuccessequal "$HEADER -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo=1.0 +$DOWNLOAD1" aptget source -q --print-uris foo=1.0 # select by release with no binary package (Bug#731102) but ensure to get # highest version +DOWNLOAD01="Need to get 0 B/25 B of source archives. +'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af +'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61" testsuccessequal "$HEADER Selected version '0.1' (wheezy) for foo -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo/wheezy +$DOWNLOAD01" aptget source -q --print-uris foo/wheezy # unavailable one testfailureequal "$HEADER @@ -78,11 +75,12 @@ E: Can not find version '9.9-not-there' of package 'foo' E: Unable to find a source package for foo" aptget source -q --print-uris foo=9.9-not-there # version and release +DOWNLOAD001="Need to get 0 B/29 B of source archives. +'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 13 MD5Sum:6c819ebf0a21b1a480e1dbf6b8edfebd +'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 16 MD5Sum:a3c7e1ac2159fc0faf522e110d6906fd" testsuccessequal "$HEADER -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_0.0.1.dsc' foo_0.0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_0.0.1.tar.gz' foo_0.0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris -t unstable foo=0.0.1 +$DOWNLOAD001" aptget source -q --print-uris -t unstable foo=0.0.1 testsuccessequal "$HEADER -Need to get 0 B of source archives. +Need to get 0 B/25 B of source archives. Fetch source foo" aptget source -q -s foo diff --git a/test/integration/test-apt-get-source-arch b/test/integration/test-apt-get-source-arch index c75798209..f54bb6012 100755 --- a/test/integration/test-apt-get-source-arch +++ b/test/integration/test-apt-get-source-arch @@ -28,31 +28,30 @@ APTARCHIVE=$(readlink -f ./aptarchive) HEADER="Reading package lists... Building dependency tree..." +DOWNLOAD10="Need to get 0 B/25 B of source archives. +'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 11 MD5Sum:b998e085e36cf162e6a33c2801318fef +'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 14 MD5Sum:d46b9a02af8487cbeb49165540c88184" # pick :amd64 testsuccessequal "$HEADER -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:amd64 +$DOWNLOAD10" aptget source -q --print-uris foo:amd64 # pick :i386 testsuccessequal "$HEADER -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386 +Need to get 0 B/25 B of source archives. +'file://${APTARCHIVE}/foo_2.0.dsc' foo_2.0.dsc 11 MD5Sum:c0de572c6f8aa576c8ff78c81132ed55 +'file://${APTARCHIVE}/foo_2.0.tar.gz' foo_2.0.tar.gz 14 MD5Sum:e10bb487c375b2b938d27bd31c2d1f5f" aptget source -q --print-uris foo:i386 # pick :i386 by release testsuccessequal "$HEADER Selected version '0.1' (oldstable) for foo -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386/oldstable +Need to get 0 B/25 B of source archives. +'file://${APTARCHIVE}/foo_0.1.dsc' foo_0.1.dsc 11 MD5Sum:0811a4d85238056c613ea897f49f01af +'file://${APTARCHIVE}/foo_0.1.tar.gz' foo_0.1.tar.gz 14 MD5Sum:fa1ecb7a1a53e8e6f6551ca7db888a61" aptget source -q --print-uris foo:i386/oldstable # pick :i386 by version testsuccessequal "$HEADER -Need to get 0 B of source archives. -'file://${APTARCHIVE}/foo_1.0.dsc' foo_1.0.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/foo_1.0.tar.gz' foo_1.0.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -q --print-uris foo:i386=1.0 +$DOWNLOAD10" aptget source -q --print-uris foo:i386=1.0 # error on unknown arch testfailureequal "$HEADER diff --git a/test/integration/test-apt-get-source-multisources b/test/integration/test-apt-get-source-multisources index 03d0400a0..887a30685 100755 --- a/test/integration/test-apt-get-source-multisources +++ b/test/integration/test-apt-get-source-multisources @@ -20,11 +20,11 @@ APTARCHIVE=$(readlink -f ./aptarchive) HEADER="Reading package lists... Building dependency tree..." testsuccessequal "$HEADER -Need to get 0 B of source archives. -'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11 +Need to get 0 B/43 B of source archives. +'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b +'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df" aptget source -qdy --print-uris --dsc-only adduser=3.113 python-fll=0.9.11 testsuccessequal "$HEADER -Need to get 0 B of source archives. -'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113 +Need to get 0 B/43 B of source archives. +'file://${APTARCHIVE}/python-fll_0.9.11.dsc' python-fll_0.9.11.dsc 21 MD5Sum:740a9dbf02a295932f15b1415d0dc0df +'file://${APTARCHIVE}/adduser_3.113+nmu3.dsc' adduser_3.113+nmu3.dsc 22 MD5Sum:255405ab5af211238ef53b7a1dd8ca4b" aptget source -qdy --print-uris --dsc-only python-fll=0.9.11 adduser=3.113 diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning index 5b81d56d2..fcabf244a 100755 --- a/test/integration/test-apt-get-update-unauth-warning +++ b/test/integration/test-apt-get-update-unauth-warning @@ -20,8 +20,10 @@ APTARCHIVE=$(readlink -f ./aptarchive) rm -f $APTARCHIVE/dists/unstable/*Release* # update without authenticated files leads to warning -testfailureequal "Ign:1 file:$APTARCHIVE unstable InRelease +testfailureequal "Get:1 file:$APTARCHIVE unstable InRelease +Ign:1 file:$APTARCHIVE unstable InRelease File not found +Get:2 file:$APTARCHIVE unstable Release Err:2 file:$APTARCHIVE unstable Release File not found W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file. This is deprecated, please contact the owner of the repository. @@ -36,10 +38,41 @@ filesize() { stat -c%s "$(aptget files --no-release-info --format '$(URI)' "Created-By: $1" | cut -d'/' -f 3- ).gz" } # allow override -testwarningequal "Ign:1 file:$APTARCHIVE unstable InRelease +#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1 +#exit +testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease +Ign:1 file:$APTARCHIVE unstable InRelease File not found +Get:2 file:$APTARCHIVE unstable Release Ign:2 file:$APTARCHIVE unstable Release File not found +Get:3 file:$APTARCHIVE unstable/main Sources +Ign:3 file:$APTARCHIVE unstable/main Sources + File not found +Get:4 file:$APTARCHIVE unstable/main i386 Packages +Ign:4 file:$APTARCHIVE unstable/main i386 Packages + File not found +Get:5 file:$APTARCHIVE unstable/main Translation-en +Ign:5 file:$APTARCHIVE unstable/main Translation-en + File not found +Get:3 file:$APTARCHIVE unstable/main Sources +Ign:3 file:$APTARCHIVE unstable/main Sources + File not found +Get:4 file:$APTARCHIVE unstable/main i386 Packages +Ign:4 file:$APTARCHIVE unstable/main i386 Packages + File not found +Get:5 file:$APTARCHIVE unstable/main Translation-en +Ign:5 file:$APTARCHIVE unstable/main Translation-en + File not found +Get:3 file:$APTARCHIVE unstable/main Sources +Ign:3 file:$APTARCHIVE unstable/main Sources + File not found +Get:4 file:$APTARCHIVE unstable/main i386 Packages +Ign:4 file:$APTARCHIVE unstable/main i386 Packages + File not found +Get:5 file:$APTARCHIVE unstable/main Translation-en +Ign:5 file:$APTARCHIVE unstable/main Translation-en + File not found Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B] Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages') B] Get:5 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B] diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd index 99b4ea050..e30d503cb 100755 --- a/test/integration/test-apt-progress-fd +++ b/test/integration/test-apt-progress-fd @@ -16,7 +16,6 @@ setupaptarchive exec 3> apt-progress.log testsuccess aptget install testing=0.1 -y -o APT::Status-Fd=3 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:0:Retrieving file 1 of 1 dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) @@ -33,7 +32,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg' exec 3> apt-progress.log testsuccess aptget install testing=0.8.15 -y -o APT::Status-Fd=3 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:0:Retrieving file 1 of 1 dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) @@ -50,7 +48,6 @@ pmstatus:dpkg-exec:83.3333:Running dpkg' exec 3> apt-progress.log testsuccess aptget install testing=0.8.15 --reinstall -y -o APT::Status-Fd=3 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:0:Retrieving file 1 of 1 dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing:0:Installing testing (amd64) @@ -77,7 +74,6 @@ pmstatus:dpkg-exec:75:Running dpkg' exec 3> apt-progress.log testsuccess aptget install testing2:i386 -y -o APT::Status-Fd=3 testfileequal './apt-progress.log' 'dlstatus:1:0:Retrieving file 1 of 1 -dlstatus:1:0:Retrieving file 1 of 1 dlstatus:1:20:Retrieving file 1 of 1 pmstatus:dpkg-exec:0:Running dpkg pmstatus:testing2:0:Installing testing2 (i386) diff --git a/test/integration/test-apt-update-ims b/test/integration/test-apt-update-ims index 2b662171c..33b4ed1b9 100755 --- a/test/integration/test-apt-update-ims +++ b/test/integration/test-apt-update-ims @@ -134,6 +134,24 @@ EXPECT="Ign:1 http://localhost:8080 unstable InRelease 404 Not Found Ign:2 http://localhost:8080 unstable Release 404 Not Found +Ign:3 http://localhost:8080 unstable/main Sources + 404 Not Found +Ign:4 http://localhost:8080 unstable/main amd64 Packages + 404 Not Found +Ign:5 http://localhost:8080 unstable/main Translation-en + 404 Not Found +Ign:3 http://localhost:8080 unstable/main Sources + 404 Not Found +Ign:4 http://localhost:8080 unstable/main amd64 Packages + 404 Not Found +Ign:5 http://localhost:8080 unstable/main Translation-en + 404 Not Found +Ign:3 http://localhost:8080 unstable/main Sources + 404 Not Found +Ign:4 http://localhost:8080 unstable/main amd64 Packages + 404 Not Found +Ign:5 http://localhost:8080 unstable/main Translation-en + 404 Not Found Hit:3 http://localhost:8080 unstable/main Sources Hit:4 http://localhost:8080 unstable/main amd64 Packages Hit:5 http://localhost:8080 unstable/main Translation-en diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet index e51d72ccd..1fa94de7d 100755 --- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet +++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet @@ -21,11 +21,11 @@ testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.d testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris -testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris +testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a +'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373" aptget source apt -qq --print-uris testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris -testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e -'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt apt2 -qq --print-uris +testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 9 MD5Sum:16ff470aaedad0f06fb951ed89ffdd3a +'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 12 MD5Sum:ab2b546f59ff9e8f5cc7a2d987ff3373 +'file://${APTARCHIVE}/apt2_1.dsc' apt2_1.dsc 10 MD5Sum:4c572ce45f1e2bedbb30da7f5e1c241c +'file://${APTARCHIVE}/apt2_1.tar.gz' apt2_1.tar.gz 13 MD5Sum:2a96fec139f8722d93312a1ff8281232" aptget source apt apt2 -qq --print-uris diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol index 539080200..e86f28824 100755 --- a/test/integration/test-bug-738785-switch-protocol +++ b/test/integration/test-bug-738785-switch-protocol @@ -39,18 +39,15 @@ cd - >/dev/null testsuccess aptget install apt -y testdpkginstalled 'apt' -# create a copy of all methods, expect https -eval `aptconfig shell METHODS Dir::Bin::Methods/d` -COPYMETHODS='usr/lib/apt/methods' -mv rootdir/${COPYMETHODS} rootdir/${COPYMETHODS}.bak -mkdir -p rootdir/$COPYMETHODS -cd rootdir/$COPYMETHODS -find $METHODS \! -type d | while read meth; do - ln -s $meth +# install a slowed down file: otherwise its to fast to reproduce combining +NEWMETHODS="$(readlink -f rootdir)/usr/lib/apt/methods" +OLDMETHODS="$(readlink -f rootdir/usr/lib/apt/methods)" +rm $NEWMETHODS +mkdir $NEWMETHODS +for METH in $(find $OLDMETHODS ! -type d); do + ln -s $OLDMETHODS/$(basename $METH) $NEWMETHODS done -rm https -cd - >/dev/null -echo "Dir::Bin::Methods \"${COPYMETHODS}\";" >> aptconfig.conf +rm $NEWMETHODS/https cd downloaded testfailureequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found. @@ -59,8 +56,7 @@ testfailure test -e apt_1.0_all.deb cd - >/dev/null # revert to all methods -rm -rf rootdir/$COPYMETHODS -mv rootdir/${COPYMETHODS}.bak rootdir/${COPYMETHODS} +ln -s $OLDMETHODS/https $NEWMETHODS # check that downgrades from https to http are not allowed webserverconfig 'aptwebserver::support::http' 'true' -- cgit v1.2.3-70-g09d2 From 533fe3d13927798c17bdef84eba60ed9441d9608 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 15 Jun 2015 23:59:14 +0200 Subject: allow ratelimiting progress reporting for testcases Progress reports once in a while which is a bit to unpredictable for testcases, so we enforce a steady progress for them in the hope that this makes the tests (mostly test-apt-progress-fd) a bit more stable. Git-Dch: Ignore --- apt-pkg/acquire.cc | 14 ++++++++----- test/integration/framework | 23 ++++++++++++---------- .../test-acquire-same-repository-multiple-times | 2 +- 3 files changed, 23 insertions(+), 16 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 34afab181..5e5bec95c 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -955,7 +955,7 @@ std::string pkgAcquire::Queue::QItem::Custom600Headers() const /*{{{*/ // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(0), Update(true), MorePulses(false) +pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(-1), Update(true), MorePulses(false) { Start(); } @@ -1054,13 +1054,17 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) Time = NewTime; } + double const OldPercent = Percent; // calculate the percentage, if we have too little data assume 1% if (TotalBytes > 0 && UnfetchedReleaseFiles) Percent = 0; - else + else // use both files and bytes because bytes can be unreliable - Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + + Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + 0.2 * (CurrentItems/float(TotalItems)*100.0)); + double const DiffPercent = Percent - OldPercent; + if (DiffPercent < 0.001 && _config->FindB("Acquire::Progress::Diffpercent", false) == true) + return true; int fd = _config->FindI("APT::Status-Fd",-1); if(fd > 0) @@ -1078,11 +1082,11 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) snprintf(msg,sizeof(msg), _("Retrieving file %li of %li (%s remaining)"), i, TotalItems, TimeToStr(ETA).c_str()); else snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems); - + // build the status str status << "dlstatus:" << i << ":" << std::setprecision(3) << Percent - << ":" << msg + << ":" << msg << endl; std::string const dlstatus = status.str(); diff --git a/test/integration/framework b/test/integration/framework index d8f7567d9..5d949009f 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -273,16 +273,19 @@ EOF chmod +x "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg" echo "Dir::Bin::dpkg \"${TMPWORKINGDIRECTORY}/rootdir/usr/bin/dpkg\";" > rootdir/etc/apt/apt.conf.d/99dpkg - if ! command dpkg --assert-multi-arch >/dev/null 2>&1; then - echo "DPKG::options:: \"--force-architecture\";" >> aptconfig.conf # Added to test multiarch before dpkg is ready for it… - fi - echo 'quiet::NoUpdate "true";' >> aptconfig.conf - echo 'quiet::NoStatistic "true";' >> aptconfig.conf - # too distracting for users, but helpful to detect changes - echo 'Acquire::Progress::Ignore::ShowErrorText "true";' >> aptconfig.conf - # in testcases, it can appear as if localhost has a rotation setup, - # hide this as we can't really deal with it properly - echo 'Acquire::Failure::ShowIP "false";' >> aptconfig.conf + { + if ! command dpkg --assert-multi-arch >/dev/null 2>&1; then + echo "DPKG::options:: \"--force-architecture\";" # Added to test multiarch before dpkg is ready for it… + fi + echo 'quiet::NoUpdate "true";' + echo 'quiet::NoStatistic "true";' + # too distracting for users, but helpful to detect changes + echo 'Acquire::Progress::Ignore::ShowErrorText "true";' + echo 'Acquire::Progress::Diffpercent "true";' + # in testcases, it can appear as if localhost has a rotation setup, + # hide this as we can't really deal with it properly + echo 'Acquire::Failure::ShowIP "false";' + } >> aptconfig.conf cp "${TESTDIRECTORY}/apt.pem" "${TMPWORKINGDIRECTORY}/rootdir/etc/webserver.pem" if [ "$(id -u)" = '0' ]; then diff --git a/test/integration/test-acquire-same-repository-multiple-times b/test/integration/test-acquire-same-repository-multiple-times index bfeaf88db..a46e0d73c 100755 --- a/test/integration/test-acquire-same-repository-multiple-times +++ b/test/integration/test-acquire-same-repository-multiple-times @@ -32,7 +32,7 @@ cat >$NEWMETHODS/file < Date: Thu, 18 Jun 2015 17:33:15 +0200 Subject: fix memory leaks reported by -fsanitize Various small leaks here and there. Nothing particularily big, but still good to fix. Found by the sanitizers while running our testcases. Reported-By: gcc -fsanitize Git-Dch: Ignore --- apt-pkg/acquire-item.cc | 57 +++++++++++++++++++--------------- apt-pkg/acquire-item.h | 45 +++++++++++++-------------- apt-pkg/acquire-worker.cc | 2 +- apt-pkg/cachefile.cc | 4 +-- apt-pkg/contrib/cdromutl.cc | 2 ++ apt-pkg/contrib/gpgv.cc | 2 ++ apt-pkg/deb/debmetaindex.cc | 2 +- apt-pkg/deb/debrecords.cc | 4 +++ apt-pkg/deb/dpkgpm.cc | 4 +++ apt-pkg/indexrecords.cc | 5 ++- apt-pkg/tagfile.cc | 13 ++++++-- apt-pkg/tagfile.h | 2 +- apt-private/private-cacheset.h | 13 +++++--- apt-private/private-output.cc | 23 ++++++++------ buildlib/environment.mak.in | 2 ++ cmdline/apt-cache.cc | 1 + cmdline/apt-get.cc | 8 ++--- ftparchive/apt-ftparchive.cc | 69 +++++++++++++++++++++--------------------- ftparchive/cachedb.cc | 1 + ftparchive/writer.cc | 31 ++++++++++++------- ftparchive/writer.h | 14 +++------ test/integration/framework | 3 ++ test/libapt/cdrom_test.cc | 7 ++--- test/libapt/file-helpers.cc | 7 ++--- test/libapt/file-helpers.h | 2 +- test/libapt/sourcelist_test.cc | 5 +-- 26 files changed, 190 insertions(+), 138 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 222ca8931..0ab52a0cd 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -109,7 +109,7 @@ static std::string GetDiffsPatchFileName(std::string const &Final) /*{{{*/ } /*}}}*/ -static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser, pkgAcqMetaBase * const TransactionManager, pkgAcquire::Item * const I) /*{{{*/ +static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I) /*{{{*/ { if(MetaIndexParser->IsAlwaysTrusted() || _config->FindB("Acquire::AllowInsecureRepositories") == true) return true; @@ -661,7 +661,7 @@ std::string pkgAcquire::Item::HashSum() const /*{{{*/ /*}}}*/ pkgAcqTransactionItem::pkgAcqTransactionItem(pkgAcquire * const Owner, /*{{{*/ - pkgAcqMetaBase * const transactionManager, IndexTarget const &target) : + pkgAcqMetaClearSig * const transactionManager, IndexTarget const &target) : pkgAcquire::Item(Owner), d(NULL), Target(target), TransactionManager(transactionManager) { if (TransactionManager != this) @@ -680,12 +680,11 @@ HashStringList pkgAcqTransactionItem::GetExpectedHashesFor(std::string const &Me // AcqMetaBase - Constructor /*{{{*/ pkgAcqMetaBase::pkgAcqMetaBase(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, std::vector const &IndexTargets, - IndexTarget const &DataTarget, - indexRecords * const MetaIndexParser) + IndexTarget const &DataTarget) : pkgAcqTransactionItem(Owner, TransactionManager, DataTarget), d(NULL), - MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL), IndexTargets(IndexTargets), + IndexTargets(IndexTargets), AuthPass(false), IMSHit(false) { } @@ -1047,7 +1046,7 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/ std::string errmsg; strprintf(errmsg, // TRANSLATOR: The first %s is the URL of the bad Release file, the second is - // the time since then the file is invalid - formated in the same way as in + // the time since then the file is invalid - formatted in the same way as in // the download progress display (e.g. 7d 3h 42min 1s) _("Release file for %s is expired (invalid since %s). " "Updates for this repository will not be applied."), @@ -1098,16 +1097,19 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/ return true; } /*}}}*/ -pkgAcqMetaBase::~pkgAcqMetaBase() {} +pkgAcqMetaBase::~pkgAcqMetaBase() +{ +} pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/ IndexTarget const &ClearsignedTarget, IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget, std::vector const &IndexTargets, indexRecords * const MetaIndexParser) : - pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget, IndexTargets, MetaIndexParser), + pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget, IndexTargets), d(NULL), ClearsignedTarget(ClearsignedTarget), - DetachedDataTarget(DetachedDataTarget) + DetachedDataTarget(DetachedDataTarget), + MetaIndexParser(MetaIndexParser), LastMetaIndexParser(NULL) { // index targets + (worst case:) Release/Release.gpg ExpectedAdditionalItems = IndexTargets.size() + 2; @@ -1116,6 +1118,10 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/ /*}}}*/ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ { + if (MetaIndexParser != NULL) + delete MetaIndexParser; + if (LastMetaIndexParser != NULL) + delete LastMetaIndexParser; } /*}}}*/ // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ @@ -1180,7 +1186,7 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c TransactionManager->TransactionStageRemoval(this, GetFinalFilename()); Status = StatDone; - new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget, IndexTargets, TransactionManager->MetaIndexParser); + new pkgAcqMetaIndex(Owner, TransactionManager, DetachedDataTarget, DetachedSigTarget, IndexTargets); } else { @@ -1240,12 +1246,11 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire * const Owner, /*{{{*/ - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &DataTarget, IndexTarget const &DetachedSigTarget, - vector const &IndexTargets, - indexRecords * const MetaIndexParser) : - pkgAcqMetaBase(Owner, TransactionManager, IndexTargets, DataTarget, MetaIndexParser), d(NULL), + vector const &IndexTargets) : + pkgAcqMetaBase(Owner, TransactionManager, IndexTargets, DataTarget), d(NULL), DetachedSigTarget(DetachedSigTarget) { if(_config->FindB("Debug::Acquire::Transaction", false) == true) @@ -1324,7 +1329,7 @@ pkgAcqMetaIndex::~pkgAcqMetaIndex() {} // AcqMetaSig::AcqMetaSig - Constructor /*{{{*/ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target, pkgAcqMetaIndex * const MetaIndex) : pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL), MetaIndex(MetaIndex) @@ -1487,7 +1492,7 @@ void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const // AcqBaseIndex - Constructor /*{{{*/ pkgAcqBaseIndex::pkgAcqBaseIndex(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) : pkgAcqTransactionItem(Owner, TransactionManager, Target), d(NULL) { @@ -1503,9 +1508,9 @@ pkgAcqBaseIndex::~pkgAcqBaseIndex() {} * the original packages file */ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) - : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL) + : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), diffs(NULL) { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -1840,7 +1845,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string const &IndexDiffFile) /*{{{*/ new pkgAcqIndexDiffs(Owner, TransactionManager, Target, available_patches); else { - std::vector *diffs = new std::vector(available_patches.size()); + diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, TransactionManager, Target, @@ -1896,7 +1901,11 @@ void pkgAcqDiffIndex::Done(string const &Message,HashStringList const &Hashes, / return; } /*}}}*/ -pkgAcqDiffIndex::~pkgAcqDiffIndex() {} +pkgAcqDiffIndex::~pkgAcqDiffIndex() +{ + if (diffs != NULL) + delete diffs; +} // AcqIndexDiffs::AcqIndexDiffs - Constructor /*{{{*/ // --------------------------------------------------------------------- @@ -1904,7 +1913,7 @@ pkgAcqDiffIndex::~pkgAcqDiffIndex() {} * for each diff and the index */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target, vector const &diffs) : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), @@ -2127,7 +2136,7 @@ pkgAcqIndexDiffs::~pkgAcqIndexDiffs() {} // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target, DiffInfo const &patch, std::vector const * const allPatches) @@ -2272,7 +2281,7 @@ pkgAcqIndexMergeDiffs::~pkgAcqIndexMergeDiffs() {} // AcqIndex::AcqIndex - Constructor /*{{{*/ pkgAcqIndex::pkgAcqIndex(pkgAcquire * const Owner, - pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target) : pkgAcqBaseIndex(Owner, TransactionManager, Target), d(NULL), Stage(STAGE_DOWNLOAD) { diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index c4bbfc7a1..4d235dce2 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -46,7 +46,8 @@ class indexRecords; class pkgRecords; class pkgSourceList; -class pkgAcqMetaBase; +class pkgAcqMetaClearSig; +class pkgAcqIndexMergeDiffs; class pkgAcquire::Item : public WeakPointable /*{{{*/ /** \brief Represents the process by which a pkgAcquire object should @@ -339,6 +340,7 @@ class pkgAcquire::Item : public WeakPointable /*{{{*/ void * const d; friend class pkgAcqMetaBase; + friend class pkgAcqMetaClearSig; }; /*}}}*/ class APT_HIDDEN pkgAcqTransactionItem: public pkgAcquire::Item /*{{{*/ @@ -356,7 +358,7 @@ class APT_HIDDEN pkgAcqTransactionItem: public pkgAcquire::Item /*{{{*/ std::string PartialFile; /** \brief TransactionManager */ - pkgAcqMetaBase * const TransactionManager; + pkgAcqMetaClearSig * const TransactionManager; enum TransactionStates { TransactionCommit, @@ -370,10 +372,11 @@ class APT_HIDDEN pkgAcqTransactionItem: public pkgAcquire::Item /*{{{*/ virtual bool HashesRequired() const; - pkgAcqTransactionItem(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, IndexTarget const &Target); + pkgAcqTransactionItem(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target); virtual ~pkgAcqTransactionItem(); friend class pkgAcqMetaBase; + friend class pkgAcqMetaClearSig; }; /*}}}*/ class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem /*{{{*/ @@ -383,12 +386,6 @@ class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem /*{{{*/ protected: std::vector Transaction; - public: - /** \brief A package-system-specific parser for the meta-index file. */ - indexRecords *MetaIndexParser; - indexRecords *LastMetaIndexParser; - protected: - /** \brief The index files which should be looked up in the meta-index * and then downloaded. */ @@ -473,10 +470,9 @@ class APT_HIDDEN pkgAcqMetaBase : public pkgAcqTransactionItem /*{{{*/ /** \brief Get the full pathname of the final file for the current URI */ virtual std::string GetFinalFilename() const; - pkgAcqMetaBase(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaBase(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, std::vector const &IndexTargets, - IndexTarget const &DataTarget, - indexRecords* const MetaIndexParser); + IndexTarget const &DataTarget); virtual ~pkgAcqMetaBase(); }; /*}}}*/ @@ -509,9 +505,9 @@ class APT_HIDDEN pkgAcqMetaIndex : public pkgAcqMetaBase virtual void Finished(); /** \brief Create a new pkgAcqMetaIndex. */ - pkgAcqMetaIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, + pkgAcqMetaIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &DataTarget, IndexTarget const &DetachedSigTarget, - std::vector const &IndexTargets, indexRecords * const MetaIndexParser); + std::vector const &IndexTargets); virtual ~pkgAcqMetaIndex(); friend class pkgAcqMetaSig; @@ -548,8 +544,8 @@ class APT_HIDDEN pkgAcqMetaSig : public pkgAcqTransactionItem pkgAcquire::MethodConfig const * const Cnf); /** \brief Create a new pkgAcqMetaSig. */ - pkgAcqMetaSig(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, IndexTarget const &Target, - pkgAcqMetaIndex * const MetaIndex); + pkgAcqMetaSig(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, + IndexTarget const &Target, pkgAcqMetaIndex * const MetaIndex); virtual ~pkgAcqMetaSig(); }; /*}}}*/ @@ -561,7 +557,11 @@ class APT_HIDDEN pkgAcqMetaClearSig : public pkgAcqMetaIndex IndexTarget const ClearsignedTarget; IndexTarget const DetachedDataTarget; -public: + public: + /** \brief A package-system-specific parser for the meta-index file. */ + indexRecords *MetaIndexParser; + indexRecords *LastMetaIndexParser; + virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf); virtual std::string Custom600Headers() const; virtual void Done(std::string const &Message, HashStringList const &Hashes, @@ -586,7 +586,7 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem /** \brief Get the full pathname of the final file for the current URI */ virtual std::string GetFinalFilename() const; - pkgAcqBaseIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, + pkgAcqBaseIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target); virtual ~pkgAcqBaseIndex(); }; @@ -603,6 +603,7 @@ class APT_HIDDEN pkgAcqBaseIndex : public pkgAcqTransactionItem class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex { void * const d; + std::vector * diffs; protected: /** \brief If \b true, debugging information will be written to std::clog. */ @@ -650,7 +651,7 @@ class APT_HIDDEN pkgAcqDiffIndex : public pkgAcqBaseIndex * * \param ShortDesc A short description of the list file to download. */ - pkgAcqDiffIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, + pkgAcqDiffIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target); virtual ~pkgAcqDiffIndex(); private: @@ -749,7 +750,7 @@ class APT_HIDDEN pkgAcqIndexMergeDiffs : public pkgAcqBaseIndex * \param allPatches contains all related items so that each item can * check if it was the last one to complete the download step */ - pkgAcqIndexMergeDiffs(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, + pkgAcqIndexMergeDiffs(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target, DiffInfo const &patch, std::vector const * const allPatches); virtual ~pkgAcqIndexMergeDiffs(); @@ -863,7 +864,7 @@ class APT_HIDDEN pkgAcqIndexDiffs : public pkgAcqBaseIndex * should be ordered so that each diff appears before any diff * that depends on it. */ - pkgAcqIndexDiffs(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, + pkgAcqIndexDiffs(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target, std::vector const &diffs=std::vector()); virtual ~pkgAcqIndexDiffs(); @@ -941,7 +942,7 @@ class APT_HIDDEN pkgAcqIndex : public pkgAcqBaseIndex virtual std::string DescURI() const {return Desc.URI;}; virtual std::string GetMetaKey() const; - pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaBase * const TransactionManager, + pkgAcqIndex(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, IndexTarget const &Target); virtual ~pkgAcqIndex(); diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 8d619e96d..c0f93f9ce 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -722,7 +722,7 @@ void pkgAcquire::Worker::PrepareFiles(char const * const caller, pkgAcquire::Que unlink(Owner->DestFile.c_str()); if (link(filename.c_str(), Owner->DestFile.c_str()) != 0) { - // diferent mounts can't happen for us as we download to lists/ by default, + // different mounts can't happen for us as we download to lists/ by default, // but if the system is reused by others the locations can potentially be on // different disks, so use symlink as poor-men replacement. // FIXME: Real copying as last fallback, but that is costly, so offload to a method preferable diff --git a/apt-pkg/cachefile.cc b/apt-pkg/cachefile.cc index ea3d45480..690776266 100644 --- a/apt-pkg/cachefile.cc +++ b/apt-pkg/cachefile.cc @@ -65,8 +65,8 @@ bool pkgCacheFile::BuildCaches(OpProgress *Progress, bool WithLock) if (_config->FindB("pkgCacheFile::Generate", true) == false) { - Map = new MMap(*new FileFd(_config->FindFile("Dir::Cache::pkgcache"), - FileFd::ReadOnly),MMap::Public|MMap::ReadOnly); + FileFd file(_config->FindFile("Dir::Cache::pkgcache"), FileFd::ReadOnly); + Map = new MMap(file, MMap::Public|MMap::ReadOnly); Cache = new pkgCache(Map); if (_error->PendingError() == true) return false; diff --git a/apt-pkg/contrib/cdromutl.cc b/apt-pkg/contrib/cdromutl.cc index 6eb917457..428ef0161 100644 --- a/apt-pkg/contrib/cdromutl.cc +++ b/apt-pkg/contrib/cdromutl.cc @@ -287,9 +287,11 @@ string FindMountPointForDevice(const char *devnode) fclose(f); // unescape the \0XXX chars in the path string mount_point = out[1]; + free(line); return DeEscapeString(mount_point); } fclose(f); + free(line); } return string(); diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc index 9d798cca9..a01e319eb 100644 --- a/apt-pkg/contrib/gpgv.cc +++ b/apt-pkg/contrib/gpgv.cc @@ -296,6 +296,8 @@ bool SplitClearSignedFile(std::string const &InFile, FileFd * const ContentFile, // all the rest is whitespace, unsigned garbage or additional message blocks we ignore } fclose(in); + if (buf != NULL) + free(buf); if (found_signature == true) return _error->Error("Signature in file %s wasn't closed", InFile.c_str()); diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 026af077f..5a517b290 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -270,7 +270,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const // special case for --print-uris std::vector const targets = GetIndexTargets(); #define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map()) - pkgAcqMetaBase * const TransactionManager = new pkgAcqMetaClearSig(Owner, + pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner, APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), targets, iR); #undef APT_TARGET diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc index 326102d08..d78a7e2e0 100644 --- a/apt-pkg/deb/debrecords.cc +++ b/apt-pkg/deb/debrecords.cc @@ -42,10 +42,14 @@ debRecordParser::debRecordParser(string FileName,pkgCache &Cache) : // RecordParser::Jump - Jump to a specific record /*{{{*/ bool debRecordParser::Jump(pkgCache::VerFileIterator const &Ver) { + if (Ver.end() == true) + return false; return Tags.Jump(Section,Ver->Offset); } bool debRecordParser::Jump(pkgCache::DescFileIterator const &Desc) { + if (Desc.end() == true) + return false; return Tags.Jump(Section,Desc->Offset); } /*}}}*/ diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 1991a4a66..3594a6efe 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1484,6 +1484,10 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress) a != Args.end(); ++a) clog << *a << ' '; clog << endl; + for (std::vector::const_iterator p = Packages.begin(); + p != Packages.end(); ++p) + free(*p); + Packages.clear(); continue; } Args.push_back(NULL); diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc index 5a93d826f..03ba59460 100644 --- a/apt-pkg/indexrecords.cc +++ b/apt-pkg/indexrecords.cc @@ -278,4 +278,7 @@ indexRecords::indexRecords(const string &ExpectedDist) : { } -indexRecords::~indexRecords() {} +indexRecords::~indexRecords() { + for (std::map::const_iterator S = Entries.begin(); S != Entries.end(); ++S) + delete S->second; +} diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 4c5505bf3..6d7d8185b 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -34,8 +34,10 @@ class pkgTagFilePrivate public: void Reset(FileFd * const pFd, unsigned long long const pSize) { - Fd = pFd; + if (Buffer != NULL) + free(Buffer); Buffer = NULL; + Fd = pFd; Start = NULL; End = NULL; Done = false; @@ -43,7 +45,7 @@ public: Size = pSize; } - pkgTagFilePrivate(FileFd * const pFd, unsigned long long const Size) + pkgTagFilePrivate(FileFd * const pFd, unsigned long long const Size) : Buffer(NULL) { Reset(pFd, Size); } @@ -54,6 +56,12 @@ public: bool Done; unsigned long long iOffset; unsigned long long Size; + + ~pkgTagFilePrivate() + { + if (Buffer != NULL) + free(Buffer); + } }; class pkgTagSectionPrivate @@ -127,7 +135,6 @@ void pkgTagFile::Init(FileFd * const pFd,unsigned long long Size) /* */ pkgTagFile::~pkgTagFile() { - free(d->Buffer); delete d; } /*}}}*/ diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index 23238d979..d0d0c7a84 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -136,7 +136,7 @@ class pkgTagSection * * @param File to write the section to * @param Order in which tags should appear in the file - * @param Rewrite is a set of tags to be renamed, rewitten and/or removed + * @param Rewrite is a set of tags to be renamed, rewritten and/or removed * @return \b true if successful, otherwise \b false */ bool Write(FileFd &File, char const * const * const Order = NULL, std::vector const &Rewrite = std::vector()) const; diff --git a/apt-private/private-cacheset.h b/apt-private/private-cacheset.h index 059c7637e..0eb22b788 100644 --- a/apt-private/private-cacheset.h +++ b/apt-private/private-cacheset.h @@ -32,10 +32,15 @@ struct VersionSortDescriptionLocality bool operator () (const pkgCache::VerIterator &v_lhs, const pkgCache::VerIterator &v_rhs) { - pkgCache::DescFile *A = v_lhs.TranslatedDescription().FileList(); - pkgCache::DescFile *B = v_rhs.TranslatedDescription().FileList(); - if (A == 0 && B == 0) - return false; + pkgCache::DescFile const *A = NULL; + pkgCache::DescFile const *B = NULL; + if (v_lhs->DescriptionList != 0) + A = v_lhs.TranslatedDescription().FileList(); + if (v_rhs->DescriptionList != 0) + B = v_rhs.TranslatedDescription().FileList(); + + if (A == 0 && B == 0) + return false; if (A == 0) return true; diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc index 4e18030ab..9944ab002 100644 --- a/apt-private/private-output.cc +++ b/apt-private/private-output.cc @@ -199,10 +199,12 @@ static std::string GetShortDescription(pkgCacheFile &CacheFile, pkgRecords &reco std::string ShortDescription = "(none)"; if(ver) { - pkgCache::DescIterator Desc = ver.TranslatedDescription(); - pkgRecords::Parser & parser = records.Lookup(Desc.FileList()); - - ShortDescription = parser.ShortDesc(); + pkgCache::DescIterator const Desc = ver.TranslatedDescription(); + if (Desc.end() == false) + { + pkgRecords::Parser & parser = records.Lookup(Desc.FileList()); + ShortDescription = parser.ShortDesc(); + } } return ShortDescription; } @@ -222,11 +224,14 @@ static std::string GetLongDescription(pkgCacheFile &CacheFile, pkgRecords &recor return EmptyDescription; pkgCache::DescIterator const Desc = ver.TranslatedDescription(); - pkgRecords::Parser & parser = records.Lookup(Desc.FileList()); - std::string const longdesc = parser.LongDesc(); - if (longdesc.empty() == true) - return EmptyDescription; - return SubstVar(longdesc, "\n ", "\n "); + if (Desc.end() == false) + { + pkgRecords::Parser & parser = records.Lookup(Desc.FileList()); + std::string const longdesc = parser.LongDesc(); + if (longdesc.empty() == false) + return SubstVar(longdesc, "\n ", "\n "); + } + return EmptyDescription; } /*}}}*/ void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records, /*{{{*/ diff --git a/buildlib/environment.mak.in b/buildlib/environment.mak.in index 0dff02e69..8ea7a05ba 100644 --- a/buildlib/environment.mak.in +++ b/buildlib/environment.mak.in @@ -16,6 +16,8 @@ CXXFLAGS+= -Wctor-dtor-privacy -Wdisabled-optimization -Winit-self -Wmissing-inc #CXXFLAGS+= -Wsuggest-attribute=pure -Wsuggest-attribute=const -Wsuggest-attribute=noreturn # gcc reports currently lots of them at the end of file - unknown reason CXXFLAGS+= -Wno-deprecated-declarations +# sanitize options to be enabled for testing +#CXXFLAGS+= -fsanitize=address -fsanitize=undefined -fno-sanitize=vptr # a bit too pedantic to be run by default #CXXFLAGS+= -Wpedantic -Wno-long-long -Wno-vla -Wno-variadic-macros NUM_PROCS = @NUM_PROCS@ diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 303605f70..9c884433c 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1484,6 +1484,7 @@ static bool Search(CommandLine &CmdL) delete [] PatternMatch; for (unsigned I = 0; I != NumPatterns; I++) regfree(&Patterns[I]); + delete [] Patterns; if (ferror(stdout)) return _error->Error("Write to stdout failed"); return true; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 184b51d23..632c7cfea 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1015,9 +1015,6 @@ static bool DoBuildDep(CommandLine &CmdL) pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers -#if APT_PKG_ABI < 413 - pkgRecords Recs(Cache); -#endif pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@ -1039,6 +1036,7 @@ static bool DoBuildDep(CommandLine &CmdL) { string Src; pkgSrcRecords::Parser *Last = 0; + SPtr LastOwner; // an unpacked debian source tree using APT::String::Startswith; @@ -1050,7 +1048,7 @@ static bool DoBuildDep(CommandLine &CmdL) std::string TypeName = "debian/control File Source Index"; pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); if(Type != NULL) - Last = Type->CreateSrcPkgParser(*I); + LastOwner = Last = Type->CreateSrcPkgParser(*I); } // if its a local file (e.g. .dsc) use this else if (FileExists(*I)) @@ -1061,7 +1059,7 @@ static bool DoBuildDep(CommandLine &CmdL) string TypeName = flExtension(*I) + " File Source Index"; pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); if(Type != NULL) - Last = Type->CreateSrcPkgParser(*I); + LastOwner = Last = Type->CreateSrcPkgParser(*I); } else { // normal case, search the cache for the source file #if APT_PKG_ABI >= 413 diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index 62108f7ca..cf667483c 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -180,7 +180,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) // Create a package writer object. MultiCompress Comp(flCombine(ArchiveDir,PkgFile), PkgCompress,Permissions); - PackagesWriter Packages(&Comp.Input, flCombine(CacheDir,BinCacheDB), + PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB), flCombine(OverrideDir,BinOverride), flCombine(OverrideDir,ExtraOverride), Arch); @@ -193,7 +193,6 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats) Packages.DirStrip = ArchiveDir; Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix); - Packages.TransWriter = TransWriter; Packages.LongDescription = LongDesc; Packages.Stats.DeLinkBytes = Stats.DeLinkBytes; @@ -457,7 +456,7 @@ bool PackageMap::GenContents(Configuration &Setup, // --------------------------------------------------------------------- /* This populates the PkgList with all the possible permutations of the section/arch lists. */ -static void LoadTree(vector &PkgList,Configuration &Setup) +static void LoadTree(vector &PkgList, std::vector &TransList, Configuration &Setup) { // Load the defaults string DDir = Setup.Find("TreeDefault::Directory", @@ -508,16 +507,7 @@ static void LoadTree(vector &PkgList,Configuration &Setup) {NULL, NULL}}; mode_t const Perms = Block.FindI("FileMode", Permissions); bool const LongDesc = Block.FindB("LongDescription", LongDescription); - TranslationWriter *TransWriter; - if (DTrans.empty() == false && LongDesc == false) - { - string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"), - SubstVar(Block.Find("Translation", DTrans.c_str()), Vars)); - string const TransCompress = Block.Find("Translation::Compress", TranslationCompress); - TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms); - } - else - TransWriter = NULL; + TranslationWriter *TransWriter = NULL; string const Tmp2 = Block.Find("Architectures"); const char *Archs = Tmp2.c_str(); @@ -546,27 +536,34 @@ static void LoadTree(vector &PkgList,Configuration &Setup) Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars); Itm.Arch = Arch; Itm.LongDesc = LongDesc; - if (TransWriter != NULL) + if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null") { - TransWriter->IncreaseRefCounter(); - Itm.TransWriter = TransWriter; + string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"), + SubstVar(Block.Find("Translation", DTrans.c_str()), Vars)); + string const TransCompress = Block.Find("Translation::Compress", TranslationCompress); + TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms); + TransList.push_back(TransWriter); } + Itm.TransWriter = TransWriter; Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars); Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars); Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars); Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars); } - Itm.GetGeneral(Setup,Block); + Itm.GetGeneral(Setup,Block); PkgList.push_back(Itm); } - // we didn't use this TransWriter, so we can release it - if (TransWriter != NULL && TransWriter->GetRefCounter() == 0) - delete TransWriter; } - + Top = Top->Next; - } + } +} + /*}}}*/ +static void UnloadTree(std::vector const &Trans) /*{{{*/ +{ + for (std::vector::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T) + delete *T; } /*}}}*/ // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/ @@ -671,7 +668,7 @@ static bool SimpleGenPackages(CommandLine &CmdL) Override = CmdL.FileList[2]; // Create a package writer object. - PackagesWriter Packages(NULL, _config->Find("APT::FTPArchive::DB"), + PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"), Override, "", _config->Find("APT::FTPArchive::Architecture")); if (_error->PendingError() == true) return false; @@ -844,12 +841,6 @@ static bool DoGeneratePackagesAndSources(Configuration &Setup, delete [] List; } - - // close the Translation master files - for (vector::reverse_iterator I = PkgList.rbegin(); I != PkgList.rend(); ++I) - if (I->TransWriter != NULL && I->TransWriter->DecreaseRefCounter() == 0) - delete I->TransWriter; - return true; } @@ -940,7 +931,8 @@ static bool Generate(CommandLine &CmdL) return false; vector PkgList; - LoadTree(PkgList,Setup); + std::vector TransList; + LoadTree(PkgList, TransList, Setup); LoadBinDir(PkgList,Setup); // Sort by cache DB to improve IO locality. @@ -951,7 +943,10 @@ static bool Generate(CommandLine &CmdL) if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false) { if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false) + { + UnloadTree(TransList); return false; + } } else { c1out << "Skipping Packages/Sources generation" << endl; } @@ -959,7 +954,10 @@ static bool Generate(CommandLine &CmdL) // do Contents if needed if (_config->FindB("APT::FTPArchive::Contents", true) == true) if (DoGenerateContents(Setup, PkgList, CmdL) == false) - return false; + { + UnloadTree(TransList); + return false; + } struct timeval NewTime; gettimeofday(&NewTime,0); @@ -968,6 +966,7 @@ static bool Generate(CommandLine &CmdL) c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages << " archives. Took " << TimeToStr((long)Delta) << endl; + UnloadTree(TransList); return true; } @@ -984,9 +983,12 @@ static bool Clean(CommandLine &CmdL) Configuration Setup; if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false) return false; + // we don't need translation creation here + Setup.Set("TreeDefault::Translation", "/dev/null"); vector PkgList; - LoadTree(PkgList,Setup); + std::vector TransList; + LoadTree(PkgList, TransList, Setup); LoadBinDir(PkgList,Setup); // Sort by cache DB to improve IO locality. @@ -1007,14 +1009,13 @@ static bool Clean(CommandLine &CmdL) _error->DumpErrors(); if (DB_SRC.Clean() == false) _error->DumpErrors(); - + string CacheDB = I->BinCacheDB; string SrcCacheDB = I->SrcCacheDB; while(I != PkgList.end() && I->BinCacheDB == CacheDB && I->SrcCacheDB == SrcCacheDB) ++I; - } diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index cc3527ea4..ce6c865f3 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -45,6 +45,7 @@ CacheDB::~CacheDB() { ReadyDB(); delete DebFile; + CloseFile(); } // CacheDB::ReadyDB - Ready the DB2 /*{{{*/ diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index 7cf7e6efc..1bc926d21 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -69,22 +69,29 @@ static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf) /*}}}*/ // FTWScanner::FTWScanner - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch): Arch(Arch), DoHashes(~0) { if (GivenOutput == NULL) { Output = new FileFd; + OwnsOutput = true; Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false); } else + { Output = GivenOutput; + OwnsOutput = false; + } ErrorPrinted = false; NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true); ConfigToDoHashes(DoHashes, "APT::FTPArchive"); } /*}}}*/ +FTWScanner::~FTWScanner() +{ + if (Output != NULL && OwnsOutput) + delete Output; +} // FTWScanner::Scanner - FTW Scanner /*{{{*/ // --------------------------------------------------------------------- /* This is the FTW scanner, it processes each directory element in the @@ -324,9 +331,10 @@ bool FTWScanner::Delink(string &FileName,const char *OriginalPath, // PackagesWriter::PackagesWriter - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -PackagesWriter::PackagesWriter(FileFd * const GivenOutput, string const &DB,string const &Overrides,string const &ExtOverrides, - string const &Arch) : - FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL) +PackagesWriter::PackagesWriter(FileFd * const GivenOutput, TranslationWriter * const transWriter, + string const &DB,string const &Overrides,string const &ExtOverrides, + string const &Arch) : + FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats), TransWriter(transWriter) { SetExts(".deb .udeb"); DeLinkLimit = 0; @@ -377,7 +385,6 @@ bool FTWScanner::SetExts(string const &Vals) return true; } - /*}}}*/ // PackagesWriter::DoPackage - Process a single package /*{{{*/ // --------------------------------------------------------------------- @@ -524,12 +531,16 @@ bool PackagesWriter::DoPackage(string FileName) return Db.Finish(); } /*}}}*/ +PackagesWriter::~PackagesWriter() /*{{{*/ +{ +} + /*}}}*/ // TranslationWriter::TranslationWriter - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Create a Translation-Master file for this Packages file */ TranslationWriter::TranslationWriter(string const &File, string const &TransCompress, - mode_t const &Permissions) : RefCounter(0) + mode_t const &Permissions) : Comp(NULL), Output(NULL) { if (File.empty() == true) return; @@ -568,10 +579,8 @@ bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc, /* */ TranslationWriter::~TranslationWriter() { - if (Comp == NULL) - return; - - delete Comp; + if (Comp != NULL) + delete Comp; } /*}}}*/ diff --git a/ftparchive/writer.h b/ftparchive/writer.h index 0ba60db5e..b9c1f672a 100644 --- a/ftparchive/writer.h +++ b/ftparchive/writer.h @@ -64,6 +64,7 @@ class FTWScanner public: FileFd *Output; + bool OwnsOutput; unsigned int DoHashes; unsigned long DeLinkLimit; @@ -79,7 +80,7 @@ class FTWScanner bool SetExts(string const &Vals); FTWScanner(FileFd * const Output, string const &Arch = string()); - virtual ~FTWScanner() {}; + virtual ~FTWScanner(); }; class MultiCompress; @@ -88,17 +89,12 @@ class TranslationWriter { MultiCompress *Comp; std::set Included; - unsigned short RefCounter; FileFd *Output; public: - void IncreaseRefCounter() { ++RefCounter; }; - unsigned short DecreaseRefCounter() { return (RefCounter == 0) ? 0 : --RefCounter; }; - unsigned short GetRefCounter() const { return RefCounter; }; bool DoPackage(string const &Pkg, string const &Desc, string const &MD5); TranslationWriter(string const &File, string const &TransCompress, mode_t const &Permissions); - TranslationWriter() : Comp(NULL), RefCounter(0) {}; ~TranslationWriter(); }; @@ -119,18 +115,18 @@ class PackagesWriter : public FTWScanner string PathPrefix; string DirStrip; struct CacheDB::Stats &Stats; - TranslationWriter *TransWriter; + TranslationWriter * const TransWriter; inline bool ReadOverride(string const &File) {return Over.ReadOverride(File);}; inline bool ReadExtraOverride(string const &File) {return Over.ReadExtraOverride(File);}; virtual bool DoPackage(string FileName); - PackagesWriter(FileFd * const Output, string const &DB, + PackagesWriter(FileFd * const Output, TranslationWriter * const TransWriter, string const &DB, string const &Overrides, string const &ExtOverrides = "", string const &Arch = ""); - virtual ~PackagesWriter() {}; + virtual ~PackagesWriter(); }; class ContentsWriter : public FTWScanner diff --git a/test/integration/framework b/test/integration/framework index 5d949009f..059cba9fb 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -143,6 +143,7 @@ gdb() { aptcache) CMD="apt-cache";; aptmark) CMD="apt-mark";; apthelper) CMD="apt-helper";; + aptftparchive) CMD="apt-ftparchive";; *) CMD="$1";; esac shift @@ -1415,6 +1416,8 @@ testfailure() { if expr match "$1" '^apt.*' >/dev/null; then if grep -q -E ' runtime error: ' "$OUTPUT"; then msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@" + elif grep -q -E '==ERROR' "$OUTPUT"; then + msgfailoutput 'compiler sanitizers reported errors' "$OUTPUT" "$@" elif ! grep -q -E '^E: ' "$OUTPUT"; then msgfailoutput "run failed with exitcode ${EXITCODE}, but with no errors" "$OUTPUT" "$@" else diff --git a/test/libapt/cdrom_test.cc b/test/libapt/cdrom_test.cc index 7257eaf1b..b4c51cdb0 100644 --- a/test/libapt/cdrom_test.cc +++ b/test/libapt/cdrom_test.cc @@ -91,7 +91,7 @@ TEST(CDROMTest,ReduceSourcelist) } TEST(CDROMTest, FindMountPointForDevice) { - char * tempfile = NULL; + std::string tempfile; FileFd fd; createTemporaryFile("mountpoints", fd, &tempfile, "rootfs / rootfs rw 0 0\n" @@ -109,7 +109,6 @@ TEST(CDROMTest, FindMountPointForDevice) EXPECT_EQ("/boot/efi", FindMountPointForDevice("/dev/sda1")); EXPECT_EQ("/tmp", FindMountPointForDevice("tmpfs")); - if (tempfile != NULL) - unlink(tempfile); - free(tempfile); + if (tempfile.empty() == false) + unlink(tempfile.c_str()); } diff --git a/test/libapt/file-helpers.cc b/test/libapt/file-helpers.cc index 5edb9a9fe..387192868 100644 --- a/test/libapt/file-helpers.cc +++ b/test/libapt/file-helpers.cc @@ -53,20 +53,19 @@ void helperCreateLink(std::string const &dir, std::string const &targetname, std link.append(linkname); ASSERT_EQ(0, symlink(target.c_str(), link.c_str())); } -void helperCreateTemporaryFile(std::string const &id, FileFd &fd, char * * const filename, char const * const content) +void helperCreateTemporaryFile(std::string const &id, FileFd &fd, std::string * const filename, char const * const content) { std::string name("apt-test-"); name.append(id).append(".XXXXXXXX"); char * tempfile = strdup(name.c_str()); + ASSERT_STRNE(NULL, tempfile); int tempfile_fd = mkstemp(tempfile); ASSERT_NE(-1, tempfile_fd); if (filename != NULL) *filename = tempfile; else - { unlink(tempfile); - free(tempfile); - } + free(tempfile); EXPECT_TRUE(fd.OpenDescriptor(tempfile_fd, FileFd::ReadWrite)); if (content != NULL) diff --git a/test/libapt/file-helpers.h b/test/libapt/file-helpers.h index e8472d503..f639c1cbc 100644 --- a/test/libapt/file-helpers.h +++ b/test/libapt/file-helpers.h @@ -24,6 +24,6 @@ void helperCreateDirectory(std::string const &dir, std::string const &name); void helperCreateLink(std::string const &dir, std::string const &targetname, std::string const &linkname); #define createTemporaryFile(id, fd, filename, content) \ ASSERT_NO_FATAL_FAILURE(helperCreateTemporaryFile(id, fd, filename, content)) -void helperCreateTemporaryFile(std::string const &id, FileFd &fd, char * * const filename, char const * const content); +void helperCreateTemporaryFile(std::string const &id, FileFd &fd, std::string * const filename, char const * const content); #endif diff --git a/test/libapt/sourcelist_test.cc b/test/libapt/sourcelist_test.cc index 747ab4957..9e6f82213 100644 --- a/test/libapt/sourcelist_test.cc +++ b/test/libapt/sourcelist_test.cc @@ -20,7 +20,7 @@ class SourceList : public pkgSourceList { TEST(SourceListTest,ParseFileDeb822) { FileFd fd; - char * tempfile = NULL; + std::string tempfile; createTemporaryFile("parsefiledeb822", fd, &tempfile, "Types: deb\n" "URIs: http://ftp.debian.org/debian\n" @@ -39,5 +39,6 @@ TEST(SourceListTest,ParseFileDeb822) EXPECT_EQ(2, sources.ParseFileDeb822(tempfile)); EXPECT_EQ(2, sources.size()); - unlink(tempfile); + if (tempfile.empty() == false) + unlink(tempfile.c_str()); } -- cgit v1.2.3-70-g09d2 From b0d408547734100bf86781615f546487ecf390d9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 24 Jun 2015 19:31:22 +0200 Subject: implement Signed-By option for sources.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Limits which key(s) can be used to sign a repository. Not immensely useful from a security perspective all by itself, but if the user has additional measures in place to confine a repository (like pinning) an attacker who gets the key for such a repository is limited to its potential and can't use the key to sign its attacks for an other (maybe less limited) repository… (yes, this is as weak as it sounds, but having the capability might come in handy for implementing other stuff later). --- apt-pkg/acquire-item.cc | 15 +++++++- apt-pkg/acquire-item.h | 1 + apt-pkg/contrib/gpgv.cc | 17 +++++++++- apt-pkg/contrib/gpgv.h | 5 ++- apt-pkg/deb/debmetaindex.cc | 35 +++++++++++++++++++ apt-pkg/deb/debmetaindex.h | 1 + apt-pkg/metaindex.cc | 4 +-- apt-pkg/metaindex.h | 4 ++- apt-pkg/sourcelist.cc | 28 ++++++++------- cmdline/apt-key.in | 22 ++++++++++-- doc/sources.list.5.xml | 24 ++++++------- methods/gpgv.cc | 18 +++++----- test/integration/framework | 23 +++++++++---- test/integration/test-apt-key | 37 ++++++++++++++++++-- test/integration/test-releasefile-verification | 47 +++++++++++++++++++++++--- 15 files changed, 225 insertions(+), 56 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index a30a5d154..01a679fe0 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -808,7 +808,6 @@ string pkgAcqMetaBase::Custom600Headers() const Header += MaximumSize; string const FinalFile = GetFinalFilename(); - struct stat Buf; if (stat(FinalFile.c_str(),&Buf) == 0) Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); @@ -1132,6 +1131,10 @@ string pkgAcqMetaClearSig::Custom600Headers() const { string Header = pkgAcqMetaBase::Custom600Headers(); Header += "\nFail-Ignore: true"; + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + key; + return Header; } /*}}}*/ @@ -1372,6 +1375,16 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner, /*}}}*/ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ { +} + /*}}}*/ +// pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ +std::string pkgAcqMetaSig::Custom600Headers() const +{ + std::string Header = pkgAcqTransactionItem::Custom600Headers(); + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + key; + return Header; } /*}}}*/ // AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 10ece76c9..1cd2a6d03 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -541,6 +541,7 @@ class APT_HIDDEN pkgAcqMetaSig : public pkgAcqTransactionItem virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf); virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const * const Cnf); + virtual std::string Custom600Headers() const; /** \brief Create a new pkgAcqMetaSig. */ pkgAcqMetaSig(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc index a01e319eb..ef84da0d8 100644 --- a/apt-pkg/contrib/gpgv.cc +++ b/apt-pkg/contrib/gpgv.cc @@ -16,6 +16,8 @@ #include #include #include + +#include #include #include #include @@ -42,7 +44,7 @@ static char * GenerateTemporaryFileTemplate(const char *basename) /*{{{*/ of the lifting in regards to merging keyrings. Fun for the whole family. */ void ExecGPGV(std::string const &File, std::string const &FileGPG, - int const &statusfd, int fd[2]) + int const &statusfd, int fd[2], std::string const &key) { #define EINTERNAL 111 std::string const aptkey = _config->FindFile("Dir::Bin::apt-key", "/usr/bin/apt-key"); @@ -55,6 +57,19 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, Args.push_back(aptkey.c_str()); Args.push_back("--quiet"); Args.push_back("--readonly"); + if (key.empty() == false) + { + if (key[0] == '/') + { + Args.push_back("--keyring"); + Args.push_back(key.c_str()); + } + else + { + Args.push_back("--keyid"); + Args.push_back(key.c_str()); + } + } Args.push_back("verify"); char statusfdstr[10]; diff --git a/apt-pkg/contrib/gpgv.h b/apt-pkg/contrib/gpgv.h index f018893fd..2a4cdad72 100644 --- a/apt-pkg/contrib/gpgv.h +++ b/apt-pkg/contrib/gpgv.h @@ -38,9 +38,12 @@ class FileFd; * * @param File is the message (unsigned or clear-signed) * @param FileSig is the signature (detached or clear-signed) + * @param statusfd is the fd given to gpgv as --status-fd + * @param fd is used as a pipe for the standard output of gpgv + * @param key is the specific one to be used instead of using all */ void ExecGPGV(std::string const &File, std::string const &FileSig, - int const &statusfd, int fd[2]) APT_NORETURN; + int const &statusfd, int fd[2], std::string const &Key = "") APT_NORETURN; inline APT_NORETURN void ExecGPGV(std::string const &File, std::string const &FileSig, int const &statusfd = -1) { int fd[2]; diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 5d7e539c7..4bb03a942 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -461,6 +461,29 @@ bool debReleaseIndex::SetValidUntilMax(time_t const Valid) else if (d->ValidUntilMax != Valid) return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Max-ValidTime", URI.c_str(), Dist.c_str()); return true; +} +bool debReleaseIndex::SetSignedBy(std::string const &pSignedBy) +{ + if (SignedBy.empty() == true && pSignedBy.empty() == false) + { + if (pSignedBy[0] == '/') // no check for existence as we could be chrooting later or such things + ; // absolute path to a keyring file + else + { + // we could go all fancy and allow short/long/string matches as gpgv/apt-key does, + // but fingerprints are harder to fake than the others and this option is set once, + // not interactively all the time so easy to type is not really a concern. + std::string finger = pSignedBy; + finger.erase(std::remove(finger.begin(), finger.end(), ' '), finger.end()); + std::transform(finger.begin(), finger.end(), finger.begin(), ::toupper); + if (finger.length() != 40 || finger.find_first_not_of("0123456789ABCDEF") != std::string::npos) + return _error->Error(_("Invalid value set for option %s concerning source %s %s (%s)"), "Signed-By", URI.c_str(), Dist.c_str(), "not a fingerprint"); + } + SignedBy = pSignedBy; + } + else if (SignedBy != pSignedBy) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Signed-By", URI.c_str(), Dist.c_str()); + return true; } /*}}}*/ // ReleaseIndex::IsTrusted /*{{{*/ @@ -706,6 +729,18 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ Deb->SetValidUntilMin(GetTimeOption(Options, "valid-until-min")) == false) return false; + std::map::const_iterator const signedby = Options.find("signed-by"); + if (signedby == Options.end()) + { + if (Deb->SetSignedBy("") == false) + return false; + } + else + { + if (Deb->SetSignedBy(signedby->second) == false) + return false; + } + return true; } diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 879eb3bfc..bf5b7c1ce 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -56,6 +56,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex bool SetCheckValidUntil(TriState const Trusted); bool SetValidUntilMin(time_t const Valid); bool SetValidUntilMax(time_t const Valid); + bool SetSignedBy(std::string const &SignedBy); virtual bool IsTrusted() const; diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc index 8bd13bb18..baf695f16 100644 --- a/apt-pkg/metaindex.cc +++ b/apt-pkg/metaindex.cc @@ -27,8 +27,7 @@ bool metaIndex::Merge(pkgCacheGenerator &Gen,OpProgress *) const metaIndex::metaIndex(std::string const &URI, std::string const &Dist, char const * const Type) : d(NULL), Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(TRI_UNSET), - LoadedSuccessfully(TRI_UNSET), - Date(0), ValidUntil(0), SupportsAcquireByHash(false) + Date(0), ValidUntil(0), SupportsAcquireByHash(false), LoadedSuccessfully(TRI_UNSET) { /* nothing */ } @@ -48,6 +47,7 @@ APT_PURE std::string metaIndex::GetURI() const { return URI; } APT_PURE std::string metaIndex::GetDist() const { return Dist; } APT_PURE const char* metaIndex::GetType() const { return Type; } APT_PURE metaIndex::TriState metaIndex::GetTrusted() const { return Trusted; } +APT_PURE std::string metaIndex::GetSignedBy() const { return SignedBy; } APT_PURE std::string metaIndex::GetCodename() const { return Codename; } APT_PURE std::string metaIndex::GetSuite() const { return Suite; } APT_PURE bool metaIndex::GetSupportsAcquireByHash() const { return SupportsAcquireByHash; } diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index 5be7397ae..d284655bf 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -52,7 +52,7 @@ protected: std::string URI; std::string Dist; TriState Trusted; - TriState LoadedSuccessfully; + std::string SignedBy; // parsed from a file std::string Suite; @@ -61,6 +61,7 @@ protected: time_t ValidUntil; bool SupportsAcquireByHash; std::map Entries; + TriState LoadedSuccessfully; public: // Various accessors @@ -68,6 +69,7 @@ public: std::string GetDist() const; const char* GetType() const; TriState GetTrusted() const; + std::string GetSignedBy() const; std::string GetCodename() const; std::string GetSuite() const; diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0d65558ed..eef0ee709 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -93,27 +93,29 @@ bool pkgSourceList::Type::ParseStanza(vector &List, /*{{{*/ if (Enabled.empty() == false && StringToBool(Enabled) == false) return true; - std::map mapping; + std::map > mapping; #define APT_PLUSMINUS(X, Y) \ - mapping.insert(std::make_pair(X, Y)); \ - mapping.insert(std::make_pair(X "Add", Y "+")); \ - mapping.insert(std::make_pair(X "Remove", Y "-")) + mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \ + mapping.insert(std::make_pair(X "Add", std::make_pair(Y "+", true))); \ + mapping.insert(std::make_pair(X "Remove", std::make_pair(Y "-", true))) APT_PLUSMINUS("Architectures", "arch"); APT_PLUSMINUS("Languages", "lang"); APT_PLUSMINUS("Targets", "target"); #undef APT_PLUSMINUS - mapping.insert(std::make_pair("Trusted", "trusted")); - mapping.insert(std::make_pair("Check-Valid-Until", "check-valid-until")); - mapping.insert(std::make_pair("Valid-Until-Min", "valid-until-min")); - mapping.insert(std::make_pair("Valid-Until-Max", "valid-until-max")); + mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false))); + mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false))); + mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false))); + mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false))); + mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false))); - for (std::map::const_iterator m = mapping.begin(); m != mapping.end(); ++m) + for (std::map >::const_iterator m = mapping.begin(); m != mapping.end(); ++m) if (Tags.Exists(m->first)) { - // for deb822 the " " is the delimiter, but the backend expects "," - std::string option = Tags.FindS(m->first); - std::replace(option.begin(), option.end(), ' ', ','); - Options[m->second] = option; + std::string option = Tags.FindS(m->first); + // for deb822 the " " is the delimiter, but the backend expects "," + if (m->second.second == true) + std::replace(option.begin(), option.end(), ' ', ','); + Options[m->second.first] = option; } // now create one item per suite/section diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 2a66ad74d..16887bd50 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -199,7 +199,7 @@ remove_key_from_keyring() { foreach_keyring_do() { local ACTION="$1" shift - # if a --keyring was given, just remove from there + # if a --keyring was given, just work on this one if [ -n "$FORCED_KEYRING" ]; then $ACTION "$FORCED_KEYRING" "$@" else @@ -279,7 +279,14 @@ merge_back_changes() { } setup_merged_keyring() { - if [ -z "$FORCED_KEYRING" ]; then + if [ -n "$FORCED_KEYID" ]; then + foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/allrings.gpg" + FORCED_KEYRING="${GPGHOMEDIR}/forcedkeyid.gpg" + TRUSTEDFILE="${FORCED_KEYRING}" + GPG="$GPG --keyring $TRUSTEDFILE" + # ignore error as this "just" means we haven't found the forced keyid and the keyring will be empty + $GPG_CMD --batch --yes --keyring "${GPGHOMEDIR}/allrings.gpg" --export "$FORCED_KEYID" | $GPG --batch --yes --import || true + elif [ -z "$FORCED_KEYRING" ]; then foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/pubring.gpg" if [ -r "${GPGHOMEDIR}/pubring.gpg" ]; then cp -a "${GPGHOMEDIR}/pubring.gpg" "${GPGHOMEDIR}/pubring.orig.gpg" @@ -328,12 +335,17 @@ while [ -n "$1" ]; do TRUSTEDFILE="$1" FORCED_KEYRING="$1" ;; + --keyid) + shift + FORCED_KEYID="$1" + ;; --secret-keyring) shift FORCED_SECRET_KEYRING="$1" ;; --readonly) merge_back_changes() { true; } + create_new_keyring() { true; } ;; --fakeroot) requires_root() { true; } @@ -460,7 +472,11 @@ case "$command" in verify) setup_merged_keyring if which gpgv >/dev/null 2>&1; then - gpgv --homedir "${GPGHOMEDIR}" --keyring "${GPGHOMEDIR}/pubring.gpg" --ignore-time-conflict "$@" + if [ -n "$FORCED_KEYRING" ]; then + gpgv --homedir "${GPGHOMEDIR}" --keyring "${FORCED_KEYRING}" --ignore-time-conflict "$@" + else + gpgv --homedir "${GPGHOMEDIR}" --keyring "${GPGHOMEDIR}/pubring.gpg" --ignore-time-conflict "$@" + fi else $GPG --verify "$@" fi diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml index aded8ecef..12a7773f5 100644 --- a/doc/sources.list.5.xml +++ b/doc/sources.list.5.xml @@ -232,18 +232,18 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [. anomalies. - () - is a tri-state value which defaults to APT deciding if a source - is considered trusted or if warnings should be raised before e.g. - packages are installed from this source. This option can be used - to override this decision either with the value yes, - which lets APT consider this source always as a trusted source - even if it has no or fails authentication checks by disabling parts - of &apt-secure; and should therefore only be used in a local and trusted - context (if at all) as otherwise security is breached. The opposite - can be achieved with the value no, which causes the source to be handled - as untrusted even if the authentication checks passed successfully. - The default value can't be set explicitly. + () + is either an absolute path to a keyring file (has to be + accessible and readable for the _apt user, + so ensure everyone has read-permissions on the file) or a + fingerprint of a key in either the + trusted.gpg keyring or in one of the + keyrings in the trusted.gpg.d/ directory + (see apt-key fingerprint). If the option is + set only the key(s) in this keyring or only the key with this + fingerprint is used for the &apt-secure; verification of this + repository. Otherwise all keys in the trusted keyrings are + considered valid signers for this repository. () diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 41f138be6..014430041 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -37,13 +37,14 @@ class GPGVMethod : public pkgAcqMethod { private: string VerifyGetSigners(const char *file, const char *outfile, - vector &GoodSigners, + std::string const &key, + vector &GoodSigners, vector &BadSigners, vector &WorthlessSigners, vector &NoPubKeySigners); protected: - virtual bool Fetch(FetchItem *Itm); + virtual bool URIAcquire(std::string const &Message, FetchItem *Itm); virtual bool Configuration(string Message); public: @@ -61,6 +62,7 @@ bool GPGVMethod::Configuration(string Message) } string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, + std::string const &key, vector &GoodSigners, vector &BadSigners, vector &WorthlessSigners, @@ -80,7 +82,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, if (pid < 0) return string("Couldn't spawn new process") + strerror(errno); else if (pid == 0) - ExecGPGV(outfile, file, 3, fd); + ExecGPGV(outfile, file, 3, fd, key); close(fd[1]); FILE *pipein = fdopen(fd[0], "r"); @@ -174,11 +176,11 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, return _("Unknown error executing apt-key"); } -bool GPGVMethod::Fetch(FetchItem *Itm) +bool GPGVMethod::URIAcquire(std::string const &Message, FetchItem *Itm) { - URI Get = Itm->Uri; - string Path = Get.Host + Get.Path; // To account for relative paths - string keyID; + URI const Get = Itm->Uri; + string const Path = Get.Host + Get.Path; // To account for relative paths + std::string const key = LookupTag(Message, "Signed-By"); vector GoodSigners; vector BadSigners; // a worthless signature is a expired or revoked one @@ -190,7 +192,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) URIStart(Res); // Run apt-key on file, extract contents and get the key ID of the signer - string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), + string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), key, GoodSigners, BadSigners, WorthlessSigners, NoPubKeySigners); if (GoodSigners.empty() || !BadSigners.empty() || !NoPubKeySigners.empty()) diff --git a/test/integration/framework b/test/integration/framework index 059cba9fb..6ae5003f7 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1414,14 +1414,23 @@ testfailure() { else local EXITCODE=$? if expr match "$1" '^apt.*' >/dev/null; then - if grep -q -E ' runtime error: ' "$OUTPUT"; then - msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@" - elif grep -q -E '==ERROR' "$OUTPUT"; then - msgfailoutput 'compiler sanitizers reported errors' "$OUTPUT" "$@" - elif ! grep -q -E '^E: ' "$OUTPUT"; then - msgfailoutput "run failed with exitcode ${EXITCODE}, but with no errors" "$OUTPUT" "$@" + if [ "$1" = 'aptkey' ]; then + if grep -q -E " Can't check signature: " "$OUTPUT" || \ + grep -q -E " BAD signature from " "$OUTPUT"; then + msgpass + else + msgfailoutput "run failed with exitcode ${EXITCODE}, but no signature error" "$OUTPUT" "$@" + fi else - msgpass + if grep -q -E ' runtime error: ' "$OUTPUT"; then + msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@" + elif grep -q -E '==ERROR' "$OUTPUT"; then + msgfailoutput 'compiler sanitizers reported errors' "$OUTPUT" "$@" + elif ! grep -q -E '^E: ' "$OUTPUT"; then + msgfailoutput "run failed with exitcode ${EXITCODE}, but with no errors" "$OUTPUT" "$@" + else + msgpass + fi fi else msgpass diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index 486acccc8..e1be08c65 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -73,7 +73,7 @@ pub 2048R/DBAC8DAE 2010-08-18' testsuccess aptkey --fakeroot del DBAC8DAE testempty aptkey list - msgtest 'Test key removal with' 'lowercase key ID' #keylength somewher between 8byte and short + msgtest 'Test key removal with' 'lowercase key ID' #keylength somewhere between 8byte and short cleanplate cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg testsuccess --nomsg aptkey --fakeroot del d141dbac8dae @@ -166,6 +166,40 @@ pub 2048R/528144E2 2011-01-16' msgtest 'Test merge-back of' 'removed duplicate keys' testsuccess --nomsg aptkey adv --batch --yes --delete-keys DBAC8DAE testaptkeys 'pub 2048R/528144E2 2011-01-16' + + cleanplate + cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg + msgtest 'Test signing a file' 'with a key' + echo 'Verify me. This is my signature.' > signature + testsuccess --nomsg aptkey --quiet --keyring keys/marvinparanoid.pub --secret-keyring keys/marvinparanoid.sec --readonly \ + adv --batch --yes --default-key 'Marvin' --armor --detach-sign --sign --output signature.gpg signature + + msgtest 'Test verify a file' 'with all keys' + testsuccess --nomsg aptkey --quiet --readonly verify signature.gpg signature + + msgtest 'Test verify a file' 'with good keyring' + testsuccess --nomsg aptkey --quiet --readonly --keyring keys/testcase-multikey.pub verify signature.gpg signature + + msgtest 'Test fail verify a file' 'with bad keyring' + testfailure --nomsg aptkey --quiet --readonly --keyring keys/joesixpack.pub verify signature.gpg signature + + msgtest 'Test fail verify a file' 'with non-existing keyring' + testfailure --nomsg aptkey --quiet --readonly --keyring keys/does-not-exist.pub verify signature.gpg signature + testfailure test -e keys/does-not-exist.pub + + msgtest 'Test verify a file' 'with good keyid' + testsuccess --nomsg aptkey --quiet --readonly --keyid 'Paranoid' verify signature.gpg signature + + msgtest 'Test fail verify a file' 'with bad keyid' + testfailure --nomsg aptkey --quiet --readonly --keyid 'Sixpack' verify signature.gpg signature + + msgtest 'Test fail verify a file' 'with non-existing keyid' + testfailure --nomsg aptkey --quiet --readonly --keyid 'Kalnischkies' verify signature.gpg signature + + msgtest 'Test verify fails on' 'bad file' + echo 'lalalalala' > signature + testfailure --nomsg aptkey --quiet --readonly verify signature.gpg signature } setupgpgcommand() { @@ -187,4 +221,3 @@ setupgpgcommand 'gpg' testrun setupgpgcommand 'gpg2' testrun - diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification index e8419524c..1c3953c8b 100755 --- a/test/integration/test-releasefile-verification +++ b/test/integration/test-releasefile-verification @@ -139,11 +139,6 @@ runtest() { failaptold prepare ${PKGFILE}-new - # weborf doesn't support If-Range - for release in $(find rootdir/var/lib/apt/lists/partial/ -name '*Release'); do - rm $release - touch $release - done signreleasefiles 'Joe Sixpack' find aptarchive/ -name "$DELETEFILE" -delete msgmsg 'Bad warm archive signed by' 'Joe Sixpack' @@ -191,6 +186,48 @@ runtest() { testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt installaptnew + + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Marvin Paranoid' + find aptarchive/ -name "$DELETEFILE" -delete + msgmsg 'Cold archive signed by good keyring' 'Marvin Paranoid' + local MARVIN="$(readlink -f keys/marvinparanoid.pub)" + sed -i "s#^\(deb\(-src\)\?\) #\1 [signed-by=$MARVIN] #" rootdir/etc/apt/sources.list.d/* + testsuccess aptget update -o Debug::pkgAcquire::Worker=1 + testsuccessequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold + + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgmsg 'Cold archive signed by bad keyring' 'Joe Sixpack' + updatewithwarnings '^W: .* NO_PUBKEY' + + sed -i "s#^\(deb\(-src\)\?\) \[signed-by=$MARVIN\] #\1 #" rootdir/etc/apt/sources.list.d/* + local MARVIN="$(aptkey --keyring $MARVIN finger | grep 'Key fingerprint' | cut -d'=' -f 2 | tr -d ' ')" + + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Marvin Paranoid' + find aptarchive/ -name "$DELETEFILE" -delete + msgmsg 'Cold archive signed by good keyid' 'Marvin Paranoid' + sed -i "s#^\(deb\(-src\)\?\) #\1 [signed-by=$MARVIN] #" rootdir/etc/apt/sources.list.d/* + cp keys/marvinparanoid.pub rootdir/etc/apt/trusted.gpg.d/marvinparanoid.gpg + testsuccess aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::gpgv=1 + testsuccessequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold + rm -f rootdir/etc/apt/trusted.gpg.d/marvinparanoid.gpg + + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgmsg 'Cold archive signed by bad keyid' 'Joe Sixpack' + updatewithwarnings '^W: .* NO_PUBKEY' + + sed -i "s#^\(deb\(-src\)\?\) \[signed-by=$MARVIN\] #\1 #" rootdir/etc/apt/sources.list.d/* } runtest2() { -- cgit v1.2.3-70-g09d2 From 5465192b9aeb1ccea778950ccf2d1b7b32f2cd91 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 18 Jul 2015 18:03:54 +0200 Subject: add volatile sources support in libapt-pkg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sources are usually defined in sources.list (and co) and are pretty stable, but once in a while a frontend might want to add an additional "source" like a local .deb file to install this package (No support for 'real' sources being added this way as this is a multistep process). We had a hack in place to allow apt-get and apt to pull this of for a short while now, but other frontends are either left in the cold by this and/or the code for it looks dirty with FIXMEs plastering it and has on top of this also some problems (like including these 'volatile' sources in the srcpkgcache.bin file). So the biggest part in this commit is actually the rewrite of the cache generation as it is now potentially a three step process. The biggest problem with adding support now through is that this makes a bunch of previously mostly unusable by externs and therefore hidden classes public, so a bit of further tuneing on this now public API is in order… --- apt-pkg/deb/debindexfile.cc | 27 +- apt-pkg/deb/debindexfile.h | 18 +- apt-pkg/deb/deblistparser.cc | 2 +- apt-pkg/deb/debmetaindex.cc | 29 --- apt-pkg/deb/debmetaindex.h | 39 --- apt-pkg/edsp/edspsystem.cc | 3 +- apt-pkg/pkgcachegen.cc | 401 ++++++++++++++---------------- apt-pkg/pkgcachegen.h | 1 - apt-pkg/pkgsystem.h | 5 +- apt-pkg/sourcelist.cc | 28 ++- apt-pkg/sourcelist.h | 19 ++ apt-private/private-cachefile.h | 21 -- apt-private/private-install.cc | 21 +- test/integration/framework | 12 +- test/integration/test-apt-get-install-deb | 45 +++- 15 files changed, 309 insertions(+), 362 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 972feba7f..e67233e5f 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -339,18 +339,16 @@ APT_CONST bool debStatusIndex::Exists() const } /*}}}*/ -// debDebPkgFile - Single .deb file /*{{{*/ -debDebPkgFileIndex::debDebPkgFileIndex(std::string DebFile) +// debDebPkgFileIndex - Single .deb file /*{{{*/ +debDebPkgFileIndex::debDebPkgFileIndex(std::string const &DebFile) : pkgIndexFile(true), d(NULL), DebFile(DebFile) { DebFileFullPath = flAbsPath(DebFile); } - std::string debDebPkgFileIndex::ArchiveURI(std::string /*File*/) const { return "file:" + DebFileFullPath; } - bool debDebPkgFileIndex::Exists() const { return FileExists(DebFile); @@ -403,6 +401,8 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const if (GetContent(content, DebFile) == false) return false; std::string const contentstr = content.str(); + if (contentstr.empty()) + return true; DebControl->Write(contentstr.c_str(), contentstr.length()); // rewind for the listparser DebControl->Seek(0); @@ -431,7 +431,7 @@ pkgCache::PkgFileIterator debDebPkgFileIndex::FindInCache(pkgCache &Cache) const return File; } - + return File; } unsigned long debDebPkgFileIndex::Size() const @@ -443,12 +443,11 @@ unsigned long debDebPkgFileIndex::Size() const } /*}}}*/ -// debDscFileIndex stuff -debDscFileIndex::debDscFileIndex(std::string &DscFile) +// debDscFileIndex - a .dsc file /*{{{*/ +debDscFileIndex::debDscFileIndex(std::string const &DscFile) : pkgIndexFile(true), d(NULL), DscFile(DscFile) { } - bool debDscFileIndex::Exists() const { return FileExists(DscFile); @@ -461,8 +460,6 @@ unsigned long debDscFileIndex::Size() const return buf.st_size; return 0; } - -// DscFileIndex::CreateSrcParser - Get a parser for the .dsc file /*{{{*/ pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const { if (!FileExists(DscFile)) @@ -471,18 +468,17 @@ pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const return new debDscRecordParser(DscFile,this); } /*}}}*/ + // Index File types for Debian /*{{{*/ class APT_HIDDEN debIFTypeSrc : public pkgIndexFile::Type { public: - debIFTypeSrc() {Label = "Debian Source Index";}; }; class APT_HIDDEN debIFTypePkg : public pkgIndexFile::Type { public: - - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debRecordParser(File.FileName(),*File.Cache()); }; @@ -496,8 +492,7 @@ class APT_HIDDEN debIFTypeTrans : public debIFTypePkg class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type { public: - - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debRecordParser(File.FileName(),*File.Cache()); }; @@ -506,7 +501,7 @@ class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type { public: - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debDebFileRecordParser(File.FileName()); }; diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index 71ca22e4d..4a818121a 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -27,7 +27,7 @@ class pkgAcquire; class pkgCacheGenerator; -class APT_HIDDEN debStatusIndex : public pkgIndexFile +class debStatusIndex : public pkgIndexFile { void * const d; protected: @@ -51,7 +51,7 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile virtual ~debStatusIndex(); }; -class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile +class debPackagesIndex : public pkgIndexTargetFile { void * const d; public: @@ -70,7 +70,7 @@ class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile virtual ~debPackagesIndex(); }; -class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile +class debTranslationsIndex : public pkgIndexTargetFile { void * const d; public: @@ -86,7 +86,7 @@ class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile virtual ~debTranslationsIndex(); }; -class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile +class debSourcesIndex : public pkgIndexTargetFile { void * const d; public: @@ -107,7 +107,7 @@ class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile virtual ~debSourcesIndex(); }; -class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile +class debDebPkgFileIndex : public pkgIndexFile { private: void * const d; @@ -141,11 +141,11 @@ class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile // Interface for acquire virtual std::string ArchiveURI(std::string /*File*/) const APT_OVERRIDE; - debDebPkgFileIndex(std::string DebFile); + debDebPkgFileIndex(std::string const &DebFile); virtual ~debDebPkgFileIndex(); }; -class APT_HIDDEN debDscFileIndex : public pkgIndexFile +class debDscFileIndex : public pkgIndexFile { private: void * const d; @@ -160,11 +160,11 @@ class APT_HIDDEN debDscFileIndex : public pkgIndexFile return DscFile; }; - debDscFileIndex(std::string &DscFile); + debDscFileIndex(std::string const &DscFile); virtual ~debDscFileIndex(); }; -class APT_HIDDEN debDebianSourceDirIndex : public debDscFileIndex +class debDebianSourceDirIndex : public debDscFileIndex { public: virtual const Type *GetType() const APT_CONST; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 1154016a9..b7988d499 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -959,7 +959,7 @@ bool debDebFileParser::UsePackage(pkgCache::PkgIterator &Pkg, bool res = debListParser::UsePackage(Pkg, Ver); // we use the full file path as a provides so that the file is found // by its name - if(NewProvidesAllArch(Ver, DebFile, Ver.VerStr(), 0) == false) + if(NewProvides(Ver, DebFile, Pkg.Cache()->NativeArch(), Ver.VerStr(), 0) == false) return false; return res; } diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 480317db3..123c91648 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -785,34 +785,5 @@ class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/ }; /*}}}*/ -debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) /*{{{*/ - : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) -{ - DebIndex = new debDebPkgFileIndex(DebFile); - Indexes = new std::vector(); - Indexes->push_back(DebIndex); -} -debDebFileMetaIndex::~debDebFileMetaIndex() {} - /*}}}*/ -class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type /*{{{*/ -{ - public: - - bool CreateItem(std::vector &List, std::string const &URI, - std::string const &/*Dist*/, std::string const &/*Section*/, - std::map const &/*Options*/) const APT_OVERRIDE - { - metaIndex *mi = new debDebFileMetaIndex(URI); - List.push_back(mi); - return true; - } - - debSLTypeDebFile() : Type("deb-file", "Debian local deb file") - { - } -}; - /*}}}*/ - APT_HIDDEN debSLTypeDeb _apt_DebType; APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType; -APT_HIDDEN debSLTypeDebFile _apt_DebFileType; diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 8c13237cb..e93959a21 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -17,7 +17,6 @@ class pkgAcquire; class pkgIndexFile; -class debDebPkgFileIndex; class IndexTarget; class pkgCacheGenerator; class OpProgress; @@ -66,42 +65,4 @@ class APT_HIDDEN debReleaseIndex : public metaIndex std::vector Languages); }; -class APT_HIDDEN debDebFileMetaIndex : public metaIndex -{ -private: - void * const d; - std::string DebFile; - debDebPkgFileIndex *DebIndex; -public: - virtual std::string ArchiveURI(std::string const& /*File*/) const APT_OVERRIDE { - return DebFile; - } - virtual bool GetIndexes(pkgAcquire* /*Owner*/, const bool& /*GetAll=false*/) APT_OVERRIDE { - return true; - } - virtual std::vector GetIndexTargets() const APT_OVERRIDE { - return std::vector(); - } - virtual std::vector *GetIndexFiles() APT_OVERRIDE { - return Indexes; - } - virtual bool IsTrusted() const APT_OVERRIDE { - return true; - } - virtual bool Load(std::string const &, std::string * const ErrorText) APT_OVERRIDE - { - LoadedSuccessfully = TRI_NO; - if (ErrorText != NULL) - strprintf(*ErrorText, "Unparseable metaindex as it represents the standalone deb file %s", DebFile.c_str()); - return false; - } - virtual metaIndex * UnloadedClone() const APT_OVERRIDE - { - return NULL; - } - debDebFileMetaIndex(std::string const &DebFile); - virtual ~debDebFileMetaIndex(); - -}; - #endif diff --git a/apt-pkg/edsp/edspsystem.cc b/apt-pkg/edsp/edspsystem.cc index 4fb34b896..f65fcc0d2 100644 --- a/apt-pkg/edsp/edspsystem.cc +++ b/apt-pkg/edsp/edspsystem.cc @@ -91,8 +91,7 @@ signed edspSystem::Score(Configuration const &Cnf) return -1000; } /*}}}*/ -// System::AddStatusFiles - Register the status files /*{{{*/ -bool edspSystem::AddStatusFiles(std::vector &List) +bool edspSystem::AddStatusFiles(std::vector &List) /*{{{*/ { if (StatusFile == 0) { diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 26a5e60a6..9529f42dc 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include @@ -1332,10 +1334,10 @@ map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, cons /* This just verifies that each file in the list of index files exists, has matching attributes with the cache and the cache does not have any extra files. */ -static bool CheckValidity(const string &CacheFile, +static bool CheckValidity(const string &CacheFile, pkgSourceList &List, - FileIterator Start, - FileIterator End, + FileIterator const Start, + FileIterator const End, MMap **OutMap = 0) { bool const Debug = _config->FindB("Debug::pkgCacheGen", false); @@ -1343,7 +1345,7 @@ static bool CheckValidity(const string &CacheFile, if (CacheFile.empty() == true || FileExists(CacheFile) == false) { if (Debug == true) - std::clog << "CacheFile doesn't exist" << std::endl; + std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl; return false; } @@ -1361,7 +1363,7 @@ static bool CheckValidity(const string &CacheFile, if (_error->PendingError() == true || Map->Size() == 0) { if (Debug == true) - std::clog << "Errors are pending or Map is empty()" << std::endl; + std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl; _error->Discard(); return false; } @@ -1385,10 +1387,9 @@ static bool CheckValidity(const string &CacheFile, if (Debug == true) std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl; - std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j) - if ((*j)->HasPackages()) - Files.push_back (*j); + std::vector const * const Indexes = (*i)->GetIndexFiles(); + std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files), + [](pkgIndexFile const * const I) { return I->HasPackages(); }); } for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I) if (RlsVisited[I] == false) @@ -1398,8 +1399,7 @@ static bool CheckValidity(const string &CacheFile, return false; } - for (; Start != End; ++Start) - Files.push_back(*Start); + std::copy(Start, End, std::back_inserter(Files)); /* Now we check every index file, see if it is in the cache, verify the IMS data and check that it is on the disk too.. */ @@ -1482,16 +1482,41 @@ static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator } /*}}}*/ // BuildCache - Merge the list of index files into the cache /*{{{*/ -// --------------------------------------------------------------------- -/* */ static bool BuildCache(pkgCacheGenerator &Gen, - OpProgress *Progress, + OpProgress * const Progress, map_filesize_t &CurrentSize,map_filesize_t TotalSize, pkgSourceList const * const List, - FileIterator Start, FileIterator End) + FileIterator const Start, FileIterator const End) { std::vector Files; bool const HasFileDeps = Gen.HasFileDeps(); + bool mergeFailure = false; + + auto const indexFileMerge = [&](pkgIndexFile * const I) { + if (HasFileDeps) + Files.push_back(I); + + if (I->HasPackages() == false || mergeFailure) + return; + + if (I->Exists() == false) + return; + + if (I->FindInCache(Gen.GetCache()).end() == false) + { + _error->Warning("Duplicate sources.list entry %s", + I->Describe().c_str()); + return; + } + + map_filesize_t const Size = I->Size(); + if (Progress != NULL) + Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists")); + CurrentSize += Size; + + if (I->Merge(Gen,Progress) == false) + mergeFailure = true; + }; if (List != NULL) { @@ -1508,63 +1533,20 @@ static bool BuildCache(pkgCacheGenerator &Gen, return false; std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I) - { - if (HasFileDeps) - Files.push_back(*I); - - if ((*I)->HasPackages() == false) - continue; - - if ((*I)->Exists() == false) - continue; - - if ((*I)->FindInCache(Gen.GetCache()).end() == false) - { - _error->Warning("Duplicate sources.list entry %s", - (*I)->Describe().c_str()); - continue; - } - - map_filesize_t Size = (*I)->Size(); - if (Progress != NULL) - Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); - CurrentSize += Size; - - if ((*I)->Merge(Gen,Progress) == false) - return false; - } + if (Indexes != NULL) + std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge); + if (mergeFailure) + return false; } } - Gen.SelectReleaseFile("", ""); - FileIterator I; - for (I = Start; I != End; ++I) + if (Start != End) { - if (HasFileDeps) - Files.push_back(*I); - - if ((*I)->HasPackages() == false) - continue; - - if ((*I)->Exists() == false) - continue; - - if ((*I)->FindInCache(Gen.GetCache()).end() == false) - { - _error->Warning("Duplicate sources.list entry %s", - (*I)->Describe().c_str()); - continue; - } - - map_filesize_t Size = (*I)->Size(); - if (Progress != NULL) - Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); - CurrentSize += Size; - - if ((*I)->Merge(Gen,Progress) == false) + Gen.SelectReleaseFile("", ""); + std::for_each(Start, End, indexFileMerge); + if (mergeFailure) return false; - } + } if (HasFileDeps == true) { @@ -1582,12 +1564,20 @@ static bool BuildCache(pkgCacheGenerator &Gen, return false; } } - + return true; } /*}}}*/ -// CacheGenerator::CreateDynamicMMap - load an mmap with configuration options /*{{{*/ -DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) { +// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ +// --------------------------------------------------------------------- +/* This makes sure that the status cache (the cache that has all + index files from the sources list and all local ones) is ready + to be mmaped. If OutMap is not zero then a MMap object representing + the cache will be stored there. This is pretty much mandetory if you + are using AllowMem. AllowMem lets the function be run as non-root + where it builds the cache 'fast' into a memory buffer. */ +static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags) +{ map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024); map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024); map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0); @@ -1599,15 +1589,42 @@ DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long else return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit); } - /*}}}*/ -// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ -// --------------------------------------------------------------------- -/* This makes sure that the status cache (the cache that has all - index files from the sources list and all local ones) is ready - to be mmaped. If OutMap is not zero then a MMap object representing - the cache will be stored there. This is pretty much mandetory if you - are using AllowMem. AllowMem lets the function be run as non-root - where it builds the cache 'fast' into a memory buffer. */ +static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map, + std::string const &FileName) +{ + FileFd SCacheF(FileName, FileFd::WriteAtomic); + if (_error->PendingError() == true) + return false; + + fchmod(SCacheF.Fd(),0644); + + // Write out the main data + if (SCacheF.Write(Map->Data(),Map->Size()) == false) + return _error->Error(_("IO Error saving source cache")); + SCacheF.Sync(); + + // Write out the proper header + Gen->GetCache().HeaderP->Dirty = false; + if (SCacheF.Seek(0) == false || + SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false) + return _error->Error(_("IO Error saving source cache")); + Gen->GetCache().HeaderP->Dirty = true; + SCacheF.Sync(); + return true; +} +static bool loadBackMMapFromFile(std::unique_ptr &Gen, + SPtr &Map, OpProgress * const Progress, std::string const &FileName) +{ + Map = CreateDynamicMMap(NULL, 0); + FileFd CacheF(FileName, FileFd::ReadOnly); + map_pointer_t const alloc = Map->RawAllocate(CacheF.Size()); + if ((alloc == 0 && _error->PendingError()) + || CacheF.Read((unsigned char *)Map->Data() + alloc, + CacheF.Size()) == false) + return false; + Gen.reset(new pkgCacheGenerator(Map.Get(),Progress)); + return true; +} APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, MMap **OutMap, bool AllowMem) { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); } @@ -1617,18 +1634,6 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress bool const Debug = _config->FindB("Debug::pkgCacheGen", false); std::vector Files; - /* - for (std::vector::const_iterator i = List.begin(); - i != List.end(); - ++i) - { - std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator j = Indexes->begin(); - j != Indexes->end(); - ++j) - Files.push_back (*j); - } -*/ if (_system->AddStatusFiles(Files) == false) return false; @@ -1649,160 +1654,130 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress CreateDirectory(dir, flNotFile(SrcCacheFile)); } - // Decide if we can write to the cache - bool Writeable = false; - if (CacheFile.empty() == false) - Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0; - else - if (SrcCacheFile.empty() == false) - Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0; - if (Debug == true) - std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl; - - if (Writeable == false && AllowMem == false && CacheFile.empty() == false) - return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str()); - if (Progress != NULL) Progress->OverallProgress(0,1,1,_("Reading package lists")); - // Cache is OK, Fin. - if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true) + bool pkgcache_fine = false; + bool srcpkgcache_fine = false; + bool volatile_fine = List.GetVolatileFiles().empty(); + + if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true) { - if (Progress != NULL) - Progress->OverallProgress(1,1,1,_("Reading package lists")); if (Debug == true) - std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl; - return true; + std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl; + pkgcache_fine = true; + srcpkgcache_fine = true; } - else if (Debug == true) - std::clog << "pkgcache.bin is NOT valid" << std::endl; - - /* At this point we know we need to reconstruct the package cache, - begin. */ - SPtr CacheF; - SPtr Map; - if (Writeable == true && CacheFile.empty() == false) + if (pkgcache_fine == false) { - _error->PushToStack(); - unlink(CacheFile.c_str()); - CacheF = new FileFd(CacheFile,FileFd::WriteAtomic); - fchmod(CacheF->Fd(),0644); - Map = CreateDynamicMMap(CacheF, MMap::Public); - if (_error->PendingError() == true) + if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true) { - delete CacheF.UnGuard(); - delete Map.UnGuard(); if (Debug == true) - std::clog << "Open filebased MMap FAILED" << std::endl; - Writeable = false; - if (AllowMem == false) - { - _error->MergeWithStack(); - return false; - } - _error->RevertToStack(); - } - else - { - _error->MergeWithStack(); - if (Debug == true) - std::clog << "Open filebased MMap" << std::endl; + std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl; + srcpkgcache_fine = true; } } - if (Writeable == false || CacheFile.empty() == true) + + if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true) { - // Just build it in memory.. - Map = CreateDynamicMMap(NULL); - if (Debug == true) - std::clog << "Open memory Map (not filebased)" << std::endl; + if (Progress != NULL) + Progress->OverallProgress(1,1,1,_("Reading package lists")); + return true; } - - // Lets try the source cache. - map_filesize_t CurrentSize = 0; - map_filesize_t TotalSize = 0; - if (CheckValidity(SrcCacheFile, List, Files.end(), - Files.end()) == true) + + bool Writeable = false; + if (srcpkgcache_fine == false || pkgcache_fine == false) { + if (CacheFile.empty() == false) + Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0; + else if (SrcCacheFile.empty() == false) + Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0; + if (Debug == true) - std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl; - // Preload the map with the source cache - FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly); - map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size()); - if ((alloc == 0 && _error->PendingError()) - || SCacheF.Read((unsigned char *)Map->Data() + alloc, - SCacheF.Size()) == false) - return false; + std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl; + + if (Writeable == false && AllowMem == false) + { + if (CacheFile.empty() == false) + return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str()); + else if (SrcCacheFile.empty() == false) + return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str()); + else + return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!"); + } + } - TotalSize = ComputeSize(NULL, Files.begin(), Files.end()); + // At this point we know we need to construct something, so get storage ready + SPtr Map = CreateDynamicMMap(NULL, 0); + if (Debug == true) + std::clog << "Open memory Map (not filebased)" << std::endl; - // Build the status cache - pkgCacheGenerator Gen(Map.Get(),Progress); - if (_error->PendingError() == true) - return false; - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL, - Files.begin(),Files.end()) == false) + std::unique_ptr Gen{nullptr}; + map_filesize_t CurrentSize = 0; + std::vector VolatileFiles = List.GetVolatileFiles(); + map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end()); + if (srcpkgcache_fine == true && pkgcache_fine == false) + { + if (Debug == true) + std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl; + if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false) return false; + srcpkgcache_fine = true; + TotalSize += ComputeSize(NULL, Files.begin(), Files.end()); } - else + else if (srcpkgcache_fine == false) { if (Debug == true) std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl; - TotalSize = ComputeSize(&List, Files.begin(),Files.end()); - - // Build the source cache - pkgCacheGenerator Gen(Map.Get(),Progress); - if (_error->PendingError() == true) - return false; - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List, - Files.end(),Files.end()) == false) + Gen.reset(new pkgCacheGenerator(Map.Get(),Progress)); + + TotalSize += ComputeSize(&List, Files.begin(),Files.end()); + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List, + Files.end(),Files.end()) == false) return false; - - // Write it back + if (Writeable == true && SrcCacheFile.empty() == false) - { - FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic); - if (_error->PendingError() == true) + if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false) return false; - - fchmod(SCacheF.Fd(),0644); - - // Write out the main data - if (SCacheF.Write(Map->Data(),Map->Size()) == false) - return _error->Error(_("IO Error saving source cache")); - SCacheF.Sync(); - - // Write out the proper header - Gen.GetCache().HeaderP->Dirty = false; - if (SCacheF.Seek(0) == false || - SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false) - return _error->Error(_("IO Error saving source cache")); - Gen.GetCache().HeaderP->Dirty = true; - SCacheF.Sync(); - } - - // Build the status cache - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL, - Files.begin(), Files.end()) == false) + } + + if (pkgcache_fine == false) + { + if (Debug == true) + std::clog << "Building status cache in pkgcache.bin now" << std::endl; + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) return false; + + if (Writeable == true && CacheFile.empty() == false) + if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false) + return false; } + if (Debug == true) - std::clog << "Caches are ready for shipping" << std::endl; + std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl; - if (_error->PendingError() == true) - return false; - if (OutMap != 0) + if (volatile_fine == false) { - if (CacheF != 0) + if (Gen == nullptr) { - delete Map.UnGuard(); - *OutMap = new MMap(*CacheF,0); + if (Debug == true) + std::clog << "Populate new MMap with cachefile contents" << std::endl; + if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false) + return false; } - else - { - *OutMap = Map.UnGuard(); - } + + Files = List.GetVolatileFiles(); + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) + return false; } - + + if (OutMap != nullptr) + *OutMap = Map.UnGuard(); + + if (Debug == true) + std::clog << "Everything is ready for shipping" << std::endl; return true; } /*}}}*/ @@ -1817,7 +1792,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O if (_system->AddStatusFiles(Files) == false) return false; - SPtr Map = CreateDynamicMMap(NULL); + SPtr Map = CreateDynamicMMap(NULL, 0); map_filesize_t CurrentSize = 0; map_filesize_t TotalSize = 0; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 0d0fb893f..4b6b91992 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -116,7 +116,6 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ APT_PUBLIC static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress, MMap **OutMap = 0,bool AllowMem = false); APT_PUBLIC static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); - APT_PUBLIC static DynamicMMap* CreateDynamicMMap(FileFd *CacheF, unsigned long Flags = 0); void ReMap(void const * const oldMap, void const * const newMap); diff --git a/apt-pkg/pkgsystem.h b/apt-pkg/pkgsystem.h index 5be93d059..5b31457e0 100644 --- a/apt-pkg/pkgsystem.h +++ b/apt-pkg/pkgsystem.h @@ -52,7 +52,7 @@ class Configuration; class pkgIndexFile; class pkgSystem -{ +{ public: // Global list of supported systems @@ -81,7 +81,8 @@ class pkgSystem virtual bool ArchiveSupported(const char *Type) = 0; // Return a list of system index files.. - virtual bool AddStatusFiles(std::vector &List) = 0; + virtual bool AddStatusFiles(std::vector &List) = 0; + virtual bool FindIndex(pkgCache::PkgFileIterator File, pkgIndexFile *&Found) const = 0; diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index eef0ee709..46e51f592 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -274,6 +274,10 @@ pkgSourceList::~pkgSourceList() { for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I) delete *I; + SrcList.clear(); + for (pkgIndexFile * const File : VolatileFiles) + delete File; + VolatileFiles.clear(); } /*}}}*/ // SourceList::ReadMainList - Read the main source list from etc /*{{{*/ @@ -339,7 +343,7 @@ bool pkgSourceList::ReadAppend(string const &File) else return ParseFileOldStyle(File); } - + /*}}}*/ // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -454,7 +458,15 @@ bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File, } } } - + for (vector::const_iterator J = VolatileFiles.begin(); + J != VolatileFiles.end(); ++J) + { + if ((*J)->FindInCache(*File.Cache()) == File) + { + Found = (*J); + return true; + } + } return false; } /*}}}*/ @@ -511,4 +523,14 @@ time_t pkgSourceList::GetLastModifiedTime() return mtime_sources; } /*}}}*/ - +std::vector pkgSourceList::GetVolatileFiles() const /*{{{*/ +{ + return VolatileFiles; +} + /*}}}*/ +void pkgSourceList::AddVolatileFile(pkgIndexFile * const File) /*{{{*/ +{ + if (File != NULL) + VolatileFiles.push_back(File); +} + /*}}}*/ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index d80131438..47a562d18 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -50,6 +50,7 @@ class metaIndex; class pkgSourceList { void * const d; + std::vector VolatileFiles; public: // List of supported source list types @@ -113,6 +114,24 @@ class pkgSourceList // query last-modified time time_t GetLastModifiedTime(); + /** \brief add file for parsing, but not to the cache + * + * pkgIndexFiles origining from pkgSourcesList are included in + * srcpkgcache, the status files added via #AddStatusFiles are + * included in pkgcache, but these files here are not included in + * any cache to have the possibility of having a file included just + * for a single run like a local .deb/.dsc file. + * + * The volatile files do not count as "normal" sourceslist entries, + * can't be iterated over with #begin and #end and can't be + * downloaded, but they can be found via #FindIndex. + * + * @param File is an index file; pointer-ownership is transferred + */ + void AddVolatileFile(pkgIndexFile * const File); + /** @return list of files registered with #AddVolatileFile */ + std::vector GetVolatileFiles() const; + pkgSourceList(); virtual ~pkgSourceList(); }; diff --git a/apt-private/private-cachefile.h b/apt-private/private-cachefile.h index ed9342db0..51703b0ad 100644 --- a/apt-private/private-cachefile.h +++ b/apt-private/private-cachefile.h @@ -11,17 +11,6 @@ #include -// FIXME: we need to find a way to export this -class APT_PUBLIC SourceList : public pkgSourceList -{ - public: - // Add custom metaIndex (e.g. local files) - void AddMetaIndex(metaIndex *mi) { - SrcList.push_back(mi); - } - -}; - // class CacheFile - Cover class for some dependency cache functions /*{{{*/ class APT_PUBLIC CacheFile : public pkgCacheFile { @@ -36,16 +25,6 @@ class APT_PUBLIC CacheFile : public pkgCacheFile return false; return true; } - // FIXME: this can go once the "libapt-pkg" pkgSourceList has a way - // to add custom metaIndexes (or custom local files or so) - bool BuildSourceList(OpProgress */*Progress*/ = NULL) { - if (SrcList != NULL) - return true; - SrcList = new SourceList(); - if (SrcList->ReadMainList() == false) - return _error->Error(_("The list of sources could not be read.")); - return true; - } bool Open(bool WithLock = true) { OpTextProgress Prog(*_config); diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index 116e01038..074874903 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -647,24 +648,8 @@ bool DoInstall(CommandLine &CmdL) // first check for local pkgs and add them to the cache for (const char **I = CmdL.FileList; *I != 0; I++) { - if(FileExists(*I)) - { - // FIXME: make this more elegant - std::string TypeStr = flExtension(*I) + "-file"; - pkgSourceList::Type *Type = pkgSourceList::Type::GetType(TypeStr.c_str()); - if(Type != 0) - { - std::vector List; - std::map Options; - if(Type->CreateItem(List, *I, "", "", Options)) - { - // we have our own CacheFile that gives us a SourceList - // with superpowerz - SourceList *sources = (SourceList*)Cache.GetSourceList(); - sources->AddMetaIndex(List[0]); - } - } - } + if(FileExists(*I) && flExtension(*I) == "deb") + Cache.GetSourceList()->AddVolatileFile(new debDebPkgFileIndex(*I)); } // then open the cache diff --git a/test/integration/framework b/test/integration/framework index 6ae5003f7..f3cc1eff9 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -99,10 +99,14 @@ msgdone() { } getaptconfig() { if [ -f ./aptconfig.conf ]; then - echo "./aptconfig.conf" + echo "$(readlink -f ./aptconfig.conf)" elif [ -f ../aptconfig.conf ]; then - echo "../aptconfig.conf" - fi + echo "$(readlink -f ../aptconfig.conf)" + elif [ -f ../../aptconfig.conf ]; then + echo "$(readlink -f ../../aptconfig.conf)" + elif [ -f "${TMPWORKINGDIRECTORY}/aptconfig.conf" ]; then + echo "$(readlink -f "${TMPWORKINGDIRECTORY}/aptconfig.conf")" + fi } runapt() { msgdebug "Executing: ${CCMD}$*${CDEBUG} " @@ -141,6 +145,8 @@ gdb() { case "$1" in aptget) CMD="apt-get";; aptcache) CMD="apt-cache";; + aptcdrom) CMD="apt-cdrom";; + aptconfig) CMD="apt-config";; aptmark) CMD="apt-mark";; apthelper) CMD="apt-helper";; aptftparchive) CMD="apt-ftparchive";; diff --git a/test/integration/test-apt-get-install-deb b/test/integration/test-apt-get-install-deb index 0f34692fe..991185dea 100755 --- a/test/integration/test-apt-get-install-deb +++ b/test/integration/test-apt-get-install-deb @@ -5,15 +5,21 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'amd64' 'i386' # regression test for #754904 testfailureequal 'E: Unable to locate package /dev/null' aptget install -qq /dev/null -# and ensure we fail for invalid debs -cat > foo.deb < foo.rpm < Date: Mon, 20 Jul 2015 10:17:29 +0200 Subject: elimate duplicated code in pkgIndexFile subclasses Trade deduplication of code for a bunch of new virtuals, so it is actually visible how the different indexes behave cleaning up the interface at large in the process. Git-Dch: Ignore --- apt-pkg/contrib/fileutl.cc | 15 +- apt-pkg/contrib/fileutl.h | 5 +- apt-pkg/deb/debindexfile.cc | 416 ++++++++---------------------- apt-pkg/deb/debindexfile.h | 84 +++--- apt-pkg/deb/deblistparser.cc | 7 +- apt-pkg/deb/deblistparser.h | 2 +- apt-pkg/edsp/edspindexfile.cc | 75 +++--- apt-pkg/edsp/edspindexfile.h | 16 +- apt-pkg/edsp/edspsystem.cc | 8 +- apt-pkg/indexfile.cc | 175 ++++++++++++- apt-pkg/indexfile.h | 79 ++++-- apt-pkg/pkgcachegen.cc | 18 +- apt-pkg/pkgcachegen.h | 17 +- cmdline/apt-internal-solver.cc | 1 + test/integration/framework | 2 +- test/integration/test-apt-get-install-deb | 12 +- 16 files changed, 460 insertions(+), 472 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index f40526b5c..3b2e06431 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -2115,28 +2115,27 @@ std::string GetTempDir() /*{{{*/ return string(tmpdir); } /*}}}*/ -FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink) /*{{{*/ +FileFd* GetTempFile(std::string const &Prefix, bool ImmediateUnlink, FileFd * const TmpFd) /*{{{*/ { char fn[512]; - FileFd *Fd = new FileFd(); + FileFd * const Fd = TmpFd == NULL ? new FileFd() : TmpFd; - std::string tempdir = GetTempDir(); - snprintf(fn, sizeof(fn), "%s/%s.XXXXXX", + std::string const tempdir = GetTempDir(); + snprintf(fn, sizeof(fn), "%s/%s.XXXXXX", tempdir.c_str(), Prefix.c_str()); - int fd = mkstemp(fn); + int const fd = mkstemp(fn); if(ImmediateUnlink) unlink(fn); - if (fd < 0) + if (fd < 0) { _error->Errno("GetTempFile",_("Unable to mkstemp %s"), fn); return NULL; } - if (!Fd->OpenDescriptor(fd, FileFd::WriteOnly, FileFd::None, true)) + if (!Fd->OpenDescriptor(fd, FileFd::ReadWrite, FileFd::None, true)) { _error->Errno("GetTempFile",_("Unable to write to %s"),fn); return NULL; } - return Fd; } /*}}}*/ diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 3a99e3e61..acfd560ab 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -159,8 +159,9 @@ time_t GetModificationTime(std::string const &Path); bool Rename(std::string From, std::string To); std::string GetTempDir(); -FileFd* GetTempFile(std::string const &Prefix = "", - bool ImmediateUnlink = true); +FileFd* GetTempFile(std::string const &Prefix = "", + bool ImmediateUnlink = true, + FileFd * const TmpFd = NULL); /** \brief Ensure the existence of the given Path * diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index e67233e5f..627cd36c5 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -36,27 +36,20 @@ #include #include #include +#include #include /*}}}*/ -using std::string; - -// SourcesIndex::debSourcesIndex - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// Sources Index /*{{{*/ debSourcesIndex::debSourcesIndex(IndexTarget const &Target,bool const Trusted) : - pkgIndexTargetFile(Target, Trusted), d(NULL) + pkgDebianIndexTargetFile(Target, Trusted), d(NULL) { } - /*}}}*/ -// SourcesIndex::SourceInfo - Short 1 liner describing a source /*{{{*/ -// --------------------------------------------------------------------- -/* The result looks like: - http://foo/debian/ stable/main src 1.1.1 (dsc) */ -string debSourcesIndex::SourceInfo(pkgSrcRecords::Parser const &Record, +std::string debSourcesIndex::SourceInfo(pkgSrcRecords::Parser const &Record, pkgSrcRecords::File const &File) const { - string Res = Target.Description; + // The result looks like: http://foo/debian/ stable/main src 1.1.1 (dsc) + std::string Res = Target.Description; Res.erase(Target.Description.rfind(' ')); Res += " "; @@ -67,294 +60,111 @@ string debSourcesIndex::SourceInfo(pkgSrcRecords::Parser const &Record, Res += " (" + File.Type + ")"; return Res; } - /*}}}*/ -// SourcesIndex::CreateSrcParser - Get a parser for the source files /*{{{*/ -// --------------------------------------------------------------------- -/* */ pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const { - string const SourcesURI = IndexFileName(); + std::string const SourcesURI = IndexFileName(); if (FileExists(SourcesURI)) return new debSrcRecordParser(SourcesURI, this); return NULL; +} +bool debSourcesIndex::OpenListFile(FileFd &, std::string const &) +{ + return true; +} +pkgCacheListParser * debSourcesIndex::CreateListParser(FileFd &) +{ + return NULL; +} +uint8_t debSourcesIndex::GetIndexFlags() const +{ + return 0; } /*}}}*/ - -// PackagesIndex::debPackagesIndex - Contructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ +// Packages Index /*{{{*/ debPackagesIndex::debPackagesIndex(IndexTarget const &Target, bool const Trusted) : - pkgIndexTargetFile(Target, Trusted), d(NULL) + pkgDebianIndexTargetFile(Target, Trusted), d(NULL) { } - /*}}}*/ -// PackagesIndex::ArchiveInfo - Short version of the archive url /*{{{*/ -// --------------------------------------------------------------------- -/* This is a shorter version that is designed to be < 60 chars or so */ -string debPackagesIndex::ArchiveInfo(pkgCache::VerIterator Ver) const +std::string debPackagesIndex::ArchiveInfo(pkgCache::VerIterator const &Ver) const { - std::string const Dist = Target.Option(IndexTarget::RELEASE); - string Res = Target.Option(IndexTarget::SITE) + " " + Dist; - std::string const Component = Target.Option(IndexTarget::COMPONENT); - if (Component.empty() == false) - Res += "/" + Component; + std::string Res = Target.Description; + Res.erase(Target.Description.rfind(' ')); Res += " "; Res += Ver.ParentPkg().Name(); Res += " "; + std::string const Dist = Target.Option(IndexTarget::RELEASE); if (Dist.empty() == false && Dist[Dist.size() - 1] != '/') Res.append(Ver.Arch()).append(" "); Res += Ver.VerStr(); return Res; } - /*}}}*/ -// PackagesIndex::Merge - Load the index file into a cache /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const -{ - string const PackageFile = IndexFileName(); - FileFd Pkg(PackageFile,FileFd::ReadOnly, FileFd::Extension); - debListParser Parser(&Pkg, Target.Option(IndexTarget::ARCHITECTURE)); - - if (_error->PendingError() == true) - return _error->Error("Problem opening %s",PackageFile.c_str()); - if (Prog != NULL) - Prog->SubProgress(0, Target.Description); - - - std::string const URI = Target.Option(IndexTarget::REPO_URI); - std::string Dist = Target.Option(IndexTarget::RELEASE); - if (Dist.empty()) - Dist = "/"; - ::URI Tmp(URI); - if (Gen.SelectFile(PackageFile, *this, Target.Option(IndexTarget::ARCHITECTURE), Target.Option(IndexTarget::COMPONENT)) == false) - return _error->Error("Problem with SelectFile %s",PackageFile.c_str()); - - // Store the IMS information - pkgCache::PkgFileIterator File = Gen.GetCurFile(); - pkgCacheGenerator::Dynamic DynFile(File); - File->Size = Pkg.FileSize(); - File->mtime = Pkg.ModificationTime(); - - if (Gen.MergeList(Parser) == false) - return _error->Error("Problem with MergeList %s",PackageFile.c_str()); - - return true; -} - /*}}}*/ -// PackagesIndex::FindInCache - Find this index /*{{{*/ -// --------------------------------------------------------------------- -/* */ -pkgCache::PkgFileIterator debPackagesIndex::FindInCache(pkgCache &Cache) const +uint8_t debPackagesIndex::GetIndexFlags() const { - string const FileName = IndexFileName(); - pkgCache::PkgFileIterator File = Cache.FileBegin(); - for (; File.end() == false; ++File) - { - if (File.FileName() == NULL || FileName != File.FileName()) - continue; - - struct stat St; - if (stat(File.FileName(),&St) != 0) - { - if (_config->FindB("Debug::pkgCacheGen", false)) - std::clog << "PackagesIndex::FindInCache - stat failed on " << File.FileName() << std::endl; - return pkgCache::PkgFileIterator(Cache); - } - if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime) - { - if (_config->FindB("Debug::pkgCacheGen", false)) - std::clog << "PackagesIndex::FindInCache - size (" << St.st_size << " <> " << File->Size - << ") or mtime (" << St.st_mtime << " <> " << File->mtime - << ") doesn't match for " << File.FileName() << std::endl; - return pkgCache::PkgFileIterator(Cache); - } - return File; - } - - return File; + return 0; } /*}}}*/ - -// TranslationsIndex::debTranslationsIndex - Contructor /*{{{*/ +// Translation-* Index /*{{{*/ debTranslationsIndex::debTranslationsIndex(IndexTarget const &Target) : - pkgIndexTargetFile(Target, true), d(NULL) + pkgDebianIndexTargetFile(Target, true), d(NULL) {} - /*}}}*/ -bool debTranslationsIndex::HasPackages() const /*{{{*/ +bool debTranslationsIndex::HasPackages() const { return Exists(); } - /*}}}*/ -// TranslationsIndex::Merge - Load the index file into a cache /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const -{ - // Check the translation file, if in use - string const TranslationFile = IndexFileName(); - if (FileExists(TranslationFile)) - { - FileFd Trans(TranslationFile,FileFd::ReadOnly, FileFd::Extension); - debTranslationsParser TransParser(&Trans); - if (_error->PendingError() == true) - return false; - - if (Prog != NULL) - Prog->SubProgress(0, Target.Description); - if (Gen.SelectFile(TranslationFile, *this, "", Target.Option(IndexTarget::COMPONENT), pkgCache::Flag::NotSource | pkgCache::Flag::NoPackages) == false) - return _error->Error("Problem with SelectFile %s",TranslationFile.c_str()); - - // Store the IMS information - pkgCache::PkgFileIterator TransFile = Gen.GetCurFile(); - TransFile->Size = Trans.FileSize(); - TransFile->mtime = Trans.ModificationTime(); - - if (Gen.MergeList(TransParser) == false) - return _error->Error("Problem with MergeList %s",TranslationFile.c_str()); - } - +bool debTranslationsIndex::OpenListFile(FileFd &Pkg, std::string const &FileName) +{ + if (FileExists(FileName)) + return pkgDebianIndexTargetFile::OpenListFile(Pkg, FileName); return true; } - /*}}}*/ -// TranslationsIndex::FindInCache - Find this index /*{{{*/ -// --------------------------------------------------------------------- -/* */ -pkgCache::PkgFileIterator debTranslationsIndex::FindInCache(pkgCache &Cache) const +uint8_t debTranslationsIndex::GetIndexFlags() const { - string FileName = IndexFileName(); - - pkgCache::PkgFileIterator File = Cache.FileBegin(); - for (; File.end() == false; ++File) - { - if (FileName != File.FileName()) - continue; - - struct stat St; - if (stat(File.FileName(),&St) != 0) - { - if (_config->FindB("Debug::pkgCacheGen", false)) - std::clog << "TranslationIndex::FindInCache - stat failed on " << File.FileName() << std::endl; - return pkgCache::PkgFileIterator(Cache); - } - if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime) - { - if (_config->FindB("Debug::pkgCacheGen", false)) - std::clog << "TranslationIndex::FindInCache - size (" << St.st_size << " <> " << File->Size - << ") or mtime (" << St.st_mtime << " <> " << File->mtime - << ") doesn't match for " << File.FileName() << std::endl; - return pkgCache::PkgFileIterator(Cache); - } - return File; - } - return File; + return pkgCache::Flag::NotSource | pkgCache::Flag::NoPackages; } - /*}}}*/ - -// StatusIndex::debStatusIndex - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ -debStatusIndex::debStatusIndex(string File) : pkgIndexFile(true), d(NULL), File(File) +std::string debTranslationsIndex::GetArchitecture() const { + return std::string(); } - /*}}}*/ -// StatusIndex::Size - Return the size of the index /*{{{*/ -// --------------------------------------------------------------------- -/* */ -unsigned long debStatusIndex::Size() const +pkgCacheListParser * debTranslationsIndex::CreateListParser(FileFd &Pkg) { - struct stat S; - if (stat(File.c_str(),&S) != 0) - return 0; - return S.st_size; -} - /*}}}*/ -// StatusIndex::Merge - Load the index file into a cache /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const -{ - FileFd Pkg(File,FileFd::ReadOnly, FileFd::Extension); - if (_error->PendingError() == true) - return false; - debListParser Parser(&Pkg); - if (_error->PendingError() == true) - return false; - - if (Prog != NULL) - Prog->SubProgress(0,File); - if (Gen.SelectFile(File, *this, "", "now", pkgCache::Flag::NotSource) == false) - return _error->Error("Problem with SelectFile %s",File.c_str()); - - // Store the IMS information - pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); - pkgCacheGenerator::Dynamic DynFile(CFile); - CFile->Size = Pkg.FileSize(); - CFile->mtime = Pkg.ModificationTime(); - - if (Gen.MergeList(Parser) == false) - return _error->Error("Problem with MergeList %s",File.c_str()); - return true; + if (Pkg.IsOpen() == false) + return NULL; + _error->PushToStack(); + pkgCacheListParser * const Parser = new debTranslationsParser(&Pkg); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? NULL : Parser; } /*}}}*/ -// StatusIndex::FindInCache - Find this index /*{{{*/ -// --------------------------------------------------------------------- -/* */ -pkgCache::PkgFileIterator debStatusIndex::FindInCache(pkgCache &Cache) const +// dpkg/status Index /*{{{*/ +debStatusIndex::debStatusIndex(std::string const &File) : pkgDebianIndexRealFile(File, true), d(NULL) { - pkgCache::PkgFileIterator File = Cache.FileBegin(); - for (; File.end() == false; ++File) - { - if (this->File != File.FileName()) - continue; - - struct stat St; - if (stat(File.FileName(),&St) != 0) - { - if (_config->FindB("Debug::pkgCacheGen", false)) - std::clog << "StatusIndex::FindInCache - stat failed on " << File.FileName() << std::endl; - return pkgCache::PkgFileIterator(Cache); - } - if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime) - { - if (_config->FindB("Debug::pkgCacheGen", false)) - std::clog << "StatusIndex::FindInCache - size (" << St.st_size << " <> " << File->Size - << ") or mtime (" << St.st_mtime << " <> " << File->mtime - << ") doesn't match for " << File.FileName() << std::endl; - return pkgCache::PkgFileIterator(Cache); - } - return File; - } - return File; } - /*}}}*/ -// StatusIndex::Exists - Check if the index is available /*{{{*/ -// --------------------------------------------------------------------- -/* */ -APT_CONST bool debStatusIndex::Exists() const +std::string debStatusIndex::GetArchitecture() const { - // Abort if the file does not exist. - return true; + return std::string(); } - /*}}}*/ - -// debDebPkgFileIndex - Single .deb file /*{{{*/ -debDebPkgFileIndex::debDebPkgFileIndex(std::string const &DebFile) - : pkgIndexFile(true), d(NULL), DebFile(DebFile) +std::string debStatusIndex::GetComponent() const { - DebFileFullPath = flAbsPath(DebFile); + return "now"; } -std::string debDebPkgFileIndex::ArchiveURI(std::string /*File*/) const +uint8_t debStatusIndex::GetIndexFlags() const { - return "file:" + DebFileFullPath; + return pkgCache::Flag::NotSource; } -bool debDebPkgFileIndex::Exists() const + /*}}}*/ +// DebPkgFile Index - a single .deb file as an index /*{{{*/ +debDebPkgFileIndex::debDebPkgFileIndex(std::string const &DebFile) + : pkgDebianIndexRealFile(DebFile, true), d(NULL), DebFile(DebFile) { - return FileExists(DebFile); } bool debDebPkgFileIndex::GetContent(std::ostream &content, std::string const &debfile) { + struct stat Buf; + if (stat(debfile.c_str(), &Buf) != 0) + return false; + // get the control data out of the deb file via dpkg-deb -I std::string dpkg = _config->Find("Dir::Bin::dpkg","dpkg-deb"); std::vector Args; @@ -381,91 +191,73 @@ bool debDebPkgFileIndex::GetContent(std::ostream &content, std::string const &de ExecWait(Child, "Popen"); content << "Filename: " << debfile << "\n"; - struct stat Buf; - if (stat(debfile.c_str(), &Buf) != 0) - return false; content << "Size: " << Buf.st_size << "\n"; return true; } -bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const +bool debDebPkgFileIndex::OpenListFile(FileFd &Pkg, std::string const &FileName) { - if(Prog) - Prog->SubProgress(0, "Reading deb file"); - // write the control data to a tempfile - SPtr DebControl = GetTempFile("deb-file-" + flNotDir(DebFile)); - if(DebControl == NULL) + if (GetTempFile("deb-file-" + flNotDir(FileName), true, &Pkg) == NULL) return false; std::ostringstream content; - if (GetContent(content, DebFile) == false) + if (GetContent(content, FileName) == false) return false; std::string const contentstr = content.str(); if (contentstr.empty()) return true; - DebControl->Write(contentstr.c_str(), contentstr.length()); - // rewind for the listparser - DebControl->Seek(0); - - // and give it to the list parser - debDebFileParser Parser(DebControl, DebFile); - if(Gen.SelectFile(DebFile, *this, "", "now", pkgCache::Flag::LocalSource) == false) - return _error->Error("Problem with SelectFile %s", DebFile.c_str()); - - pkgCache::PkgFileIterator File = Gen.GetCurFile(); - File->Size = DebControl->Size(); - File->mtime = DebControl->ModificationTime(); - - if (Gen.MergeList(Parser) == false) - return _error->Error("Problem with MergeLister for %s", DebFile.c_str()); - + if (Pkg.Write(contentstr.c_str(), contentstr.length()) == false || Pkg.Seek(0) == false) + return false; return true; } +pkgCacheListParser * debDebPkgFileIndex::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return NULL; + _error->PushToStack(); + pkgCacheListParser * const Parser = new debDebFileParser(&Pkg, DebFile); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? NULL : Parser; +} +uint8_t debDebPkgFileIndex::GetIndexFlags() const +{ + return pkgCache::Flag::LocalSource; +} +std::string debDebPkgFileIndex::GetArchitecture() const +{ + return std::string(); +} +std::string debDebPkgFileIndex::GetComponent() const +{ + return "local-deb"; +} pkgCache::PkgFileIterator debDebPkgFileIndex::FindInCache(pkgCache &Cache) const { + std::string const FileName = IndexFileName(); pkgCache::PkgFileIterator File = Cache.FileBegin(); for (; File.end() == false; ++File) { - if (File.FileName() == NULL || DebFile != File.FileName()) + if (File.FileName() == NULL || FileName != File.FileName()) continue; - - return File; + // we can't do size checks here as file size != content size + return File; } return File; } -unsigned long debDebPkgFileIndex::Size() const -{ - struct stat buf; - if(stat(DebFile.c_str(), &buf) != 0) - return 0; - return buf.st_size; -} - /*}}}*/ -// debDscFileIndex - a .dsc file /*{{{*/ + /*}}}*/ +// DscFile Index - a single .dsc file as an index /*{{{*/ debDscFileIndex::debDscFileIndex(std::string const &DscFile) - : pkgIndexFile(true), d(NULL), DscFile(DscFile) + : pkgDebianIndexRealFile(DscFile, true), d(NULL) { } -bool debDscFileIndex::Exists() const -{ - return FileExists(DscFile); -} - -unsigned long debDscFileIndex::Size() const -{ - struct stat buf; - if(stat(DscFile.c_str(), &buf) == 0) - return buf.st_size; - return 0; -} pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const { - if (!FileExists(DscFile)) + if (Exists() == false) return NULL; - - return new debDscRecordParser(DscFile,this); + return new debDscRecordParser(File, this); } /*}}}*/ @@ -478,7 +270,7 @@ class APT_HIDDEN debIFTypeSrc : public pkgIndexFile::Type class APT_HIDDEN debIFTypePkg : public pkgIndexFile::Type { public: - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &File) const APT_OVERRIDE { return new debRecordParser(File.FileName(),*File.Cache()); }; @@ -492,7 +284,7 @@ class APT_HIDDEN debIFTypeTrans : public debIFTypePkg class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type { public: - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &File) const APT_OVERRIDE { return new debRecordParser(File.FileName(),*File.Cache()); }; @@ -501,7 +293,7 @@ class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type { public: - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &File) const APT_OVERRIDE { return new debDebFileRecordParser(File.FileName()); }; @@ -510,7 +302,7 @@ class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type class APT_HIDDEN debIFTypeDscFile : public pkgIndexFile::Type { public: - virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string DscFile) const APT_OVERRIDE + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string const &DscFile) const APT_OVERRIDE { return new debDscRecordParser(DscFile, NULL); }; @@ -519,9 +311,9 @@ class APT_HIDDEN debIFTypeDscFile : public pkgIndexFile::Type class APT_HIDDEN debIFTypeDebianSourceDir : public pkgIndexFile::Type { public: - virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string SourceDir) const APT_OVERRIDE + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string const &SourceDir) const APT_OVERRIDE { - return new debDscRecordParser(SourceDir + string("/debian/control"), NULL); + return new debDscRecordParser(SourceDir + std::string("/debian/control"), NULL); }; debIFTypeDebianSourceDir() {Label = "Debian control file";}; }; diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index 4a818121a..02708b558 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -26,69 +26,73 @@ class OpProgress; class pkgAcquire; class pkgCacheGenerator; - -class debStatusIndex : public pkgIndexFile +class debStatusIndex : public pkgDebianIndexRealFile { void * const d; - protected: - std::string File; +protected: + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual std::string GetComponent() const APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; - public: +public: virtual const Type *GetType() const APT_CONST; - // Interface for acquire - virtual std::string Describe(bool /*Short*/) const APT_OVERRIDE {return File;}; - // Interface for the Cache Generator - virtual bool Exists() const APT_OVERRIDE; virtual bool HasPackages() const APT_OVERRIDE {return true;}; - virtual unsigned long Size() const APT_OVERRIDE; - virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const APT_OVERRIDE; - virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const APT_OVERRIDE; + // Abort if the file does not exist. + virtual bool Exists() const APT_OVERRIDE {return true;}; - debStatusIndex(std::string File); + debStatusIndex(std::string const &File); virtual ~debStatusIndex(); }; -class debPackagesIndex : public pkgIndexTargetFile +class debPackagesIndex : public pkgDebianIndexTargetFile { void * const d; - public: +protected: + virtual uint8_t GetIndexFlags() const; +public: virtual const Type *GetType() const APT_CONST; // Stuff for accessing files on remote items - virtual std::string ArchiveInfo(pkgCache::VerIterator Ver) const APT_OVERRIDE; + virtual std::string ArchiveInfo(pkgCache::VerIterator const &Ver) const APT_OVERRIDE; // Interface for the Cache Generator virtual bool HasPackages() const APT_OVERRIDE {return true;}; - virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const APT_OVERRIDE; - virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const APT_OVERRIDE; debPackagesIndex(IndexTarget const &Target, bool const Trusted); virtual ~debPackagesIndex(); }; -class debTranslationsIndex : public pkgIndexTargetFile +class debTranslationsIndex : public pkgDebianIndexTargetFile { void * const d; - public: +protected: + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + +public: virtual const Type *GetType() const APT_CONST; // Interface for the Cache Generator virtual bool HasPackages() const APT_OVERRIDE; - virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const APT_OVERRIDE; - virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const APT_OVERRIDE; debTranslationsIndex(IndexTarget const &Target); virtual ~debTranslationsIndex(); }; -class debSourcesIndex : public pkgIndexTargetFile +class debSourcesIndex : public pkgDebianIndexTargetFile { void * const d; + virtual uint8_t GetIndexFlags() const; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + public: virtual const Type *GetType() const APT_CONST; @@ -107,19 +111,20 @@ class debSourcesIndex : public pkgIndexTargetFile virtual ~debSourcesIndex(); }; -class debDebPkgFileIndex : public pkgIndexFile +class debDebPkgFileIndex : public pkgDebianIndexRealFile { - private: void * const d; std::string DebFile; - std::string DebFileFullPath; - public: - virtual const Type *GetType() const APT_CONST; +protected: + virtual std::string GetComponent() const APT_OVERRIDE; + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; - virtual std::string Describe(bool /*Short*/) const APT_OVERRIDE { - return DebFile; - } +public: + virtual const Type *GetType() const APT_CONST; /** get the control (file) content of the deb file * @@ -130,35 +135,22 @@ class debDebPkgFileIndex : public pkgIndexFile static bool GetContent(std::ostream &content, std::string const &debfile); // Interface for the Cache Generator - virtual bool Exists() const APT_OVERRIDE; - virtual bool HasPackages() const APT_OVERRIDE { - return true; - }; - virtual unsigned long Size() const APT_OVERRIDE; - virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const APT_OVERRIDE; + virtual bool HasPackages() const APT_OVERRIDE {return true;} virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const APT_OVERRIDE; // Interface for acquire - virtual std::string ArchiveURI(std::string /*File*/) const APT_OVERRIDE; debDebPkgFileIndex(std::string const &DebFile); virtual ~debDebPkgFileIndex(); }; -class debDscFileIndex : public pkgIndexFile +class debDscFileIndex : public pkgDebianIndexRealFile { - private: void * const d; - std::string DscFile; public: virtual const Type *GetType() const APT_CONST; virtual pkgSrcRecords::Parser *CreateSrcParser() const APT_OVERRIDE; - virtual bool Exists() const APT_OVERRIDE; virtual bool HasPackages() const APT_OVERRIDE {return false;}; - virtual unsigned long Size() const APT_OVERRIDE; - virtual std::string Describe(bool /*Short*/) const APT_OVERRIDE { - return DscFile; - }; debDscFileIndex(std::string const &DscFile); virtual ~debDscFileIndex(); diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index b7988d499..cb2b15668 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -50,8 +50,9 @@ static debListParser::WordList PrioList[] = { /* Provide an architecture and only this one and "all" will be accepted in Step(), if no Architecture is given we will accept every arch we would accept in general with checkArchitecture() */ -debListParser::debListParser(FileFd *File, string const &Arch) : d(NULL), Tags(File), - Arch(Arch) { +debListParser::debListParser(FileFd *File, string const &Arch) : + pkgCacheListParser(), d(NULL), Tags(File), Arch(Arch) +{ if (Arch == "native") this->Arch = _config->Find("APT::Architecture"); Architectures = APT::Configuration::getArchitectures(); @@ -931,7 +932,7 @@ unsigned char debListParser::GetPrio(string Str) bool debListParser::SameVersion(unsigned short const Hash, /*{{{*/ pkgCache::VerIterator const &Ver) { - if (pkgCacheGenerator::ListParser::SameVersion(Hash, Ver) == false) + if (pkgCacheListParser::SameVersion(Hash, Ver) == false) return false; // status file has no (Download)Size, but all others are fair game // status file is parsed last, so the first version we encounter is diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index 30e52718d..975620070 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -26,7 +26,7 @@ class FileFd; -class APT_HIDDEN debListParser : public pkgCacheGenerator::ListParser +class APT_HIDDEN debListParser : public pkgCacheListParser { public: diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc index 649d94b5d..a8a528131 100644 --- a/apt-pkg/edsp/edspindexfile.cc +++ b/apt-pkg/edsp/edspindexfile.cc @@ -26,59 +26,66 @@ #include /*}}}*/ -// edspIndex::edspIndex - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ -edspIndex::edspIndex(std::string File) : debStatusIndex(File), d(NULL) +// EDSP Index /*{{{*/ +edspIndex::edspIndex(std::string const &File) : pkgDebianIndexRealFile(File, true), d(NULL) { } - /*}}}*/ -// StatusIndex::Merge - Load the index file into a cache /*{{{*/ -bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const +std::string edspIndex::GetComponent() const { - FileFd Pkg; - if (File != "stdin") - Pkg.Open(File, FileFd::ReadOnly); - else - Pkg.OpenDescriptor(STDIN_FILENO, FileFd::ReadOnly); - if (_error->PendingError() == true) - return false; - edspListParser Parser(&Pkg); - if (_error->PendingError() == true) - return false; - - if (Prog != NULL) - Prog->SubProgress(0,File); - if (Gen.SelectFile(File, *this, "", "edsp") == false) - return _error->Error("Problem with SelectFile %s",File.c_str()); - - // Store the IMS information - pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); - pkgCacheGenerator::Dynamic DynFile(CFile); - CFile->Size = Pkg.FileSize(); - CFile->mtime = Pkg.ModificationTime(); - - if (Gen.MergeList(Parser) == false) - return _error->Error("Problem with MergeList %s",File.c_str()); + return "edsp"; +} +std::string edspIndex::GetArchitecture() const +{ + return std::string(); +} +bool edspIndex::HasPackages() const +{ + return true; +} +bool edspIndex::Exists() const +{ + return true; +} +uint8_t edspIndex::GetIndexFlags() const +{ + return 0; +} +bool edspIndex::OpenListFile(FileFd &Pkg, std::string const &FileName) +{ + if (FileName.empty() == false && FileName != "stdin") + return pkgDebianIndexRealFile::OpenListFile(Pkg, FileName); + if (Pkg.OpenDescriptor(STDIN_FILENO, FileFd::ReadOnly) == false) + return _error->Error("Problem opening %s",FileName.c_str()); return true; +} +pkgCacheListParser * edspIndex::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return NULL; + _error->PushToStack(); + pkgCacheListParser * const Parser = new edspListParser(&Pkg); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? NULL : Parser; } /*}}}*/ + // Index File types for APT /*{{{*/ class APT_HIDDEN edspIFType: public pkgIndexFile::Type { public: - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &) const APT_OVERRIDE { // we don't have a record parser for this type as the file is not presistent return NULL; }; edspIFType() {Label = "EDSP scenario file";}; }; -APT_HIDDEN edspIFType _apt_Universe; +APT_HIDDEN edspIFType _apt_Edsp; const pkgIndexFile::Type *edspIndex::GetType() const { - return &_apt_Universe; + return &_apt_Edsp; } /*}}}*/ diff --git a/apt-pkg/edsp/edspindexfile.h b/apt-pkg/edsp/edspindexfile.h index b2a510f14..edf799023 100644 --- a/apt-pkg/edsp/edspindexfile.h +++ b/apt-pkg/edsp/edspindexfile.h @@ -18,18 +18,24 @@ class OpProgress; class pkgCacheGenerator; -class APT_HIDDEN edspIndex : public debStatusIndex +class APT_HIDDEN edspIndex : public pkgDebianIndexRealFile { /** \brief dpointer placeholder (for later in case we need it) */ void * const d; - public: +protected: + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg) APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &File) APT_OVERRIDE; + virtual uint8_t GetIndexFlags() const APT_OVERRIDE; + virtual std::string GetComponent() const APT_OVERRIDE; + virtual std::string GetArchitecture() const APT_OVERRIDE; +public: virtual const Type *GetType() const APT_CONST; + virtual bool Exists() const APT_OVERRIDE; + virtual bool HasPackages() const APT_OVERRIDE; - virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const APT_OVERRIDE; - - edspIndex(std::string File); + edspIndex(std::string const &File); virtual ~edspIndex(); }; diff --git a/apt-pkg/edsp/edspsystem.cc b/apt-pkg/edsp/edspsystem.cc index f65fcc0d2..c4583252f 100644 --- a/apt-pkg/edsp/edspsystem.cc +++ b/apt-pkg/edsp/edspsystem.cc @@ -81,13 +81,9 @@ bool edspSystem::ArchiveSupported(const char * /*Type*/) return false; } /*}}}*/ -// System::Score - Determine if we should use the edsp system /*{{{*/ -signed edspSystem::Score(Configuration const &Cnf) +// System::Score - Never use the EDSP system automatically /*{{{*/ +signed edspSystem::Score(Configuration const &) { - if (Cnf.Find("edsp::scenario", "") == "stdin") - return 1000; - if (RealFileExists(Cnf.FindFile("edsp::scenario","")) == true) - return 1000; return -1000; } /*}}}*/ diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc index cce17403d..b592ae5a0 100644 --- a/apt-pkg/indexfile.cc +++ b/apt-pkg/indexfile.cc @@ -16,14 +16,23 @@ #include #include #include +#include #include #include +#include #include +#include + +#include +#include +#include + #include #include #include #include +#include /*}}}*/ // Global list of Item supported @@ -52,13 +61,13 @@ pkgIndexFile::Type *pkgIndexFile::Type::GetType(const char *Type) return 0; } /*}}}*/ -pkgIndexFile::pkgIndexFile(bool Trusted) : /*{{{*/ +pkgIndexFile::pkgIndexFile(bool const Trusted) : /*{{{*/ d(NULL), Trusted(Trusted) { } /*}}}*/ // IndexFile::ArchiveInfo - Stub /*{{{*/ -std::string pkgIndexFile::ArchiveInfo(pkgCache::VerIterator /*Ver*/) const +std::string pkgIndexFile::ArchiveInfo(pkgCache::VerIterator const &/*Ver*/) const { return std::string(); } @@ -89,7 +98,7 @@ bool pkgIndexFile::TranslationsAvailable() { is already done in getLanguages(). Note also that this check is rather bad (doesn't take three character like ast into account). TODO: Remove method with next API break */ -APT_DEPRECATED bool pkgIndexFile::CheckLanguageCode(const char *Lang) +APT_DEPRECATED bool pkgIndexFile::CheckLanguageCode(const char * const Lang) { if (strlen(Lang) == 2 || (strlen(Lang) == 5 && Lang[2] == '_')) return true; @@ -172,24 +181,24 @@ std::string IndexTarget::Format(std::string format) const /*{{{*/ } /*}}}*/ -pkgIndexTargetFile::pkgIndexTargetFile(IndexTarget const &Target, bool const Trusted) :/*{{{*/ - pkgIndexFile(Trusted), d(NULL), Target(Target) +pkgDebianIndexTargetFile::pkgDebianIndexTargetFile(IndexTarget const &Target, bool const Trusted) :/*{{{*/ + pkgDebianIndexFile(Trusted), d(NULL), Target(Target) { } /*}}}*/ -std::string pkgIndexTargetFile::ArchiveURI(std::string File) const/*{{{*/ +std::string pkgDebianIndexTargetFile::ArchiveURI(std::string const &File) const/*{{{*/ { return Target.Option(IndexTarget::REPO_URI) + File; } /*}}}*/ -std::string pkgIndexTargetFile::Describe(bool Short) const /*{{{*/ +std::string pkgDebianIndexTargetFile::Describe(bool const Short) const /*{{{*/ { if (Short) return Target.Description; return Target.Description + " (" + IndexFileName() + ")"; } /*}}}*/ -std::string pkgIndexTargetFile::IndexFileName() const /*{{{*/ +std::string pkgDebianIndexTargetFile::IndexFileName() const /*{{{*/ { std::string const s = Target.Option(IndexTarget::FILENAME); if (FileExists(s)) @@ -205,7 +214,7 @@ std::string pkgIndexTargetFile::IndexFileName() const /*{{{*/ return s; } /*}}}*/ -unsigned long pkgIndexTargetFile::Size() const /*{{{*/ +unsigned long pkgDebianIndexTargetFile::Size() const /*{{{*/ { unsigned long size = 0; @@ -223,11 +232,155 @@ unsigned long pkgIndexTargetFile::Size() const /*{{{*/ return size; } /*}}}*/ -bool pkgIndexTargetFile::Exists() const /*{{{*/ +bool pkgDebianIndexTargetFile::Exists() const /*{{{*/ { return FileExists(IndexFileName()); } /*}}}*/ +std::string pkgDebianIndexTargetFile::GetArchitecture() const /*{{{*/ +{ + return Target.Option(IndexTarget::ARCHITECTURE); +} + /*}}}*/ +std::string pkgDebianIndexTargetFile::GetComponent() const /*{{{*/ +{ + return Target.Option(IndexTarget::COMPONENT); +} + /*}}}*/ +bool pkgDebianIndexTargetFile::OpenListFile(FileFd &Pkg, std::string const &FileName)/*{{{*/ +{ + if (Pkg.Open(FileName, FileFd::ReadOnly, FileFd::Extension) == false) + return _error->Error("Problem opening %s",FileName.c_str()); + return true; +} + /*}}}*/ +std::string pkgDebianIndexTargetFile::GetProgressDescription() const +{ + return Target.Description; +} + +pkgDebianIndexRealFile::pkgDebianIndexRealFile(std::string const &File, bool const Trusted) :/*{{{*/ + pkgDebianIndexFile(Trusted), d(NULL), File(flAbsPath(File)) +{ +} + /*}}}*/ +// IndexRealFile::Size - Return the size of the index /*{{{*/ +unsigned long pkgDebianIndexRealFile::Size() const +{ + struct stat S; + if (stat(File.c_str(),&S) != 0) + return 0; + return S.st_size; +} + /*}}}*/ +bool pkgDebianIndexRealFile::Exists() const /*{{{*/ +{ + return FileExists(File); +} + /*}}}*/ +std::string pkgDebianIndexRealFile::Describe(bool const /*Short*/) const/*{{{*/ +{ + return File; +} + /*}}}*/ +std::string pkgDebianIndexRealFile::ArchiveURI(std::string const &/*File*/) const/*{{{*/ +{ + return "file:" + File; +} + /*}}}*/ +std::string pkgDebianIndexRealFile::IndexFileName() const /*{{{*/ +{ + return File; +} + /*}}}*/ +std::string pkgDebianIndexRealFile::GetProgressDescription() const +{ + return File; +} +bool pkgDebianIndexRealFile::OpenListFile(FileFd &Pkg, std::string const &FileName)/*{{{*/ +{ + if (Pkg.Open(FileName, FileFd::ReadOnly, FileFd::None) == false) + return _error->Error("Problem opening %s",FileName.c_str()); + return true; +} + /*}}}*/ + +pkgDebianIndexFile::pkgDebianIndexFile(bool const Trusted) : pkgIndexFile(Trusted) +{ +} +pkgDebianIndexFile::~pkgDebianIndexFile() +{ +} +pkgCacheListParser * pkgDebianIndexFile::CreateListParser(FileFd &Pkg) +{ + if (Pkg.IsOpen() == false) + return NULL; + _error->PushToStack(); + pkgCacheListParser * const Parser = new debListParser(&Pkg, GetArchitecture()); + bool const newError = _error->PendingError(); + _error->MergeWithStack(); + return newError ? NULL : Parser; +} +bool pkgDebianIndexFile::Merge(pkgCacheGenerator &Gen,OpProgress * const Prog) +{ + std::string const PackageFile = IndexFileName(); + FileFd Pkg; + if (OpenListFile(Pkg, PackageFile) == false) + return false; + _error->PushToStack(); + std::unique_ptr Parser(CreateListParser(Pkg)); + bool const newError = _error->PendingError(); + if (newError == false && Parser == nullptr) + return true; + if (Parser == NULL) + return false; + + if (Prog != NULL) + Prog->SubProgress(0, GetProgressDescription()); + + if (Gen.SelectFile(PackageFile, *this, GetArchitecture(), GetComponent(), GetIndexFlags()) == false) + return _error->Error("Problem with SelectFile %s",PackageFile.c_str()); + + // Store the IMS information + pkgCache::PkgFileIterator File = Gen.GetCurFile(); + pkgCacheGenerator::Dynamic DynFile(File); + File->Size = Pkg.FileSize(); + File->mtime = Pkg.ModificationTime(); + + if (Gen.MergeList(*Parser) == false) + return _error->Error("Problem with MergeList %s",PackageFile.c_str()); + return true; +} +pkgCache::PkgFileIterator pkgDebianIndexFile::FindInCache(pkgCache &Cache) const +{ + std::string const FileName = IndexFileName(); + pkgCache::PkgFileIterator File = Cache.FileBegin(); + for (; File.end() == false; ++File) + { + if (File.FileName() == NULL || FileName != File.FileName()) + continue; + + struct stat St; + if (stat(File.FileName(),&St) != 0) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "DebianIndexFile::FindInCache - stat failed on " << File.FileName() << std::endl; + return pkgCache::PkgFileIterator(Cache); + } + if ((map_filesize_t)St.st_size != File->Size || St.st_mtime != File->mtime) + { + if (_config->FindB("Debug::pkgCacheGen", false)) + std::clog << "DebianIndexFile::FindInCache - size (" << St.st_size << " <> " << File->Size + << ") or mtime (" << St.st_mtime << " <> " << File->mtime + << ") doesn't match for " << File.FileName() << std::endl; + return pkgCache::PkgFileIterator(Cache); + } + return File; + } + + return File; +} APT_CONST pkgIndexFile::~pkgIndexFile() {} -APT_CONST pkgIndexTargetFile::~pkgIndexTargetFile() {} +APT_CONST pkgDebianIndexTargetFile::~pkgDebianIndexTargetFile() {} +APT_CONST pkgDebianIndexRealFile::~pkgDebianIndexRealFile() {} diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h index 5be7794bf..77e80ed41 100644 --- a/apt-pkg/indexfile.h +++ b/apt-pkg/indexfile.h @@ -38,6 +38,7 @@ class pkgAcquire; #endif class pkgCacheGenerator; +class pkgCacheListParser; class OpProgress; class IndexTarget /*{{{*/ @@ -105,12 +106,12 @@ class pkgIndexFile // Global list of Items supported static Type **GlobalList; static unsigned long GlobalListLen; - static Type *GetType(const char *Type) APT_PURE; + static Type *GetType(const char * const Type) APT_PURE; const char *Label; - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator /*File*/) const {return 0;}; - virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string /*File*/) const {return 0;}; + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator const &/*File*/) const {return 0;}; + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string const &/*File*/) const {return 0;}; Type(); virtual ~Type() {}; }; @@ -118,13 +119,13 @@ class pkgIndexFile virtual const Type *GetType() const = 0; // Return descriptive strings of various sorts - virtual std::string ArchiveInfo(pkgCache::VerIterator Ver) const; + virtual std::string ArchiveInfo(pkgCache::VerIterator const &Ver) const; virtual std::string SourceInfo(pkgSrcRecords::Parser const &Record, pkgSrcRecords::File const &File) const; - virtual std::string Describe(bool Short = false) const = 0; + virtual std::string Describe(bool const Short = false) const = 0; // Interface for acquire - virtual std::string ArchiveURI(std::string /*File*/) const {return std::string();}; + virtual std::string ArchiveURI(std::string const &/*File*/) const {return std::string();}; // Interface for the record parsers virtual pkgSrcRecords::Parser *CreateSrcParser() const {return 0;}; @@ -133,40 +134,78 @@ class pkgIndexFile virtual bool Exists() const = 0; virtual bool HasPackages() const = 0; virtual unsigned long Size() const = 0; - virtual bool Merge(pkgCacheGenerator &/*Gen*/, OpProgress* /*Prog*/) const { return false; }; - APT_DEPRECATED virtual bool Merge(pkgCacheGenerator &Gen, OpProgress &Prog) const - { return Merge(Gen, &Prog); }; - virtual bool MergeFileProvides(pkgCacheGenerator &/*Gen*/,OpProgress* /*Prog*/) const {return true;}; - APT_DEPRECATED virtual bool MergeFileProvides(pkgCacheGenerator &Gen, OpProgress &Prog) const - {return MergeFileProvides(Gen, &Prog);}; + virtual bool Merge(pkgCacheGenerator &/*Gen*/, OpProgress* const /*Prog*/) { return true; }; + virtual bool MergeFileProvides(pkgCacheGenerator &/*Gen*/,OpProgress* /*Prog*/) {return true;}; virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const; static bool TranslationsAvailable(); - static bool CheckLanguageCode(const char *Lang); + static bool CheckLanguageCode(const char * const Lang); static std::string LanguageCode(); bool IsTrusted() const { return Trusted; }; - explicit pkgIndexFile(bool Trusted); + explicit pkgIndexFile(bool const Trusted); virtual ~pkgIndexFile(); }; -class pkgIndexTargetFile : public pkgIndexFile +class pkgDebianIndexFile : public pkgIndexFile +{ +protected: + virtual std::string IndexFileName() const = 0; + virtual std::string GetComponent() const = 0; + virtual std::string GetArchitecture() const = 0; + virtual std::string GetProgressDescription() const = 0; + virtual uint8_t GetIndexFlags() const = 0; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) = 0; + APT_HIDDEN virtual pkgCacheListParser * CreateListParser(FileFd &Pkg); + +public: + virtual bool Merge(pkgCacheGenerator &Gen, OpProgress* const Prog) APT_OVERRIDE; + virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const APT_OVERRIDE; + + pkgDebianIndexFile(bool const Trusted); + virtual ~pkgDebianIndexFile(); +}; + +class pkgDebianIndexTargetFile : public pkgDebianIndexFile { void * const d; protected: IndexTarget const Target; - std::string IndexFileName() const; + virtual std::string IndexFileName() const APT_OVERRIDE; + virtual std::string GetComponent() const APT_OVERRIDE; + virtual std::string GetArchitecture() const APT_OVERRIDE; + virtual std::string GetProgressDescription() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; + +public: + virtual std::string ArchiveURI(std::string const &File) const APT_OVERRIDE; + virtual std::string Describe(bool const Short = false) const APT_OVERRIDE; + virtual bool Exists() const APT_OVERRIDE; + virtual unsigned long Size() const APT_OVERRIDE; + + pkgDebianIndexTargetFile(IndexTarget const &Target, bool const Trusted); + virtual ~pkgDebianIndexTargetFile(); +}; + +class pkgDebianIndexRealFile : public pkgDebianIndexFile +{ + void * const d; +protected: + std::string File; + virtual std::string IndexFileName() const APT_OVERRIDE; + virtual std::string GetProgressDescription() const APT_OVERRIDE; + virtual bool OpenListFile(FileFd &Pkg, std::string const &FileName) APT_OVERRIDE; public: - virtual std::string ArchiveURI(std::string File) const APT_OVERRIDE; - virtual std::string Describe(bool Short = false) const APT_OVERRIDE; + virtual std::string Describe(bool const /*Short*/ = false) const APT_OVERRIDE; virtual bool Exists() const APT_OVERRIDE; virtual unsigned long Size() const APT_OVERRIDE; + virtual std::string ArchiveURI(std::string const &/*File*/) const APT_OVERRIDE; - pkgIndexTargetFile(IndexTarget const &Target, bool const Trusted); - virtual ~pkgIndexTargetFile(); + pkgDebianIndexRealFile(std::string const &File, bool const Trusted); + virtual ~pkgDebianIndexRealFile(); }; #endif diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 9529f42dc..cf845b0fb 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -1051,7 +1051,7 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, // --------------------------------------------------------------------- /* This creates a Group and the Package to link this dependency to if needed and handles also the caching of the old endpoint */ -bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver, +bool pkgCacheListParser::NewDepends(pkgCache::VerIterator &Ver, const string &PackageName, const string &Arch, const string &Version, @@ -1077,7 +1077,7 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver, if (idxVersion == 0) { - idxVersion = StoreString(VERSIONNUMBER, Version); + idxVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version); if (unlikely(idxVersion == 0)) return false; } @@ -1124,7 +1124,7 @@ bool pkgCacheGenerator::ListParser::NewDepends(pkgCache::VerIterator &Ver, } /*}}}*/ // ListParser::NewProvides - Create a Provides element /*{{{*/ -bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, +bool pkgCacheListParser::NewProvides(pkgCache::VerIterator &Ver, const string &PkgName, const string &PkgArch, const string &Version, @@ -1145,7 +1145,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, map_stringitem_t idxProvideVersion = 0; if (Version.empty() == false) { - idxProvideVersion = StoreString(VERSIONNUMBER, Version); + idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version); if (unlikely(idxProvideVersion == 0)) return false; } @@ -1178,7 +1178,7 @@ bool pkgCacheGenerator::NewProvides(pkgCache::VerIterator &Ver, } /*}}}*/ // ListParser::NewProvidesAllArch - add provides for all architectures /*{{{*/ -bool pkgCacheGenerator::ListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package, +bool pkgCacheListParser::NewProvidesAllArch(pkgCache::VerIterator &Ver, string const &Package, string const &Version, uint8_t const Flags) { pkgCache &Cache = Owner->Cache; pkgCache::GrpIterator const Grp = Cache.FindGrp(Package); @@ -1188,7 +1188,7 @@ bool pkgCacheGenerator::ListParser::NewProvidesAllArch(pkgCache::VerIterator &Ve { map_stringitem_t idxProvideVersion = 0; if (Version.empty() == false) { - idxProvideVersion = StoreString(VERSIONNUMBER, Version); + idxProvideVersion = StoreString(pkgCacheGenerator::VERSIONNUMBER, Version); if (unlikely(idxProvideVersion == 0)) return false; } @@ -1209,7 +1209,7 @@ bool pkgCacheGenerator::ListParser::NewProvidesAllArch(pkgCache::VerIterator &Ve return true; } /*}}}*/ -bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*/ +bool pkgCacheListParser::SameVersion(unsigned short const Hash, /*{{{*/ pkgCache::VerIterator const &Ver) { return Hash == Ver->Hash; @@ -1835,5 +1835,5 @@ bool pkgCacheGenerator::FinishCache(OpProgress * /*Progress*/) } /*}}}*/ -pkgCacheGenerator::ListParser::ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {} -pkgCacheGenerator::ListParser::~ListParser() {} +pkgCacheListParser::pkgCacheListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false), d(NULL) {} +pkgCacheListParser::~pkgCacheListParser() {} diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 4b6b91992..15e73627d 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -32,10 +32,10 @@ class FileFd; class pkgSourceList; class OpProgress; class pkgIndexFile; +class pkgCacheListParser; class APT_HIDDEN pkgCacheGenerator /*{{{*/ { - private: APT_HIDDEN map_stringitem_t WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; APT_HIDDEN map_stringitem_t WriteStringInMap(const char *String); APT_HIDDEN map_stringitem_t WriteStringInMap(const char *String, const unsigned long &Len); @@ -46,10 +46,10 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ std::map strPkgNames; std::map strVersions; + friend class pkgCacheListParser; + typedef pkgCacheListParser ListParser; + public: - - class ListParser; - friend class ListParser; template class Dynamic { public: @@ -138,11 +138,12 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ }; /*}}}*/ // This is the abstract package list parser class. /*{{{*/ -class APT_HIDDEN pkgCacheGenerator::ListParser +class APT_HIDDEN pkgCacheListParser { pkgCacheGenerator *Owner; friend class pkgCacheGenerator; - + template using Dynamic = pkgCacheGenerator::Dynamic; + // Some cache items pkgCache::VerIterator OldDepVer; map_pointer_t *OldDepLast; @@ -197,8 +198,8 @@ class APT_HIDDEN pkgCacheGenerator::ListParser virtual bool CollectFileProvides(pkgCache &/*Cache*/, pkgCache::VerIterator &/*Ver*/) {return true;}; - ListParser(); - virtual ~ListParser(); + pkgCacheListParser(); + virtual ~pkgCacheListParser(); }; /*}}}*/ diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index 939061b93..af301dbcd 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -109,6 +109,7 @@ int main(int argc,const char *argv[]) /*{{{*/ if (_config->FindI("quiet", 0) < 1) _config->Set("Debug::EDSP::WriteSolution", true); + _config->Set("APT::System", "Debian APT solver interface"); _config->Set("APT::Solver", "internal"); _config->Set("edsp::scenario", "stdin"); int input = STDIN_FILENO; diff --git a/test/integration/framework b/test/integration/framework index f3cc1eff9..53157e2d0 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1296,7 +1296,7 @@ testdpkgstatus() { local PKGS="$(dpkg -l "$@" 2>/dev/null | grep "^${STATE}" | wc -l)" if [ "$PKGS" != $NR ]; then echo >&2 $PKGS - dpkg -l "$@" | grep '^[a-z]' >&2 + dpkg -l "$@" | grep '^[a-z]' >&2 || true msgfail else msgpass diff --git a/test/integration/test-apt-get-install-deb b/test/integration/test-apt-get-install-deb index 991185dea..bd720bede 100755 --- a/test/integration/test-apt-get-install-deb +++ b/test/integration/test-apt-get-install-deb @@ -22,9 +22,9 @@ E: Couldn't find any package by regex './foo.rpm'" aptget install -qq ./foo.rpm mv foo.rpm foo.deb testfailure aptget install ./foo.deb testsuccess grep '^E: Sub-process Popen returned an error code' rootdir/tmp/testfailure.output -testequal 'E: Encountered a section with no Package: header -E: Problem with MergeLister for ./foo.deb -E: The package lists or status file could not be parsed or opened.' tail -n 3 rootdir/tmp/testfailure.output +testequal "E: Encountered a section with no Package: header +E: Problem with MergeList ${TMPWORKINGDIRECTORY}/foo.deb +E: The package lists or status file could not be parsed or opened." tail -n 3 rootdir/tmp/testfailure.output # fakeroot is currently not found, framwork needs updating buildsimplenativepackage 'foo' 'i386,amd64' '1.0' @@ -45,7 +45,7 @@ The following packages have unmet dependencies: E: Unable to correct problems, you have held broken packages." aptget install ./incoming/foo_1.0_i386.deb ./incoming/foo_1.0_amd64.deb -s -q=0 testdpkgnotinstalled 'foo' -testsuccess aptget install ./incoming/foo_1.0_i386.deb +testsuccess aptget install ./incoming/foo_1.0_i386.deb -o Debug::pkgCacheGen=1 testdpkginstalled 'foo' testsuccessequal "Reading package lists... @@ -58,5 +58,5 @@ The following NEW packages will be installed: foo 0 upgraded, 1 newly installed, 1 to remove and 0 not upgraded. Remv foo:i386 [1.0] -Inst foo (1.0 now [amd64]) -Conf foo (1.0 now [amd64])" aptget install ./incoming/foo_1.0_amd64.deb -s -q=0 +Inst foo (1.0 local-deb [amd64]) +Conf foo (1.0 local-deb [amd64])" aptget install ./incoming/foo_1.0_amd64.deb -s -q=0 -- cgit v1.2.3-70-g09d2 From 3cbeed985254de7e8d186ebebb69a659727eaeb0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 3 Aug 2015 07:00:33 +0200 Subject: mark again deps of pkgs in APT::Never-MarkAuto-Sections as manual MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 50ef3344c3afaaf9943142906b2f976a0337d264 (and similar for other branches), while 'fixing' the edgecase of a package being in multiple sections (e.g. moved from libs to oldlibs in newer releases) I accidently broke the feature itself completely by operating on the package itself and no longer on its dependencies… The behaviour isn't ideal in multiple ways, which we are hopefully able to fix with new ideas as mentioned in the buglog, but until then the functionality of this "hack" should be restored. Reported-By: Raphaël Hertzog Tested-By: Adam Conrad Closes: 793360 LP: 1479207 Thanks: Raphaël Hertzog and Adam Conrad for detailed reports and initial patches --- apt-pkg/depcache.cc | 9 +- test/integration/framework | 4 +- test/integration/test-apt-never-markauto-sections | 106 ++++++++++++++++++++++ 3 files changed, 116 insertions(+), 3 deletions(-) create mode 100755 test/integration/test-apt-never-markauto-sections (limited to 'test/integration/framework') diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc index 7b1448c73..e466cba28 100644 --- a/apt-pkg/depcache.cc +++ b/apt-pkg/depcache.cc @@ -1093,7 +1093,12 @@ bool pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, if (DebugMarker == true) std::clog << OutputInDepth(Depth) << "MarkInstall " << Pkg << " FU=" << FromUser << std::endl; - DepIterator Dep = P.InstVerIter(*this).DependsList(); + VerIterator const PV = P.InstVerIter(*this); + if (unlikely(PV.end() == true)) + return false; + bool const PinNeverMarkAutoSection = (PV->Section != 0 && ConfigValueInSubTree("APT::Never-MarkAuto-Sections", PV.Section())); + + DepIterator Dep = PV.DependsList(); for (; Dep.end() != true;) { // Grok or groups @@ -1206,7 +1211,7 @@ bool pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, continue; } // now check if we should consider it a automatic dependency or not - if(InstPkg->CurrentVer == 0 && InstVer->Section != 0 && ConfigValueInSubTree("APT::Never-MarkAuto-Sections", InstVer.Section())) + if(InstPkg->CurrentVer == 0 && PinNeverMarkAutoSection) { if(DebugAutoInstall == true) std::clog << OutputInDepth(Depth) << "Setting NOT as auto-installed (direct " diff --git a/test/integration/framework b/test/integration/framework index 53157e2d0..2f08c5fdc 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -570,10 +570,12 @@ echo '$NAME says \"Hello!\"'" > ${BUILDDIR}/${NAME} -- Joe Sixpack $(date -R)" > ${BUILDDIR}/debian/changelog echo "Source: $NAME -Section: $SECTION Priority: $PRIORITY Maintainer: Joe Sixpack Standards-Version: 3.9.3" > ${BUILDDIR}/debian/control + if [ "$SECTION" != '' ]; then + echo "Section: $SECTION" >> ${BUILDDIR}/debian/control + fi local BUILDDEPS="$(echo "$DEPENDENCIES" | grep '^Build-')" test -z "$BUILDDEPS" || echo "$BUILDDEPS" >> ${BUILDDIR}/debian/control echo " diff --git a/test/integration/test-apt-never-markauto-sections b/test/integration/test-apt-never-markauto-sections new file mode 100755 index 000000000..6c88c69fa --- /dev/null +++ b/test/integration/test-apt-never-markauto-sections @@ -0,0 +1,106 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'amd64' 'i386' + +aptconfig dump --no-empty --format '%v%n' APT::Never-MarkAuto-Sections > nevermarkauto.sections +testsuccess grep '^metapackages$' nevermarkauto.sections + +# this is a very crude regression test, not a "this is how it should be" test: +# In theory mydesktop-core and texteditor should be marked as manual, but +# texteditor is installed as a dependency of bad-texteditor, not of +# mydesktop-core and mydesktop-core is removed while bad-texteditor is +# installed losing the manual bit as the problem resolver will later decide to +# drop bad-texteditor and re-instate mydesktop-core which is considered an +# auto-install at that point (in theory the never-auto handling should be +# copied to this place – as to the many other places dependencies are resolved +# 'by hand' instead of via MarkInstall AutoInst… +# +# Both could be fixed if apt would figure out early that installing +# bad-texteditor is a bad idea and eventually it should (as mydesktop-core is +# a direct descendant of mydesktop which was a user-request mydesktop-core should +# be as protected from removal as mydesktop is), but this is hard in the general case +# as with more or-groups and provides you can produce 'legal' examples for this. + +buildsimplenativepackage 'mydesktop' 'all' '1' 'unstable' 'Depends: mydesktop-core, foreignpkg +Recommends: notavailable' '' 'metapackages' +buildsimplenativepackage 'mydesktop-core' 'amd64' '1' 'unstable' 'Depends: bad-texteditor | texteditor, browser (>= 42), nosection, foreignpkg +Recommends: notavailable +Multi-Arch: foreign' '' 'metapackages' +buildsimplenativepackage 'browser' 'amd64' '41' 'stable' +buildsimplenativepackage 'browser' 'amd64' '42' 'unstable' +buildsimplenativepackage 'texteditor' 'amd64' '1' 'stable' +buildsimplenativepackage 'bad-texteditor' 'amd64' '1' 'stable' 'Depends: texteditor +Conflicts: mydesktop-core' +buildsimplenativepackage 'nosection' 'amd64' '1' 'stable' '' '' '' +buildsimplenativepackage 'foreignpkg' 'i386' '1' 'stable' 'Multi-Arch: foreign' +setupaptarchive + +testsuccess aptcache show nosection +testfailure grep 'Section' rootdir/tmp/testsuccess.output +testequal 'dpkg' aptmark showmanual + +testsuccess aptget install mydesktop -y -o Debug::pkgProblemResolver=1 -o Debug::pkgDepCache::Marker=1 + +testequal 'browser +dpkg +foreignpkg:i386 +mydesktop +nosection' aptmark showmanual +testmarkedauto 'mydesktop-core' 'texteditor' + +testequal 'Reading package lists... +Building dependency tree... +Reading state information... +The following packages will be REMOVED: + mydesktop mydesktop-core texteditor +0 upgraded, 0 newly installed, 3 to remove and 0 not upgraded. +Remv mydesktop [1] +Remv mydesktop-core [1] +Remv texteditor [1]' aptget autoremove mydesktop -s + +testequal 'Reading package lists... +Building dependency tree... +Reading state information... +The following packages will be REMOVED: + mydesktop mydesktop-core texteditor +0 upgraded, 0 newly installed, 3 to remove and 0 not upgraded. +Remv mydesktop [1] +Remv mydesktop-core [1] +Remv texteditor [1]' aptget autoremove texteditor -s +testsuccess aptget autoremove texteditor -y + +testdpkgnotinstalled mydesktop mydesktop-core texteditor +testdpkginstalled browser + +testequal 'browser +dpkg +foreignpkg:i386 +nosection' aptmark showmanual +testmarkedauto + +# test that installed/upgraded auto-pkgs are not set to manual + +testsuccess aptget install browser=41 -y --force-yes + +testequal 'browser +dpkg +foreignpkg:i386 +nosection' aptmark showmanual +testmarkedauto +testsuccess aptmark auto browser +testmarkedauto 'browser' +testsuccess aptmark auto nosection +testmarkedauto 'browser' 'nosection' +testequal 'dpkg +foreignpkg:i386' aptmark showmanual + +testsuccess aptget install mydesktop -y + +testequal 'dpkg +foreignpkg:i386 +mydesktop' aptmark showmanual +testmarkedauto 'browser' 'nosection' 'mydesktop-core' 'texteditor' -- cgit v1.2.3-70-g09d2 From 7c2cc4a7bc999c8e07fba607354bfaa3b09118f9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 10 Aug 2015 14:44:14 +0200 Subject: move APT::Never-MarkAuto-Sections handling to MarkDelete Having the handling in MarkInstall means that it just effects installation of the metapackage, but if the dependencies change the new dependencies aren't protected (and the old dependencies are still protected for no 'reason'). Having it in MarkDelete means that if a metapackage is sheduled for removal all its currently installed dependencies are marked as manual, which helps against both as in this case there is no new/old and additionally if a user decides the installation of a metapackage was wrong he can just remove it explicitely avoid the manual marking entirely. --- apt-pkg/depcache.cc | 44 +++++++++++++---- test/integration/framework | 11 +++++ test/integration/test-apt-never-markauto-sections | 60 ++++++++--------------- 3 files changed, 66 insertions(+), 49 deletions(-) (limited to 'test/integration/framework') diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc index e466cba28..99e694a06 100644 --- a/apt-pkg/depcache.cc +++ b/apt-pkg/depcache.cc @@ -836,6 +836,41 @@ bool pkgDepCache::MarkDelete(PkgIterator const &Pkg, bool rPurge, ActionGroup group(*this); + if (FromUser == false) + { + VerIterator const PV = P.InstVerIter(*this); + if (PV.end() == false) + { + // removed metapackages mark their dependencies as manual to prevent in "desktop depends browser, texteditor" + // the removal of browser to suggest the removal of desktop and texteditor. + // We ignore the auto-bit here as we can't deal with metapackage cascardes otherwise. + // We do not check for or-groups here as we don't know which package takes care of + // providing the feature the user likes e.g.: browser1 | browser2 | browser3 + // Temporary removals are effected by this as well, which is bad, but unlikely in practice + bool const PinNeverMarkAutoSection = (PV->Section != 0 && ConfigValueInSubTree("APT::Never-MarkAuto-Sections", PV.Section())); + if (PinNeverMarkAutoSection) + { + for (DepIterator D = PV.DependsList(); D.end() != true; ++D) + { + if (D.IsMultiArchImplicit() == true || D.IsNegative() == true || IsImportantDep(D) == false) + continue; + + pkgCacheFile CacheFile(this); + APT::VersionList verlist = APT::VersionList::FromDependency(CacheFile, D, APT::CacheSetHelper::INSTALLED); + for (auto const &V : verlist) + { + PkgIterator const DP = V.ParentPkg(); + if(DebugAutoInstall == true) + std::clog << OutputInDepth(Depth) << "Setting " << DP.FullName(false) << " NOT as auto-installed (direct " + << D.DepType() << " of " << Pkg.FullName(false) << " which is in APT::Never-MarkAuto-Sections)" << std::endl; + + MarkAuto(DP, false); + } + } + } + } + } + if (DebugMarker == true) std::clog << OutputInDepth(Depth) << (rPurge ? "MarkPurge " : "MarkDelete ") << Pkg << " FU=" << FromUser << std::endl; @@ -1096,7 +1131,6 @@ bool pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, VerIterator const PV = P.InstVerIter(*this); if (unlikely(PV.end() == true)) return false; - bool const PinNeverMarkAutoSection = (PV->Section != 0 && ConfigValueInSubTree("APT::Never-MarkAuto-Sections", PV.Section())); DepIterator Dep = PV.DependsList(); for (; Dep.end() != true;) @@ -1210,14 +1244,6 @@ bool pkgDepCache::MarkInstall(PkgIterator const &Pkg,bool AutoInst, verlist.erase(InstVer); continue; } - // now check if we should consider it a automatic dependency or not - if(InstPkg->CurrentVer == 0 && PinNeverMarkAutoSection) - { - if(DebugAutoInstall == true) - std::clog << OutputInDepth(Depth) << "Setting NOT as auto-installed (direct " - << Start.DepType() << " of pkg in APT::Never-MarkAuto-Sections)" << std::endl; - MarkAuto(InstPkg, false); - } break; } while(true); continue; diff --git a/test/integration/framework b/test/integration/framework index 2f08c5fdc..2efe7439e 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1324,6 +1324,17 @@ testmarkedauto() { fi aptmark showauto 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail } +testmarkedmanual() { + local COMPAREFILE="${TMPWORKINGDIRECTORY}/rootdir/tmp/testmarkedmanual.comparefile" + if [ -n "$1" ]; then + msgtest 'Test for correctly marked as manually installed' "$*" + while [ -n "$1" ]; do echo "$1"; shift; done | sort > $COMPAREFILE + else + msgtest 'Test for correctly marked as manually installed' 'no package' + echo -n > $COMPAREFILE + fi + aptmark showmanual 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail +} msgfailoutput() { local MSG="$1" diff --git a/test/integration/test-apt-never-markauto-sections b/test/integration/test-apt-never-markauto-sections index 6c88c69fa..a469b4c15 100755 --- a/test/integration/test-apt-never-markauto-sections +++ b/test/integration/test-apt-never-markauto-sections @@ -9,22 +9,6 @@ configarchitecture 'amd64' 'i386' aptconfig dump --no-empty --format '%v%n' APT::Never-MarkAuto-Sections > nevermarkauto.sections testsuccess grep '^metapackages$' nevermarkauto.sections -# this is a very crude regression test, not a "this is how it should be" test: -# In theory mydesktop-core and texteditor should be marked as manual, but -# texteditor is installed as a dependency of bad-texteditor, not of -# mydesktop-core and mydesktop-core is removed while bad-texteditor is -# installed losing the manual bit as the problem resolver will later decide to -# drop bad-texteditor and re-instate mydesktop-core which is considered an -# auto-install at that point (in theory the never-auto handling should be -# copied to this place – as to the many other places dependencies are resolved -# 'by hand' instead of via MarkInstall AutoInst… -# -# Both could be fixed if apt would figure out early that installing -# bad-texteditor is a bad idea and eventually it should (as mydesktop-core is -# a direct descendant of mydesktop which was a user-request mydesktop-core should -# be as protected from removal as mydesktop is), but this is hard in the general case -# as with more or-groups and provides you can produce 'legal' examples for this. - buildsimplenativepackage 'mydesktop' 'all' '1' 'unstable' 'Depends: mydesktop-core, foreignpkg Recommends: notavailable' '' 'metapackages' buildsimplenativepackage 'mydesktop-core' 'amd64' '1' 'unstable' 'Depends: bad-texteditor | texteditor, browser (>= 42), nosection, foreignpkg @@ -45,21 +29,21 @@ testequal 'dpkg' aptmark showmanual testsuccess aptget install mydesktop -y -o Debug::pkgProblemResolver=1 -o Debug::pkgDepCache::Marker=1 -testequal 'browser -dpkg -foreignpkg:i386 -mydesktop -nosection' aptmark showmanual -testmarkedauto 'mydesktop-core' 'texteditor' +testmarkedmanual 'dpkg' 'mydesktop' +testmarkedauto 'mydesktop-core' 'foreignpkg:i386' 'texteditor' 'browser' 'nosection' +# if the remove is from a user, don't do manual-bit passing testequal 'Reading package lists... Building dependency tree... Reading state information... The following packages will be REMOVED: - mydesktop mydesktop-core texteditor -0 upgraded, 0 newly installed, 3 to remove and 0 not upgraded. + browser foreignpkg:i386 mydesktop mydesktop-core nosection texteditor +0 upgraded, 0 newly installed, 6 to remove and 0 not upgraded. Remv mydesktop [1] Remv mydesktop-core [1] +Remv browser [42] +Remv foreignpkg:i386 [1] +Remv nosection [1] Remv texteditor [1]' aptget autoremove mydesktop -s testequal 'Reading package lists... @@ -70,37 +54,33 @@ The following packages will be REMOVED: 0 upgraded, 0 newly installed, 3 to remove and 0 not upgraded. Remv mydesktop [1] Remv mydesktop-core [1] -Remv texteditor [1]' aptget autoremove texteditor -s +Remv texteditor [1]' aptget autoremove texteditor -s #-o Debug::pkgDepCache::AutoInstall=1 -o Debug::pkgProblemResolver=1 -o Debug::pkgDepCache::Marker=1 testsuccess aptget autoremove texteditor -y testdpkgnotinstalled mydesktop mydesktop-core texteditor testdpkginstalled browser -testequal 'browser -dpkg -foreignpkg:i386 -nosection' aptmark showmanual +testmarkedmanual 'browser' 'dpkg' 'foreignpkg:i386' 'nosection' testmarkedauto # test that installed/upgraded auto-pkgs are not set to manual testsuccess aptget install browser=41 -y --force-yes -testequal 'browser -dpkg -foreignpkg:i386 -nosection' aptmark showmanual +testmarkedmanual 'browser' 'dpkg' 'foreignpkg:i386' 'nosection' testmarkedauto testsuccess aptmark auto browser testmarkedauto 'browser' testsuccess aptmark auto nosection testmarkedauto 'browser' 'nosection' -testequal 'dpkg -foreignpkg:i386' aptmark showmanual +testmarkedmanual 'dpkg' 'foreignpkg:i386' + +# nosection should be auto, not manual, but is marked as such by the resolver +# removing mydesktop-core temporally… the resolver should be figuring out here +# that there is no point of removing mydesktop-core as its an unavoidable +# dependency of the user-requested mydesktop -testsuccess aptget install mydesktop -y +testsuccess aptget install mydesktop -y -o Debug::pkgProblemResolver=1 -o Debug::pkgDepCache::Marker=1 -o Debug::pkgDepCache::AutoInstall=1 -testequal 'dpkg -foreignpkg:i386 -mydesktop' aptmark showmanual -testmarkedauto 'browser' 'nosection' 'mydesktop-core' 'texteditor' +testmarkedmanual 'dpkg' 'foreignpkg:i386' 'mydesktop' 'nosection' +testmarkedauto 'browser' 'mydesktop-core' 'texteditor' -- cgit v1.2.3-70-g09d2 From 3261271e8e813b883917f0f57031ffcecbce6e20 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 10 Aug 2015 19:00:16 +0200 Subject: travis: add ppa:ubuntu-toolschain-r/test as source for gcc-5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes travis-ci able to run our tests again. Sometimes. If it doesn't spontaneously fails with internal gcc errors… Git-Dch: Ignore --- .travis.yml | 6 +++++- test/integration/framework | 13 +++++++++++-- test/integration/test-apt-update-filesize-mismatch | 2 +- test/integration/test-apt-update-hashsum-mismatch | 2 +- 4 files changed, 18 insertions(+), 5 deletions(-) (limited to 'test/integration/framework') diff --git a/.travis.yml b/.travis.yml index b449aeba5..a20018a79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: cpp before_install: + - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y - sudo apt-get update -q +install: - sudo ./prepare-release travis-ci -script: make && make test && test/integration/run-tests + - export CC=gcc-5 + - export CXX=g++-5 +script: make -j1 && make test && test/integration/run-tests diff --git a/test/integration/framework b/test/integration/framework index 2efe7439e..b443f2a7b 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1377,8 +1377,17 @@ testsuccess() { if expr match "$1" '^apt.*' >/dev/null; then if grep -q -E ' runtime error: ' "$OUTPUT"; then msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@" - elif grep -q -E '^[WE]: ' "$OUTPUT"; then - msgfailoutput 'successful run, but output contains warnings/errors' "$OUTPUT" "$@" + elif grep -E '^[WE]: ' "$OUTPUT" > "${TMPWORKINGDIRECTORY}/rootdir/tmp/checkforwarnings.output" 2>&1; then + if [ "$IGNORE_PTY_NOT_MOUNTED" = '1' ]; then + if echo 'E: Can not write log (Is /dev/pts mounted?) - posix_openpt (2: No such file or directory)' \ + | cmp - "${TMPWORKINGDIRECTORY}/rootdir/tmp/checkforwarnings.output" >/dev/null 2>&1; then + msgpass + else + msgfailoutput 'successful run, but output contains warnings/errors' "$OUTPUT" "$@" + fi + else + msgfailoutput 'successful run, but output contains warnings/errors' "$OUTPUT" "$@" + fi else msgpass fi diff --git a/test/integration/test-apt-update-filesize-mismatch b/test/integration/test-apt-update-filesize-mismatch index f78b83b5f..a23c03c3f 100755 --- a/test/integration/test-apt-update-filesize-mismatch +++ b/test/integration/test-apt-update-filesize-mismatch @@ -40,7 +40,7 @@ for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver. testfailure aptget update -o Debug::pkgAcquire::Worker=1 cp rootdir/tmp/testfailure.output rootdir/tmp/update.output - testsuccess grep -E "$(basename -s '.gz' "$COMPRESSFILE").*Hash Sum mismatch" rootdir/tmp/update.output + testsuccess grep -E "$(basename "$COMPRESSFILE" '.gz').*Hash Sum mismatch" rootdir/tmp/update.output testfailure aptcache show foo testfailure aptget install foo -s diff --git a/test/integration/test-apt-update-hashsum-mismatch b/test/integration/test-apt-update-hashsum-mismatch index c2c5b3887..4627f7afd 100755 --- a/test/integration/test-apt-update-hashsum-mismatch +++ b/test/integration/test-apt-update-hashsum-mismatch @@ -35,7 +35,7 @@ for get in $(sed -n 's#^GET /\([^ ]\+\.gz\) HTTP.\+$#\1#p' aptarchive/webserver. testfailure aptget update cp rootdir/tmp/testfailure.output rootdir/tmp/update.output - testsuccess grep -E "$(basename -s '.gz' "$get").*Hash Sum mismatch" rootdir/tmp/update.output + testsuccess grep -E "$(basename "$get" '.gz').*Hash Sum mismatch" rootdir/tmp/update.output testfailure aptcache show foo testfailure aptget install foo -s -- cgit v1.2.3-70-g09d2