From 6c55f07a5fa3612a5d59c61a17da5fe640eadc8b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 17 Jun 2015 09:29:00 +0200 Subject: make all d-pointer * const pointers Doing this disables the implicit copy assignment operator (among others) which would cause hovac if used on the classes as it would just copy the pointer, not the data the d-pointer points to. For most of the classes we don't need a copy assignment operator anyway and in many classes it was broken before as many contain a pointer of some sort. Only for our Cacheset Container interfaces we define an explicit copy assignment operator which could later be implemented to copy the data from one d-pointer to the other if we need it. Git-Dch: Ignore --- apt-pkg/sourcelist.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'apt-pkg/sourcelist.cc') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 8b960572b..99b646513 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -236,11 +236,11 @@ bool pkgSourceList::Type::ParseLine(vector &List, // SourceList::pkgSourceList - Constructors /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgSourceList::pkgSourceList() +pkgSourceList::pkgSourceList() : d(NULL) { } -pkgSourceList::pkgSourceList(string File) +pkgSourceList::pkgSourceList(string File) : d(NULL) { Read(File); } -- cgit v1.2.3-70-g09d2 From 463c8d801595ce5ac94d7c032264820be7434232 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 21 Jun 2015 16:47:53 +0200 Subject: support lang= and target= sources.list options We support arch= for a while, now we finally add lang= as well and as a first simple way of controlling which targets to acquire also target=. This asked for a redesign of the internal API of parsing and storing information about 'deb' and 'deb-src' lines. As this API isn't visible to the outside no damage done through. Beside being a nice cleanup (= it actually does more in less lines) it also provides us with a predictable order of architectures as provides in the configuration rather than based on string sorting-order, so that now the native architecture is parsed/displayed first. Observeable e.g. in apt-get output. --- apt-pkg/deb/debindexfile.cc | 6 +- apt-pkg/deb/debmetaindex.cc | 430 +++++++++------------ apt-pkg/deb/debmetaindex.h | 38 +- apt-pkg/sourcelist.cc | 40 +- apt-pkg/sourcelist.h | 8 +- cmdline/apt-get.cc | 4 +- doc/sources.list.5.xml | 16 + .../test-bug-632221-cross-dependency-satisfaction | 22 +- .../test-ignore-provides-if-versioned-breaks | 10 +- .../test-ignore-provides-if-versioned-conflicts | 10 +- .../test-sourceslist-lang-plusminus-options | 87 +++++ 11 files changed, 354 insertions(+), 317 deletions(-) create mode 100755 test/integration/test-sourceslist-lang-plusminus-options (limited to 'apt-pkg/sourcelist.cc') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 29a9a941c..c5086b04b 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -510,7 +510,7 @@ class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type { return new debDebFileRecordParser(File.FileName()); }; - debIFTypeDebPkgFile() {Label = "deb Package file";}; + debIFTypeDebPkgFile() {Label = "Debian deb file";}; }; class APT_HIDDEN debIFTypeDscFile : public pkgIndexFile::Type { @@ -519,7 +519,7 @@ class APT_HIDDEN debIFTypeDscFile : public pkgIndexFile::Type { return new debDscRecordParser(DscFile, NULL); }; - debIFTypeDscFile() {Label = "dsc File Source Index";}; + debIFTypeDscFile() {Label = "Debian dsc file";}; }; class APT_HIDDEN debIFTypeDebianSourceDir : public pkgIndexFile::Type { @@ -528,7 +528,7 @@ class APT_HIDDEN debIFTypeDebianSourceDir : public pkgIndexFile::Type { return new debDscRecordParser(SourceDir + string("/debian/control"), NULL); }; - debIFTypeDebianSourceDir() {Label = "debian/control File Source Index";}; + debIFTypeDebianSourceDir() {Label = "Debian control file";}; }; APT_HIDDEN debIFTypeSrc _apt_Src; diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 5a517b290..f690a8d64 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -29,11 +29,25 @@ #include #include -using namespace std; - -string debReleaseIndex::MetaIndexInfo(const char *Type) const +class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ +{ + public: + struct APT_HIDDEN debSectionEntry + { + std::string Name; + std::vector Targets; + std::vector Architectures; + std::vector Languages; + }; + + std::vector DebEntries; + std::vector DebSrcEntries; +}; + /*}}}*/ +// ReleaseIndex::MetaIndex* - display helpers /*{{{*/ +std::string debReleaseIndex::MetaIndexInfo(const char *Type) const { - string Info = ::URI::ArchiveOnly(URI) + ' '; + std::string Info = ::URI::ArchiveOnly(URI) + ' '; if (Dist[Dist.size() - 1] == '/') { if (Dist != "/") @@ -50,15 +64,15 @@ std::string debReleaseIndex::Describe() const return MetaIndexInfo("Release"); } -string debReleaseIndex::MetaIndexFile(const char *Type) const +std::string debReleaseIndex::MetaIndexFile(const char *Type) const { return _config->FindDir("Dir::State::lists") + URItoFileName(MetaIndexURI(Type)); } -string debReleaseIndex::MetaIndexURI(const char *Type) const +std::string debReleaseIndex::MetaIndexURI(const char *Type) const { - string Res; + std::string Res; if (Dist == "/") Res = URI; @@ -70,8 +84,8 @@ string debReleaseIndex::MetaIndexURI(const char *Type) const Res += Type; return Res; } - -std::string debReleaseIndex::LocalFileName() const + /*}}}*/ +std::string debReleaseIndex::LocalFileName() const /*{{{*/ { // see if we have a InRelease file std::string PathInRelease = MetaIndexFile("InRelease"); @@ -84,28 +98,24 @@ std::string debReleaseIndex::LocalFileName() const return ""; } - -debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) : - metaIndex(URI, Dist, "deb"), d(NULL), Trusted(CHECK_TRUST) + /*}}}*/ +// ReleaseIndex Con- and Destructors /*{{{*/ +debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist) : + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()), Trusted(CHECK_TRUST) {} - -debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist, bool const Trusted) : - metaIndex(URI, Dist, "deb"), d(NULL) { +debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted) : + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()) { SetTrusted(Trusted); } - debReleaseIndex::~debReleaseIndex() { - for (map >::const_iterator A = ArchEntries.begin(); - A != ArchEntries.end(); ++A) - for (vector::const_iterator S = A->second.begin(); - S != A->second.end(); ++S) - delete *S; + if (d != NULL) + delete d; } - -template -void foreachTarget(std::string const &URI, std::string const &Dist, - std::map > const &ArchEntries, - CallC &Call) + /*}}}*/ +// ReleaseIndex::GetIndexTargets /*{{{*/ +static void GetIndexTargetsFor(char const * const Type, std::string const &URI, std::string const &Dist, + std::vector const &entries, + std::vector &IndexTargets) { bool const flatArchive = (Dist[Dist.length() - 1] == '/'); std::string baseURI = URI; @@ -118,148 +128,101 @@ void foreachTarget(std::string const &URI, std::string const &Dist, baseURI += "dists/" + Dist + "/"; std::string const Release = (Dist == "/") ? "" : Dist; std::string const Site = ::URI::ArchiveOnly(URI); - std::vector lang = APT::Configuration::getLanguages(true); - if (lang.empty()) - lang.push_back("none"); - map >::const_iterator const src = ArchEntries.find("source"); - if (src != ArchEntries.end()) + + for (std::vector::const_iterator E = entries.begin(); E != entries.end(); ++E) { - std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb-src", "", true); - for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) + for (std::vector::const_iterator T = E->Targets.begin(); T != E->Targets.end(); ++T) { -#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb-src::") + *T + "::" + (X)) - std::string const MetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); - std::string const ShortDesc = APT_T_CONFIG("ShortDescription"); - std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); +#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::") + Type + "::" + *T + "::" + (X)) + std::string const tplMetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); + std::string const tplShortDesc = APT_T_CONFIG("ShortDescription"); + std::string const tplLongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb-src::") + *T + "::Optional", true); #undef APT_T_CONFIG - if (MetaKey.empty()) + if (tplMetaKey.empty()) continue; - vector const SectionEntries = src->second; - for (vector::const_iterator I = SectionEntries.begin(); - I != SectionEntries.end(); ++I) + for (std::vector::const_iterator L = E->Languages.begin(); L != E->Languages.end(); ++L) { - for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) + if (*L == "none" && tplMetaKey.find("$(LANGUAGE)") != std::string::npos) + continue; + + for (std::vector::const_iterator A = E->Architectures.begin(); A != E->Architectures.end(); ++A) { - if (*l == "none" && MetaKey.find("$(LANGUAGE)") != std::string::npos) - continue; std::map Options; Options.insert(std::make_pair("SITE", Site)); Options.insert(std::make_pair("RELEASE", Release)); - if (MetaKey.find("$(COMPONENT)") != std::string::npos) - Options.insert(std::make_pair("COMPONENT", (*I)->Section)); - if (MetaKey.find("$(LANGUAGE)") != std::string::npos) - Options.insert(std::make_pair("LANGUAGE", *l)); - Options.insert(std::make_pair("ARCHITECTURE", "source")); + if (tplMetaKey.find("$(COMPONENT)") != std::string::npos) + Options.insert(std::make_pair("COMPONENT", E->Name)); + if (tplMetaKey.find("$(LANGUAGE)") != std::string::npos) + Options.insert(std::make_pair("LANGUAGE", *L)); + if (tplMetaKey.find("$(ARCHITECTURE)") != std::string::npos) + Options.insert(std::make_pair("ARCHITECTURE", *A)); Options.insert(std::make_pair("BASE_URI", baseURI)); Options.insert(std::make_pair("REPO_URI", URI)); Options.insert(std::make_pair("TARGET_OF", "deb-src")); Options.insert(std::make_pair("CREATED_BY", *T)); - Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); - if (MetaKey.find("$(LANGUAGE)") == std::string::npos) + std::string MetaKey = tplMetaKey; + std::string ShortDesc = tplShortDesc; + std::string LongDesc = tplLongDesc; + for (std::map::const_iterator O = Options.begin(); O != Options.end(); ++O) + { + MetaKey = SubstVar(MetaKey, std::string("$(") + O->first + ")", O->second); + ShortDesc = SubstVar(ShortDesc, std::string("$(") + O->first + ")", O->second); + LongDesc = SubstVar(LongDesc, std::string("$(") + O->first + ")", O->second); + } + IndexTarget Target( + MetaKey, + ShortDesc, + LongDesc, + Options.find("BASE_URI")->second + MetaKey, + IsOptional, + Options + ); + IndexTargets.push_back(Target); + + if (tplMetaKey.find("$(ARCHITECTURE)") == std::string::npos) break; - } - - if (MetaKey.find("$(COMPONENT)") == std::string::npos) - break; - } - } - } - - std::vector const targets = _config->FindVector("APT::Acquire::Targets::deb", "", true); - for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T) - { -#define APT_T_CONFIG(X) _config->Find(std::string("APT::Acquire::Targets::deb::") + *T + "::" + (X)) - std::string const MetaKey = APT_T_CONFIG(flatArchive ? "flatMetaKey" : "MetaKey"); - std::string const ShortDesc = APT_T_CONFIG("ShortDescription"); - std::string const LongDesc = APT_T_CONFIG(flatArchive ? "flatDescription" : "Description"); - bool const IsOptional = _config->FindB(std::string("APT::Acquire::Targets::deb::") + *T + "::Optional", true); -#undef APT_T_CONFIG - if (MetaKey.empty()) - continue; - - for (map >::const_iterator a = ArchEntries.begin(); - a != ArchEntries.end(); ++a) - { - if (a->first == "source") - continue; - - for (vector ::const_iterator I = a->second.begin(); - I != a->second.end(); ++I) { - - for (vector::const_iterator l = lang.begin(); l != lang.end(); ++l) - { - if (*l == "none" && MetaKey.find("$(LANGUAGE)") != std::string::npos) - continue; - - std::map Options; - Options.insert(std::make_pair("SITE", Site)); - Options.insert(std::make_pair("RELEASE", Release)); - if (MetaKey.find("$(COMPONENT)") != std::string::npos) - Options.insert(std::make_pair("COMPONENT", (*I)->Section)); - if (MetaKey.find("$(LANGUAGE)") != std::string::npos) - Options.insert(std::make_pair("LANGUAGE", *l)); - if (MetaKey.find("$(ARCHITECTURE)") != std::string::npos) - Options.insert(std::make_pair("ARCHITECTURE", a->first)); - Options.insert(std::make_pair("BASE_URI", baseURI)); - Options.insert(std::make_pair("REPO_URI", URI)); - Options.insert(std::make_pair("TARGET_OF", "deb")); - Options.insert(std::make_pair("CREATED_BY", *T)); - Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options); - if (MetaKey.find("$(LANGUAGE)") == std::string::npos) - break; } - if (MetaKey.find("$(COMPONENT)") == std::string::npos) + if (tplMetaKey.find("$(LANGUAGE)") == std::string::npos) break; + } - if (MetaKey.find("$(ARCHITECTURE)") == std::string::npos) - break; } } } - - -struct ComputeIndexTargetsClass -{ - vector IndexTargets; - - void operator()(std::string MetaKey, std::string ShortDesc, std::string LongDesc, - bool const IsOptional, std::map Options) - { - for (std::map::const_iterator O = Options.begin(); O != Options.end(); ++O) - { - MetaKey = SubstVar(MetaKey, std::string("$(") + O->first + ")", O->second); - ShortDesc = SubstVar(ShortDesc, std::string("$(") + O->first + ")", O->second); - LongDesc = SubstVar(LongDesc, std::string("$(") + O->first + ")", O->second); - } - IndexTarget Target( - MetaKey, - ShortDesc, - LongDesc, - Options.find("BASE_URI")->second + MetaKey, - IsOptional, - Options - ); - IndexTargets.push_back(Target); - } -}; - std::vector debReleaseIndex::GetIndexTargets() const { - ComputeIndexTargetsClass comp; - foreachTarget(URI, Dist, ArchEntries, comp); - return comp.IndexTargets; + std::vector IndexTargets; + GetIndexTargetsFor("deb-src", URI, Dist, d->DebSrcEntries, IndexTargets); + GetIndexTargetsFor("deb", URI, Dist, d->DebEntries, IndexTargets); + return IndexTargets; } + /*}}}*/ +void debReleaseIndex::AddComponent(bool const isSrc, std::string const &Name,/*{{{*/ + std::vector const &Targets, + std::vector const &Architectures, + std::vector Languages) +{ + if (Languages.empty() == true) + Languages.push_back("none"); + debReleaseIndexPrivate::debSectionEntry const entry = { + Name, Targets, Architectures, Languages + }; + if (isSrc) + d->DebSrcEntries.push_back(entry); + else + d->DebEntries.push_back(entry); +} + /*}}}*/ - /*}}}*/ -bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const +bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const/*{{{*/ { indexRecords * const iR = new indexRecords(Dist); if (Trusted == ALWAYS_TRUSTED) @@ -282,7 +245,8 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const return true; } - + /*}}}*/ +// ReleaseIndex::*Trusted setters and checkers /*{{{*/ void debReleaseIndex::SetTrusted(bool const Trusted) { if (Trusted == true) @@ -290,7 +254,6 @@ void debReleaseIndex::SetTrusted(bool const Trusted) else this->Trusted = NEVER_TRUSTED; } - bool debReleaseIndex::IsTrusted() const { if (Trusted == ALWAYS_TRUSTED) @@ -303,19 +266,13 @@ bool debReleaseIndex::IsTrusted() const if(URI.substr(0,strlen("cdrom:")) == "cdrom:") return true; - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(MetaIndexURI("Release")) + ".gpg"; - - if (FileExists(VerifiedSigFile)) + if (FileExists(MetaIndexFile("Release.gpg"))) return true; - VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(MetaIndexURI("InRelease")); - - return FileExists(VerifiedSigFile); + return FileExists(MetaIndexFile("InRelease")); } - -std::vector *debReleaseIndex::GetIndexFiles() + /*}}}*/ +std::vector *debReleaseIndex::GetIndexFiles() /*{{{*/ { if (Indexes != NULL) return Indexes; @@ -335,23 +292,9 @@ std::vector *debReleaseIndex::GetIndexFiles() } return Indexes; } + /*}}}*/ -void debReleaseIndex::PushSectionEntry(vector const &Archs, const debSectionEntry *Entry) { - for (vector::const_iterator a = Archs.begin(); - a != Archs.end(); ++a) - ArchEntries[*a].push_back(new debSectionEntry(Entry->Section, Entry->IsSrc)); - delete Entry; -} - -void debReleaseIndex::PushSectionEntry(string const &Arch, const debSectionEntry *Entry) { - ArchEntries[Arch].push_back(Entry); -} - -debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section, - bool const &IsSrc): Section(Section), IsSrc(IsSrc) -{} - -static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile) +static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile)/*{{{*/ { ReleaseFile = That->MetaIndexFile("InRelease"); bool releaseExists = false; @@ -365,7 +308,7 @@ static bool ReleaseFileName(debReleaseIndex const * const That, std::string &Rel } return releaseExists; } - + /*}}}*/ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/ { std::string ReleaseFile; @@ -459,46 +402,46 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool con } /*}}}*/ -debDebFileMetaIndex::~debDebFileMetaIndex() {} - -class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type +static std::vector parsePlusMinusOptions(std::string const &Name, /*{{{*/ + std::map const &Options, std::vector const &defaultValues) { - protected: + std::map::const_iterator val = Options.find(Name); + std::vector Values; + if (val != Options.end()) + Values = VectorizeString(val->second, ','); + else + Values = defaultValues; - bool CreateItemInternal(vector &List, string const &URI, - string const &Dist, string const &Section, - bool const &IsSrc, map const &Options) const + if ((val = Options.find(Name + "+")) != Options.end()) { - // parse arch=, arch+= and arch-= settings - map::const_iterator arch = Options.find("arch"); - vector Archs; - if (arch != Options.end()) - Archs = VectorizeString(arch->second, ','); - else - Archs = APT::Configuration::getArchitectures(); - - if ((arch = Options.find("arch+")) != Options.end()) - { - std::vector const plusArch = VectorizeString(arch->second, ','); - for (std::vector::const_iterator plus = plusArch.begin(); plus != plusArch.end(); ++plus) - if (std::find(Archs.begin(), Archs.end(), *plus) == Archs.end()) - Archs.push_back(*plus); - } - if ((arch = Options.find("arch-")) != Options.end()) + std::vector const plusArch = VectorizeString(val->second, ','); + for (std::vector::const_iterator plus = plusArch.begin(); plus != plusArch.end(); ++plus) + if (std::find(Values.begin(), Values.end(), *plus) == Values.end()) + Values.push_back(*plus); + } + if ((val = Options.find(Name + "-")) != Options.end()) + { + std::vector const minusArch = VectorizeString(val->second, ','); + for (std::vector::const_iterator minus = minusArch.begin(); minus != minusArch.end(); ++minus) { - std::vector const minusArch = VectorizeString(arch->second, ','); - for (std::vector::const_iterator minus = minusArch.begin(); minus != minusArch.end(); ++minus) - { - std::vector::iterator kill = std::find(Archs.begin(), Archs.end(), *minus); - if (kill != Archs.end()) - Archs.erase(kill); - } + std::vector::iterator kill = std::find(Values.begin(), Values.end(), *minus); + if (kill != Values.end()) + Values.erase(kill); } + } + return Values; +} + /*}}}*/ +class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ +{ + protected: - map::const_iterator const trusted = Options.find("trusted"); - + bool CreateItemInternal(std::vector &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + bool const &IsSrc, std::map const &Options) const + { debReleaseIndex *Deb = NULL; - for (vector::const_iterator I = List.begin(); + for (std::vector::const_iterator I = List.begin(); I != List.end(); ++I) { // We only worry about debian entries here @@ -523,87 +466,86 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type List.push_back(Deb); } - if (IsSrc == true) - Deb->PushSectionEntry ("source", new debReleaseIndex::debSectionEntry(Section, IsSrc)); - else - { - if (Dist[Dist.size() - 1] == '/') - Deb->PushSectionEntry ("any", new debReleaseIndex::debSectionEntry(Section, IsSrc)); - else - Deb->PushSectionEntry (Archs, new debReleaseIndex::debSectionEntry(Section, IsSrc)); - } + Deb->AddComponent( + IsSrc, + Section, + parsePlusMinusOptions("target", Options, _config->FindVector(std::string("APT::Acquire::Targets::") + Name, "", true)), + parsePlusMinusOptions("arch", Options, APT::Configuration::getArchitectures()), + parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)) + ); + std::map::const_iterator const trusted = Options.find("trusted"); if (trusted != Options.end()) Deb->SetTrusted(StringToBool(trusted->second, false)); return true; } -}; - -debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) - : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) -{ - DebIndex = new debDebPkgFileIndex(DebFile); - Indexes = new vector(); - Indexes->push_back(DebIndex); -} - -class APT_HIDDEN debSLTypeDeb : public debSLTypeDebian + debSLTypeDebian(char const * const Name, char const * const Label) : Type(Name, Label) + { + } +}; + /*}}}*/ +class APT_HIDDEN debSLTypeDeb : public debSLTypeDebian /*{{{*/ { public: - bool CreateItem(vector &List, string const &URI, - string const &Dist, string const &Section, - std::map const &Options) const + bool CreateItem(std::vector &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + std::map const &Options) const { return CreateItemInternal(List, URI, Dist, Section, false, Options); } - debSLTypeDeb() + debSLTypeDeb() : debSLTypeDebian("deb", "Debian binary tree") { - Name = "deb"; - Label = "Standard Debian binary tree"; - } + } }; - -class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian + /*}}}*/ +class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/ { public: - bool CreateItem(vector &List, string const &URI, - string const &Dist, string const &Section, - std::map const &Options) const + bool CreateItem(std::vector &List, std::string const &URI, + std::string const &Dist, std::string const &Section, + std::map const &Options) const { return CreateItemInternal(List, URI, Dist, Section, true, Options); } - - debSLTypeDebSrc() + + debSLTypeDebSrc() : debSLTypeDebian("deb-src", "Debian source tree") { - Name = "deb-src"; - Label = "Standard Debian source tree"; - } + } }; + /*}}}*/ -class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type +debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) /*{{{*/ + : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) +{ + DebIndex = new debDebPkgFileIndex(DebFile); + Indexes = new std::vector(); + Indexes->push_back(DebIndex); +} +debDebFileMetaIndex::~debDebFileMetaIndex() {} + /*}}}*/ +class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type /*{{{*/ { public: - bool CreateItem(vector &List, string const &URI, - string const &/*Dist*/, string const &/*Section*/, - std::map const &/*Options*/) const + bool CreateItem(std::vector &List, std::string const &URI, + std::string const &/*Dist*/, std::string const &/*Section*/, + std::map const &/*Options*/) const { metaIndex *mi = new debDebFileMetaIndex(URI); List.push_back(mi); return true; } - - debSLTypeDebFile() + + debSLTypeDebFile() : Type("deb-file", "Debian local deb file") { - Name = "deb-file"; - Label = "Debian Deb File"; - } + } }; + /*}}}*/ APT_HIDDEN debSLTypeDeb _apt_DebType; APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType; diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 648c22436..9b60b6137 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -1,4 +1,3 @@ -// ijones, walters #ifndef PKGLIB_DEBMETAINDEX_H #define PKGLIB_DEBMETAINDEX_H @@ -22,26 +21,20 @@ class debDebPkgFileIndex; class IndexTarget; class pkgCacheGenerator; class OpProgress; +class debReleaseIndexPrivate; -class APT_HIDDEN debReleaseIndex : public metaIndex { - public: +class APT_HIDDEN debReleaseIndex : public metaIndex +{ + debReleaseIndexPrivate * const d; - class debSectionEntry - { - public: - debSectionEntry (std::string const &Section, bool const &IsSrc); - std::string const Section; - bool const IsSrc; - }; - - private: - /** \brief dpointer placeholder (for later in case we need it) */ - void * const d; - std::map > ArchEntries; enum APT_HIDDEN { ALWAYS_TRUSTED, NEVER_TRUSTED, CHECK_TRUST } Trusted; public: + APT_HIDDEN std::string MetaIndexInfo(const char *Type) const; + APT_HIDDEN std::string MetaIndexFile(const char *Types) const; + APT_HIDDEN std::string MetaIndexURI(const char *Type) const; + debReleaseIndex(std::string const &URI, std::string const &Dist); debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted); virtual ~debReleaseIndex(); @@ -54,22 +47,17 @@ class APT_HIDDEN debReleaseIndex : public metaIndex { virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const; virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; - std::string MetaIndexInfo(const char *Type) const; - std::string MetaIndexFile(const char *Types) const; - std::string MetaIndexURI(const char *Type) const; - -#if APT_PKG_ABI >= 413 - virtual -#endif - std::string LocalFileName() const; + virtual std::string LocalFileName() const; virtual std::vector *GetIndexFiles(); void SetTrusted(bool const Trusted); virtual bool IsTrusted() const; - void PushSectionEntry(std::vector const &Archs, const debSectionEntry *Entry); - void PushSectionEntry(std::string const &Arch, const debSectionEntry *Entry); + void AddComponent(bool const isSrc, std::string const &Name, + std::vector const &Targets, + std::vector const &Architectures, + std::vector Languages); }; class APT_HIDDEN debDebFileMetaIndex : public metaIndex diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 99b646513..6ef99863d 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -37,25 +37,26 @@ using namespace std; // Global list of Items supported -static pkgSourceList::Type *ItmList[10]; +static pkgSourceList::Type *ItmList[10]; pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList; unsigned long pkgSourceList::Type::GlobalListLen = 0; // Type::Type - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Link this to the global list of items*/ -pkgSourceList::Type::Type() : Name(NULL), Label(NULL) +pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel) { ItmList[GlobalListLen] = this; - GlobalListLen++; + ++GlobalListLen; } +pkgSourceList::Type::~Type() {} /*}}}*/ // Type::GetType - Get a specific meta for a given type /*{{{*/ // --------------------------------------------------------------------- /* */ pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type) { - for (unsigned I = 0; I != GlobalListLen; I++) + for (unsigned I = 0; I != GlobalListLen; ++I) if (strcmp(GlobalList[I]->Name,Type) == 0) return GlobalList[I]; return 0; @@ -91,23 +92,26 @@ bool pkgSourceList::Type::ParseStanza(vector &List, string Enabled = Tags.FindS("Enabled"); if (Enabled.size() > 0 && StringToBool(Enabled) == false) return true; - - // Define external/internal options - const char* option_deb822[] = { - "Architectures", "Architectures-Add", "Architectures-Remove", "Trusted", - }; - const char* option_internal[] = { - "arch", "arch+", "arch-", "trusted", - }; - for (unsigned int j=0; j < sizeof(option_deb822)/sizeof(char*); j++) - if (Tags.Exists(option_deb822[j])) + + std::map mapping; +#define APT_PLUSMINUS(X, Y) \ + mapping.insert(std::make_pair(X, Y)); \ + mapping.insert(std::make_pair(X "Add", Y "+")); \ + mapping.insert(std::make_pair(X "Remove", Y "-")) + APT_PLUSMINUS("Architectures", "arch"); + APT_PLUSMINUS("Languages", "lang"); + APT_PLUSMINUS("Targets", "target"); +#undef APT_PLUSMINUS + mapping.insert(std::make_pair("Trusted", "trusted")); + for (std::map::const_iterator m = mapping.begin(); m != mapping.end(); ++m) + if (Tags.Exists(m->first)) { // for deb822 the " " is the delimiter, but the backend expects "," - std::string option = Tags.FindS(option_deb822[j]); + std::string option = Tags.FindS(m->first); std::replace(option.begin(), option.end(), ' ', ','); - Options[option_internal[j]] = option; + Options[m->second] = option; } - + // now create one item per suite/section string Suite = Tags.FindS("Suites"); Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture")); @@ -209,7 +213,7 @@ bool pkgSourceList::Type::ParseLine(vector &List, if (FixupURI(URI) == false) return _error->Error(_("Malformed line %lu in source list %s (URI parse)"),CurLine,File.c_str()); - + // Check for an absolute dists specification. if (Dist.empty() == false && Dist[Dist.size() - 1] == '/') { diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index e17ad6a9a..4f42b3e91 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -66,8 +66,8 @@ class pkgSourceList static unsigned long GlobalListLen; static Type *GetType(const char *Type) APT_PURE; - const char *Name; - const char *Label; + char const * const Name; + char const * const Label; bool FixupURI(std::string &URI) const; virtual bool ParseStanza(std::vector &List, @@ -80,8 +80,8 @@ class pkgSourceList virtual bool CreateItem(std::vector &List,std::string const &URI, std::string const &Dist,std::string const &Section, std::map const &Options) const = 0; - Type(); - virtual ~Type() {}; + Type(char const * const Name, char const * const Label); + virtual ~Type(); }; typedef std::vector::const_iterator const_iterator; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 632c7cfea..500a0a3c5 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1045,7 +1045,7 @@ static bool DoBuildDep(CommandLine &CmdL) { ioprintf(c1out, _("Note, using directory '%s' to get the build dependencies\n"), *I); // FIXME: how can we make this more elegant? - std::string TypeName = "debian/control File Source Index"; + std::string TypeName = "Debian control file"; pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); if(Type != NULL) LastOwner = Last = Type->CreateSrcPkgParser(*I); @@ -1056,7 +1056,7 @@ static bool DoBuildDep(CommandLine &CmdL) ioprintf(c1out, _("Note, using file '%s' to get the build dependencies\n"), *I); // see if we can get a parser for this pkgIndexFile type - string TypeName = flExtension(*I) + " File Source Index"; + string TypeName = "Debian " + flExtension(*I) + " file"; pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); if(Type != NULL) LastOwner = Last = Type->CreateSrcPkgParser(*I); diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml index da4f571b5..6ebba3528 100644 --- a/doc/sources.list.5.xml +++ b/doc/sources.list.5.xml @@ -137,9 +137,25 @@ can be used to specify for which architectures information should be downloaded. If this option is not set all architectures defined by the APT::Architectures option will be downloaded. + arch+=arch1,arch2,… and arch-=arch1,arch2,… which can be used to add/remove architectures from the set which will be downloaded. + + lang=lang1,lang2,…, + lang+=lang1,lang2,… and + lang-=lang1,lang2,… functioning in + the same way as the arch-options described before. They can be used to specify for + which languages apt will acquire metadata, like translated package descriptions, for. If not specified, the + default set is defined by the Acquire::Languages config option. + + target=target1,target2,…, + target+=target1,target2,… and + target-=target1,target2,… again functioning in + the same way as the arch-options described before. They can be used to specify which + targets apt will try to acquire from this source. If not specified, the default set is defined by + the APT::Acquire::Targets configuration scope. + trusted=yes can be set to indicate that packages from this source are always authenticated even if the Release file is not signed or the signature can't be checked. This disables parts of &apt-secure; diff --git a/test/integration/test-bug-632221-cross-dependency-satisfaction b/test/integration/test-bug-632221-cross-dependency-satisfaction index 563821173..12704e5de 100755 --- a/test/integration/test-bug-632221-cross-dependency-satisfaction +++ b/test/integration/test-bug-632221-cross-dependency-satisfaction @@ -167,17 +167,17 @@ Conf libfwibble-dev (1.0 unstable [armel])' aptget build-dep apt -s testsuccessequal 'Reading package lists... Building dependency tree... The following NEW packages will be installed: - amdboot:amd64 cool doxygen foreigner libc6:amd64 libc6 libc6-dev:amd64 - libc6-dev libfwibble-dev:amd64 libfwibble1:amd64 linux-stuff:amd64 + amdboot:amd64 cool doxygen foreigner libc6 libc6:amd64 libc6-dev + libc6-dev:amd64 libfwibble-dev:amd64 libfwibble1:amd64 linux-stuff:amd64 0 upgraded, 11 newly installed, 0 to remove and 0 not upgraded. Inst amdboot:amd64 (1.0 unstable [amd64]) Inst cool (1.0 unstable [armel]) Inst doxygen (1.0 unstable [armel]) Inst foreigner (1.0 unstable [armel]) -Inst libc6:amd64 (1.0 unstable [amd64]) Inst libc6 (1.0 unstable [armel]) -Inst libc6-dev:amd64 (1.0 unstable [amd64]) +Inst libc6:amd64 (1.0 unstable [amd64]) Inst libc6-dev (1.0 unstable [armel]) +Inst libc6-dev:amd64 (1.0 unstable [amd64]) Inst libfwibble1:amd64 (1.0 unstable [amd64]) Inst libfwibble-dev:amd64 (1.0 unstable [amd64]) Inst linux-stuff:amd64 (1.0 unstable [amd64]) @@ -185,10 +185,10 @@ Conf amdboot:amd64 (1.0 unstable [amd64]) Conf cool (1.0 unstable [armel]) Conf doxygen (1.0 unstable [armel]) Conf foreigner (1.0 unstable [armel]) -Conf libc6:amd64 (1.0 unstable [amd64]) Conf libc6 (1.0 unstable [armel]) -Conf libc6-dev:amd64 (1.0 unstable [amd64]) +Conf libc6:amd64 (1.0 unstable [amd64]) Conf libc6-dev (1.0 unstable [armel]) +Conf libc6-dev:amd64 (1.0 unstable [amd64]) Conf libfwibble1:amd64 (1.0 unstable [amd64]) Conf libfwibble-dev:amd64 (1.0 unstable [amd64]) Conf linux-stuff:amd64 (1.0 unstable [amd64])' aptget build-dep apt -s -a amd64 @@ -275,24 +275,24 @@ Conf libfwibble-dev (1.0 unstable [armel])' aptget build-dep apt -s testsuccessequal 'Reading package lists... Building dependency tree... The following NEW packages will be installed: - amdboot:amd64 doxygen libc6:amd64 libc6 libc6-dev:amd64 libc6-dev + amdboot:amd64 doxygen libc6 libc6:amd64 libc6-dev libc6-dev:amd64 libfwibble-dev:amd64 libfwibble1:amd64 linux-stuff:amd64 0 upgraded, 9 newly installed, 0 to remove and 2 not upgraded. Inst amdboot:amd64 (1.0 unstable [amd64]) Inst doxygen (1.0 unstable [armel]) -Inst libc6:amd64 (1.0 unstable [amd64]) Inst libc6 (1.0 unstable [armel]) -Inst libc6-dev:amd64 (1.0 unstable [amd64]) +Inst libc6:amd64 (1.0 unstable [amd64]) Inst libc6-dev (1.0 unstable [armel]) +Inst libc6-dev:amd64 (1.0 unstable [amd64]) Inst libfwibble1:amd64 (1.0 unstable [amd64]) Inst libfwibble-dev:amd64 (1.0 unstable [amd64]) Inst linux-stuff:amd64 (1.0 unstable [amd64]) Conf amdboot:amd64 (1.0 unstable [amd64]) Conf doxygen (1.0 unstable [armel]) -Conf libc6:amd64 (1.0 unstable [amd64]) Conf libc6 (1.0 unstable [armel]) -Conf libc6-dev:amd64 (1.0 unstable [amd64]) +Conf libc6:amd64 (1.0 unstable [amd64]) Conf libc6-dev (1.0 unstable [armel]) +Conf libc6-dev:amd64 (1.0 unstable [amd64]) Conf libfwibble1:amd64 (1.0 unstable [amd64]) Conf libfwibble-dev:amd64 (1.0 unstable [amd64]) Conf linux-stuff:amd64 (1.0 unstable [amd64])' aptget build-dep apt -s -a amd64 diff --git a/test/integration/test-ignore-provides-if-versioned-breaks b/test/integration/test-ignore-provides-if-versioned-breaks index 20424b942..4d6114637 100755 --- a/test/integration/test-ignore-provides-if-versioned-breaks +++ b/test/integration/test-ignore-provides-if-versioned-breaks @@ -134,17 +134,17 @@ Conf foo-same-provider (1.0 unstable [i386])' aptget install foo-same-provider f testsuccessequal 'Reading package lists... Building dependency tree... The following extra packages will be installed: - foo-same:amd64 foo-same + foo-same foo-same:amd64 The following NEW packages will be installed: foo-same-breaker-3 foo-same-provider The following packages will be upgraded: - foo-same:amd64 foo-same + foo-same foo-same:amd64 2 upgraded, 2 newly installed, 0 to remove and 2 not upgraded. -Inst foo-same:amd64 [2.0] (4.0 unstable [amd64]) [foo-same:amd64 on foo-same:i386] [foo-same:i386 on foo-same:amd64] [foo-same:i386 ] -Inst foo-same [2.0] (4.0 unstable [i386]) +Inst foo-same [2.0] (4.0 unstable [i386]) [foo-same:i386 on foo-same:amd64] [foo-same:amd64 on foo-same:i386] [foo-same:amd64 ] +Inst foo-same:amd64 [2.0] (4.0 unstable [amd64]) Inst foo-same-breaker-3 (1.0 unstable [i386]) Inst foo-same-provider (1.0 unstable [i386]) -Conf foo-same (4.0 unstable [i386]) Conf foo-same:amd64 (4.0 unstable [amd64]) +Conf foo-same (4.0 unstable [i386]) Conf foo-same-breaker-3 (1.0 unstable [i386]) Conf foo-same-provider (1.0 unstable [i386])' aptget install foo-same-provider foo-same-breaker-3 -s diff --git a/test/integration/test-ignore-provides-if-versioned-conflicts b/test/integration/test-ignore-provides-if-versioned-conflicts index a781d8e44..6a0c924e2 100755 --- a/test/integration/test-ignore-provides-if-versioned-conflicts +++ b/test/integration/test-ignore-provides-if-versioned-conflicts @@ -134,17 +134,17 @@ Conf foo-same-provider (1.0 unstable [i386])' aptget install foo-same-provider f testsuccessequal 'Reading package lists... Building dependency tree... The following extra packages will be installed: - foo-same:amd64 foo-same + foo-same foo-same:amd64 The following NEW packages will be installed: foo-same-breaker-3 foo-same-provider The following packages will be upgraded: - foo-same:amd64 foo-same + foo-same foo-same:amd64 2 upgraded, 2 newly installed, 0 to remove and 2 not upgraded. -Inst foo-same:amd64 [2.0] (4.0 unstable [amd64]) [foo-same:amd64 on foo-same:i386] [foo-same:i386 on foo-same:amd64] [foo-same:i386 ] -Inst foo-same [2.0] (4.0 unstable [i386]) +Inst foo-same [2.0] (4.0 unstable [i386]) [foo-same:i386 on foo-same:amd64] [foo-same:amd64 on foo-same:i386] [foo-same:amd64 ] +Inst foo-same:amd64 [2.0] (4.0 unstable [amd64]) Inst foo-same-breaker-3 (1.0 unstable [i386]) Inst foo-same-provider (1.0 unstable [i386]) -Conf foo-same (4.0 unstable [i386]) Conf foo-same:amd64 (4.0 unstable [amd64]) +Conf foo-same (4.0 unstable [i386]) Conf foo-same-breaker-3 (1.0 unstable [i386]) Conf foo-same-provider (1.0 unstable [i386])' aptget install foo-same-provider foo-same-breaker-3 -s diff --git a/test/integration/test-sourceslist-lang-plusminus-options b/test/integration/test-sourceslist-lang-plusminus-options new file mode 100755 index 000000000..28d5f9e06 --- /dev/null +++ b/test/integration/test-sourceslist-lang-plusminus-options @@ -0,0 +1,87 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'native' + +testlangs() { + msgtest 'Test acquired languages for' "$1" + local LANGS="$2" + shift 2 + rm -f gotlangs.list + aptget files --no-release-info 'Created-By: Translations' "$@" --format '$(LANGUAGE)' | sort -u > gotlangs.list + if [ -z "$LANGS" ]; then + echo -n | tr ',' '\n' | sort | checkdiff - gotlangs.list && msgpass || msgfail + else + echo -n "$LANGS" | tr ',' '\n' | sort | checkdiff - gotlangs.list && msgpass || msgfail + fi +} +echo 'deb http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'default' 'en' + +echo 'Acquire::Languages "environment,en";' > rootdir/etc/apt/apt.conf.d/langs.conf +testlangs 'default config' 'en' + +echo 'Acquire::Languages "en,en,en";' > rootdir/etc/apt/apt.conf.d/langs.conf +testlangs 'duplicated config' 'en' + +echo 'Acquire::Languages "none";' > rootdir/etc/apt/apt.conf.d/langs.conf +testlangs 'none config' '' + +echo 'Acquire::Languages "en,none,de,de_DE";' > rootdir/etc/apt/apt.conf.d/langs.conf +testlangs 'english + german config' 'en,de,de_DE' + +echo 'deb [lang=pt] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang=pt' 'pt' + +echo 'deb [lang=en] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang=en' 'en' + +echo 'deb [lang=de_DE] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang=de_DE' 'de_DE' + +echo 'deb [lang=none] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang=none' '' +testequal 'amd64' aptget files --no-release-info 'Created-By: Packages' --format '$(ARCHITECTURE)' + +echo 'deb [lang+=pt] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang+=pt' 'en,de,de_DE,pt' + +echo 'deb [lang+=en] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang+=en' 'en,de,de_DE' + +echo 'deb [lang+=de_DE] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang+=de_DE' 'en,de,de_DE' + +echo 'deb [lang-=pt] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang-=pt' 'en,de,de_DE' + +echo 'deb [lang-=en] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang-=en' 'de,de_DE' + +echo 'deb [lang-=de_DE] http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list +testlangs 'lang-=de_DE' 'en,de' + +echo 'deb http://example.org/debian stable rocks +deb http://example.org/debian stable solid' > rootdir/etc/apt/sources.list +testlangs 'english + german config multicomponent' 'en,de,de_DE' + +echo 'deb http://example.org/debian stable rocks +deb [lang=pt] http://example.org/debian stable solid' > rootdir/etc/apt/sources.list +testlangs 'multicomponent one lang= combined' 'en,de,de_DE,pt' +testlangs 'multicomponent one lang= rocks' 'en,de,de_DE' 'Component: rocks' +testlangs 'multicomponent one lang= solid' 'pt' 'Component: solid' + +echo 'deb [lang=pt] http://example.org/debian stable rocks +deb [lang=de] http://example.org/debian stable solid' > rootdir/etc/apt/sources.list +testlangs 'multicomponent different lang= combined' 'de,pt' +testlangs 'multicomponent different lang= rocks' 'pt' 'Component: rocks' +testlangs 'multicomponent different lang= solid' 'de' 'Component: solid' + +echo 'deb [lang+=pt] http://example.org/debian stable rocks +deb [lang-=de] http://example.org/debian stable solid' > rootdir/etc/apt/sources.list +testlangs 'multicomponent different lang+-= combined' 'en,de,de_DE,pt' +testlangs 'multicomponent different lang+-= rocks' 'en,de,de_DE,pt' 'Component: rocks' +testlangs 'multicomponent different lang+-= solid' 'en,de_DE' 'Component: solid' -- cgit v1.2.3-70-g09d2 From 81460e32961bb0b9922bf8a1a27d87705d8c3e51 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 21 Jun 2015 23:12:24 +0200 Subject: bring back deb822 sources.list entries as .sources Having two different formats in the same file is very dirty and causes external tools to fail hard trying to parse them. It is probably not a good idea for them to parse them in the first place, but they do and we shouldn't break them if there is a better way. So we solve this issue for now by giving our deb822 format a new filename extension ".sources" which unsupporting applications are likely to ignore an can begin gradually moving forward rather than waiting for the unknown applications to catch up. Currently and for the forseeable future apt is going to support both with the same feature set as documented in the manpage, with the longtime plan of adopting the 'new' format as default, but that is a long way to go and might get going more from having an easier time setting options than from us pushing it explicitely. --- apt-pkg/policy.cc | 19 +- apt-pkg/sourcelist.cc | 228 ++++++++++----------- apt-pkg/sourcelist.h | 16 +- apt-pkg/tagfile.cc | 8 + apt-pkg/tagfile.h | 8 + doc/sources.list.5.xml | 337 +++++++++++++++++++++---------- test/integration/test-apt-sources-deb822 | 75 +++++-- test/libapt/file-helpers.cc | 11 +- test/libapt/sourcelist_test.cc | 15 +- vendor/README | 8 +- vendor/blankon/apt-vendor.ent | 9 + vendor/debian/apt-vendor.ent | 12 ++ vendor/debian/sources.list.in | 3 +- vendor/raspbian/apt-vendor.ent | 6 + vendor/steamos/apt-vendor.ent | 7 + vendor/steamos/sources.list.in | 4 +- vendor/tanglu/apt-vendor.ent | 6 + vendor/ubuntu/apt-vendor.ent | 13 ++ 18 files changed, 500 insertions(+), 285 deletions(-) (limited to 'apt-pkg/sourcelist.cc') diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc index cd48e040c..170da7c63 100644 --- a/apt-pkg/policy.cc +++ b/apt-pkg/policy.cc @@ -396,21 +396,6 @@ APT_PURE signed short pkgPolicy::GetPriority(pkgCache::PkgFileIterator const &Fi return PFPriority[File->ID]; } /*}}}*/ -// PreferenceSection class - Overriding the default TrimRecord method /*{{{*/ -// --------------------------------------------------------------------- -/* The preference file is a user generated file so the parser should - therefore be a bit more friendly by allowing comments and new lines - all over the place rather than forcing a special format */ -class PreferenceSection : public pkgTagSection -{ - void TrimRecord(bool /*BeforeRecord*/, const char* &End) - { - for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r' || Stop[0] == '#'); Stop++) - if (Stop[0] == '#') - Stop = (const char*) memchr(Stop,'\n',End-Stop); - } -}; - /*}}}*/ // ReadPinDir - Load the pin files from this dir into a Policy /*{{{*/ // --------------------------------------------------------------------- /* This will load each pin file in the given dir into a Policy. If the @@ -455,8 +440,8 @@ bool ReadPinFile(pkgPolicy &Plcy,string File) pkgTagFile TF(&Fd); if (_error->PendingError() == true) return false; - - PreferenceSection Tags; + + pkgUserTagSection Tags; while (TF.Step(Tags) == true) { // can happen when there are only comments in a record diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 6ef99863d..69f7ac043 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -82,15 +82,15 @@ bool pkgSourceList::Type::FixupURI(string &URI) const return true; } /*}}}*/ -bool pkgSourceList::Type::ParseStanza(vector &List, +bool pkgSourceList::Type::ParseStanza(vector &List, /*{{{*/ pkgTagSection &Tags, - int i, + unsigned int const i, FileFd &Fd) { map Options; string Enabled = Tags.FindS("Enabled"); - if (Enabled.size() > 0 && StringToBool(Enabled) == false) + if (Enabled.empty() == false && StringToBool(Enabled) == false) return true; std::map mapping; @@ -115,46 +115,63 @@ bool pkgSourceList::Type::ParseStanza(vector &List, // now create one item per suite/section string Suite = Tags.FindS("Suites"); Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture")); - string const Section = Tags.FindS("Sections"); - string URIS = Tags.FindS("URIs"); + string const Component = Tags.FindS("Components"); + string const URIS = Tags.FindS("URIs"); + + std::vector const list_uris = VectorizeString(URIS, ' '); + std::vector const list_suite = VectorizeString(Suite, ' '); + std::vector const list_comp = VectorizeString(Component, ' '); + + if (list_uris.empty()) + // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI"); - std::vector list_uris = StringSplit(URIS, " "); - std::vector list_dist = StringSplit(Suite, " "); - std::vector list_section = StringSplit(Section, " "); - for (std::vector::const_iterator U = list_uris.begin(); U != list_uris.end(); ++U) { - std::string URI = (*U); - if (!FixupURI(URI)) - { - _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str()); - return false; - } + std::string URI = *U; + if (U->empty() || FixupURI(URI) == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse"); + + if (list_suite.empty()) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite"); - for (std::vector::const_iterator I = list_dist.begin(); - I != list_dist.end(); ++I) + for (std::vector::const_iterator S = list_suite.begin(); + S != list_suite.end(); ++S) { - for (std::vector::const_iterator J = list_section.begin(); - J != list_section.end(); ++J) - { - if (CreateItem(List, URI, (*I), (*J), Options) == false) - { - return false; - } - } + if (S->empty() == false && (*S)[S->size() - 1] == '/') + { + if (list_comp.empty() == false) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component"); + if (CreateItem(List, URI, *S, "", Options) == false) + return false; + } + else + { + if (list_comp.empty()) + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component"); + + for (std::vector::const_iterator C = list_comp.begin(); + C != list_comp.end(); ++C) + { + if (CreateItem(List, URI, *S, *C, Options) == false) + { + return false; + } + } + } } } return true; } - + /*}}}*/ // Type::ParseLine - Parse a single line /*{{{*/ // --------------------------------------------------------------------- /* This is a generic one that is the 'usual' format for sources.list Weird types may override this. */ bool pkgSourceList::Type::ParseLine(vector &List, const char *Buffer, - unsigned long const &CurLine, + unsigned int const CurLine, string const &File) const { for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces @@ -171,10 +188,10 @@ bool pkgSourceList::Type::ParseLine(vector &List, // get one option, e.g. option1=value1 string option; if (ParseQuoteWord(Buffer,option) == false) - return _error->Error(_("Malformed line %lu in source list %s ([option] unparseable)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable"); if (option.length() < 3) - return _error->Error(_("Malformed line %lu in source list %s ([option] too short)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short"); // accept options even if the last has no space before the ]-end marker if (option.at(option.length()-1) == ']') @@ -185,16 +202,16 @@ bool pkgSourceList::Type::ParseLine(vector &List, size_t const needle = option.find('='); if (needle == string::npos) - return _error->Error(_("Malformed line %lu in source list %s ([%s] is not an assignment)"),CurLine,File.c_str(), option.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment"); string const key = string(option, 0, needle); string const value = string(option, needle + 1, option.length()); if (key.empty() == true) - return _error->Error(_("Malformed line %lu in source list %s ([%s] has no key)"),CurLine,File.c_str(), option.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key"); if (value.empty() == true) - return _error->Error(_("Malformed line %lu in source list %s ([%s] key %s has no value)"),CurLine,File.c_str(),option.c_str(),key.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value"); Options[key] = value; } @@ -207,33 +224,33 @@ bool pkgSourceList::Type::ParseLine(vector &List, string Section; if (ParseQuoteWord(Buffer,URI) == false) - return _error->Error(_("Malformed line %lu in source list %s (URI)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI"); if (ParseQuoteWord(Buffer,Dist) == false) - return _error->Error(_("Malformed line %lu in source list %s (dist)"),CurLine,File.c_str()); - + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite"); + if (FixupURI(URI) == false) - return _error->Error(_("Malformed line %lu in source list %s (URI parse)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse"); // Check for an absolute dists specification. if (Dist.empty() == false && Dist[Dist.size() - 1] == '/') { if (ParseQuoteWord(Buffer,Section) == true) - return _error->Error(_("Malformed line %lu in source list %s (absolute dist)"),CurLine,File.c_str()); + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component"); Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture")); return CreateItem(List, URI, Dist, Section, Options); } - + // Grab the rest of the dists if (ParseQuoteWord(Buffer,Section) == false) - return _error->Error(_("Malformed line %lu in source list %s (dist parse)"),CurLine,File.c_str()); - + return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component"); + do { if (CreateItem(List, URI, Dist, Section, Options) == false) return false; } while (ParseQuoteWord(Buffer,Section) == true); - + return true; } /*}}}*/ @@ -242,11 +259,6 @@ bool pkgSourceList::Type::ParseLine(vector &List, /* */ pkgSourceList::pkgSourceList() : d(NULL) { -} - -pkgSourceList::pkgSourceList(string File) : d(NULL) -{ - Read(File); } /*}}}*/ // SourceList::~pkgSourceList - Destructor /*{{{*/ @@ -305,7 +317,7 @@ void pkgSourceList::Reset() // SourceList::Read - Parse the sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgSourceList::Read(string File) +bool pkgSourceList::Read(string const &File) { Reset(); return ReadAppend(File); @@ -314,71 +326,63 @@ bool pkgSourceList::Read(string File) // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgSourceList::ReadAppend(string File) +bool pkgSourceList::ReadAppend(string const &File) { - if (_config->FindB("APT::Sources::Use-Deb822", false) == true) - { - int lines_parsed =ParseFileDeb822(File); - if (lines_parsed < 0) - return false; - else if (lines_parsed > 0) - return true; - // no lines parsed ... fall through and use old style parser - } - return ParseFileOldStyle(File); + if (flExtension(File) == "sources") + return ParseFileDeb822(File); + else + return ParseFileOldStyle(File); } // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgSourceList::ParseFileOldStyle(string File) +bool pkgSourceList::ParseFileOldStyle(std::string const &File) { // Open the stream for reading ifstream F(File.c_str(),ios::in /*| ios::nocreate*/); if (F.fail() == true) return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str()); - // CNC:2003-12-10 - 300 is too short. - char Buffer[1024]; - - int CurLine = 0; - while (F.eof() == false) + std::string Buffer; + for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine) { - F.getline(Buffer,sizeof(Buffer)); - CurLine++; - _strtabexpand(Buffer,sizeof(Buffer)); - if (F.fail() && !F.eof()) - return _error->Error(_("Line %u too long in source list %s."), - CurLine,File.c_str()); - - - char *I; - // CNC:2003-02-20 - Do not break if '#' is inside []. - for (I = Buffer; *I != 0 && *I != '#'; I++) - if (*I == '[') - { - char *b_end = strchr(I + 1, ']'); - if (b_end != NULL) - I = b_end; - } - *I = 0; - - const char *C = _strstrip(Buffer); - - // Comment or blank - if (C[0] == '#' || C[0] == 0) + // remove comments + size_t curpos = 0; + while ((curpos = Buffer.find('#', curpos)) != std::string::npos) + { + size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '['); + size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']'); + if (openbrackets > closedbrackets) + { + // a # in an option, unlikely, but oh well, it was supported so stick to it + ++curpos; + continue; + } + Buffer.erase(curpos); + break; + } + // remove spaces before/after + curpos = Buffer.find_first_not_of(" \t\r"); + if (curpos != 0) + Buffer.erase(0, curpos); + curpos = Buffer.find_last_not_of(" \t\r"); + if (curpos != std::string::npos) + Buffer.erase(curpos + 1); + + if (Buffer.empty()) continue; - + // Grok it - string LineType; - if (ParseQuoteWord(C,LineType) == false) + std::string const LineType = Buffer.substr(0, Buffer.find(' ')); + if (LineType.empty() || LineType == Buffer) return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str()); Type *Parse = Type::GetType(LineType.c_str()); if (Parse == 0) return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str()); - - if (Parse->ParseLine(SrcList, C, CurLine, File) == false) + + if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false) return false; } return true; @@ -387,30 +391,25 @@ bool pkgSourceList::ParseFileOldStyle(string File) // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/ // --------------------------------------------------------------------- /* Returns: the number of stanzas parsed*/ -int pkgSourceList::ParseFileDeb822(string File) +bool pkgSourceList::ParseFileDeb822(string const &File) { - pkgTagSection Tags; - unsigned int i=0; + pkgUserTagSection Tags; + unsigned int i = 1; // see if we can read the file - _error->PushToStack(); FileFd Fd(File, FileFd::ReadOnly); pkgTagFile Sources(&Fd); if (_error->PendingError() == true) - { - _error->RevertToStack(); - return 0; - } - _error->MergeWithStack(); - + return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str()); + // read step by step while (Sources.Step(Tags) == true) { - if(!Tags.Exists("Types")) - continue; + if(Tags.Exists("Types") == false) + return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str()); string const types = Tags.FindS("Types"); - std::vector list_types = StringSplit(types, " "); + std::vector const list_types = VectorizeString(types, ' '); for (std::vector::const_iterator I = list_types.begin(); I != list_types.end(); ++I) { @@ -418,18 +417,16 @@ int pkgSourceList::ParseFileDeb822(string File) if (Parse == 0) { _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str()); - return -1; + return false; } - + if (!Parse->ParseStanza(SrcList, Tags, i, Fd)) - return -1; + return false; - i++; + ++i; } } - - // we are done, return the number of stanzas read - return i; + return true; } /*}}}*/ // SourceList::FindIndex - Get the index associated with a file /*{{{*/ @@ -471,9 +468,12 @@ bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const // Based on ReadConfigDir() /*{{{*/ // --------------------------------------------------------------------- /* */ -bool pkgSourceList::ReadSourceDir(string Dir) +bool pkgSourceList::ReadSourceDir(string const &Dir) { - vector const List = GetListOfFilesInDir(Dir, "list", true); + std::vector ext; + ext.push_back("list"); + ext.push_back("sources"); + std::vector const List = GetListOfFilesInDir(Dir, ext, true); // Read the files for (vector::const_iterator I = List.begin(); I != List.end(); ++I) diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 4f42b3e91..079f3cb3d 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -72,11 +72,11 @@ class pkgSourceList bool FixupURI(std::string &URI) const; virtual bool ParseStanza(std::vector &List, pkgTagSection &Tags, - int stanza_n, + unsigned int const stanza_n, FileFd &Fd); virtual bool ParseLine(std::vector &List, const char *Buffer, - unsigned long const &CurLine,std::string const &File) const; + unsigned int const CurLine,std::string const &File) const; virtual bool CreateItem(std::vector &List,std::string const &URI, std::string const &Dist,std::string const &Section, std::map const &Options) const = 0; @@ -90,18 +90,19 @@ class pkgSourceList std::vector SrcList; - int ParseFileDeb822(std::string File); - bool ParseFileOldStyle(std::string File); + private: + APT_HIDDEN bool ParseFileDeb822(std::string const &File); + APT_HIDDEN bool ParseFileOldStyle(std::string const &File); public: bool ReadMainList(); - bool Read(std::string File); + bool Read(std::string const &File); // CNC:2003-03-03 void Reset(); - bool ReadAppend(std::string File); - bool ReadSourceDir(std::string Dir); + bool ReadAppend(std::string const &File); + bool ReadSourceDir(std::string const &Dir); // List accessors inline const_iterator begin() const {return SrcList.begin();}; @@ -117,7 +118,6 @@ class pkgSourceList time_t GetLastModifiedTime(); pkgSourceList(); - explicit pkgSourceList(std::string File); virtual ~pkgSourceList(); }; diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 6d7d8185b..cc63b213f 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -775,6 +775,14 @@ bool pkgTagSection::Write(FileFd &File, char const * const * const Order, std::v } /*}}}*/ +void pkgUserTagSection::TrimRecord(bool /*BeforeRecord*/, const char* &End)/*{{{*/ +{ + for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r' || Stop[0] == '#'); Stop++) + if (Stop[0] == '#') + Stop = (const char*) memchr(Stop,'\n',End-Stop); +} + /*}}}*/ + #include "tagfile-order.c" // TFRewrite - Rewrite a control record /*{{{*/ diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index d0d0c7a84..81fff89f0 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -142,6 +142,14 @@ class pkgTagSection bool Write(FileFd &File, char const * const * const Order = NULL, std::vector const &Rewrite = std::vector()) const; }; + +/* For user generated file the parser should be a bit more relaxed in exchange + for being a bit slower to allow comments and new lines all over the place */ +class pkgUserTagSection : public pkgTagSection +{ + virtual void TrimRecord(bool BeforeRecord, const char* &End); +}; + class pkgTagFilePrivate; class pkgTagFile { diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml index 6ebba3528..8506017ad 100644 --- a/doc/sources.list.5.xml +++ b/doc/sources.list.5.xml @@ -31,37 +31,99 @@ Description - The source list /etc/apt/sources.list is designed to support - any number of active sources and a variety of source media. The file lists one - source per line, with the most preferred source listed first. The information available - from the configured sources is acquired by apt-get update - (or by an equivalent command from another APT front-end). - - - Each line specifying a source starts with type (e.g. deb-src) - followed by options and arguments for this type. - Individual entries cannot be continued onto a following line. Empty lines - are ignored, and a # character anywhere on a line marks - the remainder of that line as a comment. + The source list /etc/apt/sources.list and the the + files contained in /etc/apt/sources.list.d/ are + designed to support any number of active sources and a variety of source + media. The files list one source per line (one line style) or contain multiline + stanzas defining one or more sources per stanza (deb822 style), with the + most preferred source listed first. The information available from the + configured sources is acquired by apt-get update (or + by an equivalent command from another APT front-end). sources.list.d - The /etc/apt/sources.list.d directory provides - a way to add sources.list entries in separate files. - The format is the same as for the regular sources.list file. - File names need to end with - .list and may only contain letters (a-z and A-Z), - digits (0-9), underscore (_), hyphen (-) and period (.) characters. - Otherwise APT will print a notice that it has ignored a file, unless that - file matches a pattern in the Dir::Ignore-Files-Silently - configuration list - in which case it will be silently ignored. + The /etc/apt/sources.list.d directory provides + a way to add sources.list entries in separate files. + Two different file formats are allowed as described in the next two sections. + Filenames need to have either the extension .list or + .sources depending on the contained format. + The filenames may only contain letters (a-z and A-Z), + digits (0-9), underscore (_), hyphen (-) and period (.) characters. + Otherwise APT will print a notice that it has ignored a file, unless that + file matches a pattern in the Dir::Ignore-Files-Silently + configuration list - in which case it will be silently ignored. - The deb and deb-src types + one line style format + + Files in this format have the extension .list. + Each line specifying a source starts with a type (e.g. deb-src) + followed by options and arguments for this type. + + Individual entries cannot be continued onto a following line. Empty lines + are ignored, and a # character anywhere on a line marks + the remainder of that line as a comment. Consequently an entry can be + disabled by commenting out the entire line. + + If options should be provided they are separated by spaces and all of + them together are enclosed by square brackets ([]) + included in the line after the type separated from it with a space. + If an option allows multiple values these are separated from each other + with a comma (,). An option name is separated from its + value(s) by a equal sign (=). Multivalue options have + also -= and += as separator which + instead of replacing the default with the given value(s) modify the default + value(s) to remove or include the given values. + + This is the traditional format and supported by all apt versions. + Note that not all options as described below are supported by all apt versions. + Note also that some older applications parsing this format on its own might not + expect to encounter options as they were uncommon before the introduction of + multi-architecture support. + + + + deb822 style format + + Files in this format have the extension .sources. + The format is similar in syntax to other files used by Debian and its + derivatives, like the metadata itself apt will download from the configured + sources or the debian/control file in a Debian source package. + + Individual entries are separated by an empty line, additional empty + lines are ignored, and a # character at the start of + the line marks the entire line as a comment. An entry can hence be + disabled by commenting out each line belonging to the stanza, but it is + usually easier to add the field "Enabled: no" to the stanza to disable + the entry. Removing the field or setting it to yes reenables it. + + Options have the same syntax as every other field: A fieldname separated by + a colon (:) and optionally spaces from its value(s). + Note especially that multiple values are separated by spaces, not by + commas as in the one line format. Multivalue fields like Architectures + also have Architectures-Add and Architectures-Remove + to modify the default value rather than replacing it. + + This is a new format supported by apt itself since version 1.1. Previous + versions ignore such files with a notice message as described earlier. + It is intended to make this format gradually the default format and + deprecating the previously described one line style format as it is + easier to create, extend and modify by humans and machines alike + especially if a lot of sources and/or options are involved. + + Developers who are working with and/or parsing apt sources are highly + encouraged to add support for this format and to contact the APT team + to coordinate and share this work. Users can freely adopt this format + already, but could encounter problems with software not supporting + the format yet. + + + + The deb and deb-src types: General Format The deb type references a typical two-level Debian archive, distribution/component. The - distribution is generally an archive name like + distribution is generally a suite name like stable or testing or a codename like &stable-codename; or &testing-codename; while component is one of main, contrib or @@ -70,42 +132,33 @@ code in the same form as the deb type. A deb-src line is required to fetch source indexes. - The format for a sources.list entry using the + The format for two one line style entries using the deb and deb-src types is: - deb [ options ] uri suite [component1] [component2] [...] + deb [ option1=value1 option2=value2 ] uri suite [component1] [component2] [...] +deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [...] - Alternatively a rfc822 style format is also supported: + Alternatively the equivalent entry in deb822 style looks like this: Types: deb deb-src - URIs: http://example.com - Suites: stable testing - Sections: component1 component2 - Description: short - long long long - [option1]: [option1-value] - - Types: deb - URIs: http://another.example.com - Suites: experimental - Sections: component1 component2 - Enabled: no - Description: short - long long long - [option1]: [option1-value] + URIs: uri + Suites: suite + Components: [component1] [component2] [...] + option1: value1 + option2: value2 The URI for the deb type must specify the base of the - Debian distribution, from which APT will find the information it needs. - suite can specify an exact path, in which case the + Debian distribution, from which APT will find the information it needs. + suite can specify an exact path, in which case the components must be omitted and suite must end with a slash (/). This is useful for the case when only a - particular sub-section of the archive denoted by the URI is of interest. + particular sub-directory of the archive denoted by the URI is of interest. If suite does not specify an exact path, at least one component must be present. - suite may also contain a variable, + suite may also contain a variable, $(ARCH) which expands to the Debian architecture (such as amd64 or armel) used on the system. This permits architecture-independent @@ -113,67 +166,80 @@ of interest when specifying an exact path, APT will automatically generate a URI with the current architecture otherwise. - In the traditional style sources.list format since only one - distribution can be specified per line it may be necessary to have - multiple lines for the same URI, if a subset of all available - distributions or components at that location is desired. APT will - sort the URI list after it has generated a complete set internally, - and will collapse multiple references to the same Internet host, - for instance, into a single connection, so that it does not - inefficiently establish an FTP connection, close it, do something - else, and then re-establish a connection to that same host. This - feature is useful for accessing busy FTP sites with limits on the - number of simultaneous anonymous users. APT also parallelizes - connections to different hosts to more effectively deal with sites - with low bandwidth. - - options is always optional and needs to be surrounded by - square brackets. It can consist of multiple settings in the form - setting=value. - Multiple settings are separated by spaces. The following settings are supported by APT - (note however that unsupported settings will be ignored silently): - - arch=arch1,arch2,… - can be used to specify for which architectures information should - be downloaded. If this option is not set all architectures defined by the - APT::Architectures option will be downloaded. - - arch+=arch1,arch2,… - and arch-=arch1,arch2,… - which can be used to add/remove architectures from the set which will be downloaded. - - lang=lang1,lang2,…, - lang+=lang1,lang2,… and - lang-=lang1,lang2,… functioning in - the same way as the arch-options described before. They can be used to specify for - which languages apt will acquire metadata, like translated package descriptions, for. If not specified, the - default set is defined by the Acquire::Languages config option. - - target=target1,target2,…, - target+=target1,target2,… and - target-=target1,target2,… again functioning in - the same way as the arch-options described before. They can be used to specify which - targets apt will try to acquire from this source. If not specified, the default set is defined by - the APT::Acquire::Targets configuration scope. - - trusted=yes can be set to indicate that packages - from this source are always authenticated even if the Release file - is not signed or the signature can't be checked. This disables parts of &apt-secure; - and should therefore only be used in a local and trusted context. trusted=no - is the opposite which handles even correctly authenticated sources as not authenticated. - + Especially in the one line style format since only one distribution + can be specified per line it may be necessary to have multiple lines for + the same URI, if a subset of all available distributions or components at + that location is desired. APT will sort the URI list after it has + generated a complete set internally, and will collapse multiple + references to the same Internet host, for instance, into a single + connection, so that it does not inefficiently establish a + connection, close it, do something else, and then re-establish a + connection to that same host. APT also parallelizes connections to + different hosts to more effectively deal with sites with low + bandwidth. It is important to list sources in order of preference, with the most preferred source listed first. Typically this will result in sorting by speed from fastest to slowest (CD-ROM followed by hosts on a local network, followed by distant Internet hosts, for example). - Some examples: - -deb http://ftp.debian.org/debian &stable-codename; main contrib non-free -deb http://security.debian.org/ &stable-codename;/updates main contrib non-free - + As an example, the sources for your distribution could look like this + in one line style format: + &sourceslist-list-format; or like this in + deb822 style format: + &sourceslist-sources-format; + + The deb and deb-src types: Options + Each source entry can have options specified modifying which and how + the source is accessed and data acquired from it. Format, syntax and names + of the options varies between the two formats one line and deb822 style + as described, but they have both the same options available. For simplicity + we list the deb822 fieldname and provide the one line name in brackets. + Remember that beside setting multivalue options explicitly, there is also + the option to modify them based on the default, but we aren't listing those + names explicitly here. Unsupported options are silently ignored by all + APT versions. + + + Architectures + (arch) is a multivalue option defining for + which architectures information should be downloaded. If this + option isn't set the default is all architectures as defined by + the APT::Architectures config option. + + + Languages + (lang) is a multivalue option defining for + which languages information like translated package + descriptions should be downloaded. If this option isn't set + the default is all languages as defined by the + Acquire::Languages config option. + + + Targets + (target) is a multivalue option defining + which download targets apt will try to acquire from this + source. If not specified, the default set is defined by the + APT::Acquire::Targets configuration scope. + + + Trusted (trusted) + is a tri-state value which defaults to APT deciding if a source + is considered trusted or if warnings should be raised before e.g. + packages are installed from this source. This option can be used + to override this decision either with the value yes, + which lets APT consider this source always as a trusted source + even if it has no or fails authentication checks by disabling parts + of &apt-secure; and should therefore only be used in a local and trusted + context (if at all) as otherwise security is breached. The opposite + can be achieved with the value no, which causes the source to be handled + as untrusted even if the authentication checks passed successfully. + The default value can't be set explicitly. + + + + URI specification @@ -247,34 +313,70 @@ deb http://security.debian.org/ &stable-codename;/updates main contrib non-free Examples - Uses the archive stored locally (or NFS mounted) at /home/jason/debian + Uses the archive stored locally (or NFS mounted) at /home/apt/debian for stable/main, stable/contrib, and stable/non-free. - deb file:/home/jason/debian stable main contrib non-free + deb file:/home/apt/debian stable main contrib non-free + Types: deb +URIs: file:/home/apt/debian +Suites: stable +Components: main contrib non-free As above, except this uses the unstable (development) distribution. - deb file:/home/jason/debian unstable main contrib non-free + deb file:/home/apt/debian unstable main contrib non-free + Types: deb +URIs: file:/home/apt/debian +Suites: unstable +Components: main contrib non-free Source line for the above - deb-src file:/home/jason/debian unstable main contrib non-free + deb-src file:/home/apt/debian unstable main contrib non-free + Types: deb-src +URIs: file:/home/apt/debian +Suites: unstable +Components: main contrib non-free + The first line gets package information for the architectures in APT::Architectures while the second always retrieves amd64 and armel. - deb http://ftp.debian.org/debian &stable-codename; main -deb [ arch=amd64,armel ] http://ftp.debian.org/debian &stable-codename; main + deb http://httpredir.debian.org/debian &stable-codename; main +deb [ arch=amd64,armel ] http://httpredir.debian.org/debian &stable-codename; main + Types: deb +URIs: http://httpredir.debian.org/debian +Suites: &stable-codename; +Components: main + +Types: deb +URIs: http://httpredir.debian.org/debian +Suites: &stable-codename; +Components: main +Architectures: amd64 armel + Uses HTTP to access the archive at archive.debian.org, and uses only the hamm/main area. deb http://archive.debian.org/debian-archive hamm main + Types: deb +URIs: http://archive.debian.org/debian-archive +Suites: hamm +Components: main Uses FTP to access the archive at ftp.debian.org, under the debian directory, and uses only the &stable-codename;/contrib area. deb ftp://ftp.debian.org/debian &stable-codename; contrib + Types: deb +URIs: ftp://ftp.debian.org/debian +Suites: &stable-codename; +Components: contrib Uses FTP to access the archive at ftp.debian.org, under the debian directory, and uses only the unstable/contrib area. If this line appears as well as the one in the previous example in sources.list a single FTP session will be used for both resource lines. deb ftp://ftp.debian.org/debian unstable contrib + Types: deb +URIs: ftp://ftp.debian.org/debian +Suites: unstable +Components: contrib Uses HTTP to access the archive at ftp.tlh.debian.org, under the universe directory, and uses only files found under @@ -284,15 +386,32 @@ deb [ arch=amd64,armel ] http://ftp.debian.org/debian &stable-codename; maindeb http://ftp.tlh.debian.org/universe unstable/binary-$(ARCH)/ + Types: deb +URIs: http://ftp.tlh.debian.org/universe +Suites: unstable/binary-$(ARCH)/ + + Uses HTTP to get binary packages as well as sources from the stable, testing and unstable + suites and the components main and contrib. + deb http://httpredir.debian.org/debian stable main contrib +deb-src http://httpredir.debian.org/debian stable main contrib +deb http://httpredir.debian.org/debian testing main contrib +deb-src http://httpredir.debian.org/debian testing main contrib +deb http://httpredir.debian.org/debian unstable main contrib +deb-src http://httpredir.debian.org/debian unstable main contrib + Types: deb deb-src +URIs: http://httpredir.debian.org/debian +Suites: stable testing unstable +Components: main contrib + + - + See Also - &apt-cache; &apt-conf; + &apt-get;, &apt-conf; &manbugs; - - + diff --git a/test/integration/test-apt-sources-deb822 b/test/integration/test-apt-sources-deb822 index 51fe7bcfe..adfe0e003 100755 --- a/test/integration/test-apt-sources-deb822 +++ b/test/integration/test-apt-sources-deb822 @@ -7,9 +7,8 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture 'i386' -echo 'APT::Sources::Use-Deb822 "true";' > rootdir/etc/apt/apt.conf.d/00enabledeb822 - -SOURCES='rootdir/etc/apt/sources.list' +LISTS='rootdir/etc/apt/sources.list.d/test.list' +SOURCES='rootdir/etc/apt/sources.list.d/test.sources' BASE='# some comment # that contains a : as well #Types: meep @@ -17,23 +16,48 @@ BASE='# some comment Types: deb URIs: http://ftp.debian.org/debian Suites: stable -Sections: main +Components: main Description: summay and the long part' -msgtest 'Test sources.list' 'old style' -echo "deb http://ftp.debian.org/debian stable main" > $SOURCES +msgcleantest() { + rm -f $LISTS $SOURCES + msgtest "$@" +} + +msgcleantest 'Test sources.list' 'old style' +echo "deb http://ftp.debian.org/debian stable main" > $LISTS +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 +'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris + +msgcleantest 'Test sources.list' 'old style with options' +echo "deb [trusted=yes arch+=armel,powerpc] http://ftp.debian.org/debian stable main" > $LISTS +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 +'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/binary-powerpc/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-powerpc_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris + +msgcleantest 'Test sources.list' 'old style with comments' +echo "deb http://ftp.debian.org/debian stable main # non-free" > $LISTS +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 +'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris + +msgcleantest 'Test sources.list' 'old style with option comments' +echo "deb [trusted=yes#yeahreally] http://ftp.debian.org/debian stable main # non-free" > $LISTS testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris -msgtest 'Test sources.list' 'simple deb822' +msgcleantest 'Test sources.list' 'simple deb822' echo "$BASE" > $SOURCES testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris -msgtest 'Test deb822 with' 'two entries' +msgcleantest 'Test deb822 with' 'two entries' # Two entries echo "$BASE" > $SOURCES echo "" >> $SOURCES @@ -46,7 +70,7 @@ testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.deb 'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris # two suite entries -msgtest 'Test deb822 with' 'two Suite entries' +msgcleantest 'Test deb822 with' 'two Suite entries' echo "$BASE" | sed -e "s/stable/stable unstable/" > $SOURCES testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 @@ -55,7 +79,7 @@ testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.deb 'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 " aptget update --print-uris -msgtest 'Test deb822' 'architecture option' +msgcleantest 'Test deb822' 'architecture option' echo "$BASE" > $SOURCES echo "Architectures: amd64 armel" >> $SOURCES testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 @@ -63,17 +87,30 @@ testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.deb 'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris - -msgtest 'Test old-style sources.list file which has' 'malformed dist' -echo "deb http://ftp.debian.org" > $SOURCES -testequal --nomsg "E: Malformed line 1 in source list $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list (dist) +msgcleantest 'Test old-style' 'suite arch variable' +echo 'deb http://ftp.tlh.debian.org/universe unstable/binary-$(ARCH)/' > $LISTS +testequal --nomsg "'http://ftp.tlh.debian.org/universe/unstable/binary-i386/InRelease' ftp.tlh.debian.org_universe_unstable_binary-i386_InRelease 0 +'http://ftp.tlh.debian.org/universe/unstable/binary-i386/Packages.bz2' ftp.tlh.debian.org_universe_unstable_binary-i386_Packages 0 +'http://ftp.tlh.debian.org/universe/unstable/binary-i386/en.bz2' ftp.tlh.debian.org_universe_unstable_binary-i386_en 0 " aptget update --print-uris + +msgcleantest 'Test deb822' 'suite arch variable' +echo 'Types: deb +URIs: http://ftp.tlh.debian.org/universe +Suites: stable/binary-$(ARCH)/' > $SOURCES +testequal --nomsg "'http://ftp.tlh.debian.org/universe/stable/binary-i386/InRelease' ftp.tlh.debian.org_universe_stable_binary-i386_InRelease 0 +'http://ftp.tlh.debian.org/universe/stable/binary-i386/Packages.bz2' ftp.tlh.debian.org_universe_stable_binary-i386_Packages 0 +'http://ftp.tlh.debian.org/universe/stable/binary-i386/en.bz2' ftp.tlh.debian.org_universe_stable_binary-i386_en 0 " aptget update --print-uris + +msgcleantest 'Test old-style sources.list file which has' 'malformed dist' +echo "deb http://ftp.debian.org" > $LISTS +testequal --nomsg "E: Malformed entry 1 in list file $TMPWORKINGDIRECTORY/$LISTS (Suite) E: The list of sources could not be read." aptget update --print-uris -msgtest 'Test deb822 sources.list file which has' 'malformed URI' +msgcleantest 'Test deb822 sources.list file which has' 'malformed URI' echo "Types: deb Suites: stable " > $SOURCES -testequal --nomsg "E: Malformed stanza 0 in source list $TMPWORKINGDIRECTORY/rootdir/etc/apt/sources.list (URI parse) +testequal --nomsg "E: Malformed entry 1 in sources file $TMPWORKINGDIRECTORY/$SOURCES (URI) E: The list of sources could not be read." aptget update --print-uris # with Enabled: false @@ -82,7 +119,7 @@ echo "Enabled: no" >> $SOURCES testempty aptget update --print-uris # multiple URIs -msgtest 'Test deb822 sources.list file which has' 'Multiple URIs work' +msgcleantest 'Test deb822 sources.list file which has' 'Multiple URIs work' echo "$BASE" | sed -e 's#http://ftp.debian.org/debian#http://ftp.debian.org/debian http://ftp.de.debian.org/debian#' > $SOURCES testequal --nomsg "'http://ftp.de.debian.org/debian/dists/stable/InRelease' ftp.de.debian.org_debian_dists_stable_InRelease 0 'http://ftp.de.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.de.debian.org_debian_dists_stable_main_binary-i386_Packages 0 @@ -92,7 +129,7 @@ testequal --nomsg "'http://ftp.de.debian.org/debian/dists/stable/InRelease' ftp 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris # multiple Type in one field -msgtest 'Test deb822 sources.list file which has' 'Multiple Types work' +msgcleantest 'Test deb822 sources.list file which has' 'Multiple Types work' echo "$BASE" | sed -e 's#Types: deb#Types: deb deb-src#' > $SOURCES testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 'http://ftp.debian.org/debian/dists/stable/main/source/Sources.bz2' ftp.debian.org_debian_dists_stable_main_source_Sources 0 @@ -100,7 +137,7 @@ testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.deb 'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris # a Suite -msgtest 'Test deb822 sources.list file which has' 'a exact path and no sections' +msgcleantest 'Test deb822 sources.list file which has' 'an exact path and no sections' cat > $SOURCES < + + + + diff --git a/vendor/debian/apt-vendor.ent b/vendor/debian/apt-vendor.ent index 6cda5995c..8b26da385 100644 --- a/vendor/debian/apt-vendor.ent +++ b/vendor/debian/apt-vendor.ent @@ -5,3 +5,15 @@ /usr/share/keyrings/debian-archive-removed-keys.gpg"> + + + diff --git a/vendor/debian/sources.list.in b/vendor/debian/sources.list.in index 2e430296a..c36fb16a7 100644 --- a/vendor/debian/sources.list.in +++ b/vendor/debian/sources.list.in @@ -1,7 +1,6 @@ # See sources.list(5) manpage for more information # Remember that CD-ROMs, DVDs and such are managed through the apt-cdrom tool. -deb http://ftp.us.debian.org/debian &debian-stable-codename; main contrib non-free -deb http://security.debian.org &debian-stable-codename;/updates main contrib non-free +&sourceslist-list-format; # Uncomment if you want the apt-get source function to work #deb-src http://ftp.us.debian.org/debian &debian-stable-codename; main contrib non-free diff --git a/vendor/raspbian/apt-vendor.ent b/vendor/raspbian/apt-vendor.ent index e359d2001..dc69f3c03 100644 --- a/vendor/raspbian/apt-vendor.ent +++ b/vendor/raspbian/apt-vendor.ent @@ -5,3 +5,9 @@ /usr/share/keyrings/raspbian-archive-removed-keys.gpg"> + + + diff --git a/vendor/steamos/apt-vendor.ent b/vendor/steamos/apt-vendor.ent index dc1b798e6..7cf100fc4 100644 --- a/vendor/steamos/apt-vendor.ent +++ b/vendor/steamos/apt-vendor.ent @@ -6,3 +6,10 @@ + + + diff --git a/vendor/steamos/sources.list.in b/vendor/steamos/sources.list.in index 69174d090..246d2e336 100644 --- a/vendor/steamos/sources.list.in +++ b/vendor/steamos/sources.list.in @@ -1,5 +1,3 @@ # See sources.list(5) manpage for more information # Remember that CD-ROMs, DVDs and such are managed through the apt-cdrom tool. - -deb http://repo.steampowered.com/steamos ¤t-codename; main contrib non-free -deb-src http://repo.steampowered.com/steamos ¤t-codename; main contrib non-free +&sourceslist-list-format; diff --git a/vendor/tanglu/apt-vendor.ent b/vendor/tanglu/apt-vendor.ent index d2442209c..f5cd813bf 100644 --- a/vendor/tanglu/apt-vendor.ent +++ b/vendor/tanglu/apt-vendor.ent @@ -6,3 +6,9 @@ + + + diff --git a/vendor/ubuntu/apt-vendor.ent b/vendor/ubuntu/apt-vendor.ent index caa532699..dcebc9209 100644 --- a/vendor/ubuntu/apt-vendor.ent +++ b/vendor/ubuntu/apt-vendor.ent @@ -5,3 +5,16 @@ /usr/share/keyrings/ubuntu-archive-removed-keys.gpg"> + + + -- cgit v1.2.3-70-g09d2 From 5ad0096a4e19e191b59634e8a8817995ec4045ad Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 23 Jun 2015 15:16:08 +0200 Subject: merge indexRecords into metaIndex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit indexRecords was used to parse the Release file – mostly the hashes – while metaIndex deals with downloading the Release file, storing all indexes coming from this release and … parsing the Release file, but this time mostly for the other fields. That wasn't a problem in metaIndex as this was done in the type specific subclass, but indexRecords while allowing to override the parsing method did expect by default a specific format. APT isn't really supporting different types at the moment, but this is a violation of the abstraction we have everywhere else and, which is the actual reason for this merge: Options e.g. coming from the sources.list come to metaIndex naturally, which needs to wrap them up and bring them into indexRecords, so the acquire system is told about it as they don't get to see the metaIndex, but they don't really belong in indexRecords as this is just for storing data loaded from the Release file… the result is a complete mess. I am not saying it is a lot prettier after the merge, but at least adding new options is now slightly easier and there is just one place responsible for parsing the Release file. That can't hurt. --- apt-pkg/acquire-item.cc | 113 +++++++------- apt-pkg/acquire-item.h | 9 +- apt-pkg/deb/debmetaindex.cc | 231 +++++++++++++++++++++++----- apt-pkg/deb/debmetaindex.h | 29 ++-- apt-pkg/indexcopy.cc | 18 ++- apt-pkg/indexcopy.h | 4 +- apt-pkg/indexrecords.cc | 284 ----------------------------------- apt-pkg/indexrecords.h | 88 ----------- apt-pkg/metaindex.cc | 76 ++++++++-- apt-pkg/metaindex.h | 66 ++++++-- apt-pkg/sourcelist.cc | 1 + cmdline/apt-get.cc | 35 ++--- test/integration/test-apt-cli-update | 4 +- 13 files changed, 423 insertions(+), 535 deletions(-) delete mode 100644 apt-pkg/indexrecords.cc delete mode 100644 apt-pkg/indexrecords.h (limited to 'apt-pkg/sourcelist.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 0ab52a0cd..100199bc1 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include #include @@ -109,9 +109,9 @@ static std::string GetDiffsPatchFileName(std::string const &Final) /*{{{*/ } /*}}}*/ -static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I) /*{{{*/ +static bool AllowInsecureRepositories(metaIndex const * const MetaIndexParser, pkgAcqMetaClearSig * const TransactionManager, pkgAcquire::Item * const I) /*{{{*/ { - if(MetaIndexParser->IsAlwaysTrusted() || _config->FindB("Acquire::AllowInsecureRepositories") == true) + if(MetaIndexParser->GetTrusted() == metaIndex::TRI_YES || _config->FindB("Acquire::AllowInsecureRepositories") == true) return true; _error->Error(_("Use --allow-insecure-repositories to force the update")); @@ -120,11 +120,11 @@ static bool AllowInsecureRepositories(indexRecords const * const MetaIndexParser return false; } /*}}}*/ -static HashStringList GetExpectedHashesFromFor(indexRecords * const Parser, std::string const &MetaKey)/*{{{*/ +static HashStringList GetExpectedHashesFromFor(metaIndex * const Parser, std::string const &MetaKey)/*{{{*/ { if (Parser == NULL) return HashStringList(); - indexRecords::checkSum * const R = Parser->Lookup(MetaKey); + metaIndex::checkSum * const R = Parser->Lookup(MetaKey); if (R == NULL) return HashStringList(); return R->Hashes; @@ -144,7 +144,8 @@ APT_CONST bool pkgAcqTransactionItem::HashesRequired() const we can at least trust them for integrity of the download itself. Only repositories without a Release file can (obviously) not have hashes – and they are very uncommon and strongly discouraged */ - return TransactionManager->MetaIndexParser != NULL; + return TransactionManager->MetaIndexParser != NULL && + TransactionManager->MetaIndexParser->GetLoadedSuccessfully() != metaIndex::TRI_UNSET; } HashStringList pkgAcqTransactionItem::GetExpectedHashes() const { @@ -900,26 +901,28 @@ bool pkgAcqMetaBase::CheckAuthDone(string const &Message) /*{{{*/ } if (RealFileExists(FinalInRelease) || RealFileExists(FinalRelease)) { - TransactionManager->LastMetaIndexParser = new indexRecords; - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); } - _error->RevertToStack(); } } - if (TransactionManager->MetaIndexParser->Load(DestFile) == false) + if (TransactionManager->MetaIndexParser->Load(DestFile, &ErrorText) == false) { Status = StatAuthError; - ErrorText = TransactionManager->MetaIndexParser->ErrorText; return false; } @@ -1065,14 +1068,16 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/ TransactionManager->IMSHit = true; unlink(DestFile.c_str()); PartialFile = DestFile = GetFinalFilename(); - delete TransactionManager->MetaIndexParser; - TransactionManager->MetaIndexParser = TransactionManager->LastMetaIndexParser; + // load the 'old' file in the 'new' one instead of flipping pointers as + // the new one isn't owned by us, while the old one is so cleanup would be confused. + TransactionManager->MetaIndexParser->swapLoad(TransactionManager->LastMetaIndexParser); + delete TransactionManager->LastMetaIndexParser; TransactionManager->LastMetaIndexParser = NULL; } if (_config->FindB("Debug::pkgAcquire::Auth", false)) { - std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetDist() << std::endl; + std::cerr << "Got Codename: " << TransactionManager->MetaIndexParser->GetCodename() << std::endl; std::cerr << "Expecting Dist: " << TransactionManager->MetaIndexParser->GetExpectedDist() << std::endl; std::cerr << "Transformed Dist: " << Transformed << std::endl; } @@ -1083,14 +1088,14 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/ // Status = StatAuthError; // ErrorText = "Conflicting distribution; expected " // + MetaIndexParser->GetExpectedDist() + " but got " -// + MetaIndexParser->GetDist(); +// + MetaIndexParser->GetCodename(); // return false; if (!Transformed.empty()) { _error->Warning(_("Conflicting distribution: %s (expected %s but got %s)"), Desc.Description.c_str(), Transformed.c_str(), - TransactionManager->MetaIndexParser->GetDist().c_str()); + TransactionManager->MetaIndexParser->GetCodename().c_str()); } } @@ -1105,7 +1110,7 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/ IndexTarget const &ClearsignedTarget, IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget, std::vector const &IndexTargets, - indexRecords * const MetaIndexParser) : + metaIndex * const MetaIndexParser) : pkgAcqMetaIndex(Owner, this, ClearsignedTarget, DetachedSigTarget, IndexTargets), d(NULL), ClearsignedTarget(ClearsignedTarget), DetachedDataTarget(DetachedDataTarget), @@ -1118,8 +1123,6 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire * const Owner, /*{{{*/ /*}}}*/ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ { - if (MetaIndexParser != NULL) - delete MetaIndexParser; if (LastMetaIndexParser != NULL) delete LastMetaIndexParser; } @@ -1218,25 +1221,28 @@ void pkgAcqMetaClearSig::Failed(string const &Message,pkgAcquire::MethodConfig c // open the last Release if we have it if (TransactionManager->IMSHit == false) { - TransactionManager->LastMetaIndexParser = new indexRecords; - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); } - _error->RevertToStack(); } } // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him - if (TransactionManager->MetaIndexParser->Load(PartialRelease) == false || VerifyVendor(Message) == false) + if (TransactionManager->MetaIndexParser->Load(PartialRelease, &ErrorText) == false || VerifyVendor(Message) == false) /* expired Release files are still a problem you need extra force for */; else QueueIndexes(true); @@ -1303,8 +1309,6 @@ void pkgAcqMetaIndex::Failed(string const &Message, { // ensure old Release files are removed TransactionManager->TransactionStageRemoval(this, GetFinalFilename()); - delete TransactionManager->MetaIndexParser; - TransactionManager->MetaIndexParser = NULL; // queue without any kind of hashsum support QueueIndexes(false); @@ -1453,25 +1457,28 @@ void pkgAcqMetaSig::Failed(string const &Message,pkgAcquire::MethodConfig const // open the last Release if we have it if (TransactionManager->IMSHit == false) { - TransactionManager->LastMetaIndexParser = new indexRecords; - _error->PushToStack(); - if (RealFileExists(FinalInRelease)) - TransactionManager->LastMetaIndexParser->Load(FinalInRelease); - else - TransactionManager->LastMetaIndexParser->Load(FinalRelease); - // its unlikely to happen, but if what we have is bad ignore it - if (_error->PendingError()) + TransactionManager->LastMetaIndexParser = TransactionManager->MetaIndexParser->UnloadedClone(); + if (TransactionManager->LastMetaIndexParser != NULL) { - delete TransactionManager->LastMetaIndexParser; - TransactionManager->LastMetaIndexParser = NULL; + _error->PushToStack(); + if (RealFileExists(FinalInRelease)) + TransactionManager->LastMetaIndexParser->Load(FinalInRelease, NULL); + else + TransactionManager->LastMetaIndexParser->Load(FinalRelease, NULL); + // its unlikely to happen, but if what we have is bad ignore it + if (_error->PendingError()) + { + delete TransactionManager->LastMetaIndexParser; + TransactionManager->LastMetaIndexParser = NULL; + } + _error->RevertToStack(); } - _error->RevertToStack(); } } // we parse the indexes here because at this point the user wanted // a repository that may potentially harm him - if (TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile) == false || MetaIndex->VerifyVendor(Message) == false) + if (TransactionManager->MetaIndexParser->Load(MetaIndex->DestFile, &ErrorText) == false || MetaIndex->VerifyVendor(Message) == false) /* expired Release files are still a problem you need extra force for */; else MetaIndex->QueueIndexes(true); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 4d235dce2..10ece76c9 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -34,7 +34,6 @@ #include #include #include -#include #endif /** \addtogroup acquire @@ -43,11 +42,11 @@ * \file acquire-item.h */ -class indexRecords; class pkgRecords; class pkgSourceList; class pkgAcqMetaClearSig; class pkgAcqIndexMergeDiffs; +class metaIndex; class pkgAcquire::Item : public WeakPointable /*{{{*/ /** \brief Represents the process by which a pkgAcquire object should @@ -559,8 +558,8 @@ class APT_HIDDEN pkgAcqMetaClearSig : public pkgAcqMetaIndex public: /** \brief A package-system-specific parser for the meta-index file. */ - indexRecords *MetaIndexParser; - indexRecords *LastMetaIndexParser; + metaIndex *MetaIndexParser; + metaIndex *LastMetaIndexParser; virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf); virtual std::string Custom600Headers() const; @@ -573,7 +572,7 @@ class APT_HIDDEN pkgAcqMetaClearSig : public pkgAcqMetaIndex IndexTarget const &DetachedDataTarget, IndexTarget const &DetachedSigTarget, std::vector const &IndexTargets, - indexRecords * const MetaIndexParser); + metaIndex * const MetaIndexParser); virtual ~pkgAcqMetaClearSig(); }; /*}}}*/ diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 1f725ba05..f0b859eb4 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -45,10 +44,7 @@ class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ std::vector DebEntries; std::vector DebSrcEntries; - debReleaseIndex::TriState Trusted; - - debReleaseIndexPrivate() : Trusted(debReleaseIndex::TRI_UNSET) {} - debReleaseIndexPrivate(bool const pTrusted) : Trusted(pTrusted ? debReleaseIndex::TRI_YES : debReleaseIndex::TRI_NO) {} + debReleaseIndexPrivate() {} }; /*}}}*/ // ReleaseIndex::MetaIndex* - display helpers /*{{{*/ @@ -92,27 +88,15 @@ std::string debReleaseIndex::MetaIndexURI(const char *Type) const return Res; } /*}}}*/ -std::string debReleaseIndex::LocalFileName() const /*{{{*/ -{ - // see if we have a InRelease file - std::string PathInRelease = MetaIndexFile("InRelease"); - if (FileExists(PathInRelease)) - return PathInRelease; - - // and if not return the normal one - if (FileExists(PathInRelease)) - return MetaIndexFile("Release"); - - return ""; -} - /*}}}*/ // ReleaseIndex Con- and Destructors /*{{{*/ debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist) : metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()) {} -debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, bool const Trusted) : - metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate(Trusted)) -{} +debReleaseIndex::debReleaseIndex(std::string const &URI, std::string const &Dist, bool const pTrusted) : + metaIndex(URI, Dist, "deb"), d(new debReleaseIndexPrivate()) +{ + Trusted = pTrusted ? TRI_YES : TRI_NO; +} debReleaseIndex::~debReleaseIndex() { if (d != NULL) delete d; @@ -227,22 +211,197 @@ void debReleaseIndex::AddComponent(bool const isSrc, std::string const &Name,/*{ } /*}}}*/ +bool debReleaseIndex::Load(std::string const &Filename, std::string * const ErrorText)/*{{{*/ +{ + LoadedSuccessfully = TRI_NO; + FileFd Fd; + if (OpenMaybeClearSignedFile(Filename, Fd) == false) + return false; + + pkgTagFile TagFile(&Fd, Fd.Size()); + if (_error->PendingError() == true) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Unable to parse Release file %s"),Filename.c_str()); + return false; + } + + pkgTagSection Section; + const char *Start, *End; + if (TagFile.Step(Section) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("No sections in Release file %s"), Filename.c_str()); + return false; + } + // FIXME: find better tag name + SupportsAcquireByHash = Section.FindB("Acquire-By-Hash", false); + + Suite = Section.FindS("Suite"); + Codename = Section.FindS("Codename"); + + bool FoundHashSum = false; + for (int i=0;HashString::SupportedHashes()[i] != NULL; i++) + { + if (!Section.Find(HashString::SupportedHashes()[i], Start, End)) + continue; + + std::string Name; + std::string Hash; + unsigned long long Size; + while (Start < End) + { + if (!parseSumData(Start, End, Name, Hash, Size)) + return false; + + if (Entries.find(Name) == Entries.end()) + { + metaIndex::checkSum *Sum = new metaIndex::checkSum; + Sum->MetaKeyFilename = Name; + Sum->Size = Size; + Sum->Hashes.FileSize(Size); + APT_IGNORE_DEPRECATED(Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash);) + Entries[Name] = Sum; + } + Entries[Name]->Hashes.push_back(HashString(HashString::SupportedHashes()[i],Hash)); + FoundHashSum = true; + } + } + + if(FoundHashSum == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("No Hash entry in Release file %s"), Filename.c_str()); + return false; + } + + std::string const StrDate = Section.FindS("Date"); + if (RFC1123StrToTime(StrDate.c_str(), Date) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Invalid 'Date' entry in Release file %s"), Filename.c_str()); + return false; + } -bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const/*{{{*/ + std::string const Label = Section.FindS("Label"); + std::string const StrValidUntil = Section.FindS("Valid-Until"); + + // if we have a Valid-Until header in the Release file, use it as default + if (StrValidUntil.empty() == false) + { + if(RFC1123StrToTime(StrValidUntil.c_str(), ValidUntil) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Invalid 'Valid-Until' entry in Release file %s"), Filename.c_str()); + return false; + } + } + // get the user settings for this archive and use what expires earlier + int MaxAge = _config->FindI("Acquire::Max-ValidTime", 0); + if (Label.empty() == false) + MaxAge = _config->FindI(("Acquire::Max-ValidTime::" + Label).c_str(), MaxAge); + int MinAge = _config->FindI("Acquire::Min-ValidTime", 0); + if (Label.empty() == false) + MinAge = _config->FindI(("Acquire::Min-ValidTime::" + Label).c_str(), MinAge); + + LoadedSuccessfully = TRI_YES; + if(MaxAge == 0 && + (MinAge == 0 || ValidUntil == 0)) // No user settings, use the one from the Release file + return true; + + if (MinAge != 0 && ValidUntil != 0) { + time_t const min_date = Date + MinAge; + if (ValidUntil < min_date) + ValidUntil = min_date; + } + if (MaxAge != 0) { + time_t const max_date = Date + MaxAge; + if (ValidUntil == 0 || ValidUntil > max_date) + ValidUntil = max_date; + } + + return true; +} + /*}}}*/ +metaIndex * debReleaseIndex::UnloadedClone() const /*{{{*/ +{ + if (Trusted == TRI_NO) + return new debReleaseIndex(URI, Dist, false); + else if (Trusted == TRI_YES) + return new debReleaseIndex(URI, Dist, true); + else + return new debReleaseIndex(URI, Dist); +} + /*}}}*/ +bool debReleaseIndex::parseSumData(const char *&Start, const char *End, /*{{{*/ + std::string &Name, std::string &Hash, unsigned long long &Size) { - indexRecords * const iR = new indexRecords(Dist); - if (d->Trusted == TRI_YES) - iR->SetTrusted(true); - else if (d->Trusted == TRI_NO) - iR->SetTrusted(false); + Name = ""; + Hash = ""; + Size = 0; + /* Skip over the first blank */ + while ((*Start == '\t' || *Start == ' ' || *Start == '\n' || *Start == '\r') + && Start < End) + Start++; + if (Start >= End) + return false; - // special case for --print-uris + /* Move EntryEnd to the end of the first entry (the hash) */ + const char *EntryEnd = Start; + while ((*EntryEnd != '\t' && *EntryEnd != ' ') + && EntryEnd < End) + EntryEnd++; + if (EntryEnd == End) + return false; + + Hash.append(Start, EntryEnd-Start); + + /* Skip over intermediate blanks */ + Start = EntryEnd; + while (*Start == '\t' || *Start == ' ') + Start++; + if (Start >= End) + return false; + + EntryEnd = Start; + /* Find the end of the second entry (the size) */ + while ((*EntryEnd != '\t' && *EntryEnd != ' ' ) + && EntryEnd < End) + EntryEnd++; + if (EntryEnd == End) + return false; + + Size = strtoull (Start, NULL, 10); + + /* Skip over intermediate blanks */ + Start = EntryEnd; + while (*Start == '\t' || *Start == ' ') + Start++; + if (Start >= End) + return false; + + EntryEnd = Start; + /* Find the end of the third entry (the filename) */ + while ((*EntryEnd != '\t' && *EntryEnd != ' ' && + *EntryEnd != '\n' && *EntryEnd != '\r') + && EntryEnd < End) + EntryEnd++; + + Name.append(Start, EntryEnd-Start); + Start = EntryEnd; //prepare for the next round + return true; +} + /*}}}*/ + +bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/ +{ std::vector const targets = GetIndexTargets(); #define APT_TARGET(X) IndexTarget("", X, MetaIndexInfo(X), MetaIndexURI(X), false, std::map()) pkgAcqMetaClearSig * const TransactionManager = new pkgAcqMetaClearSig(Owner, APT_TARGET("InRelease"), APT_TARGET("Release"), APT_TARGET("Release.gpg"), - targets, iR); + targets, this); #undef APT_TARGET + // special case for --print-uris if (GetAll) { for (std::vector::const_iterator Target = targets.begin(); Target != targets.end(); ++Target) @@ -253,20 +412,20 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const/*{ } /*}}}*/ // ReleaseIndex::IsTrusted /*{{{*/ -bool debReleaseIndex::SetTrusted(TriState const Trusted) +bool debReleaseIndex::SetTrusted(TriState const pTrusted) { - if (d->Trusted == TRI_UNSET) - d->Trusted = Trusted; - else if (d->Trusted != Trusted) + if (Trusted == TRI_UNSET) + Trusted = pTrusted; + else if (Trusted != pTrusted) // TRANSLATOR: The first is an option name from sources.list manpage, the other two URI and Suite return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Trusted", URI.c_str(), Dist.c_str()); return true; } bool debReleaseIndex::IsTrusted() const { - if (d->Trusted == TRI_YES) + if (Trusted == TRI_YES) return true; - else if (d->Trusted == TRI_NO) + else if (Trusted == TRI_NO) return false; diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index a6db4e287..19fe6806c 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -27,6 +27,8 @@ class APT_HIDDEN debReleaseIndex : public metaIndex { debReleaseIndexPrivate * const d; + APT_HIDDEN bool parseSumData(const char *&Start, const char *End, std::string &Name, + std::string &Hash, unsigned long long &Size); public: APT_HIDDEN std::string MetaIndexInfo(const char *Type) const; @@ -38,20 +40,18 @@ class APT_HIDDEN debReleaseIndex : public metaIndex virtual ~debReleaseIndex(); virtual std::string ArchiveURI(std::string const &File) const {return URI + File;}; - virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const; + virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false); virtual std::vector GetIndexTargets() const; virtual std::string Describe() const; virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const; virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; - virtual std::string LocalFileName() const; + virtual bool Load(std::string const &Filename, std::string * const ErrorText); + virtual metaIndex * UnloadedClone() const; virtual std::vector *GetIndexFiles(); - enum APT_HIDDEN TriState { - TRI_YES, TRI_DONTCARE, TRI_NO, TRI_UNSET - }; bool SetTrusted(TriState const Trusted); virtual bool IsTrusted() const; @@ -64,15 +64,15 @@ class APT_HIDDEN debReleaseIndex : public metaIndex class APT_HIDDEN debDebFileMetaIndex : public metaIndex { - private: - void * const d; +private: + void * const d; std::string DebFile; debDebPkgFileIndex *DebIndex; - public: +public: virtual std::string ArchiveURI(std::string const& /*File*/) const { return DebFile; } - virtual bool GetIndexes(pkgAcquire* /*Owner*/, const bool& /*GetAll=false*/) const { + virtual bool GetIndexes(pkgAcquire* /*Owner*/, const bool& /*GetAll=false*/) { return true; } virtual std::vector GetIndexTargets() const { @@ -84,6 +84,17 @@ class APT_HIDDEN debDebFileMetaIndex : public metaIndex virtual bool IsTrusted() const { return true; } + virtual bool Load(std::string const &, std::string * const ErrorText) + { + LoadedSuccessfully = TRI_NO; + if (ErrorText != NULL) + strprintf(*ErrorText, "Unparseable metaindex as it represents the standalone deb file %s", DebFile.c_str()); + return false; + } + virtual metaIndex * UnloadedClone() const + { + return NULL; + } debDebFileMetaIndex(std::string const &DebFile); virtual ~debDebFileMetaIndex(); diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index 6d210e65b..f9adb2fb8 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -19,10 +19,11 @@ #include #include #include -#include +#include #include #include #include +#include #include #include @@ -476,9 +477,9 @@ bool SourceCopy::RewriteEntry(FileFd &Target, std::string const &File) } /*}}}*/ // SigVerify::Verify - Verify a files md5sum against its metaindex /*{{{*/ -bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex) +bool SigVerify::Verify(string prefix, string file, metaIndex *MetaIndex) { - const indexRecords::checkSum *Record = MetaIndex->Lookup(file); + const metaIndex::checkSum *Record = MetaIndex->Lookup(file); bool const Debug = _config->FindB("Debug::aptcdrom",false); // we skip non-existing files in the verifcation of the Release file @@ -545,11 +546,11 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, // Read all Release files for (vector::iterator I = SigList.begin(); I != SigList.end(); ++I) - { + { if(Debug) cout << "Signature verify for: " << *I << endl; - indexRecords *MetaIndex = new indexRecords; + metaIndex *MetaIndex = new debReleaseIndex("",""); string prefix = *I; string const releasegpg = *I+"Release.gpg"; @@ -591,12 +592,13 @@ bool SigVerify::CopyAndVerify(string CDROM,string Name,vector &SigList, } // Open the Release file and add it to the MetaIndex - if(!MetaIndex->Load(release)) + std::string ErrorText; + if(MetaIndex->Load(release, &ErrorText) == false) { - _error->Error("%s",MetaIndex->ErrorText.c_str()); + _error->Error("%s", ErrorText.c_str()); return false; } - + // go over the Indexfiles and see if they verify // if so, remove them from our copy of the lists vector keys = MetaIndex->MetaKeys(); diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h index 4f4c47169..6eecad028 100644 --- a/apt-pkg/indexcopy.h +++ b/apt-pkg/indexcopy.h @@ -25,9 +25,9 @@ using std::vector; #endif class pkgTagSection; -class indexRecords; class pkgCdromStatus; class FileFd; +class metaIndex; class IndexCopy /*{{{*/ { @@ -106,7 +106,7 @@ class SigVerify /*{{{*/ /** \brief dpointer placeholder (for later in case we need it) */ void * const d; - APT_HIDDEN bool Verify(std::string prefix,std::string file, indexRecords *records); + APT_HIDDEN bool Verify(std::string prefix,std::string file, metaIndex *records); APT_HIDDEN bool CopyMetaIndex(std::string CDROM, std::string CDName, std::string prefix, std::string file); diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc deleted file mode 100644 index 03ba59460..000000000 --- a/apt-pkg/indexrecords.cc +++ /dev/null @@ -1,284 +0,0 @@ -// -*- mode: cpp; mode: fold -*- -// Description /*{{{*/ -// $Id: indexrecords.cc,v 1.1.2.4 2003/12/30 02:11:43 mdz Exp $ - /*}}}*/ -// Include Files /*{{{*/ -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - /*}}}*/ - -using std::string; - -APT_PURE string indexRecords::GetDist() const -{ - return this->Dist; -} - -APT_PURE string indexRecords::GetSuite() const -{ - return this->Suite; -} - -APT_PURE bool indexRecords::GetSupportsAcquireByHash() const -{ - return this->SupportsAcquireByHash; -} - -APT_PURE bool indexRecords::CheckDist(string const &MaybeDist) const -{ - return (this->Dist == MaybeDist - || this->Suite == MaybeDist); -} - -APT_PURE string indexRecords::GetExpectedDist() const -{ - return this->ExpectedDist; -} - -APT_PURE time_t indexRecords::GetValidUntil() const -{ - return this->ValidUntil; -} - -APT_PURE time_t indexRecords::GetDate() const -{ - return this->Date; -} - -APT_PURE indexRecords::checkSum *indexRecords::Lookup(string const &MetaKey) -{ - std::map::const_iterator sum = Entries.find(MetaKey); - if (sum == Entries.end()) - return NULL; - return sum->second; -} - -APT_PURE bool indexRecords::Exists(string const &MetaKey) const -{ - return Entries.find(MetaKey) != Entries.end(); -} - -bool indexRecords::Load(string const &Filename) /*{{{*/ -{ - FileFd Fd; - if (OpenMaybeClearSignedFile(Filename, Fd) == false) - return false; - - pkgTagFile TagFile(&Fd, Fd.Size()); - if (_error->PendingError() == true) - { - strprintf(ErrorText, _("Unable to parse Release file %s"),Filename.c_str()); - return false; - } - - pkgTagSection Section; - const char *Start, *End; - if (TagFile.Step(Section) == false) - { - strprintf(ErrorText, _("No sections in Release file %s"), Filename.c_str()); - return false; - } - // FIXME: find better tag name - SupportsAcquireByHash = Section.FindB("Acquire-By-Hash", false); - - Suite = Section.FindS("Suite"); - Dist = Section.FindS("Codename"); - - bool FoundHashSum = false; - for (int i=0;HashString::SupportedHashes()[i] != NULL; i++) - { - if (!Section.Find(HashString::SupportedHashes()[i], Start, End)) - continue; - - string Name; - string Hash; - unsigned long long Size; - while (Start < End) - { - if (!parseSumData(Start, End, Name, Hash, Size)) - return false; - - if (Entries.find(Name) == Entries.end()) - { - indexRecords::checkSum *Sum = new indexRecords::checkSum; - Sum->MetaKeyFilename = Name; - Sum->Size = Size; - Sum->Hashes.FileSize(Size); - APT_IGNORE_DEPRECATED(Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash);) - Entries[Name] = Sum; - } - Entries[Name]->Hashes.push_back(HashString(HashString::SupportedHashes()[i],Hash)); - FoundHashSum = true; - } - } - - if(FoundHashSum == false) - { - strprintf(ErrorText, _("No Hash entry in Release file %s"), Filename.c_str()); - return false; - } - - string const StrDate = Section.FindS("Date"); - if (RFC1123StrToTime(StrDate.c_str(), Date) == false) - { - strprintf(ErrorText, _("Invalid 'Date' entry in Release file %s"), Filename.c_str()); - return false; - } - - string const Label = Section.FindS("Label"); - string const StrValidUntil = Section.FindS("Valid-Until"); - - // if we have a Valid-Until header in the Release file, use it as default - if (StrValidUntil.empty() == false) - { - if(RFC1123StrToTime(StrValidUntil.c_str(), ValidUntil) == false) - { - strprintf(ErrorText, _("Invalid 'Valid-Until' entry in Release file %s"), Filename.c_str()); - return false; - } - } - // get the user settings for this archive and use what expires earlier - int MaxAge = _config->FindI("Acquire::Max-ValidTime", 0); - if (Label.empty() == false) - MaxAge = _config->FindI(("Acquire::Max-ValidTime::" + Label).c_str(), MaxAge); - int MinAge = _config->FindI("Acquire::Min-ValidTime", 0); - if (Label.empty() == false) - MinAge = _config->FindI(("Acquire::Min-ValidTime::" + Label).c_str(), MinAge); - - if(MaxAge == 0 && - (MinAge == 0 || ValidUntil == 0)) // No user settings, use the one from the Release file - return true; - - if (MinAge != 0 && ValidUntil != 0) { - time_t const min_date = Date + MinAge; - if (ValidUntil < min_date) - ValidUntil = min_date; - } - if (MaxAge != 0) { - time_t const max_date = Date + MaxAge; - if (ValidUntil == 0 || ValidUntil > max_date) - ValidUntil = max_date; - } - - return true; -} - /*}}}*/ -std::vector indexRecords::MetaKeys() /*{{{*/ -{ - std::vector keys; - std::map::iterator I = Entries.begin(); - while(I != Entries.end()) { - keys.push_back((*I).first); - ++I; - } - return keys; -} - /*}}}*/ -bool indexRecords::parseSumData(const char *&Start, const char *End, /*{{{*/ - string &Name, string &Hash, unsigned long long &Size) -{ - Name = ""; - Hash = ""; - Size = 0; - /* Skip over the first blank */ - while ((*Start == '\t' || *Start == ' ' || *Start == '\n' || *Start == '\r') - && Start < End) - Start++; - if (Start >= End) - return false; - - /* Move EntryEnd to the end of the first entry (the hash) */ - const char *EntryEnd = Start; - while ((*EntryEnd != '\t' && *EntryEnd != ' ') - && EntryEnd < End) - EntryEnd++; - if (EntryEnd == End) - return false; - - Hash.append(Start, EntryEnd-Start); - - /* Skip over intermediate blanks */ - Start = EntryEnd; - while (*Start == '\t' || *Start == ' ') - Start++; - if (Start >= End) - return false; - - EntryEnd = Start; - /* Find the end of the second entry (the size) */ - while ((*EntryEnd != '\t' && *EntryEnd != ' ' ) - && EntryEnd < End) - EntryEnd++; - if (EntryEnd == End) - return false; - - Size = strtoull (Start, NULL, 10); - - /* Skip over intermediate blanks */ - Start = EntryEnd; - while (*Start == '\t' || *Start == ' ') - Start++; - if (Start >= End) - return false; - - EntryEnd = Start; - /* Find the end of the third entry (the filename) */ - while ((*EntryEnd != '\t' && *EntryEnd != ' ' && - *EntryEnd != '\n' && *EntryEnd != '\r') - && EntryEnd < End) - EntryEnd++; - - Name.append(Start, EntryEnd-Start); - Start = EntryEnd; //prepare for the next round - return true; -} - /*}}}*/ - -APT_PURE bool indexRecords::IsAlwaysTrusted() const -{ - if (Trusted == ALWAYS_TRUSTED) - return true; - return false; -} -APT_PURE bool indexRecords::IsNeverTrusted() const -{ - if (Trusted == NEVER_TRUSTED) - return true; - return false; -} -void indexRecords::SetTrusted(bool const Trusted) -{ - if (Trusted == true) - this->Trusted = ALWAYS_TRUSTED; - else - this->Trusted = NEVER_TRUSTED; -} - -indexRecords::indexRecords(const string &ExpectedDist) : - Trusted(CHECK_TRUST), d(NULL), ExpectedDist(ExpectedDist), ValidUntil(0), - SupportsAcquireByHash(false) -{ -} - -indexRecords::~indexRecords() { - for (std::map::const_iterator S = Entries.begin(); S != Entries.end(); ++S) - delete S->second; -} diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h deleted file mode 100644 index 683247e42..000000000 --- a/apt-pkg/indexrecords.h +++ /dev/null @@ -1,88 +0,0 @@ -// -*- mode: cpp; mode: fold -*- -#ifndef PKGLIB_INDEXRECORDS_H -#define PKGLIB_INDEXRECORDS_H - -#include - -#include -#include -#include -#include - -#ifndef APT_8_CLEANER_HEADERS -#include -#endif -#ifndef APT_10_CLEANER_HEADERS -#include -#endif - -class indexRecords -{ - APT_HIDDEN bool parseSumData(const char *&Start, const char *End, std::string &Name, - std::string &Hash, unsigned long long &Size); - public: - struct checkSum; - std::string ErrorText; - - private: - enum APT_HIDDEN { ALWAYS_TRUSTED, NEVER_TRUSTED, CHECK_TRUST } Trusted; - // dpointer (for later) - void * const d; - - protected: - std::string Dist; - std::string Suite; - std::string ExpectedDist; - time_t Date; - time_t ValidUntil; - bool SupportsAcquireByHash; - - std::map Entries; - - public: - explicit indexRecords(const std::string &ExpectedDist = ""); - - // Lookup function - virtual checkSum *Lookup(std::string const &MetaKey); - /** \brief tests if a checksum for this file is available */ - bool Exists(std::string const &MetaKey) const; - std::vector MetaKeys(); - - virtual bool Load(std::string const &Filename); - virtual bool CheckDist(std::string const &MaybeDist) const; - - std::string GetDist() const; - std::string GetSuite() const; - bool GetSupportsAcquireByHash() const; - time_t GetValidUntil() const; - time_t GetDate() const; - std::string GetExpectedDist() const; - - /** \brief check if source is marked as always trusted */ - bool IsAlwaysTrusted() const; - /** \brief check if source is marked as never trusted */ - bool IsNeverTrusted() const; - - /** \brief sets an explicit trust value - * - * \b true means that the source should always be considered trusted, - * while \b false marks a source as always untrusted, even if we have - * a valid signature and everything. - */ - void SetTrusted(bool const Trusted); - - virtual ~indexRecords(); -}; - -APT_IGNORE_DEPRECATED_PUSH -struct indexRecords::checkSum -{ - std::string MetaKeyFilename; - HashStringList Hashes; - unsigned long long Size; - - APT_DEPRECATED HashString Hash; -}; -APT_IGNORE_DEPRECATED_POP - -#endif diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc index 0c88ee9cd..8bd13bb18 100644 --- a/apt-pkg/metaindex.cc +++ b/apt-pkg/metaindex.cc @@ -9,20 +9,6 @@ #include /*}}}*/ -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) -std::string metaIndex::LocalFileName() const { return ""; } -#else -#include -std::string metaIndex::LocalFileName() const -{ - debReleaseIndex const * deb = dynamic_cast(this); - if (deb != NULL) - return deb->LocalFileName(); - - return ""; -} -#endif - std::string metaIndex::Describe() const { return "Release"; @@ -38,10 +24,11 @@ bool metaIndex::Merge(pkgCacheGenerator &Gen,OpProgress *) const return Gen.SelectReleaseFile("", ""); } - metaIndex::metaIndex(std::string const &URI, std::string const &Dist, char const * const Type) -: d(NULL), Indexes(NULL), Type(Type), URI(URI), Dist(Dist) +: d(NULL), Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(TRI_UNSET), + LoadedSuccessfully(TRI_UNSET), + Date(0), ValidUntil(0), SupportsAcquireByHash(false) { /* nothing */ } @@ -55,3 +42,60 @@ metaIndex::~metaIndex() delete *I; delete Indexes; } + +// one line Getters for public fields /*{{{*/ +APT_PURE std::string metaIndex::GetURI() const { return URI; } +APT_PURE std::string metaIndex::GetDist() const { return Dist; } +APT_PURE const char* metaIndex::GetType() const { return Type; } +APT_PURE metaIndex::TriState metaIndex::GetTrusted() const { return Trusted; } +APT_PURE std::string metaIndex::GetCodename() const { return Codename; } +APT_PURE std::string metaIndex::GetSuite() const { return Suite; } +APT_PURE bool metaIndex::GetSupportsAcquireByHash() const { return SupportsAcquireByHash; } +APT_PURE time_t metaIndex::GetValidUntil() const { return ValidUntil; } +APT_PURE time_t metaIndex::GetDate() const { return this->Date; } +APT_PURE metaIndex::TriState metaIndex::GetLoadedSuccessfully() const { return LoadedSuccessfully; } + +APT_PURE bool metaIndex::CheckDist(string const &MaybeDist) const +{ + return (this->Codename == MaybeDist + || this->Suite == MaybeDist); +} +APT_PURE std::string metaIndex::GetExpectedDist() const +{ + // TODO: Used to be an explicit value set in the constructor + return ""; +} + /*}}}*/ +APT_PURE metaIndex::checkSum *metaIndex::Lookup(string const &MetaKey) const /*{{{*/ +{ + std::map::const_iterator sum = Entries.find(MetaKey); + if (sum == Entries.end()) + return NULL; + return sum->second; +} + /*}}}*/ +APT_PURE bool metaIndex::Exists(string const &MetaKey) const /*{{{*/ +{ + return Entries.find(MetaKey) != Entries.end(); +} + /*}}}*/ +std::vector metaIndex::MetaKeys() const /*{{{*/ +{ + std::vector keys; + std::map::const_iterator I = Entries.begin(); + while(I != Entries.end()) { + keys.push_back((*I).first); + ++I; + } + return keys; +} + /*}}}*/ +void metaIndex::swapLoad(metaIndex * const OldMetaIndex) /*{{{*/ +{ + std::swap(Entries, OldMetaIndex->Entries); + std::swap(Date, OldMetaIndex->Date); + std::swap(ValidUntil, OldMetaIndex->ValidUntil); + std::swap(SupportsAcquireByHash, OldMetaIndex->SupportsAcquireByHash); + std::swap(LoadedSuccessfully, OldMetaIndex->LoadedSuccessfully); +} + /*}}}*/ diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index 9667e1c92..5be7397ae 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -28,35 +28,81 @@ class OpProgress; class metaIndex { +public: + APT_IGNORE_DEPRECATED_PUSH + struct checkSum + { + std::string MetaKeyFilename; + HashStringList Hashes; + unsigned long long Size; + + APT_DEPRECATED HashString Hash; + }; + APT_IGNORE_DEPRECATED_POP + + enum APT_HIDDEN TriState { + TRI_YES, TRI_DONTCARE, TRI_NO, TRI_UNSET + }; +private: void * const d; - protected: +protected: std::vector *Indexes; + // parsed from the sources.list const char *Type; std::string URI; std::string Dist; + TriState Trusted; + TriState LoadedSuccessfully; - public: + // parsed from a file + std::string Suite; + std::string Codename; + time_t Date; + time_t ValidUntil; + bool SupportsAcquireByHash; + std::map Entries; +public: // Various accessors - virtual std::string GetURI() const {return URI;} - virtual std::string GetDist() const {return Dist;} - virtual const char* GetType() const {return Type;} + std::string GetURI() const; + std::string GetDist() const; + const char* GetType() const; + TriState GetTrusted() const; - // interface to to query it - /** \return the path of the local file (or "" if its not available) */ - virtual std::string LocalFileName() const; + std::string GetCodename() const; + std::string GetSuite() const; + bool GetSupportsAcquireByHash() const; + time_t GetValidUntil() const; + time_t GetDate() const; + + std::string GetExpectedDist() const; + bool CheckDist(std::string const &MaybeDist) const; // Interface for acquire + virtual std::string Describe() const; virtual std::string ArchiveURI(std::string const& File) const = 0; - virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const = 0; + virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) = 0; virtual std::vector GetIndexTargets() const = 0; virtual std::vector *GetIndexFiles() = 0; virtual bool IsTrusted() const = 0; + virtual bool Load(std::string const &Filename, std::string * const ErrorText) = 0; + /** @return a new metaIndex object based on this one, but without information from #Load */ + virtual metaIndex * UnloadedClone() const = 0; + // the given metaIndex is potentially invalid after this call and should be deleted + void swapLoad(metaIndex * const OldMetaIndex); - virtual std::string Describe() const; + // Lookup functions for parsed Hashes + checkSum *Lookup(std::string const &MetaKey) const; + /** \brief tests if a checksum for this file is available */ + bool Exists(std::string const &MetaKey) const; + std::vector MetaKeys() const; + TriState GetLoadedSuccessfully() const; + + // Interfaces for pkgCacheGen virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const; virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const; + metaIndex(std::string const &URI, std::string const &Dist, char const * const Type); virtual ~metaIndex(); diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 69f7ac043..0502f0e1d 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -103,6 +103,7 @@ bool pkgSourceList::Type::ParseStanza(vector &List, /*{{{*/ APT_PLUSMINUS("Targets", "target"); #undef APT_PLUSMINUS mapping.insert(std::make_pair("Trusted", "trusted")); + for (std::map::const_iterator m = mapping.begin(); m != mapping.end(); ++m) if (Tags.Exists(m->first)) { diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 500a0a3c5..10d4b3cc5 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -137,11 +137,9 @@ static bool TryToInstallBuildDep(pkgCache::PkgIterator Pkg,pkgCacheFile &Cache, return true; } /*}}}*/ -// GetReleaseForSourceRecord - Return Suite for the given srcrecord /*{{{*/ -// --------------------------------------------------------------------- -/* */ -static std::string GetReleaseForSourceRecord(pkgSourceList *SrcList, - pkgSrcRecords::Parser *Parse) +// GetReleaseFileForSourceRecord - Return Suite for the given srcrecord /*{{{*/ +static pkgCache::RlsFileIterator GetReleaseFileForSourceRecord(CacheFile &CacheFile, + pkgSourceList *SrcList, pkgSrcRecords::Parser *Parse) { // try to find release const pkgIndexFile& CurrentIndexFile = Parse->Index(); @@ -154,18 +152,10 @@ static std::string GetReleaseForSourceRecord(pkgSourceList *SrcList, IF != Indexes->end(); ++IF) { if (&CurrentIndexFile == (*IF)) - { - std::string const path = (*S)->LocalFileName(); - if (path != "") - { - indexRecords records; - records.Load(path); - return records.GetSuite(); - } - } + return (*S)->FindInCache(CacheFile, false); } } - return ""; + return pkgCache::RlsFileIterator(CacheFile); } /*}}}*/ // FindSrc - Find a source record /*{{{*/ @@ -379,13 +369,16 @@ static pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, // See if we need to look for a specific release tag if (RelTag != "" && UserRequestedVerTag == "") { - const string Rel = GetReleaseForSourceRecord(SrcList, Parse); - - if (Rel == RelTag) + pkgCache::RlsFileIterator const Rls = GetReleaseFileForSourceRecord(CacheFile, SrcList, Parse); + if (Rls.end() == false) { - Last = Parse; - Offset = Parse->Offset(); - Version = Ver; + if ((Rls->Archive != 0 && RelTag == Rls.Archive()) || + (Rls->Codename != 0 && RelTag == Rls.Codename())) + { + Last = Parse; + Offset = Parse->Offset(); + Version = Ver; + } } } diff --git a/test/integration/test-apt-cli-update b/test/integration/test-apt-cli-update index d68ab25e4..dad365f7e 100755 --- a/test/integration/test-apt-cli-update +++ b/test/integration/test-apt-cli-update @@ -13,9 +13,7 @@ insertinstalledpackage 'foo' 'all' '1.0' setupaptarchive --no-update -APTARCHIVE=$(readlink -f ./aptarchive) - -testfailureequal 'E: The update command takes no arguments' apt update -q arguments +testfailuremsg 'E: The update command takes no arguments' apt update arguments testsuccessequal "1 package can be upgraded. Run 'apt list --upgradable' to see it." apt update -q -- cgit v1.2.3-70-g09d2 From 0741daeb7ab870b4dd62a93fa12a1cf6330f9a72 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 23 Jun 2015 17:26:57 +0200 Subject: add sources.list Check-Valid-Until and Valid-Until-{Max,Min} options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These options could be set via configuration before, but the connection to the actual sources is so strong that they should really be set in the sources.list instead – especially as this can be done a lot more specific rather than e.g. disabling Valid-Until for all sources at once. Valid-Until-* names are chosen instead of the Min/Max-ValidTime as this seems like a better name and their use in the wild is probably low enough that this isn't going to confuse anyone if we have to names for the same thing in different areas. In the longrun, the config options should be removed, but for now documentation hinting at the new options is good enough as these are the kind of options you set once across many systems with different apt versions, so the new way should work everywhere first before we deprecate the old way. --- apt-pkg/acquire-item.cc | 4 +- apt-pkg/deb/debmetaindex.cc | 134 ++++++++++++++++++-------- apt-pkg/deb/debmetaindex.h | 3 + apt-pkg/sourcelist.cc | 3 + doc/apt-get.8.xml | 10 +- doc/apt.conf.5.xml | 8 +- doc/sources.list.5.xml | 55 +++++++++-- test/integration/test-releasefile-valid-until | 9 ++ 8 files changed, 170 insertions(+), 56 deletions(-) (limited to 'apt-pkg/sourcelist.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 100199bc1..a30a5d154 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1041,8 +1041,8 @@ bool pkgAcqMetaBase::VerifyVendor(string const &Message) /*{{{*/ Transformed = ""; } - if (_config->FindB("Acquire::Check-Valid-Until", true) == true && - TransactionManager->MetaIndexParser->GetValidUntil() > 0) { + if (TransactionManager->MetaIndexParser->GetValidUntil() > 0) + { time_t const invalid_since = time(NULL) - TransactionManager->MetaIndexParser->GetValidUntil(); if (invalid_since > 0) { diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index f0b859eb4..5d7e539c7 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -44,7 +44,11 @@ class APT_HIDDEN debReleaseIndexPrivate /*{{{*/ std::vector DebEntries; std::vector DebSrcEntries; - debReleaseIndexPrivate() {} + metaIndex::TriState CheckValidUntil; + time_t ValidUntilMin; + time_t ValidUntilMax; + + debReleaseIndexPrivate() : CheckValidUntil(metaIndex::TRI_UNSET), ValidUntilMin(0), ValidUntilMax(0) {} }; /*}}}*/ // ReleaseIndex::MetaIndex* - display helpers /*{{{*/ @@ -283,43 +287,56 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro return false; } - std::string const Label = Section.FindS("Label"); - std::string const StrValidUntil = Section.FindS("Valid-Until"); + bool CheckValidUntil = _config->FindB("Acquire::Check-Valid-Until", true); + if (d->CheckValidUntil == metaIndex::TRI_NO) + CheckValidUntil = false; + else if (d->CheckValidUntil == metaIndex::TRI_YES) + CheckValidUntil = true; - // if we have a Valid-Until header in the Release file, use it as default - if (StrValidUntil.empty() == false) + if (CheckValidUntil == true) { - if(RFC1123StrToTime(StrValidUntil.c_str(), ValidUntil) == false) + std::string const Label = Section.FindS("Label"); + std::string const StrValidUntil = Section.FindS("Valid-Until"); + + // if we have a Valid-Until header in the Release file, use it as default + if (StrValidUntil.empty() == false) { - if (ErrorText != NULL) - strprintf(*ErrorText, _("Invalid 'Valid-Until' entry in Release file %s"), Filename.c_str()); - return false; + if(RFC1123StrToTime(StrValidUntil.c_str(), ValidUntil) == false) + { + if (ErrorText != NULL) + strprintf(*ErrorText, _("Invalid 'Valid-Until' entry in Release file %s"), Filename.c_str()); + return false; + } + } + // get the user settings for this archive and use what expires earlier + time_t MaxAge = d->ValidUntilMax; + if (MaxAge == 0) + { + MaxAge = _config->FindI("Acquire::Max-ValidTime", 0); + if (Label.empty() == false) + MaxAge = _config->FindI(("Acquire::Max-ValidTime::" + Label).c_str(), MaxAge); + } + time_t MinAge = d->ValidUntilMin; + if (MinAge == 0) + { + MinAge = _config->FindI("Acquire::Min-ValidTime", 0); + if (Label.empty() == false) + MinAge = _config->FindI(("Acquire::Min-ValidTime::" + Label).c_str(), MinAge); } - } - // get the user settings for this archive and use what expires earlier - int MaxAge = _config->FindI("Acquire::Max-ValidTime", 0); - if (Label.empty() == false) - MaxAge = _config->FindI(("Acquire::Max-ValidTime::" + Label).c_str(), MaxAge); - int MinAge = _config->FindI("Acquire::Min-ValidTime", 0); - if (Label.empty() == false) - MinAge = _config->FindI(("Acquire::Min-ValidTime::" + Label).c_str(), MinAge); - - LoadedSuccessfully = TRI_YES; - if(MaxAge == 0 && - (MinAge == 0 || ValidUntil == 0)) // No user settings, use the one from the Release file - return true; - if (MinAge != 0 && ValidUntil != 0) { - time_t const min_date = Date + MinAge; - if (ValidUntil < min_date) - ValidUntil = min_date; - } - if (MaxAge != 0) { - time_t const max_date = Date + MaxAge; - if (ValidUntil == 0 || ValidUntil > max_date) - ValidUntil = max_date; + if (MinAge != 0 && ValidUntil != 0) { + time_t const min_date = Date + MinAge; + if (ValidUntil < min_date) + ValidUntil = min_date; + } + if (MaxAge != 0) { + time_t const max_date = Date + MaxAge; + if (ValidUntil == 0 || ValidUntil > max_date) + ValidUntil = max_date; + } } + LoadedSuccessfully = TRI_YES; return true; } /*}}}*/ @@ -411,7 +428,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll)/*{{{*/ return true; } /*}}}*/ -// ReleaseIndex::IsTrusted /*{{{*/ +// ReleaseIndex::Set* TriState options /*{{{*/ bool debReleaseIndex::SetTrusted(TriState const pTrusted) { if (Trusted == TRI_UNSET) @@ -421,6 +438,32 @@ bool debReleaseIndex::SetTrusted(TriState const pTrusted) return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Trusted", URI.c_str(), Dist.c_str()); return true; } +bool debReleaseIndex::SetCheckValidUntil(TriState const pCheckValidUntil) +{ + if (d->CheckValidUntil == TRI_UNSET) + d->CheckValidUntil = pCheckValidUntil; + else if (d->CheckValidUntil != pCheckValidUntil) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Check-Valid-Until", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetValidUntilMin(time_t const Valid) +{ + if (d->ValidUntilMin == 0) + d->ValidUntilMin = Valid; + else if (d->ValidUntilMin != Valid) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Min-ValidTime", URI.c_str(), Dist.c_str()); + return true; +} +bool debReleaseIndex::SetValidUntilMax(time_t const Valid) +{ + if (d->ValidUntilMax == 0) + d->ValidUntilMax = Valid; + else if (d->ValidUntilMax != Valid) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Max-ValidTime", URI.c_str(), Dist.c_str()); + return true; +} + /*}}}*/ +// ReleaseIndex::IsTrusted /*{{{*/ bool debReleaseIndex::IsTrusted() const { if (Trusted == TRI_YES) @@ -601,6 +644,22 @@ static std::vector parsePlusMinusOptions(std::string const &Name, / /*}}}*/ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ { + metaIndex::TriState GetTriStateOption(std::mapconst &Options, char const * const name) const + { + std::map::const_iterator const opt = Options.find(name); + if (opt != Options.end()) + return StringToBool(opt->second, false) ? metaIndex::TRI_YES : metaIndex::TRI_NO; + return metaIndex::TRI_DONTCARE; + } + + time_t GetTimeOption(std::mapconst &Options, char const * const name) const + { + std::map::const_iterator const opt = Options.find(name); + if (opt == Options.end()) + return 0; + return strtoull(opt->second.c_str(), NULL, 10); + } + protected: bool CreateItemInternal(std::vector &List, std::string const &URI, @@ -641,13 +700,10 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ parsePlusMinusOptions("lang", Options, APT::Configuration::getLanguages(true)) ); - std::map::const_iterator const trusted = Options.find("trusted"); - if (trusted != Options.end()) - { - if (Deb->SetTrusted(StringToBool(trusted->second, false) ? debReleaseIndex::TRI_YES : debReleaseIndex::TRI_NO) == false) - return false; - } - else if (Deb->SetTrusted(debReleaseIndex::TRI_DONTCARE) == false) + if (Deb->SetTrusted(GetTriStateOption(Options, "trusted")) == false || + Deb->SetCheckValidUntil(GetTriStateOption(Options, "check-valid-until")) == false || + Deb->SetValidUntilMax(GetTimeOption(Options, "valid-until-max")) == false || + Deb->SetValidUntilMin(GetTimeOption(Options, "valid-until-min")) == false) return false; return true; diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 19fe6806c..879eb3bfc 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -53,6 +53,9 @@ class APT_HIDDEN debReleaseIndex : public metaIndex virtual std::vector *GetIndexFiles(); bool SetTrusted(TriState const Trusted); + bool SetCheckValidUntil(TriState const Trusted); + bool SetValidUntilMin(time_t const Valid); + bool SetValidUntilMax(time_t const Valid); virtual bool IsTrusted() const; diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0502f0e1d..0d65558ed 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -103,6 +103,9 @@ bool pkgSourceList::Type::ParseStanza(vector &List, /*{{{*/ APT_PLUSMINUS("Targets", "target"); #undef APT_PLUSMINUS mapping.insert(std::make_pair("Trusted", "trusted")); + mapping.insert(std::make_pair("Check-Valid-Until", "check-valid-until")); + mapping.insert(std::make_pair("Valid-Until-Min", "valid-until-min")); + mapping.insert(std::make_pair("Valid-Until-Max", "valid-until-max")); for (std::map::const_iterator m = mapping.begin(); m != mapping.end(); ++m) if (Tags.Exists(m->first)) diff --git a/doc/apt-get.8.xml b/doc/apt-get.8.xml index 5b6788ed4..b0fe390df 100644 --- a/doc/apt-get.8.xml +++ b/doc/apt-get.8.xml @@ -531,9 +531,13 @@ - Ignore if packages can't be authenticated and don't prompt about it. - This is useful for tools like pbuilder. - Configuration Item: APT::Get::AllowUnauthenticated. + Ignore if packages can't be authenticated and don't prompt + about it. This can be useful while working with local repositories, + but is a huge security risk if data authenticity isn't ensured in + another way by the user itself. The usage of the + option for &sources-list; entries should + usually be preferred over this global override. Configuration Item: + APT::Get::AllowUnauthenticated. diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index 7d5f7e9b3..103d0622c 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -301,6 +301,8 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; Valid-Until header, but if they don't or a stricter value is desired the Max-ValidTime option below can be used. + The option of &sources-list; entries should be + preferred to disable the check selectively instead of using this global override. @@ -312,7 +314,8 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; the earlier date of the two is used as the expiration date. The default value is 0 which stands for "valid forever". Archive specific settings can be made by appending the label of the archive - to the option name. + to the option name. Preferably, the same can be achieved for specific + &sources-list; entries by using the option there. @@ -324,7 +327,8 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; frequently updated archive with a Valid-Until header instead of completely disabling the expiration date checking. Archive specific settings can and should be used by appending the label of - the archive to the option name. + the archive to the option name. Preferably, the same can be achieved for specific + &sources-list; entries by using the option there. diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml index f87dcda23..aded8ecef 100644 --- a/doc/sources.list.5.xml +++ b/doc/sources.list.5.xml @@ -202,26 +202,26 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [. APT versions. - Architectures - (arch) is a multivalue option defining for + + () is a multivalue option defining for which architectures information should be downloaded. If this option isn't set the default is all architectures as defined by - the APT::Architectures config option. + the config option. - Languages - (lang) is a multivalue option defining for + + () is a multivalue option defining for which languages information like translated package descriptions should be downloaded. If this option isn't set the default is all languages as defined by the - Acquire::Languages config option. + config option. - Targets - (target) is a multivalue option defining + + () is a multivalue option defining which download targets apt will try to acquire from this source. If not specified, the default set is defined by the - APT::Acquire::Targets configuration scope. + configuration scope. @@ -232,7 +232,7 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [. anomalies. - Trusted (trusted) + () is a tri-state value which defaults to APT deciding if a source is considered trusted or if warnings should be raised before e.g. packages are installed from this source. This option can be used @@ -245,6 +245,41 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [. as untrusted even if the authentication checks passed successfully. The default value can't be set explicitly. + + () + is a yes/no value which controls if APT should try to detect + replay attacks. A repository creator can declare until then the + data provided in the repository should be considered valid and + if this time is reached, but no new data is provided the data + is considered expired and an error is raised. Beside + increasing security as a malicious attacker can't sent old data + forever denying a user to be able to upgrade to a new version, + this also helps users identify mirrors which are no longer + updated. Some repositories like historic archives aren't + updated anymore by design through, so this check can be + disabled by setting this option to no. + Defaults to the value of configuration option + which itself + defaults to yes. + + + + () and + + () can be used to raise or + lower the time period in seconds in which the data from this + repository is considered valid. -Max can be especially useful + if the repository provides no Valid-Until field on its Release + file to set your own value, while -Min can be used to increase + the valid time on seldomly updated (local) mirrors of a more + frequently updated but less accessible archive (which is in the + sources.list as well) instead of disabling the check entirely. + Default to the value of the configuration options + and + which are both unset by + default. + + diff --git a/test/integration/test-releasefile-valid-until b/test/integration/test-releasefile-valid-until index e000abf5d..43574ec3e 100755 --- a/test/integration/test-releasefile-valid-until +++ b/test/integration/test-releasefile-valid-until @@ -46,3 +46,12 @@ runtest 'accepted' 'good Min-Valid (bad Until, good Max-Valid) <' 'now - 7 days' runtest 'rejected' 'bad Max-Valid (bad Until, good Min-Valid) >' 'now - 7 days' 'now - 2 days' -o Acquire::Max-ValidTime=12096 -o Acquire::Min-ValidTime=2419200 runtest 'rejected' 'bad Max-Valid (bad Until, bad Min-Valid) <' 'now - 7 days' 'now - 2 days' -o Acquire::Min-ValidTime=12096 -o Acquire::Max-ValidTime=241920 runtest 'rejected' 'bad Max-Valid (bad Until, bad Min-Valid) >' 'now - 7 days' 'now - 2 days' -o Acquire::Max-ValidTime=12096 -o Acquire::Min-ValidTime=241920 + +sed -i -e 's#\(deb\(-src\)\?\) #\1 [check-valid-until=no] #' rootdir/etc/apt/sources.list.d/* +runtest 'accepted' 'bad Until but overriden by sources option' 'now - 7 days' 'now - 4 days' + +sed -i -e 's#\(deb\(-src\)\?\) \[check-valid-until=no\] #\1 [valid-until-max=86400] #' rootdir/etc/apt/sources.list.d/* +runtest 'rejected' 'bad Max-Valid (good Until) via sources option' 'now - 7 days' 'now + 4 days' + +sed -i -e 's#\(deb\(-src\)\?\) \[valid-until-max=86400\] #\1 [valid-until-min=1209600] #' rootdir/etc/apt/sources.list.d/* +runtest 'accepted' 'good Min-Valid (bad Until) via sources option' 'now - 7 days' 'now - 4 days' -- cgit v1.2.3-70-g09d2 From b0d408547734100bf86781615f546487ecf390d9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 24 Jun 2015 19:31:22 +0200 Subject: implement Signed-By option for sources.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Limits which key(s) can be used to sign a repository. Not immensely useful from a security perspective all by itself, but if the user has additional measures in place to confine a repository (like pinning) an attacker who gets the key for such a repository is limited to its potential and can't use the key to sign its attacks for an other (maybe less limited) repository… (yes, this is as weak as it sounds, but having the capability might come in handy for implementing other stuff later). --- apt-pkg/acquire-item.cc | 15 +++++++- apt-pkg/acquire-item.h | 1 + apt-pkg/contrib/gpgv.cc | 17 +++++++++- apt-pkg/contrib/gpgv.h | 5 ++- apt-pkg/deb/debmetaindex.cc | 35 +++++++++++++++++++ apt-pkg/deb/debmetaindex.h | 1 + apt-pkg/metaindex.cc | 4 +-- apt-pkg/metaindex.h | 4 ++- apt-pkg/sourcelist.cc | 28 ++++++++------- cmdline/apt-key.in | 22 ++++++++++-- doc/sources.list.5.xml | 24 ++++++------- methods/gpgv.cc | 18 +++++----- test/integration/framework | 23 +++++++++---- test/integration/test-apt-key | 37 ++++++++++++++++++-- test/integration/test-releasefile-verification | 47 +++++++++++++++++++++++--- 15 files changed, 225 insertions(+), 56 deletions(-) (limited to 'apt-pkg/sourcelist.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index a30a5d154..01a679fe0 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -808,7 +808,6 @@ string pkgAcqMetaBase::Custom600Headers() const Header += MaximumSize; string const FinalFile = GetFinalFilename(); - struct stat Buf; if (stat(FinalFile.c_str(),&Buf) == 0) Header += "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); @@ -1132,6 +1131,10 @@ string pkgAcqMetaClearSig::Custom600Headers() const { string Header = pkgAcqMetaBase::Custom600Headers(); Header += "\nFail-Ignore: true"; + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + key; + return Header; } /*}}}*/ @@ -1372,6 +1375,16 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire * const Owner, /*}}}*/ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ { +} + /*}}}*/ +// pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ +std::string pkgAcqMetaSig::Custom600Headers() const +{ + std::string Header = pkgAcqTransactionItem::Custom600Headers(); + std::string const key = TransactionManager->MetaIndexParser->GetSignedBy(); + if (key.empty() == false) + Header += "\nSigned-By: " + key; + return Header; } /*}}}*/ // AcqMetaSig::Done - The signature was downloaded/verified /*{{{*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 10ece76c9..1cd2a6d03 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -541,6 +541,7 @@ class APT_HIDDEN pkgAcqMetaSig : public pkgAcqTransactionItem virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf); virtual void Done(std::string const &Message, HashStringList const &Hashes, pkgAcquire::MethodConfig const * const Cnf); + virtual std::string Custom600Headers() const; /** \brief Create a new pkgAcqMetaSig. */ pkgAcqMetaSig(pkgAcquire * const Owner, pkgAcqMetaClearSig * const TransactionManager, diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc index a01e319eb..ef84da0d8 100644 --- a/apt-pkg/contrib/gpgv.cc +++ b/apt-pkg/contrib/gpgv.cc @@ -16,6 +16,8 @@ #include #include #include + +#include #include #include #include @@ -42,7 +44,7 @@ static char * GenerateTemporaryFileTemplate(const char *basename) /*{{{*/ of the lifting in regards to merging keyrings. Fun for the whole family. */ void ExecGPGV(std::string const &File, std::string const &FileGPG, - int const &statusfd, int fd[2]) + int const &statusfd, int fd[2], std::string const &key) { #define EINTERNAL 111 std::string const aptkey = _config->FindFile("Dir::Bin::apt-key", "/usr/bin/apt-key"); @@ -55,6 +57,19 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, Args.push_back(aptkey.c_str()); Args.push_back("--quiet"); Args.push_back("--readonly"); + if (key.empty() == false) + { + if (key[0] == '/') + { + Args.push_back("--keyring"); + Args.push_back(key.c_str()); + } + else + { + Args.push_back("--keyid"); + Args.push_back(key.c_str()); + } + } Args.push_back("verify"); char statusfdstr[10]; diff --git a/apt-pkg/contrib/gpgv.h b/apt-pkg/contrib/gpgv.h index f018893fd..2a4cdad72 100644 --- a/apt-pkg/contrib/gpgv.h +++ b/apt-pkg/contrib/gpgv.h @@ -38,9 +38,12 @@ class FileFd; * * @param File is the message (unsigned or clear-signed) * @param FileSig is the signature (detached or clear-signed) + * @param statusfd is the fd given to gpgv as --status-fd + * @param fd is used as a pipe for the standard output of gpgv + * @param key is the specific one to be used instead of using all */ void ExecGPGV(std::string const &File, std::string const &FileSig, - int const &statusfd, int fd[2]) APT_NORETURN; + int const &statusfd, int fd[2], std::string const &Key = "") APT_NORETURN; inline APT_NORETURN void ExecGPGV(std::string const &File, std::string const &FileSig, int const &statusfd = -1) { int fd[2]; diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 5d7e539c7..4bb03a942 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -461,6 +461,29 @@ bool debReleaseIndex::SetValidUntilMax(time_t const Valid) else if (d->ValidUntilMax != Valid) return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Max-ValidTime", URI.c_str(), Dist.c_str()); return true; +} +bool debReleaseIndex::SetSignedBy(std::string const &pSignedBy) +{ + if (SignedBy.empty() == true && pSignedBy.empty() == false) + { + if (pSignedBy[0] == '/') // no check for existence as we could be chrooting later or such things + ; // absolute path to a keyring file + else + { + // we could go all fancy and allow short/long/string matches as gpgv/apt-key does, + // but fingerprints are harder to fake than the others and this option is set once, + // not interactively all the time so easy to type is not really a concern. + std::string finger = pSignedBy; + finger.erase(std::remove(finger.begin(), finger.end(), ' '), finger.end()); + std::transform(finger.begin(), finger.end(), finger.begin(), ::toupper); + if (finger.length() != 40 || finger.find_first_not_of("0123456789ABCDEF") != std::string::npos) + return _error->Error(_("Invalid value set for option %s concerning source %s %s (%s)"), "Signed-By", URI.c_str(), Dist.c_str(), "not a fingerprint"); + } + SignedBy = pSignedBy; + } + else if (SignedBy != pSignedBy) + return _error->Error(_("Conflicting values set for option %s concerning source %s %s"), "Signed-By", URI.c_str(), Dist.c_str()); + return true; } /*}}}*/ // ReleaseIndex::IsTrusted /*{{{*/ @@ -706,6 +729,18 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ Deb->SetValidUntilMin(GetTimeOption(Options, "valid-until-min")) == false) return false; + std::map::const_iterator const signedby = Options.find("signed-by"); + if (signedby == Options.end()) + { + if (Deb->SetSignedBy("") == false) + return false; + } + else + { + if (Deb->SetSignedBy(signedby->second) == false) + return false; + } + return true; } diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 879eb3bfc..bf5b7c1ce 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -56,6 +56,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex bool SetCheckValidUntil(TriState const Trusted); bool SetValidUntilMin(time_t const Valid); bool SetValidUntilMax(time_t const Valid); + bool SetSignedBy(std::string const &SignedBy); virtual bool IsTrusted() const; diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc index 8bd13bb18..baf695f16 100644 --- a/apt-pkg/metaindex.cc +++ b/apt-pkg/metaindex.cc @@ -27,8 +27,7 @@ bool metaIndex::Merge(pkgCacheGenerator &Gen,OpProgress *) const metaIndex::metaIndex(std::string const &URI, std::string const &Dist, char const * const Type) : d(NULL), Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(TRI_UNSET), - LoadedSuccessfully(TRI_UNSET), - Date(0), ValidUntil(0), SupportsAcquireByHash(false) + Date(0), ValidUntil(0), SupportsAcquireByHash(false), LoadedSuccessfully(TRI_UNSET) { /* nothing */ } @@ -48,6 +47,7 @@ APT_PURE std::string metaIndex::GetURI() const { return URI; } APT_PURE std::string metaIndex::GetDist() const { return Dist; } APT_PURE const char* metaIndex::GetType() const { return Type; } APT_PURE metaIndex::TriState metaIndex::GetTrusted() const { return Trusted; } +APT_PURE std::string metaIndex::GetSignedBy() const { return SignedBy; } APT_PURE std::string metaIndex::GetCodename() const { return Codename; } APT_PURE std::string metaIndex::GetSuite() const { return Suite; } APT_PURE bool metaIndex::GetSupportsAcquireByHash() const { return SupportsAcquireByHash; } diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index 5be7397ae..d284655bf 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -52,7 +52,7 @@ protected: std::string URI; std::string Dist; TriState Trusted; - TriState LoadedSuccessfully; + std::string SignedBy; // parsed from a file std::string Suite; @@ -61,6 +61,7 @@ protected: time_t ValidUntil; bool SupportsAcquireByHash; std::map Entries; + TriState LoadedSuccessfully; public: // Various accessors @@ -68,6 +69,7 @@ public: std::string GetDist() const; const char* GetType() const; TriState GetTrusted() const; + std::string GetSignedBy() const; std::string GetCodename() const; std::string GetSuite() const; diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 0d65558ed..eef0ee709 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -93,27 +93,29 @@ bool pkgSourceList::Type::ParseStanza(vector &List, /*{{{*/ if (Enabled.empty() == false && StringToBool(Enabled) == false) return true; - std::map mapping; + std::map > mapping; #define APT_PLUSMINUS(X, Y) \ - mapping.insert(std::make_pair(X, Y)); \ - mapping.insert(std::make_pair(X "Add", Y "+")); \ - mapping.insert(std::make_pair(X "Remove", Y "-")) + mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \ + mapping.insert(std::make_pair(X "Add", std::make_pair(Y "+", true))); \ + mapping.insert(std::make_pair(X "Remove", std::make_pair(Y "-", true))) APT_PLUSMINUS("Architectures", "arch"); APT_PLUSMINUS("Languages", "lang"); APT_PLUSMINUS("Targets", "target"); #undef APT_PLUSMINUS - mapping.insert(std::make_pair("Trusted", "trusted")); - mapping.insert(std::make_pair("Check-Valid-Until", "check-valid-until")); - mapping.insert(std::make_pair("Valid-Until-Min", "valid-until-min")); - mapping.insert(std::make_pair("Valid-Until-Max", "valid-until-max")); + mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false))); + mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false))); + mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false))); + mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false))); + mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false))); - for (std::map::const_iterator m = mapping.begin(); m != mapping.end(); ++m) + for (std::map >::const_iterator m = mapping.begin(); m != mapping.end(); ++m) if (Tags.Exists(m->first)) { - // for deb822 the " " is the delimiter, but the backend expects "," - std::string option = Tags.FindS(m->first); - std::replace(option.begin(), option.end(), ' ', ','); - Options[m->second] = option; + std::string option = Tags.FindS(m->first); + // for deb822 the " " is the delimiter, but the backend expects "," + if (m->second.second == true) + std::replace(option.begin(), option.end(), ' ', ','); + Options[m->second.first] = option; } // now create one item per suite/section diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 2a66ad74d..16887bd50 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -199,7 +199,7 @@ remove_key_from_keyring() { foreach_keyring_do() { local ACTION="$1" shift - # if a --keyring was given, just remove from there + # if a --keyring was given, just work on this one if [ -n "$FORCED_KEYRING" ]; then $ACTION "$FORCED_KEYRING" "$@" else @@ -279,7 +279,14 @@ merge_back_changes() { } setup_merged_keyring() { - if [ -z "$FORCED_KEYRING" ]; then + if [ -n "$FORCED_KEYID" ]; then + foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/allrings.gpg" + FORCED_KEYRING="${GPGHOMEDIR}/forcedkeyid.gpg" + TRUSTEDFILE="${FORCED_KEYRING}" + GPG="$GPG --keyring $TRUSTEDFILE" + # ignore error as this "just" means we haven't found the forced keyid and the keyring will be empty + $GPG_CMD --batch --yes --keyring "${GPGHOMEDIR}/allrings.gpg" --export "$FORCED_KEYID" | $GPG --batch --yes --import || true + elif [ -z "$FORCED_KEYRING" ]; then foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/pubring.gpg" if [ -r "${GPGHOMEDIR}/pubring.gpg" ]; then cp -a "${GPGHOMEDIR}/pubring.gpg" "${GPGHOMEDIR}/pubring.orig.gpg" @@ -328,12 +335,17 @@ while [ -n "$1" ]; do TRUSTEDFILE="$1" FORCED_KEYRING="$1" ;; + --keyid) + shift + FORCED_KEYID="$1" + ;; --secret-keyring) shift FORCED_SECRET_KEYRING="$1" ;; --readonly) merge_back_changes() { true; } + create_new_keyring() { true; } ;; --fakeroot) requires_root() { true; } @@ -460,7 +472,11 @@ case "$command" in verify) setup_merged_keyring if which gpgv >/dev/null 2>&1; then - gpgv --homedir "${GPGHOMEDIR}" --keyring "${GPGHOMEDIR}/pubring.gpg" --ignore-time-conflict "$@" + if [ -n "$FORCED_KEYRING" ]; then + gpgv --homedir "${GPGHOMEDIR}" --keyring "${FORCED_KEYRING}" --ignore-time-conflict "$@" + else + gpgv --homedir "${GPGHOMEDIR}" --keyring "${GPGHOMEDIR}/pubring.gpg" --ignore-time-conflict "$@" + fi else $GPG --verify "$@" fi diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml index aded8ecef..12a7773f5 100644 --- a/doc/sources.list.5.xml +++ b/doc/sources.list.5.xml @@ -232,18 +232,18 @@ deb-src [ option1=value1 option2=value2 ] uri suite [component1] [component2] [. anomalies. - () - is a tri-state value which defaults to APT deciding if a source - is considered trusted or if warnings should be raised before e.g. - packages are installed from this source. This option can be used - to override this decision either with the value yes, - which lets APT consider this source always as a trusted source - even if it has no or fails authentication checks by disabling parts - of &apt-secure; and should therefore only be used in a local and trusted - context (if at all) as otherwise security is breached. The opposite - can be achieved with the value no, which causes the source to be handled - as untrusted even if the authentication checks passed successfully. - The default value can't be set explicitly. + () + is either an absolute path to a keyring file (has to be + accessible and readable for the _apt user, + so ensure everyone has read-permissions on the file) or a + fingerprint of a key in either the + trusted.gpg keyring or in one of the + keyrings in the trusted.gpg.d/ directory + (see apt-key fingerprint). If the option is + set only the key(s) in this keyring or only the key with this + fingerprint is used for the &apt-secure; verification of this + repository. Otherwise all keys in the trusted keyrings are + considered valid signers for this repository. () diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 41f138be6..014430041 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -37,13 +37,14 @@ class GPGVMethod : public pkgAcqMethod { private: string VerifyGetSigners(const char *file, const char *outfile, - vector &GoodSigners, + std::string const &key, + vector &GoodSigners, vector &BadSigners, vector &WorthlessSigners, vector &NoPubKeySigners); protected: - virtual bool Fetch(FetchItem *Itm); + virtual bool URIAcquire(std::string const &Message, FetchItem *Itm); virtual bool Configuration(string Message); public: @@ -61,6 +62,7 @@ bool GPGVMethod::Configuration(string Message) } string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, + std::string const &key, vector &GoodSigners, vector &BadSigners, vector &WorthlessSigners, @@ -80,7 +82,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, if (pid < 0) return string("Couldn't spawn new process") + strerror(errno); else if (pid == 0) - ExecGPGV(outfile, file, 3, fd); + ExecGPGV(outfile, file, 3, fd, key); close(fd[1]); FILE *pipein = fdopen(fd[0], "r"); @@ -174,11 +176,11 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, return _("Unknown error executing apt-key"); } -bool GPGVMethod::Fetch(FetchItem *Itm) +bool GPGVMethod::URIAcquire(std::string const &Message, FetchItem *Itm) { - URI Get = Itm->Uri; - string Path = Get.Host + Get.Path; // To account for relative paths - string keyID; + URI const Get = Itm->Uri; + string const Path = Get.Host + Get.Path; // To account for relative paths + std::string const key = LookupTag(Message, "Signed-By"); vector GoodSigners; vector BadSigners; // a worthless signature is a expired or revoked one @@ -190,7 +192,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) URIStart(Res); // Run apt-key on file, extract contents and get the key ID of the signer - string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), + string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), key, GoodSigners, BadSigners, WorthlessSigners, NoPubKeySigners); if (GoodSigners.empty() || !BadSigners.empty() || !NoPubKeySigners.empty()) diff --git a/test/integration/framework b/test/integration/framework index 059cba9fb..6ae5003f7 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1414,14 +1414,23 @@ testfailure() { else local EXITCODE=$? if expr match "$1" '^apt.*' >/dev/null; then - if grep -q -E ' runtime error: ' "$OUTPUT"; then - msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@" - elif grep -q -E '==ERROR' "$OUTPUT"; then - msgfailoutput 'compiler sanitizers reported errors' "$OUTPUT" "$@" - elif ! grep -q -E '^E: ' "$OUTPUT"; then - msgfailoutput "run failed with exitcode ${EXITCODE}, but with no errors" "$OUTPUT" "$@" + if [ "$1" = 'aptkey' ]; then + if grep -q -E " Can't check signature: " "$OUTPUT" || \ + grep -q -E " BAD signature from " "$OUTPUT"; then + msgpass + else + msgfailoutput "run failed with exitcode ${EXITCODE}, but no signature error" "$OUTPUT" "$@" + fi else - msgpass + if grep -q -E ' runtime error: ' "$OUTPUT"; then + msgfailoutput 'compiler detected undefined behavior' "$OUTPUT" "$@" + elif grep -q -E '==ERROR' "$OUTPUT"; then + msgfailoutput 'compiler sanitizers reported errors' "$OUTPUT" "$@" + elif ! grep -q -E '^E: ' "$OUTPUT"; then + msgfailoutput "run failed with exitcode ${EXITCODE}, but with no errors" "$OUTPUT" "$@" + else + msgpass + fi fi else msgpass diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index 486acccc8..e1be08c65 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -73,7 +73,7 @@ pub 2048R/DBAC8DAE 2010-08-18' testsuccess aptkey --fakeroot del DBAC8DAE testempty aptkey list - msgtest 'Test key removal with' 'lowercase key ID' #keylength somewher between 8byte and short + msgtest 'Test key removal with' 'lowercase key ID' #keylength somewhere between 8byte and short cleanplate cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg testsuccess --nomsg aptkey --fakeroot del d141dbac8dae @@ -166,6 +166,40 @@ pub 2048R/528144E2 2011-01-16' msgtest 'Test merge-back of' 'removed duplicate keys' testsuccess --nomsg aptkey adv --batch --yes --delete-keys DBAC8DAE testaptkeys 'pub 2048R/528144E2 2011-01-16' + + cleanplate + cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg + msgtest 'Test signing a file' 'with a key' + echo 'Verify me. This is my signature.' > signature + testsuccess --nomsg aptkey --quiet --keyring keys/marvinparanoid.pub --secret-keyring keys/marvinparanoid.sec --readonly \ + adv --batch --yes --default-key 'Marvin' --armor --detach-sign --sign --output signature.gpg signature + + msgtest 'Test verify a file' 'with all keys' + testsuccess --nomsg aptkey --quiet --readonly verify signature.gpg signature + + msgtest 'Test verify a file' 'with good keyring' + testsuccess --nomsg aptkey --quiet --readonly --keyring keys/testcase-multikey.pub verify signature.gpg signature + + msgtest 'Test fail verify a file' 'with bad keyring' + testfailure --nomsg aptkey --quiet --readonly --keyring keys/joesixpack.pub verify signature.gpg signature + + msgtest 'Test fail verify a file' 'with non-existing keyring' + testfailure --nomsg aptkey --quiet --readonly --keyring keys/does-not-exist.pub verify signature.gpg signature + testfailure test -e keys/does-not-exist.pub + + msgtest 'Test verify a file' 'with good keyid' + testsuccess --nomsg aptkey --quiet --readonly --keyid 'Paranoid' verify signature.gpg signature + + msgtest 'Test fail verify a file' 'with bad keyid' + testfailure --nomsg aptkey --quiet --readonly --keyid 'Sixpack' verify signature.gpg signature + + msgtest 'Test fail verify a file' 'with non-existing keyid' + testfailure --nomsg aptkey --quiet --readonly --keyid 'Kalnischkies' verify signature.gpg signature + + msgtest 'Test verify fails on' 'bad file' + echo 'lalalalala' > signature + testfailure --nomsg aptkey --quiet --readonly verify signature.gpg signature } setupgpgcommand() { @@ -187,4 +221,3 @@ setupgpgcommand 'gpg' testrun setupgpgcommand 'gpg2' testrun - diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification index e8419524c..1c3953c8b 100755 --- a/test/integration/test-releasefile-verification +++ b/test/integration/test-releasefile-verification @@ -139,11 +139,6 @@ runtest() { failaptold prepare ${PKGFILE}-new - # weborf doesn't support If-Range - for release in $(find rootdir/var/lib/apt/lists/partial/ -name '*Release'); do - rm $release - touch $release - done signreleasefiles 'Joe Sixpack' find aptarchive/ -name "$DELETEFILE" -delete msgmsg 'Bad warm archive signed by' 'Joe Sixpack' @@ -191,6 +186,48 @@ runtest() { testsuccessequal "$(cat ${PKGFILE}-new) " aptcache show apt installaptnew + + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Marvin Paranoid' + find aptarchive/ -name "$DELETEFILE" -delete + msgmsg 'Cold archive signed by good keyring' 'Marvin Paranoid' + local MARVIN="$(readlink -f keys/marvinparanoid.pub)" + sed -i "s#^\(deb\(-src\)\?\) #\1 [signed-by=$MARVIN] #" rootdir/etc/apt/sources.list.d/* + testsuccess aptget update -o Debug::pkgAcquire::Worker=1 + testsuccessequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold + + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgmsg 'Cold archive signed by bad keyring' 'Joe Sixpack' + updatewithwarnings '^W: .* NO_PUBKEY' + + sed -i "s#^\(deb\(-src\)\?\) \[signed-by=$MARVIN\] #\1 #" rootdir/etc/apt/sources.list.d/* + local MARVIN="$(aptkey --keyring $MARVIN finger | grep 'Key fingerprint' | cut -d'=' -f 2 | tr -d ' ')" + + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Marvin Paranoid' + find aptarchive/ -name "$DELETEFILE" -delete + msgmsg 'Cold archive signed by good keyid' 'Marvin Paranoid' + sed -i "s#^\(deb\(-src\)\?\) #\1 [signed-by=$MARVIN] #" rootdir/etc/apt/sources.list.d/* + cp keys/marvinparanoid.pub rootdir/etc/apt/trusted.gpg.d/marvinparanoid.gpg + testsuccess aptget update -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::gpgv=1 + testsuccessequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold + rm -f rootdir/etc/apt/trusted.gpg.d/marvinparanoid.gpg + + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgmsg 'Cold archive signed by bad keyid' 'Joe Sixpack' + updatewithwarnings '^W: .* NO_PUBKEY' + + sed -i "s#^\(deb\(-src\)\?\) \[signed-by=$MARVIN\] #\1 #" rootdir/etc/apt/sources.list.d/* } runtest2() { -- cgit v1.2.3-70-g09d2 From 5465192b9aeb1ccea778950ccf2d1b7b32f2cd91 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 18 Jul 2015 18:03:54 +0200 Subject: add volatile sources support in libapt-pkg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sources are usually defined in sources.list (and co) and are pretty stable, but once in a while a frontend might want to add an additional "source" like a local .deb file to install this package (No support for 'real' sources being added this way as this is a multistep process). We had a hack in place to allow apt-get and apt to pull this of for a short while now, but other frontends are either left in the cold by this and/or the code for it looks dirty with FIXMEs plastering it and has on top of this also some problems (like including these 'volatile' sources in the srcpkgcache.bin file). So the biggest part in this commit is actually the rewrite of the cache generation as it is now potentially a three step process. The biggest problem with adding support now through is that this makes a bunch of previously mostly unusable by externs and therefore hidden classes public, so a bit of further tuneing on this now public API is in order… --- apt-pkg/deb/debindexfile.cc | 27 +- apt-pkg/deb/debindexfile.h | 18 +- apt-pkg/deb/deblistparser.cc | 2 +- apt-pkg/deb/debmetaindex.cc | 29 --- apt-pkg/deb/debmetaindex.h | 39 --- apt-pkg/edsp/edspsystem.cc | 3 +- apt-pkg/pkgcachegen.cc | 401 ++++++++++++++---------------- apt-pkg/pkgcachegen.h | 1 - apt-pkg/pkgsystem.h | 5 +- apt-pkg/sourcelist.cc | 28 ++- apt-pkg/sourcelist.h | 19 ++ apt-private/private-cachefile.h | 21 -- apt-private/private-install.cc | 21 +- test/integration/framework | 12 +- test/integration/test-apt-get-install-deb | 45 +++- 15 files changed, 309 insertions(+), 362 deletions(-) (limited to 'apt-pkg/sourcelist.cc') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 972feba7f..e67233e5f 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -339,18 +339,16 @@ APT_CONST bool debStatusIndex::Exists() const } /*}}}*/ -// debDebPkgFile - Single .deb file /*{{{*/ -debDebPkgFileIndex::debDebPkgFileIndex(std::string DebFile) +// debDebPkgFileIndex - Single .deb file /*{{{*/ +debDebPkgFileIndex::debDebPkgFileIndex(std::string const &DebFile) : pkgIndexFile(true), d(NULL), DebFile(DebFile) { DebFileFullPath = flAbsPath(DebFile); } - std::string debDebPkgFileIndex::ArchiveURI(std::string /*File*/) const { return "file:" + DebFileFullPath; } - bool debDebPkgFileIndex::Exists() const { return FileExists(DebFile); @@ -403,6 +401,8 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const if (GetContent(content, DebFile) == false) return false; std::string const contentstr = content.str(); + if (contentstr.empty()) + return true; DebControl->Write(contentstr.c_str(), contentstr.length()); // rewind for the listparser DebControl->Seek(0); @@ -431,7 +431,7 @@ pkgCache::PkgFileIterator debDebPkgFileIndex::FindInCache(pkgCache &Cache) const return File; } - + return File; } unsigned long debDebPkgFileIndex::Size() const @@ -443,12 +443,11 @@ unsigned long debDebPkgFileIndex::Size() const } /*}}}*/ -// debDscFileIndex stuff -debDscFileIndex::debDscFileIndex(std::string &DscFile) +// debDscFileIndex - a .dsc file /*{{{*/ +debDscFileIndex::debDscFileIndex(std::string const &DscFile) : pkgIndexFile(true), d(NULL), DscFile(DscFile) { } - bool debDscFileIndex::Exists() const { return FileExists(DscFile); @@ -461,8 +460,6 @@ unsigned long debDscFileIndex::Size() const return buf.st_size; return 0; } - -// DscFileIndex::CreateSrcParser - Get a parser for the .dsc file /*{{{*/ pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const { if (!FileExists(DscFile)) @@ -471,18 +468,17 @@ pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const return new debDscRecordParser(DscFile,this); } /*}}}*/ + // Index File types for Debian /*{{{*/ class APT_HIDDEN debIFTypeSrc : public pkgIndexFile::Type { public: - debIFTypeSrc() {Label = "Debian Source Index";}; }; class APT_HIDDEN debIFTypePkg : public pkgIndexFile::Type { public: - - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debRecordParser(File.FileName(),*File.Cache()); }; @@ -496,8 +492,7 @@ class APT_HIDDEN debIFTypeTrans : public debIFTypePkg class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type { public: - - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debRecordParser(File.FileName(),*File.Cache()); }; @@ -506,7 +501,7 @@ class APT_HIDDEN debIFTypeStatus : public pkgIndexFile::Type class APT_HIDDEN debIFTypeDebPkgFile : public pkgIndexFile::Type { public: - virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE + virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator File) const APT_OVERRIDE { return new debDebFileRecordParser(File.FileName()); }; diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index 71ca22e4d..4a818121a 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -27,7 +27,7 @@ class pkgAcquire; class pkgCacheGenerator; -class APT_HIDDEN debStatusIndex : public pkgIndexFile +class debStatusIndex : public pkgIndexFile { void * const d; protected: @@ -51,7 +51,7 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile virtual ~debStatusIndex(); }; -class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile +class debPackagesIndex : public pkgIndexTargetFile { void * const d; public: @@ -70,7 +70,7 @@ class APT_HIDDEN debPackagesIndex : public pkgIndexTargetFile virtual ~debPackagesIndex(); }; -class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile +class debTranslationsIndex : public pkgIndexTargetFile { void * const d; public: @@ -86,7 +86,7 @@ class APT_HIDDEN debTranslationsIndex : public pkgIndexTargetFile virtual ~debTranslationsIndex(); }; -class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile +class debSourcesIndex : public pkgIndexTargetFile { void * const d; public: @@ -107,7 +107,7 @@ class APT_HIDDEN debSourcesIndex : public pkgIndexTargetFile virtual ~debSourcesIndex(); }; -class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile +class debDebPkgFileIndex : public pkgIndexFile { private: void * const d; @@ -141,11 +141,11 @@ class APT_HIDDEN debDebPkgFileIndex : public pkgIndexFile // Interface for acquire virtual std::string ArchiveURI(std::string /*File*/) const APT_OVERRIDE; - debDebPkgFileIndex(std::string DebFile); + debDebPkgFileIndex(std::string const &DebFile); virtual ~debDebPkgFileIndex(); }; -class APT_HIDDEN debDscFileIndex : public pkgIndexFile +class debDscFileIndex : public pkgIndexFile { private: void * const d; @@ -160,11 +160,11 @@ class APT_HIDDEN debDscFileIndex : public pkgIndexFile return DscFile; }; - debDscFileIndex(std::string &DscFile); + debDscFileIndex(std::string const &DscFile); virtual ~debDscFileIndex(); }; -class APT_HIDDEN debDebianSourceDirIndex : public debDscFileIndex +class debDebianSourceDirIndex : public debDscFileIndex { public: virtual const Type *GetType() const APT_CONST; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 1154016a9..b7988d499 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -959,7 +959,7 @@ bool debDebFileParser::UsePackage(pkgCache::PkgIterator &Pkg, bool res = debListParser::UsePackage(Pkg, Ver); // we use the full file path as a provides so that the file is found // by its name - if(NewProvidesAllArch(Ver, DebFile, Ver.VerStr(), 0) == false) + if(NewProvides(Ver, DebFile, Pkg.Cache()->NativeArch(), Ver.VerStr(), 0) == false) return false; return res; } diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 480317db3..123c91648 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -785,34 +785,5 @@ class APT_HIDDEN debSLTypeDebSrc : public debSLTypeDebian /*{{{*/ }; /*}}}*/ -debDebFileMetaIndex::debDebFileMetaIndex(std::string const &DebFile) /*{{{*/ - : metaIndex(DebFile, "local-uri", "deb-dist"), d(NULL), DebFile(DebFile) -{ - DebIndex = new debDebPkgFileIndex(DebFile); - Indexes = new std::vector(); - Indexes->push_back(DebIndex); -} -debDebFileMetaIndex::~debDebFileMetaIndex() {} - /*}}}*/ -class APT_HIDDEN debSLTypeDebFile : public pkgSourceList::Type /*{{{*/ -{ - public: - - bool CreateItem(std::vector &List, std::string const &URI, - std::string const &/*Dist*/, std::string const &/*Section*/, - std::map const &/*Options*/) const APT_OVERRIDE - { - metaIndex *mi = new debDebFileMetaIndex(URI); - List.push_back(mi); - return true; - } - - debSLTypeDebFile() : Type("deb-file", "Debian local deb file") - { - } -}; - /*}}}*/ - APT_HIDDEN debSLTypeDeb _apt_DebType; APT_HIDDEN debSLTypeDebSrc _apt_DebSrcType; -APT_HIDDEN debSLTypeDebFile _apt_DebFileType; diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 8c13237cb..e93959a21 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -17,7 +17,6 @@ class pkgAcquire; class pkgIndexFile; -class debDebPkgFileIndex; class IndexTarget; class pkgCacheGenerator; class OpProgress; @@ -66,42 +65,4 @@ class APT_HIDDEN debReleaseIndex : public metaIndex std::vector Languages); }; -class APT_HIDDEN debDebFileMetaIndex : public metaIndex -{ -private: - void * const d; - std::string DebFile; - debDebPkgFileIndex *DebIndex; -public: - virtual std::string ArchiveURI(std::string const& /*File*/) const APT_OVERRIDE { - return DebFile; - } - virtual bool GetIndexes(pkgAcquire* /*Owner*/, const bool& /*GetAll=false*/) APT_OVERRIDE { - return true; - } - virtual std::vector GetIndexTargets() const APT_OVERRIDE { - return std::vector(); - } - virtual std::vector *GetIndexFiles() APT_OVERRIDE { - return Indexes; - } - virtual bool IsTrusted() const APT_OVERRIDE { - return true; - } - virtual bool Load(std::string const &, std::string * const ErrorText) APT_OVERRIDE - { - LoadedSuccessfully = TRI_NO; - if (ErrorText != NULL) - strprintf(*ErrorText, "Unparseable metaindex as it represents the standalone deb file %s", DebFile.c_str()); - return false; - } - virtual metaIndex * UnloadedClone() const APT_OVERRIDE - { - return NULL; - } - debDebFileMetaIndex(std::string const &DebFile); - virtual ~debDebFileMetaIndex(); - -}; - #endif diff --git a/apt-pkg/edsp/edspsystem.cc b/apt-pkg/edsp/edspsystem.cc index 4fb34b896..f65fcc0d2 100644 --- a/apt-pkg/edsp/edspsystem.cc +++ b/apt-pkg/edsp/edspsystem.cc @@ -91,8 +91,7 @@ signed edspSystem::Score(Configuration const &Cnf) return -1000; } /*}}}*/ -// System::AddStatusFiles - Register the status files /*{{{*/ -bool edspSystem::AddStatusFiles(std::vector &List) +bool edspSystem::AddStatusFiles(std::vector &List) /*{{{*/ { if (StatusFile == 0) { diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 26a5e60a6..9529f42dc 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include @@ -1332,10 +1334,10 @@ map_stringitem_t pkgCacheGenerator::StoreString(enum StringType const type, cons /* This just verifies that each file in the list of index files exists, has matching attributes with the cache and the cache does not have any extra files. */ -static bool CheckValidity(const string &CacheFile, +static bool CheckValidity(const string &CacheFile, pkgSourceList &List, - FileIterator Start, - FileIterator End, + FileIterator const Start, + FileIterator const End, MMap **OutMap = 0) { bool const Debug = _config->FindB("Debug::pkgCacheGen", false); @@ -1343,7 +1345,7 @@ static bool CheckValidity(const string &CacheFile, if (CacheFile.empty() == true || FileExists(CacheFile) == false) { if (Debug == true) - std::clog << "CacheFile doesn't exist" << std::endl; + std::clog << "CacheFile " << CacheFile << " doesn't exist" << std::endl; return false; } @@ -1361,7 +1363,7 @@ static bool CheckValidity(const string &CacheFile, if (_error->PendingError() == true || Map->Size() == 0) { if (Debug == true) - std::clog << "Errors are pending or Map is empty()" << std::endl; + std::clog << "Errors are pending or Map is empty() for " << CacheFile << std::endl; _error->Discard(); return false; } @@ -1385,10 +1387,9 @@ static bool CheckValidity(const string &CacheFile, if (Debug == true) std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl; - std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j) - if ((*j)->HasPackages()) - Files.push_back (*j); + std::vector const * const Indexes = (*i)->GetIndexFiles(); + std::copy_if(Indexes->begin(), Indexes->end(), std::back_inserter(Files), + [](pkgIndexFile const * const I) { return I->HasPackages(); }); } for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I) if (RlsVisited[I] == false) @@ -1398,8 +1399,7 @@ static bool CheckValidity(const string &CacheFile, return false; } - for (; Start != End; ++Start) - Files.push_back(*Start); + std::copy(Start, End, std::back_inserter(Files)); /* Now we check every index file, see if it is in the cache, verify the IMS data and check that it is on the disk too.. */ @@ -1482,16 +1482,41 @@ static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator } /*}}}*/ // BuildCache - Merge the list of index files into the cache /*{{{*/ -// --------------------------------------------------------------------- -/* */ static bool BuildCache(pkgCacheGenerator &Gen, - OpProgress *Progress, + OpProgress * const Progress, map_filesize_t &CurrentSize,map_filesize_t TotalSize, pkgSourceList const * const List, - FileIterator Start, FileIterator End) + FileIterator const Start, FileIterator const End) { std::vector Files; bool const HasFileDeps = Gen.HasFileDeps(); + bool mergeFailure = false; + + auto const indexFileMerge = [&](pkgIndexFile * const I) { + if (HasFileDeps) + Files.push_back(I); + + if (I->HasPackages() == false || mergeFailure) + return; + + if (I->Exists() == false) + return; + + if (I->FindInCache(Gen.GetCache()).end() == false) + { + _error->Warning("Duplicate sources.list entry %s", + I->Describe().c_str()); + return; + } + + map_filesize_t const Size = I->Size(); + if (Progress != NULL) + Progress->OverallProgress(CurrentSize, TotalSize, Size, _("Reading package lists")); + CurrentSize += Size; + + if (I->Merge(Gen,Progress) == false) + mergeFailure = true; + }; if (List != NULL) { @@ -1508,63 +1533,20 @@ static bool BuildCache(pkgCacheGenerator &Gen, return false; std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I) - { - if (HasFileDeps) - Files.push_back(*I); - - if ((*I)->HasPackages() == false) - continue; - - if ((*I)->Exists() == false) - continue; - - if ((*I)->FindInCache(Gen.GetCache()).end() == false) - { - _error->Warning("Duplicate sources.list entry %s", - (*I)->Describe().c_str()); - continue; - } - - map_filesize_t Size = (*I)->Size(); - if (Progress != NULL) - Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); - CurrentSize += Size; - - if ((*I)->Merge(Gen,Progress) == false) - return false; - } + if (Indexes != NULL) + std::for_each(Indexes->begin(), Indexes->end(), indexFileMerge); + if (mergeFailure) + return false; } } - Gen.SelectReleaseFile("", ""); - FileIterator I; - for (I = Start; I != End; ++I) + if (Start != End) { - if (HasFileDeps) - Files.push_back(*I); - - if ((*I)->HasPackages() == false) - continue; - - if ((*I)->Exists() == false) - continue; - - if ((*I)->FindInCache(Gen.GetCache()).end() == false) - { - _error->Warning("Duplicate sources.list entry %s", - (*I)->Describe().c_str()); - continue; - } - - map_filesize_t Size = (*I)->Size(); - if (Progress != NULL) - Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); - CurrentSize += Size; - - if ((*I)->Merge(Gen,Progress) == false) + Gen.SelectReleaseFile("", ""); + std::for_each(Start, End, indexFileMerge); + if (mergeFailure) return false; - } + } if (HasFileDeps == true) { @@ -1582,12 +1564,20 @@ static bool BuildCache(pkgCacheGenerator &Gen, return false; } } - + return true; } /*}}}*/ -// CacheGenerator::CreateDynamicMMap - load an mmap with configuration options /*{{{*/ -DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) { +// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ +// --------------------------------------------------------------------- +/* This makes sure that the status cache (the cache that has all + index files from the sources list and all local ones) is ready + to be mmaped. If OutMap is not zero then a MMap object representing + the cache will be stored there. This is pretty much mandetory if you + are using AllowMem. AllowMem lets the function be run as non-root + where it builds the cache 'fast' into a memory buffer. */ +static DynamicMMap* CreateDynamicMMap(FileFd * const CacheF, unsigned long Flags) +{ map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024); map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024); map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0); @@ -1599,15 +1589,42 @@ DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long else return new DynamicMMap(Flags, MapStart, MapGrow, MapLimit); } - /*}}}*/ -// CacheGenerator::MakeStatusCache - Construct the status cache /*{{{*/ -// --------------------------------------------------------------------- -/* This makes sure that the status cache (the cache that has all - index files from the sources list and all local ones) is ready - to be mmaped. If OutMap is not zero then a MMap object representing - the cache will be stored there. This is pretty much mandetory if you - are using AllowMem. AllowMem lets the function be run as non-root - where it builds the cache 'fast' into a memory buffer. */ +static bool writeBackMMapToFile(pkgCacheGenerator * const Gen, DynamicMMap * const Map, + std::string const &FileName) +{ + FileFd SCacheF(FileName, FileFd::WriteAtomic); + if (_error->PendingError() == true) + return false; + + fchmod(SCacheF.Fd(),0644); + + // Write out the main data + if (SCacheF.Write(Map->Data(),Map->Size()) == false) + return _error->Error(_("IO Error saving source cache")); + SCacheF.Sync(); + + // Write out the proper header + Gen->GetCache().HeaderP->Dirty = false; + if (SCacheF.Seek(0) == false || + SCacheF.Write(Map->Data(),sizeof(*Gen->GetCache().HeaderP)) == false) + return _error->Error(_("IO Error saving source cache")); + Gen->GetCache().HeaderP->Dirty = true; + SCacheF.Sync(); + return true; +} +static bool loadBackMMapFromFile(std::unique_ptr &Gen, + SPtr &Map, OpProgress * const Progress, std::string const &FileName) +{ + Map = CreateDynamicMMap(NULL, 0); + FileFd CacheF(FileName, FileFd::ReadOnly); + map_pointer_t const alloc = Map->RawAllocate(CacheF.Size()); + if ((alloc == 0 && _error->PendingError()) + || CacheF.Read((unsigned char *)Map->Data() + alloc, + CacheF.Size()) == false) + return false; + Gen.reset(new pkgCacheGenerator(Map.Get(),Progress)); + return true; +} APT_DEPRECATED bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress, MMap **OutMap, bool AllowMem) { return pkgCacheGenerator::MakeStatusCache(List, &Progress, OutMap, AllowMem); } @@ -1617,18 +1634,6 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress bool const Debug = _config->FindB("Debug::pkgCacheGen", false); std::vector Files; - /* - for (std::vector::const_iterator i = List.begin(); - i != List.end(); - ++i) - { - std::vector *Indexes = (*i)->GetIndexFiles(); - for (std::vector::const_iterator j = Indexes->begin(); - j != Indexes->end(); - ++j) - Files.push_back (*j); - } -*/ if (_system->AddStatusFiles(Files) == false) return false; @@ -1649,160 +1654,130 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress CreateDirectory(dir, flNotFile(SrcCacheFile)); } - // Decide if we can write to the cache - bool Writeable = false; - if (CacheFile.empty() == false) - Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0; - else - if (SrcCacheFile.empty() == false) - Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0; - if (Debug == true) - std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl; - - if (Writeable == false && AllowMem == false && CacheFile.empty() == false) - return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str()); - if (Progress != NULL) Progress->OverallProgress(0,1,1,_("Reading package lists")); - // Cache is OK, Fin. - if (CheckValidity(CacheFile, List, Files.begin(),Files.end(),OutMap) == true) + bool pkgcache_fine = false; + bool srcpkgcache_fine = false; + bool volatile_fine = List.GetVolatileFiles().empty(); + + if (CheckValidity(CacheFile, List, Files.begin(), Files.end(), volatile_fine ? OutMap : NULL) == true) { - if (Progress != NULL) - Progress->OverallProgress(1,1,1,_("Reading package lists")); if (Debug == true) - std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl; - return true; + std::clog << "pkgcache.bin is valid - no need to build any cache" << std::endl; + pkgcache_fine = true; + srcpkgcache_fine = true; } - else if (Debug == true) - std::clog << "pkgcache.bin is NOT valid" << std::endl; - - /* At this point we know we need to reconstruct the package cache, - begin. */ - SPtr CacheF; - SPtr Map; - if (Writeable == true && CacheFile.empty() == false) + if (pkgcache_fine == false) { - _error->PushToStack(); - unlink(CacheFile.c_str()); - CacheF = new FileFd(CacheFile,FileFd::WriteAtomic); - fchmod(CacheF->Fd(),0644); - Map = CreateDynamicMMap(CacheF, MMap::Public); - if (_error->PendingError() == true) + if (CheckValidity(SrcCacheFile, List, Files.end(), Files.end()) == true) { - delete CacheF.UnGuard(); - delete Map.UnGuard(); if (Debug == true) - std::clog << "Open filebased MMap FAILED" << std::endl; - Writeable = false; - if (AllowMem == false) - { - _error->MergeWithStack(); - return false; - } - _error->RevertToStack(); - } - else - { - _error->MergeWithStack(); - if (Debug == true) - std::clog << "Open filebased MMap" << std::endl; + std::clog << "srcpkgcache.bin is valid - it can be reused" << std::endl; + srcpkgcache_fine = true; } } - if (Writeable == false || CacheFile.empty() == true) + + if (volatile_fine == true && srcpkgcache_fine == true && pkgcache_fine == true) { - // Just build it in memory.. - Map = CreateDynamicMMap(NULL); - if (Debug == true) - std::clog << "Open memory Map (not filebased)" << std::endl; + if (Progress != NULL) + Progress->OverallProgress(1,1,1,_("Reading package lists")); + return true; } - - // Lets try the source cache. - map_filesize_t CurrentSize = 0; - map_filesize_t TotalSize = 0; - if (CheckValidity(SrcCacheFile, List, Files.end(), - Files.end()) == true) + + bool Writeable = false; + if (srcpkgcache_fine == false || pkgcache_fine == false) { + if (CacheFile.empty() == false) + Writeable = access(flNotFile(CacheFile).c_str(),W_OK) == 0; + else if (SrcCacheFile.empty() == false) + Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0; + if (Debug == true) - std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl; - // Preload the map with the source cache - FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly); - map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size()); - if ((alloc == 0 && _error->PendingError()) - || SCacheF.Read((unsigned char *)Map->Data() + alloc, - SCacheF.Size()) == false) - return false; + std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl; + + if (Writeable == false && AllowMem == false) + { + if (CacheFile.empty() == false) + return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str()); + else if (SrcCacheFile.empty() == false) + return _error->Error(_("Unable to write to %s"),flNotFile(SrcCacheFile).c_str()); + else + return _error->Error("Unable to create caches as file usage is disabled, but memory not allowed either!"); + } + } - TotalSize = ComputeSize(NULL, Files.begin(), Files.end()); + // At this point we know we need to construct something, so get storage ready + SPtr Map = CreateDynamicMMap(NULL, 0); + if (Debug == true) + std::clog << "Open memory Map (not filebased)" << std::endl; - // Build the status cache - pkgCacheGenerator Gen(Map.Get(),Progress); - if (_error->PendingError() == true) - return false; - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL, - Files.begin(),Files.end()) == false) + std::unique_ptr Gen{nullptr}; + map_filesize_t CurrentSize = 0; + std::vector VolatileFiles = List.GetVolatileFiles(); + map_filesize_t TotalSize = ComputeSize(NULL, VolatileFiles.begin(), VolatileFiles.end()); + if (srcpkgcache_fine == true && pkgcache_fine == false) + { + if (Debug == true) + std::clog << "srcpkgcache.bin was valid - populate MMap with it" << std::endl; + if (loadBackMMapFromFile(Gen, Map, Progress, SrcCacheFile) == false) return false; + srcpkgcache_fine = true; + TotalSize += ComputeSize(NULL, Files.begin(), Files.end()); } - else + else if (srcpkgcache_fine == false) { if (Debug == true) std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl; - TotalSize = ComputeSize(&List, Files.begin(),Files.end()); - - // Build the source cache - pkgCacheGenerator Gen(Map.Get(),Progress); - if (_error->PendingError() == true) - return false; - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List, - Files.end(),Files.end()) == false) + Gen.reset(new pkgCacheGenerator(Map.Get(),Progress)); + + TotalSize += ComputeSize(&List, Files.begin(),Files.end()); + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, &List, + Files.end(),Files.end()) == false) return false; - - // Write it back + if (Writeable == true && SrcCacheFile.empty() == false) - { - FileFd SCacheF(SrcCacheFile,FileFd::WriteAtomic); - if (_error->PendingError() == true) + if (writeBackMMapToFile(Gen.get(), Map.Get(), SrcCacheFile) == false) return false; - - fchmod(SCacheF.Fd(),0644); - - // Write out the main data - if (SCacheF.Write(Map->Data(),Map->Size()) == false) - return _error->Error(_("IO Error saving source cache")); - SCacheF.Sync(); - - // Write out the proper header - Gen.GetCache().HeaderP->Dirty = false; - if (SCacheF.Seek(0) == false || - SCacheF.Write(Map->Data(),sizeof(*Gen.GetCache().HeaderP)) == false) - return _error->Error(_("IO Error saving source cache")); - Gen.GetCache().HeaderP->Dirty = true; - SCacheF.Sync(); - } - - // Build the status cache - if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL, - Files.begin(), Files.end()) == false) + } + + if (pkgcache_fine == false) + { + if (Debug == true) + std::clog << "Building status cache in pkgcache.bin now" << std::endl; + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) return false; + + if (Writeable == true && CacheFile.empty() == false) + if (writeBackMMapToFile(Gen.get(), Map.Get(), CacheFile) == false) + return false; } + if (Debug == true) - std::clog << "Caches are ready for shipping" << std::endl; + std::clog << "Caches done. Now bring in the volatile files (if any)" << std::endl; - if (_error->PendingError() == true) - return false; - if (OutMap != 0) + if (volatile_fine == false) { - if (CacheF != 0) + if (Gen == nullptr) { - delete Map.UnGuard(); - *OutMap = new MMap(*CacheF,0); + if (Debug == true) + std::clog << "Populate new MMap with cachefile contents" << std::endl; + if (loadBackMMapFromFile(Gen, Map, Progress, CacheFile) == false) + return false; } - else - { - *OutMap = Map.UnGuard(); - } + + Files = List.GetVolatileFiles(); + if (BuildCache(*Gen, Progress, CurrentSize, TotalSize, NULL, + Files.begin(), Files.end()) == false) + return false; } - + + if (OutMap != nullptr) + *OutMap = Map.UnGuard(); + + if (Debug == true) + std::clog << "Everything is ready for shipping" << std::endl; return true; } /*}}}*/ @@ -1817,7 +1792,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O if (_system->AddStatusFiles(Files) == false) return false; - SPtr Map = CreateDynamicMMap(NULL); + SPtr Map = CreateDynamicMMap(NULL, 0); map_filesize_t CurrentSize = 0; map_filesize_t TotalSize = 0; diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 0d0fb893f..4b6b91992 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -116,7 +116,6 @@ class APT_HIDDEN pkgCacheGenerator /*{{{*/ APT_PUBLIC static bool MakeStatusCache(pkgSourceList &List,OpProgress *Progress, MMap **OutMap = 0,bool AllowMem = false); APT_PUBLIC static bool MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap); - APT_PUBLIC static DynamicMMap* CreateDynamicMMap(FileFd *CacheF, unsigned long Flags = 0); void ReMap(void const * const oldMap, void const * const newMap); diff --git a/apt-pkg/pkgsystem.h b/apt-pkg/pkgsystem.h index 5be93d059..5b31457e0 100644 --- a/apt-pkg/pkgsystem.h +++ b/apt-pkg/pkgsystem.h @@ -52,7 +52,7 @@ class Configuration; class pkgIndexFile; class pkgSystem -{ +{ public: // Global list of supported systems @@ -81,7 +81,8 @@ class pkgSystem virtual bool ArchiveSupported(const char *Type) = 0; // Return a list of system index files.. - virtual bool AddStatusFiles(std::vector &List) = 0; + virtual bool AddStatusFiles(std::vector &List) = 0; + virtual bool FindIndex(pkgCache::PkgFileIterator File, pkgIndexFile *&Found) const = 0; diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index eef0ee709..46e51f592 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -274,6 +274,10 @@ pkgSourceList::~pkgSourceList() { for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I) delete *I; + SrcList.clear(); + for (pkgIndexFile * const File : VolatileFiles) + delete File; + VolatileFiles.clear(); } /*}}}*/ // SourceList::ReadMainList - Read the main source list from etc /*{{{*/ @@ -339,7 +343,7 @@ bool pkgSourceList::ReadAppend(string const &File) else return ParseFileOldStyle(File); } - + /*}}}*/ // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -454,7 +458,15 @@ bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File, } } } - + for (vector::const_iterator J = VolatileFiles.begin(); + J != VolatileFiles.end(); ++J) + { + if ((*J)->FindInCache(*File.Cache()) == File) + { + Found = (*J); + return true; + } + } return false; } /*}}}*/ @@ -511,4 +523,14 @@ time_t pkgSourceList::GetLastModifiedTime() return mtime_sources; } /*}}}*/ - +std::vector pkgSourceList::GetVolatileFiles() const /*{{{*/ +{ + return VolatileFiles; +} + /*}}}*/ +void pkgSourceList::AddVolatileFile(pkgIndexFile * const File) /*{{{*/ +{ + if (File != NULL) + VolatileFiles.push_back(File); +} + /*}}}*/ diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index d80131438..47a562d18 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -50,6 +50,7 @@ class metaIndex; class pkgSourceList { void * const d; + std::vector VolatileFiles; public: // List of supported source list types @@ -113,6 +114,24 @@ class pkgSourceList // query last-modified time time_t GetLastModifiedTime(); + /** \brief add file for parsing, but not to the cache + * + * pkgIndexFiles origining from pkgSourcesList are included in + * srcpkgcache, the status files added via #AddStatusFiles are + * included in pkgcache, but these files here are not included in + * any cache to have the possibility of having a file included just + * for a single run like a local .deb/.dsc file. + * + * The volatile files do not count as "normal" sourceslist entries, + * can't be iterated over with #begin and #end and can't be + * downloaded, but they can be found via #FindIndex. + * + * @param File is an index file; pointer-ownership is transferred + */ + void AddVolatileFile(pkgIndexFile * const File); + /** @return list of files registered with #AddVolatileFile */ + std::vector GetVolatileFiles() const; + pkgSourceList(); virtual ~pkgSourceList(); }; diff --git a/apt-private/private-cachefile.h b/apt-private/private-cachefile.h index ed9342db0..51703b0ad 100644 --- a/apt-private/private-cachefile.h +++ b/apt-private/private-cachefile.h @@ -11,17 +11,6 @@ #include -// FIXME: we need to find a way to export this -class APT_PUBLIC SourceList : public pkgSourceList -{ - public: - // Add custom metaIndex (e.g. local files) - void AddMetaIndex(metaIndex *mi) { - SrcList.push_back(mi); - } - -}; - // class CacheFile - Cover class for some dependency cache functions /*{{{*/ class APT_PUBLIC CacheFile : public pkgCacheFile { @@ -36,16 +25,6 @@ class APT_PUBLIC CacheFile : public pkgCacheFile return false; return true; } - // FIXME: this can go once the "libapt-pkg" pkgSourceList has a way - // to add custom metaIndexes (or custom local files or so) - bool BuildSourceList(OpProgress */*Progress*/ = NULL) { - if (SrcList != NULL) - return true; - SrcList = new SourceList(); - if (SrcList->ReadMainList() == false) - return _error->Error(_("The list of sources could not be read.")); - return true; - } bool Open(bool WithLock = true) { OpTextProgress Prog(*_config); diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index 116e01038..074874903 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -647,24 +648,8 @@ bool DoInstall(CommandLine &CmdL) // first check for local pkgs and add them to the cache for (const char **I = CmdL.FileList; *I != 0; I++) { - if(FileExists(*I)) - { - // FIXME: make this more elegant - std::string TypeStr = flExtension(*I) + "-file"; - pkgSourceList::Type *Type = pkgSourceList::Type::GetType(TypeStr.c_str()); - if(Type != 0) - { - std::vector List; - std::map Options; - if(Type->CreateItem(List, *I, "", "", Options)) - { - // we have our own CacheFile that gives us a SourceList - // with superpowerz - SourceList *sources = (SourceList*)Cache.GetSourceList(); - sources->AddMetaIndex(List[0]); - } - } - } + if(FileExists(*I) && flExtension(*I) == "deb") + Cache.GetSourceList()->AddVolatileFile(new debDebPkgFileIndex(*I)); } // then open the cache diff --git a/test/integration/framework b/test/integration/framework index 6ae5003f7..f3cc1eff9 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -99,10 +99,14 @@ msgdone() { } getaptconfig() { if [ -f ./aptconfig.conf ]; then - echo "./aptconfig.conf" + echo "$(readlink -f ./aptconfig.conf)" elif [ -f ../aptconfig.conf ]; then - echo "../aptconfig.conf" - fi + echo "$(readlink -f ../aptconfig.conf)" + elif [ -f ../../aptconfig.conf ]; then + echo "$(readlink -f ../../aptconfig.conf)" + elif [ -f "${TMPWORKINGDIRECTORY}/aptconfig.conf" ]; then + echo "$(readlink -f "${TMPWORKINGDIRECTORY}/aptconfig.conf")" + fi } runapt() { msgdebug "Executing: ${CCMD}$*${CDEBUG} " @@ -141,6 +145,8 @@ gdb() { case "$1" in aptget) CMD="apt-get";; aptcache) CMD="apt-cache";; + aptcdrom) CMD="apt-cdrom";; + aptconfig) CMD="apt-config";; aptmark) CMD="apt-mark";; apthelper) CMD="apt-helper";; aptftparchive) CMD="apt-ftparchive";; diff --git a/test/integration/test-apt-get-install-deb b/test/integration/test-apt-get-install-deb index 0f34692fe..991185dea 100755 --- a/test/integration/test-apt-get-install-deb +++ b/test/integration/test-apt-get-install-deb @@ -5,15 +5,21 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'amd64' 'i386' # regression test for #754904 testfailureequal 'E: Unable to locate package /dev/null' aptget install -qq /dev/null -# and ensure we fail for invalid debs -cat > foo.deb < foo.rpm < Date: Tue, 11 Aug 2015 11:05:57 +0200 Subject: Fix an obscure warning from GCC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It complained about the previous code: apt-pkg/sourcelist.cc: In destructor ‘pkgSourceList::~pkgSourceList()’: apt-pkg/sourcelist.cc:278:4: warning: cannot optimize loop, the loop counter may overflow [-Wunsafe-loop-optimizations] for (pkgIndexFile * const File : VolatileFiles) ^ There really cannot be an overflow, though. Rewriting it like this seems to fix it. --- apt-pkg/sourcelist.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'apt-pkg/sourcelist.cc') diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc index 46e51f592..3e714667c 100644 --- a/apt-pkg/sourcelist.cc +++ b/apt-pkg/sourcelist.cc @@ -275,8 +275,8 @@ pkgSourceList::~pkgSourceList() for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I) delete *I; SrcList.clear(); - for (pkgIndexFile * const File : VolatileFiles) - delete File; + for (auto F = VolatileFiles.begin(); F != VolatileFiles.end(); ++F) + delete (*F); VolatileFiles.clear(); } /*}}}*/ -- cgit v1.2.3-70-g09d2