From f105aaba433f5a8b9c4326dd0d704501bf07d1e5 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 6 Nov 2014 12:53:59 +0100 Subject: better non-virtual metaIndex.LocalFileName() implementation We can't add a new virtual method without breaking the ABI, but we can freely add new methods, so for older ABIs we just implement this method with a dynamic_cast, so that clients can be more ignorant about the API here and especially don't need to pull a very dirty trick by assuming internal knowledge (like apt-get did here). --- cmdline/apt-get.cc | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index b6786faf8..e9e38debc 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -136,28 +136,6 @@ static bool TryToInstallBuildDep(pkgCache::PkgIterator Pkg,pkgCacheFile &Cache, return true; } /*}}}*/ - - -// helper that can go wit hthe next ABI break -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR < 13) -static std::string MetaIndexFileNameOnDisk(metaIndex *metaindex) -{ - // FIXME: this cast is the horror, the horror - debReleaseIndex *r = (debReleaseIndex*)metaindex; - - // see if we have a InRelease file - std::string PathInRelease = r->MetaIndexFile("InRelease"); - if (FileExists(PathInRelease)) - return PathInRelease; - - // and if not return the normal one - if (FileExists(PathInRelease)) - return r->MetaIndexFile("Release"); - - return ""; -} -#endif - // GetReleaseForSourceRecord - Return Suite for the given srcrecord /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -176,12 +154,8 @@ static std::string GetReleaseForSourceRecord(pkgSourceList *SrcList, { if (&CurrentIndexFile == (*IF)) { -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR < 13) - std::string path = MetaIndexFileNameOnDisk(*S); -#else - std::string path = (*S)->LocalFileName(); -#endif - if (path != "") + std::string const path = (*S)->LocalFileName(); + if (path != "") { indexRecords records; records.Load(path); -- cgit v1.2.3-70-g09d2 From 765190e493645e13b5651625d87fd9c8db910a85 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 7 Nov 2014 16:45:18 +0100 Subject: guard ABI changes for SourcePkg/Ver in pkgCache Git-Dch: Ignore --- apt-pkg/cacheiterators.h | 2 ++ apt-pkg/deb/deblistparser.cc | 2 ++ apt-pkg/deb/dpkgpm.cc | 11 +++++++++- apt-pkg/edsp.cc | 7 ++++++ apt-pkg/pkgcache.h | 2 ++ cmdline/apt-cache.cc | 2 ++ cmdline/apt-get.cc | 52 ++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 73 insertions(+), 5 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index b0c02d4a2..f8f4c05ce 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -215,12 +215,14 @@ class pkgCache::VerIterator : public Iterator { // Accessors inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;} inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;} +#if APT_PKG_ABI >= 413 /** \brief source package name this version comes from Always contains the name, even if it is the same as the binary name */ inline const char *SourcePkgName() const {return Owner->StrP + S->SourcePkgName;} /** \brief source version this version comes from Always contains the version string, even if it is the same as the binary version */ inline const char *SourceVerStr() const {return Owner->StrP + S->SourceVerStr;} +#endif inline const char *Arch() const { if ((S->MultiArch & pkgCache::Version::All) == pkgCache::Version::All) return "all"; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 42db341b3..462818a03 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -141,6 +141,7 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) map_stringitem_t const idx = StoreString(pkgCacheGenerator::SECTION, Start, Stop - Start); Ver->Section = idx; } +#if APT_PKG_ABI >= 413 // Parse the source package name pkgCache::GrpIterator const G = Ver.ParentPkg().Group(); Ver->SourcePkgName = G->Name; @@ -192,6 +193,7 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) } } } +#endif Ver->MultiArch = ParseMultiArch(true); // Archive Size diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 0cded32e1..3f6039e0d 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1684,7 +1684,7 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) if (apportPkg.end() == true || apportPkg->CurrentVer == 0) return; - string pkgname, reportfile, srcpkgname, pkgver, arch; + string pkgname, reportfile, pkgver, arch; string::size_type pos; FILE *report; @@ -1823,7 +1823,16 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) time_t now = time(NULL); fprintf(report, "Date: %s" , ctime(&now)); fprintf(report, "Package: %s %s\n", pkgname.c_str(), pkgver.c_str()); +#if APT_PKG_ABI >= 413 fprintf(report, "SourcePackage: %s\n", Ver.SourcePkgName()); +#else + pkgRecords Recs(Cache); + pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); + std::string srcpkgname = Parse.SourcePkg(); + if(srcpkgname.empty()) + srcpkgname = pkgname; + fprintf(report, "SourcePackage: %s\n", srcpkgname.c_str()); +#endif fprintf(report, "ErrorMessage:\n %s\n", errormsg); // ensure that the log is flushed diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc index 2ba914b16..3c6a7e30f 100644 --- a/apt-pkg/edsp.cc +++ b/apt-pkg/edsp.cc @@ -96,7 +96,14 @@ void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgI pkgCache::VerIterator const &Ver) { fprintf(output, "Package: %s\n", Pkg.Name()); +#if APT_PKG_ABI >= 413 fprintf(output, "Source: %s\n", Ver.SourcePkgName()); +#else + pkgRecords Recs(Cache); + pkgRecords::Parser &rec = Recs.Lookup(Ver.FileList()); + string srcpkg = rec.SourcePkg().empty() ? Pkg.Name() : rec.SourcePkg(); + fprintf(output, "Source: %s\n", srcpkg.c_str()); +#endif fprintf(output, "Architecture: %s\n", Ver.Arch()); fprintf(output, "Version: %s\n", Ver.VerStr()); if (Pkg.CurrentVer() == Ver) diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 4f8568205..a7b7fa539 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -510,12 +510,14 @@ struct pkgCache::Version map_stringitem_t VerStr; /** \brief section this version is filled in */ map_stringitem_t Section; +#if APT_PKG_ABI >= 413 /** \brief source package name this version comes from Always contains the name, even if it is the same as the binary name */ map_stringitem_t SourcePkgName; /** \brief source version this version comes from Always contains the version string, even if it is the same as the binary version */ map_stringitem_t SourceVerStr; +#endif /** \brief Multi-Arch capabilities of a package version */ enum VerMultiArch { None = 0, /*!< is the default and doesn't trigger special behaviour */ diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 0f4f7e1ce..1bd75dfba 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -389,8 +389,10 @@ static bool Stats(CommandLine &) stritems.insert(V->VerStr); if (V->Section != 0) stritems.insert(V->Section); +#if APT_PKG_ABI >= 413 stritems.insert(V->SourcePkgName); stritems.insert(V->SourceVerStr); +#endif for (pkgCache::DepIterator D = V.DependsList(); D.end() == false; ++D) { if (D->Version != 0) diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index e9e38debc..eca4a723b 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -170,7 +170,11 @@ static std::string GetReleaseForSourceRecord(pkgSourceList *SrcList, // FindSrc - Find a source record /*{{{*/ // --------------------------------------------------------------------- /* */ +#if APT_PKG_ABI >= 413 static pkgSrcRecords::Parser *FindSrc(const char *Name, +#else +static pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, +#endif pkgSrcRecords &SrcRecs,string &Src, CacheFile &CacheFile) { @@ -278,10 +282,21 @@ static pkgSrcRecords::Parser *FindSrc(const char *Name, (VF.File().Archive() != 0 && VF.File().Archive() == RelTag) || (VF.File().Codename() != 0 && VF.File().Codename() == RelTag)) { - Src = Ver.SourcePkgName(); // the Version we have is possibly fuzzy or includes binUploads, - // so we use the Version of the SourcePkg + // so we use the Version of the SourcePkg (empty if same as package) +#if APT_PKG_ABI >= 413 + Src = Ver.SourcePkgName(); VerTag = Ver.SourceVerStr(); +#else + pkgRecords::Parser &Parse = Recs.Lookup(VF); + Src = Parse.SourcePkg(); + // no SourcePkg name, so it is the "binary" name + if (Src.empty() == true) + Src = TmpSrc; + VerTag = Parse.SourceVer(); + if (VerTag.empty() == true) + VerTag = Ver.VerStr(); +#endif break; } } @@ -312,10 +327,17 @@ static pkgSrcRecords::Parser *FindSrc(const char *Name, pkgCache::VerIterator Ver = Cache->GetCandidateVer(Pkg); if (Ver.end() == false) { +#if APT_PKG_ABI >= 413 if (strcmp(Ver.SourcePkgName(),Ver.ParentPkg().Name()) != 0) Src = Ver.SourcePkgName(); if (VerTag.empty() == true && strcmp(Ver.SourceVerStr(),Ver.VerStr()) != 0) VerTag = Ver.SourceVerStr(); +#else + pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); + Src = Parse.SourcePkg(); + if (VerTag.empty() == true) + VerTag = Parse.SourceVer(); +#endif } } } @@ -717,6 +739,9 @@ static bool DoSource(CommandLine &CmdL) pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers +#if APT_PKG_ABI < 413 + pkgRecords Recs(Cache); +#endif pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@ -744,8 +769,11 @@ static bool DoSource(CommandLine &CmdL) for (const char **I = CmdL.FileList + 1; *I != 0; I++, J++) { string Src; +#if APT_PKG_ABI >= 413 pkgSrcRecords::Parser *Last = FindSrc(*I,SrcRecs,Src,Cache); - +#else + pkgSrcRecords::Parser *Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); +#endif if (Last == 0) { return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); } @@ -1004,6 +1032,9 @@ static bool DoBuildDep(CommandLine &CmdL) pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers +#if APT_PKG_ABI < 413 + pkgRecords Recs(Cache); +#endif pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@ -1050,7 +1081,11 @@ static bool DoBuildDep(CommandLine &CmdL) Last = Type->CreateSrcPkgParser(*I); } else { // normal case, search the cache for the source file - Last = FindSrc(*I,SrcRecs,Src,Cache); +#if APT_PKG_ABI >= 413 + Last = FindSrc(*I,SrcRecs,Src,Cache); +#else + Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); +#endif } if (Last == 0) @@ -1407,9 +1442,18 @@ static string GetChangelogPath(CacheFile &Cache, pkgRecords Recs(Cache); pkgRecords::Parser &rec=Recs.Lookup(Ver.FileList()); string path = flNotFile(rec.FileName()); +#if APT_PKG_ABI >= 413 path.append(Ver.SourcePkgName()); path.append("_"); path.append(StripEpoch(Ver.SourceVerStr())); +#else + string srcpkg = rec.SourcePkg().empty() ? Ver.ParentPkg().Name() : rec.SourcePkg(); + string ver = Ver.VerStr(); + // if there is a source version it always wins + if (rec.SourceVer() != "") + ver = rec.SourceVer(); + path += srcpkg + "_" + StripEpoch(ver); +#endif return path; } /*}}}*/ -- cgit v1.2.3-70-g09d2 From 32ab4bd05cb298f6bf1f9574f5b20570beaae429 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 7 Nov 2014 19:18:21 +0100 Subject: guard pkg/grp hashtable creation changes The change itself is no problem ABI wise, but the remove of the old undynamic hashtables is, so we bring it back for older abis and happily use the now available free space to backport more recent additions like the dynamic hashtable itself. Git-Dch: Ignore --- apt-pkg/pkgcache.cc | 39 ++++++++++++--------- apt-pkg/pkgcache.h | 93 ++++++++++++++++++++++++++++++++++++++++++-------- apt-pkg/pkgcachegen.cc | 18 +++++----- cmdline/apt-cache.cc | 13 ++++--- 4 files changed, 118 insertions(+), 45 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 572685ba5..8d6e242b1 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -82,10 +82,13 @@ pkgCache::Header::Header() MaxDescFileSize = 0; FileList = 0; +#if APT_PKG_ABI < 413 + APT_IGNORE_DEPRECATED(StringList = 0;) +#endif VerSysName = 0; Architecture = 0; - Architectures = 0; - HashTableSize = _config->FindI("APT::Cache-HashTableSize", 10 * 1048); + SetArchitectures(0); + SetHashTableSize(_config->FindI("APT::Cache-HashTableSize", 10 * 1048)); memset(Pools,0,sizeof(Pools)); CacheFileSize = 0; @@ -114,6 +117,7 @@ bool pkgCache::Header::CheckSizes(Header &Against) const // Cache::pkgCache - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ +APT_IGNORE_DEPRECATED_PUSH pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map) { // call getArchitectures() with cached=false to ensure that the @@ -123,6 +127,7 @@ pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map) if (DoMap == true) ReMap(); } +APT_IGNORE_DEPRECATED_POP /*}}}*/ // Cache::ReMap - Reopen the cache file /*{{{*/ // --------------------------------------------------------------------- @@ -162,7 +167,7 @@ bool pkgCache::ReMap(bool const &Errorchecks) if (Map.Size() < HeaderP->CacheFileSize) return _error->Error(_("The package cache file is corrupted, it is too small")); - if (HeaderP->VerSysName == 0 || HeaderP->Architecture == 0 || HeaderP->Architectures == 0) + if (HeaderP->VerSysName == 0 || HeaderP->Architecture == 0 || HeaderP->GetArchitectures() == 0) return _error->Error(_("The package cache file is corrupted")); // Locate our VS.. @@ -176,8 +181,8 @@ bool pkgCache::ReMap(bool const &Errorchecks) for (++a; a != archs.end(); ++a) list.append(",").append(*a); if (_config->Find("APT::Architecture") != StrP + HeaderP->Architecture || - list != StrP + HeaderP->Architectures) - return _error->Error(_("The package cache was built for different architectures: %s vs %s"), StrP + HeaderP->Architectures, list.c_str()); + list != StrP + HeaderP->GetArchitectures()) + return _error->Error(_("The package cache was built for different architectures: %s vs %s"), StrP + HeaderP->GetArchitectures(), list.c_str()); return true; } @@ -192,7 +197,7 @@ map_id_t pkgCache::sHash(const string &Str) const unsigned long Hash = 0; for (string::const_iterator I = Str.begin(); I != Str.end(); ++I) Hash = 41 * Hash + tolower_ascii(*I); - return Hash % HeaderP->HashTableSize; + return Hash % HeaderP->GetHashTableSize(); } map_id_t pkgCache::sHash(const char *Str) const @@ -200,7 +205,7 @@ map_id_t pkgCache::sHash(const char *Str) const unsigned long Hash = tolower_ascii(*Str); for (const char *I = Str + 1; *I != 0; ++I) Hash = 41 * Hash + tolower_ascii(*I); - return Hash % HeaderP->HashTableSize; + return Hash % HeaderP->GetHashTableSize(); } /*}}}*/ // Cache::SingleArchFindPkg - Locate a package by name /*{{{*/ @@ -211,8 +216,8 @@ map_id_t pkgCache::sHash(const char *Str) const pkgCache::PkgIterator pkgCache::SingleArchFindPkg(const string &Name) { // Look at the hash bucket - Package *Pkg = PkgP + HeaderP->PkgHashTable()[Hash(Name)]; - for (; Pkg != PkgP; Pkg = PkgP + Pkg->Next) + Package *Pkg = PkgP + HeaderP->PkgHashTableP()[Hash(Name)]; + for (; Pkg != PkgP; Pkg = PkgP + Pkg->NextPackage) { int const cmp = strcmp(Name.c_str(), StrP + (GrpP + Pkg->Group)->Name); if (cmp == 0) @@ -273,7 +278,7 @@ pkgCache::GrpIterator pkgCache::FindGrp(const string &Name) { return GrpIterator(*this,0); // Look at the hash bucket for the group - Group *Grp = GrpP + HeaderP->GrpHashTable()[sHash(Name)]; + Group *Grp = GrpP + HeaderP->GrpHashTableP()[sHash(Name)]; for (; Grp != GrpP; Grp = GrpP + Grp->Next) { int const cmp = strcmp(Name.c_str(), StrP + Grp->Name); if (cmp == 0) @@ -359,7 +364,7 @@ pkgCache::PkgIterator pkgCache::GrpIterator::FindPkg(string Arch) const { // Iterate over the list to find the matching arch for (pkgCache::Package *Pkg = PackageList(); Pkg != Owner->PkgP; - Pkg = Owner->PkgP + Pkg->Next) { + Pkg = Owner->PkgP + Pkg->NextPackage) { if (stringcmp(Arch, Owner->StrP + Pkg->Arch) == 0) return PkgIterator(*Owner, Pkg); if ((Owner->PkgP + S->LastPackage) == Pkg) @@ -407,7 +412,7 @@ pkgCache::PkgIterator pkgCache::GrpIterator::NextPkg(pkgCache::PkgIterator const if (S->LastPackage == LastPkg.Index()) return PkgIterator(*Owner, 0); - return PkgIterator(*Owner, Owner->PkgP + LastPkg->Next); + return PkgIterator(*Owner, Owner->PkgP + LastPkg->NextPackage); } /*}}}*/ // GrpIterator::operator ++ - Postfix incr /*{{{*/ @@ -420,10 +425,10 @@ void pkgCache::GrpIterator::operator ++(int) S = Owner->GrpP + S->Next; // Follow the hash table - while (S == Owner->GrpP && (HashIndex+1) < (signed)Owner->HeaderP->HashTableSize) + while (S == Owner->GrpP && (HashIndex+1) < (signed)Owner->HeaderP->GetHashTableSize()) { HashIndex++; - S = Owner->GrpP + Owner->HeaderP->GrpHashTable()[HashIndex]; + S = Owner->GrpP + Owner->HeaderP->GrpHashTableP()[HashIndex]; } } /*}}}*/ @@ -434,13 +439,13 @@ void pkgCache::PkgIterator::operator ++(int) { // Follow the current links if (S != Owner->PkgP) - S = Owner->PkgP + S->Next; + S = Owner->PkgP + S->NextPackage; // Follow the hash table - while (S == Owner->PkgP && (HashIndex+1) < (signed)Owner->HeaderP->HashTableSize) + while (S == Owner->PkgP && (HashIndex+1) < (signed)Owner->HeaderP->GetHashTableSize()) { HashIndex++; - S = Owner->PkgP + Owner->HeaderP->PkgHashTable()[HashIndex]; + S = Owner->PkgP + Owner->HeaderP->PkgHashTableP()[HashIndex]; } } /*}}}*/ diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index a7b7fa539..2ba23c5c0 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -85,16 +85,45 @@ using std::string; #endif +#if APT_PKG_ABI >= 413 // storing file sizes of indexes, which are way below 4 GB for now typedef uint32_t map_filesize_t; +typedef map_filesize_t should_be_map_filesize_t; +#else +typedef unsigned long map_filesize_t; +typedef unsigned int should_be_map_filesize_t; +#endif +#if APT_PKG_ABI >= 413 // each package/group/dependency gets an id typedef uint32_t map_id_t; +typedef map_id_t should_be_map_id_t; +#else +typedef unsigned long map_id_t; +typedef unsigned int should_be_map_id_t; +#endif +#if APT_PKG_ABI >= 413 // some files get an id, too, but in far less absolute numbers typedef uint16_t map_fileid_t; +typedef map_fileid_t should_be_map_fileid_t; +#else +typedef unsigned long map_fileid_t; +typedef unsigned int should_be_map_fileid_t; +#endif +#if APT_PKG_ABI >= 413 // relative pointer from cache start typedef uint32_t map_pointer_t; +#else +typedef unsigned int map_pointer_t; +#endif // same as the previous, but documented to be to a string item typedef map_pointer_t map_stringitem_t; +#if APT_PKG_ABI >= 413 +typedef uint64_t should_be_uint64_t; +typedef uint64_t should_be_uint64_small_t; +#else +typedef unsigned long long should_be_uint64_t; +typedef unsigned long should_be_uint64_small_t; +#endif class pkgVersioningSystem; class pkgCache /*{{{*/ @@ -109,6 +138,7 @@ class pkgCache /*{{{*/ struct Description; struct Provides; struct Dependency; + struct StringItem; struct VerFile; struct DescFile; @@ -185,6 +215,7 @@ class pkgCache /*{{{*/ Description *DescP; Provides *ProvideP; Dependency *DepP; + APT_DEPRECATED StringItem *StringItemP; char *StrP; virtual bool ReMap(bool const &Errorchecks = true); @@ -288,12 +319,17 @@ struct pkgCache::Header The PackageFile structures are singly linked lists that represent all package files that have been merged into the cache. */ map_pointer_t FileList; +#if APT_PKG_ABI < 413 + APT_DEPRECATED map_pointer_t StringList; +#endif /** \brief String representing the version system used */ map_pointer_t VerSysName; /** \brief native architecture the cache was built against */ map_pointer_t Architecture; +#if APT_PKG_ABI >= 413 /** \brief all architectures the cache was built against */ map_pointer_t Architectures; +#endif /** \brief The maximum size of a raw entry from the original Package file */ map_filesize_t MaxVerFileSize; /** \brief The maximum size of a raw entry from the original Translation file */ @@ -319,12 +355,26 @@ struct pkgCache::Header In the PkgHashTable is it possible that multiple packages have the same name - these packages are stored as a sequence in the list. The size of both tables is the same. */ +#if APT_PKG_ABI >= 413 unsigned int HashTableSize; - map_pointer_t * PkgHashTable() const { return (map_pointer_t*) (this + 1); } - map_pointer_t * GrpHashTable() const { return PkgHashTable() + HashTableSize; } + unsigned int GetHashTableSize() const { return HashTableSize; } + void SetHashTableSize(unsigned int const sz) { HashTableSize = sz; } + map_pointer_t GetArchitectures() const { return Architectures; } + void SetArchitectures(map_pointer_t const idx) { Architectures = idx; } +#else + // BEWARE: these tables are pretty much empty and just here for abi compat + map_ptrloc PkgHashTable[2*1048]; + map_ptrloc GrpHashTable[2*1048]; + unsigned int GetHashTableSize() const { return PkgHashTable[0]; } + void SetHashTableSize(unsigned int const sz) { PkgHashTable[0] = sz; } + map_pointer_t GetArchitectures() const { return PkgHashTable[1]; } + void SetArchitectures(map_pointer_t const idx) { PkgHashTable[1] = idx; } +#endif + map_pointer_t * PkgHashTableP() const { return (map_pointer_t*) (this + 1); } + map_pointer_t * GrpHashTableP() const { return PkgHashTableP() + GetHashTableSize(); } /** \brief Size of the complete cache file */ - unsigned long long CacheFileSize; + should_be_uint64_small_t CacheFileSize; bool CheckSizes(Header &Against) const APT_PURE; Header(); @@ -350,7 +400,7 @@ struct pkgCache::Group /** \brief Link to the next Group */ map_pointer_t Next; // Group /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; }; /*}}}*/ @@ -387,12 +437,18 @@ struct pkgCache::Package map_pointer_t VersionList; // Version /** \brief index to the installed version */ map_pointer_t CurrentVer; // Version + /** \brief indicates nothing (consistently) + This field used to contain ONE section the package belongs to, + if those differs between versions it is a RANDOM one. + The Section() method tries to reproduce it, but the only sane + thing to do is use the Section field from the version! */ + APT_DEPRECATED map_ptrloc Section; // StringItem /** \brief index of the group this package belongs to */ map_pointer_t Group; // Group the Package belongs to // Linked list /** \brief Link to the next package in the same bucket */ - map_pointer_t Next; // Package + map_pointer_t NextPackage; // Package /** \brief List of all dependencies on this package */ map_pointer_t RevDepends; // Dependency /** \brief List of all "packages" this package provide */ @@ -416,7 +472,7 @@ struct pkgCache::Package This allows clients to create an array of size PackageCount and use it to store state information for the package map. For instance the status file emitter uses this to track which packages have been emitted already. */ - map_id_t ID; + should_be_map_id_t ID; /** \brief some useful indicators of the package's state */ unsigned long Flags; }; @@ -464,7 +520,7 @@ struct pkgCache::PackageFile /** \brief Link to the next PackageFile in the Cache */ map_pointer_t NextFile; // PackageFile /** \brief unique sequel ID */ - map_fileid_t ID; + should_be_map_fileid_t ID; }; /*}}}*/ // VerFile structure /*{{{*/ @@ -479,7 +535,7 @@ struct pkgCache::VerFile /** \brief next step in the linked list */ map_pointer_t NextFile; // PkgVerFile /** \brief position in the package file */ - map_filesize_t Offset; // File offset + should_be_map_filesize_t Offset; // File offset /** @TODO document pkgCache::VerFile::Size */ map_filesize_t Size; }; @@ -493,7 +549,7 @@ struct pkgCache::DescFile /** \brief next step in the linked list */ map_pointer_t NextFile; // PkgVerFile /** \brief position in the file */ - map_filesize_t Offset; // File offset + should_be_map_filesize_t Offset; // File offset /** @TODO document pkgCache::DescFile::Size */ map_filesize_t Size; }; @@ -556,16 +612,16 @@ struct pkgCache::Version /** \brief archive size for this version For Debian this is the size of the .deb file. */ - uint64_t Size; // These are the .deb size + should_be_uint64_t Size; // These are the .deb size /** \brief uncompressed size for this version */ - uint64_t InstalledSize; + should_be_uint64_t InstalledSize; /** \brief characteristic value representing this version No two packages in existence should have the same VerStr and Hash with different contents. */ unsigned short Hash; /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; /** \brief parsed priority value */ unsigned char Priority; }; @@ -593,7 +649,7 @@ struct pkgCache::Description map_pointer_t ParentPkg; // Package /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; }; /*}}}*/ // Dependency structure /*{{{*/ @@ -620,7 +676,7 @@ struct pkgCache::Dependency map_pointer_t ParentVer; // Version /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; /** \brief Dependency type - Depends, Recommends, Conflicts, etc */ unsigned char Type; /** \brief comparison operator specified on the depends line @@ -656,6 +712,15 @@ struct pkgCache::Provides map_pointer_t NextPkgProv; // Provides }; /*}}}*/ +// UNUSED StringItem structure /*{{{*/ +struct APT_DEPRECATED pkgCache::StringItem +{ + /** \brief string this refers to */ + map_ptrloc String; // StringItem + /** \brief Next link in the chain */ + map_ptrloc NextItem; // StringItem +}; + /*}}}*/ inline char const * pkgCache::NativeArch() { return StrP + HeaderP->Architecture; } diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index a2c1c8760..ba454f057 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -74,7 +74,7 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : *Cache.HeaderP = pkgCache::Header(); // make room for the hashtables for packages and groups - if (Map.RawAllocate(2 * (Cache.HeaderP->HashTableSize * sizeof(map_pointer_t))) == 0) + if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0) return; map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label); @@ -96,10 +96,10 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : map_stringitem_t const idxArchitectures = WriteStringInMap(list); if (unlikely(idxArchitectures == 0)) return; - Cache.HeaderP->Architectures = idxArchitectures; + Cache.HeaderP->SetArchitectures(idxArchitectures); } else - Cache.HeaderP->Architectures = idxArchitecture; + Cache.HeaderP->SetArchitectures(idxArchitecture); Cache.ReMap(); } @@ -616,7 +616,7 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) // Insert it into the hash table unsigned long const Hash = Cache.Hash(Name); - map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTable()[Hash]; + map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash]; while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0) insertAt = &(Cache.GrpP + *insertAt)->Next; Grp->Next = *insertAt; @@ -652,18 +652,18 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name Grp->FirstPackage = Package; // Insert it into the hash table map_id_t const Hash = Cache.Hash(Name); - map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTable()[Hash]; + map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash]; while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0) - insertAt = &(Cache.PkgP + *insertAt)->Next; - Pkg->Next = *insertAt; + insertAt = &(Cache.PkgP + *insertAt)->NextPackage; + Pkg->NextPackage = *insertAt; *insertAt = Package; } else // Group the Packages together { // this package is the new last package pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage); - Pkg->Next = LastPkg->Next; - LastPkg->Next = Package; + Pkg->NextPackage = LastPkg->NextPackage; + LastPkg->NextPackage = Package; } Grp->LastPackage = Package; diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 1bd75dfba..9bac45029 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -267,11 +267,14 @@ static bool DumpPackage(CommandLine &CmdL) // ShowHashTableStats - Show stats about a hashtable /*{{{*/ // --------------------------------------------------------------------- /* */ +static map_pointer_t PackageNext(pkgCache::Package const * const P) { return P->NextPackage; } +static map_pointer_t GroupNext(pkgCache::Group const * const G) { return G->Next; } template static void ShowHashTableStats(std::string Type, T *StartP, map_pointer_t *Hashtable, - unsigned long Size) + unsigned long Size, + map_pointer_t(*Next)(T const * const)) { // hashtable stats for the HashTable unsigned long NumBuckets = Size; @@ -290,7 +293,7 @@ static void ShowHashTableStats(std::string Type, } ++UsedBuckets; unsigned long ThisBucketSize = 0; - for (; P != StartP; P = StartP + P->Next) + for (; P != StartP; P = StartP + Next(P)) ++ThisBucketSize; Entries += ThisBucketSize; LongestBucket = std::max(ThisBucketSize, LongestBucket); @@ -447,13 +450,13 @@ static bool Stats(CommandLine &) APT_CACHESIZE(VerFileCount, VerFileSz) + APT_CACHESIZE(DescFileCount, DescFileSz) + APT_CACHESIZE(ProvidesCount, ProvidesSz) + - (2 * Cache->Head().HashTableSize * sizeof(map_id_t)); + (2 * Cache->Head().GetHashTableSize() * sizeof(map_id_t)); cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; #undef APT_CACHESIZE // hashtable stats - ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->Head().PkgHashTable(), Cache->Head().HashTableSize); - ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->Head().GrpHashTable(), Cache->Head().HashTableSize); + ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->Head().PkgHashTableP(), Cache->Head().GetHashTableSize(), PackageNext); + ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->Head().GrpHashTableP(), Cache->Head().GetHashTableSize(), GroupNext); return true; } -- cgit v1.2.3-70-g09d2 From ad7e0941b376d792911f240377094a2e78ca8756 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 18:14:46 +0100 Subject: streamline display of --help in all tools By convention, if I run a tool with --help or --version I expect it to exit successfully with the usage, while if I do call it wrong (like without any parameters) I expect the usage message shown with a non-zero exit. --- apt-pkg/contrib/cmndline.cc | 3 ++ apt-pkg/contrib/cmndline.h | 1 + apt-private/private-cmndline.cc | 33 ++++++++++++++++++ apt-private/private-cmndline.h | 6 ++++ cmdline/apt-cache.cc | 18 ++-------- cmdline/apt-cdrom.cc | 15 ++------- cmdline/apt-config.cc | 15 ++------- cmdline/apt-extracttemplates.cc | 32 +++++++----------- cmdline/apt-get.cc | 22 ++---------- cmdline/apt-helper.cc | 21 ++---------- cmdline/apt-internal-solver.cc | 17 +++------- cmdline/apt-mark.cc | 22 ++---------- cmdline/apt-sortpkgs.cc | 24 +++++-------- cmdline/apt.cc | 20 ++--------- cmdline/makefile | 8 ++--- debian/tests/run-tests | 1 + ftparchive/apt-ftparchive.cc | 19 +++-------- ftparchive/makefile | 4 +-- prepare-release | 4 +-- test/integration/framework | 2 ++ test/integration/test-00-commands-have-help | 52 +++++++++++++++++++++++++++++ 21 files changed, 149 insertions(+), 190 deletions(-) create mode 100755 test/integration/test-00-commands-have-help (limited to 'cmdline') diff --git a/apt-pkg/contrib/cmndline.cc b/apt-pkg/contrib/cmndline.cc index 93c1f4664..ff8b09ebc 100644 --- a/apt-pkg/contrib/cmndline.cc +++ b/apt-pkg/contrib/cmndline.cc @@ -33,6 +33,9 @@ using namespace std; CommandLine::CommandLine(Args *AList,Configuration *Conf) : ArgList(AList), Conf(Conf), FileList(0) { +} +CommandLine::CommandLine() : ArgList(NULL), Conf(NULL), FileList(0) +{ } /*}}}*/ // CommandLine::~CommandLine - Destructor /*{{{*/ diff --git a/apt-pkg/contrib/cmndline.h b/apt-pkg/contrib/cmndline.h index 143df58b2..58cbaa8c3 100644 --- a/apt-pkg/contrib/cmndline.h +++ b/apt-pkg/contrib/cmndline.h @@ -91,6 +91,7 @@ class CommandLine static CommandLine::Args MakeArgs(char ShortOpt, char const *LongOpt, char const *ConfName, unsigned long Flags) APT_CONST; + CommandLine(); CommandLine(Args *AList,Configuration *Conf); ~CommandLine(); }; diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc index aa01be757..bb9a00803 100644 --- a/apt-private/private-cmndline.cc +++ b/apt-private/private-cmndline.cc @@ -2,12 +2,17 @@ #include #include +#include +#include +#include +#include #include #include #include #include +#include #include /*}}}*/ @@ -287,3 +292,31 @@ std::vector getCommandArgs(char const * const Program, char c /*}}}*/ #undef CmdMatches #undef addArg +void ParseCommandLine(CommandLine &CmdL, CommandLine::Dispatch * const Cmds, CommandLine::Args * const Args,/*{{{*/ + Configuration * const * const Cnf, pkgSystem ** const Sys, int const argc, const char *argv[], bool(*ShowHelp)(CommandLine &CmdL)) +{ + CmdL = CommandLine(Args,_config); + if ((Cnf != NULL && pkgInitConfig(**Cnf) == false) || + CmdL.Parse(argc,argv) == false || + (Sys != NULL && pkgInitSystem(*_config, *Sys) == false)) + { + if (_config->FindB("version") == true) + ShowHelp(CmdL); + + _error->DumpErrors(); + exit(100); + } + + // See if the help should be shown + if (_config->FindB("help") == true || _config->FindB("version") == true) + { + ShowHelp(CmdL); + exit(0); + } + if (Cmds != NULL && CmdL.FileSize() == 0) + { + ShowHelp(CmdL); + exit(1); + } +} + /*}}}*/ diff --git a/apt-private/private-cmndline.h b/apt-private/private-cmndline.h index d0af16782..7b468456b 100644 --- a/apt-private/private-cmndline.h +++ b/apt-private/private-cmndline.h @@ -6,6 +6,12 @@ #include +class Configuration; +class pkgSystem; + APT_PUBLIC std::vector getCommandArgs(char const * const Program, char const * const Cmd); +APT_PUBLIC void ParseCommandLine(CommandLine &CmdL, CommandLine::Dispatch * const Cmds, CommandLine::Args * const Args, + Configuration * const * const Cnf, pkgSystem ** const Sys, int const argc, const char * argv[], + bool(*ShowHelp)(CommandLine &CmdL)); #endif diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 9bac45029..12ed4f719 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1896,23 +1896,9 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - // Deal with stdout not being a tty if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) _config->Set("quiet","1"); diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc index 53efe65b8..8ac73fd7e 100644 --- a/cmdline/apt-cdrom.cc +++ b/cmdline/apt-cdrom.cc @@ -249,19 +249,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || _config->FindB("version") == true || - CmdL.FileSize() == 0) - return ShowHelp(CmdL); + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // Deal with stdout not being a tty if (isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) diff --git a/cmdline/apt-config.cc b/cmdline/apt-config.cc index 40ba468eb..e0b8a624e 100644 --- a/cmdline/apt-config.cc +++ b/cmdline/apt-config.cc @@ -115,19 +115,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(CmdL); + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); std::vector const langs = APT::Configuration::getLanguages(true); _config->Clear("Acquire::Languages"); diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc index f95b9e5ba..5211ee027 100644 --- a/cmdline/apt-extracttemplates.cc +++ b/cmdline/apt-extracttemplates.cc @@ -33,6 +33,8 @@ #include #include +#include + #include #include #include @@ -215,15 +217,15 @@ bool DebFile::ParseInfo() // ShowHelp - show a short help text /*{{{*/ // --------------------------------------------------------------------- /* */ -static int ShowHelp(void) +static bool ShowHelp(CommandLine &) { - ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, + ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, COMMON_ARCH,__DATE__,__TIME__); - if (_config->FindB("version") == true) - return 0; + if (_config->FindB("version") == true) + return true; - cout << + cout << _("Usage: apt-extracttemplates file1 [file2 ...]\n" "\n" "apt-extracttemplates is a tool to extract config and template info\n" @@ -234,7 +236,7 @@ static int ShowHelp(void) " -t Set the temp dir\n" " -c=? Read this configuration file\n" " -o=? Set an arbitrary configuration option, eg -o dir::cache=/tmp\n"); - return 0; + return true; } /*}}}*/ // WriteFile - write the contents of the passed string to a file /*{{{*/ @@ -356,20 +358,10 @@ int main(int argc, const char **argv) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(); - + CommandLine::Dispatch Cmds[] = {{NULL, NULL}}; + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args, &_config, &_system, argc, argv, ShowHelp); + Go(CmdL); // Print any errors or warnings found during operation diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index eca4a723b..a9053bdfd 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1746,26 +1746,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // see if we are in simulate mode CheckSimulateMode(CmdL); diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index 27abb2013..1b832f165 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -108,25 +108,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); InitOutput(); diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index 92a4429e5..4fabeb02f 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -24,7 +24,9 @@ #include #include #include + #include +#include #include #include @@ -79,19 +81,8 @@ int main(int argc,const char *argv[]) /*{{{*/ // we really don't need anything DropPrivileges(); - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false) { - _error->DumpErrors(); - return 2; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true) { - ShowHelp(CmdL); - return 1; - } + CommandLine CmdL; + ParseCommandLine(CmdL, NULL, Args, &_config, NULL, argc, argv, ShowHelp); if (CmdL.FileList[0] != 0 && strcmp(CmdL.FileList[0], "scenario") == 0) { diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index ed348358a..487f3d8a1 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -441,26 +441,8 @@ int main(int argc,const char *argv[]) /*{{{*/ setlocale(LC_ALL,""); textdomain(PACKAGE); - // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // Deal with stdout not being a tty if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc index c2b11890a..9b66ad4db 100644 --- a/cmdline/apt-sortpkgs.cc +++ b/cmdline/apt-sortpkgs.cc @@ -23,6 +23,8 @@ #include #include +#include + #include #include #include @@ -142,12 +144,12 @@ static bool DoIt(string InFile) // ShowHelp - Show the help text /*{{{*/ // --------------------------------------------------------------------- /* */ -static int ShowHelp() +static bool ShowHelp(CommandLine &) { ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, COMMON_ARCH,__DATE__,__TIME__); if (_config->FindB("version") == true) - return 0; + return true; cout << _("Usage: apt-sortpkgs [options] file1 [file2 ...]\n" @@ -161,7 +163,7 @@ static int ShowHelp() " -c=? Read this configuration file\n" " -o=? Set an arbitrary configuration option, eg -o dir::cache=/tmp\n"); - return 0; + return true; } /*}}}*/ int main(int argc,const char *argv[]) /*{{{*/ @@ -179,19 +181,9 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(); + CommandLine::Dispatch Cmds[] = {{NULL, NULL}}; + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args, &_config, &_system, argc, argv, ShowHelp); // Match the operation for (unsigned int I = 0; I != CmdL.FileSize(); I++) diff --git a/cmdline/apt.cc b/cmdline/apt.cc index 2cfdf8e8e..056cd213f 100644 --- a/cmdline/apt.cc +++ b/cmdline/apt.cc @@ -119,15 +119,10 @@ int main(int argc, const char *argv[]) /*{{{*/ _config->CndSet("APT::Cmd::Show-Update-Stats", true); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(), _config); - if (CmdL.Parse(argc, argv) == false || - pkgInitSystem(*_config, _system) == false) - { - _error->DumpErrors(); - return 100; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), NULL, &_system, argc, argv, ShowHelp); - if(!isatty(STDOUT_FILENO) && + if(!isatty(STDOUT_FILENO) && _config->FindB("Apt::Cmd::Disable-Script-Warning", false) == false) { std::cerr << std::endl @@ -138,15 +133,6 @@ int main(int argc, const char *argv[]) /*{{{*/ << std::endl; } - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - // see if we are in simulate mode CheckSimulateMode(CmdL); diff --git a/cmdline/makefile b/cmdline/makefile index b7c35ddd1..816038c3b 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -67,15 +67,15 @@ APT_DOMAIN:=apt-utils # The apt-sortpkgs program PROGRAM=apt-sortpkgs -SLIBS = -lapt-pkg $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-sortpkgs.cc include $(PROGRAM_H) # The apt-extracttemplates program PROGRAM=apt-extracttemplates -SLIBS = -lapt-pkg -lapt-inst $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile apt-inst/makefile +SLIBS = -lapt-pkg -lapt-inst -lapt-private $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-inst/makefile apt-private/makefile SOURCE = apt-extracttemplates.cc include $(PROGRAM_H) diff --git a/debian/tests/run-tests b/debian/tests/run-tests index 9d7c484fc..7cc37f618 100644 --- a/debian/tests/run-tests +++ b/debian/tests/run-tests @@ -15,6 +15,7 @@ APT_INTEGRATION_TESTS_WEBSERVER_BIN_DIR=$(pwd)/build/bin \ APT_INTEGRATION_TESTS_METHODS_DIR=/usr/lib/apt/methods \ APT_INTEGRATION_TESTS_LIBEXEC_DIR=/usr/lib/apt/ \ APT_INTEGRATION_TESTS_INTERNAL_SOLVER=/usr/lib/apt/solvers/apt \ +APT_INTEGRATION_TESTS_DUMP_SOLVER=/usr/lib/apt/solvers/dump \ APT_INTEGRATION_TESTS_BUILD_DIR=/usr/bin \ APT_INTEGRATION_TESTS_LIBRARY_PATH=/dev/null/does/not/exist \ ./test/integration/run-tests diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index ebf99a8f8..7d9af4178 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -19,6 +19,8 @@ #include #include +#include + #include #include #include @@ -1060,21 +1062,8 @@ int main(int argc, const char *argv[]) // Parse the command line and initialize the package library CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - + ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp); + // Setup the output streams c0out.rdbuf(clog.rdbuf()); c1out.rdbuf(clog.rdbuf()); diff --git a/ftparchive/makefile b/ftparchive/makefile index d1ffe182a..e67272e1e 100644 --- a/ftparchive/makefile +++ b/ftparchive/makefile @@ -9,8 +9,8 @@ include ../buildlib/defaults.mak ifdef BDBLIB APT_DOMAIN:=apt-utils PROGRAM=apt-ftparchive -SLIBS = -lapt-pkg -lapt-inst $(BDBLIB) $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile apt-inst/makefile +SLIBS = -lapt-pkg -lapt-inst -lapt-private $(BDBLIB) $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-inst/makefile apt-private/makefile SOURCE = apt-ftparchive.cc cachedb.cc writer.cc contents.cc override.cc \ multicompress.cc sources.cc include $(PROGRAM_H) diff --git a/prepare-release b/prepare-release index e61266eef..67f4638d6 100755 --- a/prepare-release +++ b/prepare-release @@ -130,8 +130,8 @@ elif [ "$1" = 'coverage' ]; then LCOVRC='--rc geninfo_checksum=1 --rc lcov_branch_coverage=1' mkdir "$DIR" lcov --no-external --directory . --capture --initial --output-file "${DIR}/apt.coverage.init" ${LCOVRC} - make test - ./test/integration/run-tests -q + make test || true + ./test/integration/run-tests -q || true lcov --no-external --directory . --capture --output-file "${DIR}/apt.coverage.run" ${LCOVRC} lcov -a "${DIR}/apt.coverage.init" -a "${DIR}/apt.coverage.run" -o "${DIR}/apt.coverage.total" ${LCOVRC} cp "${DIR}/apt.coverage.total" "${DIR}/apt.coverage.fixed" diff --git a/test/integration/framework b/test/integration/framework index 153c5bb25..dd66f2a0c 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -128,6 +128,7 @@ aptwebserver() { runapt "${APTWEBSERVERBINDIR}/aptwebserver" "$@"; } aptitude() { runapt aptitude "$@"; } aptextracttemplates() { runapt apt-extracttemplates "$@"; } aptinternalsolver() { runapt "${APTINTERNALSOLVER}" "$@"; } +aptdumpsolver() { runapt "${APTDUMPSOLVER}" "$@"; } dpkg() { command dpkg --root=${TMPWORKINGDIRECTORY}/rootdir --force-not-root --force-bad-path --log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log "$@" @@ -194,6 +195,7 @@ setupenvironment() { APTHELPERBINDIR=${APT_INTEGRATION_TESTS_LIBEXEC_DIR:-"${BUILDDIRECTORY}"} APTWEBSERVERBINDIR=${APT_INTEGRATION_TESTS_WEBSERVER_BIN_DIR:-"${BUILDDIRECTORY}"} APTINTERNALSOLVER=${APT_INTEGRATION_TESTS_INTERNAL_SOLVER:-"${BUILDDIRECTORY}/apt-internal-solver"} + APTDUMPSOLVER=${APT_INTEGRATION_TESTS_DUMP_SOLVER:-"${BUILDDIRECTORY}/apt-dump-solver"} test -x "${BUILDDIRECTORY}/apt-get" || msgdie "You need to build tree first" # ----- diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help new file mode 100755 index 000000000..ebf8b8cfa --- /dev/null +++ b/test/integration/test-00-commands-have-help @@ -0,0 +1,52 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' + +# this test does double duty: The obvious is checking for --help and co, +# but it also checks if the binary can find all methods in the library. +# The later is quite handy for manual testing of non-abibreaking changes +export LD_BIND_NOW=1 + +checkversionmessage() { + testsuccess grep '^apt .* compiled on ' ${1}-help.output +} + +checkhelpmessage() { + checkversionmessage "$1" + testsuccess grep '^Usage:' ${1}-help.output +} + +checkoptions() { + testsuccess $1 --help + cp -f rootdir/tmp/testsuccess.output ${1}-help.output + checkhelpmessage "$1" + + testsuccess $1 --version + cp -f rootdir/tmp/testsuccess.output ${1}-help.output + checkversionmessage "$1" +} + +for CMD in 'apt-cache' 'apt-cdrom' 'apt-config' \ + 'apt-extracttemplates' 'apt-get' 'apt-helper' \ + 'apt-mark' 'apt-sortpkgs' 'apt' 'apt-ftparchive'; do + cmd="$(echo "$CMD" | tr -d '-')" + msgtest 'Test for failure with no parameters calling' "$CMD" + if $cmd > ${cmd}-help.output 2>&1; then + echo + cat ${cmd}-help.output + msgfail 'zero exit' + else + msgpass + fi + checkhelpmessage "$cmd" + checkoptions "$cmd" +done + +for CMD in 'apt-dump-solver' 'apt-internal-solver'; do + checkoptions "$(echo "$CMD" | tr -d '-')" +done -- cgit v1.2.3-70-g09d2 From d9e518c6f7dc0ad464495b586d1b8e115d54d41a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 20:44:44 +0100 Subject: use the same code to detect quiet setting in all tools Git-Dch: Ignore --- apt-private/private-output.cc | 8 ++-- apt-private/private-output.h | 3 +- cmdline/apt-cache.cc | 4 +- cmdline/apt-cdrom.cc | 7 ++- cmdline/apt-mark.cc | 18 +------ ftparchive/apt-ftparchive.cc | 19 ++------ test/integration/test-apt-ftparchive-cachedb | 30 +++++++----- test/integration/test-apt-ftparchive-src-cachedb | 60 +++++++++++------------- 8 files changed, 63 insertions(+), 86 deletions(-) (limited to 'cmdline') diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc index fc76a05bc..4e18030ab 100644 --- a/apt-private/private-output.cc +++ b/apt-private/private-output.cc @@ -52,14 +52,14 @@ static void SigWinch(int) #endif } /*}}}*/ -bool InitOutput() /*{{{*/ +bool InitOutput(std::basic_streambuf * const out) /*{{{*/ { if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) _config->Set("quiet","1"); - c0out.rdbuf(cout.rdbuf()); - c1out.rdbuf(cout.rdbuf()); - c2out.rdbuf(cout.rdbuf()); + c0out.rdbuf(out); + c1out.rdbuf(out); + c2out.rdbuf(out); if (_config->FindI("quiet",0) > 0) c0out.rdbuf(devnull.rdbuf()); if (_config->FindI("quiet",0) > 1) diff --git a/apt-private/private-output.h b/apt-private/private-output.h index e0dc9bf62..d5b57adec 100644 --- a/apt-private/private-output.h +++ b/apt-private/private-output.h @@ -6,6 +6,7 @@ #include #include +#include // forward declaration class pkgCacheFile; @@ -20,7 +21,7 @@ APT_PUBLIC extern std::ostream c2out; APT_PUBLIC extern std::ofstream devnull; APT_PUBLIC extern unsigned int ScreenWidth; -APT_PUBLIC bool InitOutput(); +APT_PUBLIC bool InitOutput(std::basic_streambuf * const out = std::cout.rdbuf()); void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records, pkgCache::VerIterator const &V, std::ostream &out, diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 12ed4f719..342ad1858 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1899,9 +1899,7 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); + InitOutput(); if (_config->Exists("APT::Cache::Generate") == true) _config->Set("pkgCacheFile::Generate", _config->FindB("APT::Cache::Generate", true)); diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc index 8ac73fd7e..327039e00 100644 --- a/cmdline/apt-cdrom.cc +++ b/cmdline/apt-cdrom.cc @@ -31,6 +31,7 @@ #include #include +#include #include /*}}}*/ @@ -252,10 +253,8 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); - + InitOutput(); + // Match the operation bool returned = CmdL.DispatchArg(Cmds); diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 487f3d8a1..2702dbbd3 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -41,10 +42,6 @@ /*}}}*/ using namespace std; -ostream c0out(0); -ostream c1out(0); -ostream c2out(0); -ofstream devnull("/dev/null"); /* DoAuto - mark packages as automatically/manually installed {{{*/ static bool DoAuto(CommandLine &CmdL) { @@ -444,18 +441,7 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); - - // Setup the output streams - c0out.rdbuf(cout.rdbuf()); - c1out.rdbuf(cout.rdbuf()); - c2out.rdbuf(cout.rdbuf()); - if (_config->FindI("quiet",0) > 0) - c0out.rdbuf(devnull.rdbuf()); - if (_config->FindI("quiet",0) > 1) - c1out.rdbuf(devnull.rdbuf()); + InitOutput(); // Match the operation CmdL.DispatchArg(Cmds); diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index 7d9af4178..adf1b6d73 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -42,11 +43,7 @@ #include /*}}}*/ -using namespace std; -ostream c0out(0); -ostream c1out(0); -ostream c2out(0); -ofstream devnull("/dev/null"); +using namespace std; unsigned Quiet = 0; // struct PackageMap - List of all package files in the config file /*{{{*/ @@ -1064,16 +1061,10 @@ int main(int argc, const char *argv[]) CommandLine CmdL(Args,_config); ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp); - // Setup the output streams - c0out.rdbuf(clog.rdbuf()); - c1out.rdbuf(clog.rdbuf()); - c2out.rdbuf(clog.rdbuf()); + _config->CndSet("quiet",0); Quiet = _config->FindI("quiet",0); - if (Quiet > 0) - c0out.rdbuf(devnull.rdbuf()); - if (Quiet > 1) - c1out.rdbuf(devnull.rdbuf()); - + InitOutput(clog.rdbuf()); + // Match the operation CmdL.DispatchArg(Cmds); diff --git a/test/integration/test-apt-ftparchive-cachedb b/test/integration/test-apt-ftparchive-cachedb index 0e1986bcd..866e5a469 100755 --- a/test/integration/test-apt-ftparchive-cachedb +++ b/test/integration/test-apt-ftparchive-cachedb @@ -69,32 +69,38 @@ buildsimplenativepackage 'foo' 'i386' '1' 'test' mv incoming/* aptarchive/pool/main/ # generate (empty cachedb) -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 2 - dists/test/Contents-i386: New 402 B Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 2 + dists/test/Contents-i386: New 402 B Misses in Cache: 0' # generate again -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 0 - dists/test/Contents-i386: Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 0 + dists/test/Contents-i386: Misses in Cache: 0' # and again (with removing the Packages file) rm -f ./aptarchive/dists/test/main/binary-i386/* rm -f ./aptarchive/dists/test/Contents-i386 -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 0 - dists/test/Contents-i386: New 402 B Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 0 + dists/test/Contents-i386: New 402 B Misses in Cache: 0' # and clean rm -rf aptarchive/pool/main/* -testequal "packages-main-i386.db" aptftparchive clean ftparchive.conf -aptftparchive clean ftparchive.conf -o Debug::APT::FTPArchive::Clean=1 > clean-out.txt 2>&1 +testequal "packages-main-i386.db" aptftparchive clean ftparchive.conf -q=0 +testsuccess aptftparchive clean ftparchive.conf -q=0 -o Debug::APT::FTPArchive::Clean=1 +cp rootdir/tmp/testsuccess.output clean-out.txt testequal "0 Number of unique keys in the tree" grep unique clean-out.txt testequal "packages-main-i386.db" grep packages-main-i386.db clean-out.txt - diff --git a/test/integration/test-apt-ftparchive-src-cachedb b/test/integration/test-apt-ftparchive-src-cachedb index e7b148530..6e857c0cb 100755 --- a/test/integration/test-apt-ftparchive-src-cachedb +++ b/test/integration/test-apt-ftparchive-src-cachedb @@ -106,28 +106,22 @@ mkdir -p aptarchive/dists/test/main/source/ mkdir aptarchive-overrides mkdir aptarchive-cache - - -# generate with --db option -(cd aptarchive && aptftparchive --db ./test.db sources pool/main/ \ - -o APT::FTPArchive::ShowCacheMisses=1 \ - > dists/test/main/source/Sources \ - 2> stats-out.txt - testequal " Misses in Cache: 2" grep Misses stats-out.txt -) +msgtest 'generate with --db option' +cd aptarchive +aptftparchive --db ./test.db sources pool/main/ -q=0 -o APT::FTPArchive::ShowCacheMisses=1 > dists/test/main/source/Sources 2>stats-out.txt && msgpass || msgfail +testsuccess grep Misses stats-out.txt +testfileequal '../rootdir/tmp/testsuccess.output' ' Misses in Cache: 2' +cd .. assert_correct_sources_file -# generate with --db option (again to ensure its in the cache) -(cd aptarchive && aptftparchive --db ./test.db sources pool/main/ \ - -o APT::FTPArchive::ShowCacheMisses=1 \ - > dists/test/main/source/Sources \ - 2> stats-out.txt - testequal " Misses in Cache: 0" grep Misses stats-out.txt -) +msgtest 'generate with --db option (again to ensure its in the cache)' +cd aptarchive +aptftparchive --db ./test.db sources pool/main/ -q=0 -o APT::FTPArchive::ShowCacheMisses=1 > dists/test/main/source/Sources 2>stats-out.txt && msgpass || msgfail +testsuccess grep Misses stats-out.txt +testfileequal '../rootdir/tmp/testsuccess.output' ' Misses in Cache: 0' +cd .. assert_correct_sources_file - - # get ready for the "apt-ftparchive generate" command cat > apt-ftparchive.conf <<"EOF" Dir { @@ -159,20 +153,21 @@ Tree "dists/test" { }; EOF -# generate (empty cachedb) -aptftparchive generate apt-ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt -testequal " Misses in Cache: 2" grep Misses stats-out.txt +msgtest 'generate (empty cachedb)' +testsuccess aptftparchive generate apt-ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal rootdir/tmp/testsuccess.output ' Misses in Cache: 2' assert_correct_sources_file - -# generate again out of the cache +msgtest 'generate again out of the cache' rm -f ./aptarchive/dists/test/main/source/Sources -aptftparchive generate apt-ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt -testequal " Misses in Cache: 0" grep Misses stats-out.txt +testsuccess aptftparchive generate apt-ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal rootdir/tmp/testsuccess.output ' Misses in Cache: 0' assert_correct_sources_file - - # generate invalid files mkdir aptarchive/pool/invalid printf "meep" > aptarchive/pool/invalid/invalid_1.0.dsc @@ -191,8 +186,9 @@ E: DSC file 'aptarchive/pool/invalid/toobig_1.0.dsc' is too large!" aptftparchiv # ensure clean works rm -f aptarchive/pool/main/* -aptftparchive clean apt-ftparchive.conf -o Debug::APT::FTPArchive::Clean=1 > clean-out.txt 2>&1 -testequal "0 Number of unique keys in the tree" grep unique clean-out.txt -testequal "sources-main.db" grep sources-main.db clean-out.txt - - +testsuccess aptftparchive clean apt-ftparchive.conf -q=0 -o Debug::APT::FTPArchive::Clean=1 +cp rootdir/tmp/testsuccess.output clean-out.txt +testsuccess grep unique clean-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' "0 Number of unique keys in the tree" +testsuccess grep sources-main.db clean-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' "sources-main.db" -- cgit v1.2.3-70-g09d2 From 374f8492e6f109e8427816a8f513e5e8feda9049 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 15:40:19 +0100 Subject: allow uninstalled packages to be put on hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dpkg wants to know about a package before it can be put on hold, so we have to at least hint about its existance in the available file it "maintaince" to know about such stuff. The simple thing would probably be to just feed all Packages files into dpkg as well, but what would be the point really? Exactly, so we take a shortcut here and just create dummies in the available file if we need to which isn't going to be that common as usually you are holding packages back and not off. Who would have thought that a simple feature like setting a package on hold requires more than 200 lines of code… at least with the testcase it is now explicitly tested code. --- cmdline/apt-mark.cc | 63 ++++++++++++++++++++++++++++++++---- test/integration/test-apt-mark | 72 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 6 deletions(-) create mode 100755 test/integration/test-apt-mark (limited to 'cmdline') diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 2702dbbd3..860982f9e 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -275,10 +275,64 @@ static bool DoHold(CommandLine &CmdL) } Args.erase(Args.begin() + BaseArgs, Args.end()); - Args.push_back("--set-selections"); + Args.push_back("--merge-avail"); + Args.push_back("-"); Args.push_back(NULL); int external[2] = {-1, -1}; + if (pipe(external) != 0) + return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); + + pid_t dpkgMergeAvail = ExecFork(); + if (dpkgMergeAvail == 0) + { + close(external[1]); + std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); + if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) + _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); + dup2(external[0], STDIN_FILENO); + int const nullfd = open("/dev/null", O_RDONLY); + dup2(nullfd, STDOUT_FILENO); + execvp(Args[0], (char**) &Args[0]); + _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); + _exit(2); + } + + FILE* dpkg = fdopen(external[1], "w"); + for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) + { + if (Pkg->CurrentVer != 0) + continue; + char const * Arch; + if (Pkg->VersionList != 0) + Arch = Pkg.VersionList().Arch(); + else + Arch = Pkg.Arch(); + fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" + "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + } + fclose(dpkg); + + if (dpkgMergeAvail > 0) + { + int Status = 0; + while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + { + if (errno == EINTR) + continue; + _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); + break; + } + if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) + return _error->Error(_("Executing dpkg failed. Are you root?")); + } + + Args.erase(Args.begin() + BaseArgs, Args.end()); + Args.push_back("--set-selections"); + Args.push_back(NULL); + + external[0] = -1; + external[1] = -1; if (pipe(external) != 0) return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --set-selections"); @@ -289,16 +343,13 @@ static bool DoHold(CommandLine &CmdL) std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --set-selections", chrootDir.c_str()); - int const nullfd = open("/dev/null", O_RDONLY); dup2(external[0], STDIN_FILENO); - dup2(nullfd, STDOUT_FILENO); - dup2(nullfd, STDERR_FILENO); execvp(Args[0], (char**) &Args[0]); - _error->WarningE("dpkgGo", "Can't detect if dpkg supports multi-arch!"); + _error->WarningE("dpkgGo", "Can't get dpkg --set-selections running!"); _exit(2); } - FILE* dpkg = fdopen(external[1], "w"); + dpkg = fdopen(external[1], "w"); for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (dpkgMultiArch == false) diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark new file mode 100755 index 000000000..69e0f933d --- /dev/null +++ b/test/integration/test-apt-mark @@ -0,0 +1,72 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'amd64' 'i386' + +insertpackage 'unstable' 'bar' 'amd64,i386' '1' +insertpackage 'unstable' 'uninstalled' 'all' '1' +insertpackage 'unstable' 'uninstalled-native' 'amd64' '1' + +insertinstalledpackage 'foo' 'all' '1' +insertinstalledpackage 'bar' 'amd64' '1' + +setupaptarchive + +# dpkg is "installed" by our test framework +testdpkginstalled dpkg + +testnoautopkg() { + testempty aptmark showauto + testequal 'bar +dpkg +foo' aptmark showmanual + testequal 'bar +foo' aptmark showmanual bar foo uninstalled +} +testmarkonpkgasauto() { + testsuccess aptmark $1 foo + testequal 'foo' aptmark showauto + testequal 'foo' aptmark showauto foo + testequal 'bar +dpkg' aptmark showmanual + testequal 'bar' aptmark showmanual bar + + testsuccess aptmark $2 foo + testnoautopkg +} + +testequal 'E: No packages found' aptmark auto +testequal 'E: No packages found' aptmark manual + +testnoautopkg +testmarkonpkgasauto 'auto' 'manual' +testmarkonpkgasauto 'markauto' 'unmarkauto' + +testnoholdpkg() { + testempty aptmark showhold + testempty aptmark showholds # typical "typo" + testempty aptmark showhold dpkg + testempty aptmark showholds dpkg +} +testmarkonepkgashold() { + testsuccess aptmark hold $1 + testequal "$1" aptmark showhold + testequal "$1" aptmark showholds + testsuccess aptmark unhold $1 + testnoholdpkg +} + +testequal 'E: No packages found' aptmark hold +testequal 'E: No packages found' aptmark unhold + +testnoholdpkg +testmarkonepkgashold 'foo' +testmarkonepkgashold 'bar' +testmarkonepkgashold 'uninstalled' +testmarkonepkgashold 'uninstalled-native' + +testequal 'uninstalled set on hold.' aptmark hold uninstalled +testequal 'uninstalled-native set on hold.' aptmark hold uninstalled-native -- cgit v1.2.3-70-g09d2 From 7e04a6bf23d857db60afd2ec3d0f4a8271b1c597 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 19:04:54 +0100 Subject: use pkgAcquire::GetLock instead of own code Do the same with less code in apt-get. This especially ensures that the lock file (and the parent directories) exist before we are trying to lock. It also means that clean now creates the directories if they are missing so we returned to a proper clean state now. Git-Dch: Ignore --- apt-pkg/acquire.cc | 2 ++ cmdline/apt-get.cc | 22 ++-------------------- 2 files changed, 4 insertions(+), 20 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 2c89c2dea..07c4646f5 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -152,6 +152,8 @@ bool pkgAcquire::GetLock(std::string const &Lock) return true; // Lock the directory this acquire object will work in + if (LockFD != -1) + close(LockFD); LockFD = ::GetLock(flCombine(Lock, "lock")); if (LockFD == -1) return _error->Error(_("Unable to lock directory %s"), Lock.c_str()); diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index a9053bdfd..eab792264 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -558,30 +558,12 @@ static bool DoClean(CommandLine &) return true; } - bool const NoLocking = _config->FindB("Debug::NoLocking",false); - // Lock the archive directory - FileFd Lock; - if (NoLocking == false) - { - int lock_fd = GetLock(archivedir + "lock"); - if (lock_fd < 0) - return _error->Error(_("Unable to lock directory %s"), archivedir.c_str()); - Lock.Fd(lock_fd); - } - pkgAcquire Fetcher; + Fetcher.GetLock(archivedir); Fetcher.Clean(archivedir); Fetcher.Clean(archivedir + "partial/"); - if (NoLocking == false) - { - Lock.Close(); - int lock_fd = GetLock(listsdir + "lock"); - if (lock_fd < 0) - return _error->Error(_("Unable to lock directory %s"), listsdir.c_str()); - Lock.Fd(lock_fd); - } - + Fetcher.GetLock(listsdir); Fetcher.Clean(listsdir + "partial/"); pkgCacheFile::RemoveCaches(); -- cgit v1.2.3-70-g09d2 From 3a2b39ee602dd5a98b8fdaee2f1c8e0b13a276e2 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 18 Aug 2013 23:27:24 +0200 Subject: use 'best' hash for source authentication Collect all hashes we can get from the source record and put them into a HashStringList so that 'apt-get source' can use it instead of using always the MD5sum. We therefore also deprecate the MD5 struct member in favor of the list. While at it, the parsing of the Files is enhanced so that records which miss "Files" (aka MD5 checksums) are still searched for other checksums as they include just as much data, just not with a nice and catchy name. This is a cherry-pick of 1262d35 with some dirty tricks to preserve ABI. LP: 1098738 --- apt-pkg/deb/debsrcrecords.cc | 162 +++++++++---- apt-pkg/deb/debsrcrecords.h | 1 + apt-pkg/srcrecords.cc | 31 ++- apt-pkg/srcrecords.h | 21 +- cmdline/apt-get.cc | 28 ++- debian/libapt-pkg4.12.symbols | 2 + .../test-ubuntu-bug-1098738-apt-get-source-md5sum | 260 +++++++++++++++++++++ 7 files changed, 445 insertions(+), 60 deletions(-) create mode 100755 test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum (limited to 'cmdline') diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc index a444cbe4d..49a348dd4 100644 --- a/apt-pkg/deb/debsrcrecords.cc +++ b/apt-pkg/deb/debsrcrecords.cc @@ -118,13 +118,32 @@ bool debSrcRecordParser::BuildDepends(std::vector &List) +bool debSrcRecordParser::Files(std::vector &F) { - List.erase(List.begin(),List.end()); - - string Files = Sect.FindS("Files"); - if (Files.empty() == true) + std::vector F2; + if (Files2(F2) == false) return false; + for (std::vector::const_iterator f2 = F2.begin(); f2 != F2.end(); ++f2) + { + pkgSrcRecords::File2 f; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + f.MD5Hash = f2->MD5Hash; + f.Size = f2->Size; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + f.Path = f2->Path; + f.Type = f2->Type; + F.push_back(f); + } + return true; +} +bool debSrcRecordParser::Files2(std::vector &List) +{ + List.clear(); // Stash the / terminated directory prefix string Base = Sect.FindS("Directory"); @@ -133,51 +152,106 @@ bool debSrcRecordParser::Files(std::vector &List) std::vector const compExts = APT::Configuration::getCompressorExtensions(); - // Iterate over the entire list grabbing each triplet - const char *C = Files.c_str(); - while (*C != 0) - { - pkgSrcRecords::File F; - string Size; - - // Parse each of the elements - if (ParseQuoteWord(C,F.MD5Hash) == false || - ParseQuoteWord(C,Size) == false || - ParseQuoteWord(C,F.Path) == false) - return _error->Error("Error parsing file record"); - - // Parse the size and append the directory - F.Size = atoi(Size.c_str()); - F.Path = Base + F.Path; - - // Try to guess what sort of file it is we are getting. - string::size_type Pos = F.Path.length()-1; - while (1) + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + // derive field from checksum type + std::string checksumField("Checksums-"); + if (strcmp(*type, "MD5Sum") == 0) + checksumField = "Files"; // historic name for MD5 checksums + else + checksumField.append(*type); + + string const Files = Sect.FindS(checksumField.c_str()); + if (Files.empty() == true) + continue; + + // Iterate over the entire list grabbing each triplet + const char *C = Files.c_str(); + while (*C != 0) { - string::size_type Tmp = F.Path.rfind('.',Pos); - if (Tmp == string::npos) - break; - if (F.Type == "tar") { - // source v3 has extension 'debian.tar.*' instead of 'diff.*' - if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") - F.Type = "diff"; - break; - } - F.Type = string(F.Path,Tmp+1,Pos-Tmp); - - if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || - F.Type == "tar") + string hash, size, path; + + // Parse each of the elements + if (ParseQuoteWord(C, hash) == false || + ParseQuoteWord(C, size) == false || + ParseQuoteWord(C, path) == false) + return _error->Error("Error parsing file record in %s of source package %s", checksumField.c_str(), Package().c_str()); + + HashString const hashString(*type, hash); + if (Base.empty() == false) + path = Base + path; + + // look if we have a record for this file already + std::vector::iterator file = List.begin(); + for (; file != List.end(); ++file) + if (file->Path == path) + break; + + // we have it already, store the new hash and be done + if (file != List.end()) { - Pos = Tmp-1; +#if __GNUC__ >= 4 + // set for compatibility only, so warn users not us + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + if (checksumField == "Files") + file->MD5Hash = hash; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + // an error here indicates that we have two different hashes for the same file + if (file->Hashes.push_back(hashString) == false) + return _error->Error("Error parsing checksum in %s of source package %s", checksumField.c_str(), Package().c_str()); continue; } - - break; + + // we haven't seen this file yet + pkgSrcRecords::File2 F; + F.Path = path; + F.FileSize = strtoull(size.c_str(), NULL, 10); + F.Hashes.push_back(hashString); + +#if __GNUC__ >= 4 + // set for compatibility only, so warn users not us + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + F.Size = F.FileSize; + if (checksumField == "Files") + F.MD5Hash = hash; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + + // Try to guess what sort of file it is we are getting. + string::size_type Pos = F.Path.length()-1; + while (1) + { + string::size_type Tmp = F.Path.rfind('.',Pos); + if (Tmp == string::npos) + break; + if (F.Type == "tar") { + // source v3 has extension 'debian.tar.*' instead of 'diff.*' + if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") + F.Type = "diff"; + break; + } + F.Type = string(F.Path,Tmp+1,Pos-Tmp); + + if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || + F.Type == "tar") + { + Pos = Tmp-1; + continue; + } + + break; + } + List.push_back(F); } - - List.push_back(F); } - + return true; } /*}}}*/ diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h index b65d1480b..2a3fc86c9 100644 --- a/apt-pkg/deb/debsrcrecords.h +++ b/apt-pkg/deb/debsrcrecords.h @@ -53,6 +53,7 @@ class debSrcRecordParser : public pkgSrcRecords::Parser return std::string(Start,Stop); }; virtual bool Files(std::vector &F); + bool Files2(std::vector &F); debSrcRecordParser(std::string const &File,pkgIndexFile const *Index) : Parser(Index), Fd(File,FileFd::ReadOnly, FileFd::Extension), Tags(&Fd,102400), diff --git a/apt-pkg/srcrecords.cc b/apt-pkg/srcrecords.cc index 81b1c545d..3175ee75f 100644 --- a/apt-pkg/srcrecords.cc +++ b/apt-pkg/srcrecords.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -147,5 +148,33 @@ const char *pkgSrcRecords::Parser::BuildDepType(unsigned char const &Type) return fields[Type]; } /*}}}*/ +bool pkgSrcRecords::Parser::Files2(std::vector &F2)/*{{{*/ +{ + debSrcRecordParser * const deb = dynamic_cast(this); + if (deb != NULL) + return deb->Files2(F2); - + std::vector F; + if (Files(F) == false) + return false; + for (std::vector::const_iterator f = F.begin(); f != F.end(); ++f) + { + pkgSrcRecords::File2 f2; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + f2.MD5Hash = f->MD5Hash; + f2.Size = f->Size; + f2.Hashes.push_back(HashString("MD5Sum", f->MD5Hash)); + f2.FileSize = f->Size; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + f2.Path = f->Path; + f2.Type = f->Type; + F2.push_back(f2); + } + return true; +} + /*}}}*/ diff --git a/apt-pkg/srcrecords.h b/apt-pkg/srcrecords.h index e000e176a..dde22bd65 100644 --- a/apt-pkg/srcrecords.h +++ b/apt-pkg/srcrecords.h @@ -14,6 +14,7 @@ #define PKGLIB_SRCRECORDS_H #include +#include #include #include @@ -29,15 +30,28 @@ class pkgSrcRecords { public: +#if __GNUC__ >= 4 + // ensure that con- & de-structor don't trigger this warning + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif // Describes a single file struct File { - std::string MD5Hash; - unsigned long Size; + APT_DEPRECATED std::string MD5Hash; + APT_DEPRECATED unsigned long Size; std::string Path; std::string Type; }; - + struct File2 : public File + { + unsigned long long FileSize; + HashStringList Hashes; + }; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + // Abstract parser for each source record class Parser { @@ -77,6 +91,7 @@ class pkgSrcRecords static const char *BuildDepType(unsigned char const &Type) APT_PURE; virtual bool Files(std::vector &F) = 0; + bool Files2(std::vector &F); Parser(const pkgIndexFile *Index) : iIndex(Index) {}; virtual ~Parser() {}; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index cfa79339b..a28537712 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -797,13 +797,13 @@ static bool DoSource(CommandLine &CmdL) } // Back track - vector Lst; - if (Last->Files(Lst) == false) { + vector Lst; + if (Last->Files2(Lst) == false) { return false; } // Load them into the fetcher - for (vector::const_iterator I = Lst.begin(); + for (vector::const_iterator I = Lst.begin(); I != Lst.end(); ++I) { // Try to guess what sort of file it is we are getting. @@ -832,22 +832,26 @@ static bool DoSource(CommandLine &CmdL) queued.insert(Last->Index().ArchiveURI(I->Path)); // check if we have a file with that md5 sum already localy - if(!I->MD5Hash.empty() && FileExists(flNotDir(I->Path))) - { - FileFd Fd(flNotDir(I->Path), FileFd::ReadOnly); - MD5Summation sum; - sum.AddFD(Fd.Fd(), Fd.Size()); - Fd.Close(); - if((string)sum.Result() == I->MD5Hash) + std::string localFile = flNotDir(I->Path); + if (FileExists(localFile) == true) + if(I->Hashes.VerifyFile(localFile) == true) { ioprintf(c1out,_("Skipping already downloaded file '%s'\n"), - flNotDir(I->Path).c_str()); + localFile.c_str()); continue; } + + // see if we have a hash (Acquire::ForceHash is the only way to have none) + HashString const * const hs = I->Hashes.find(NULL); + if (hs == NULL && _config->FindB("APT::Get::AllowUnauthenticated",false) == false) + { + ioprintf(c1out, "Skipping download of file '%s' as requested hashsum is not available for authentication\n", + localFile.c_str()); + continue; } new pkgAcqFile(&Fetcher,Last->Index().ArchiveURI(I->Path), - I->MD5Hash,I->Size, + hs != NULL ? hs->toStr() : "", I->FileSize, Last->Index().SourceInfo(*Last,*I),Src); } } diff --git a/debian/libapt-pkg4.12.symbols b/debian/libapt-pkg4.12.symbols index d89f07bb5..d481e51ed 100644 --- a/debian/libapt-pkg4.12.symbols +++ b/debian/libapt-pkg4.12.symbols @@ -1587,6 +1587,8 @@ libapt-pkg.so.4.12 libapt-pkg4.12 #MINVER# (c++)"HashStringList::VerifyFile(std::basic_string, std::allocator >) const@Base" 1.0.9.4 (c++)"HashString::operator==(HashString const&) const@Base" 1.0.9.4 (c++)"HashString::operator!=(HashString const&) const@Base" 1.0.9.4 + (c++)"pkgSrcRecords::Parser::Files2(std::vector >&)@Base" 1.0.9.4 + (c++)"debSrcRecordParser::Files2(std::vector >&)@Base" 1.0.9.4 ### demangle strangeness - buildd report it as MISSING and as new… (c++)"pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire*, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::vector > const*, indexRecords*)@Base" 0.8.0 ### gcc-4.6 artefacts diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum new file mode 100755 index 000000000..9bdc81264 --- /dev/null +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -0,0 +1,260 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'native' + +cat > aptarchive/Sources < +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.tar.gz + +Package: pkg-sha256-ok +Binary: pkg-sha256-ok +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.tar.gz + +Package: pkg-sha256-bad +Binary: pkg-sha256-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.tar.gz +Checksums-Sha256: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-bad_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-bad_1.0.tar.gz + +Package: pkg-no-md5 +Binary: pkg-no-md5 +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.tar.gz + +Package: pkg-mixed-ok +Binary: pkg-mixed-ok +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-ok_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-ok_1.0.dsc + +Package: pkg-mixed-sha1-bad +Binary: pkg-mixed-sha1-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-mixed-sha1-bad_1.0.dsc +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-sha1-bad_1.0.tar.gz + +Package: pkg-mixed-sha2-bad +Binary: pkg-mixed-sha2-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-sha2-bad_1.0.dsc +Checksums-Sha256: + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-mixed-sha2-bad_1.0.tar.gz + +Package: pkg-md5-disagree +Binary: pkg-md5-disagree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.tar.gz + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-md5-disagree_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-md5-disagree_1.0.tar.gz + +Package: pkg-md5-agree +Binary: pkg-md5-agree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc + +Package: pkg-sha256-disagree +Binary: pkg-sha256-disagree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.tar.gz + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-disagree_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-disagree_1.0.tar.gz +EOF + +# create fetchable files +for x in 'pkg-md5-ok' 'pkg-sha256-ok' 'pkg-sha256-bad' 'pkg-no-md5' \ + 'pkg-mixed-ok' 'pkg-mixed-sha1-bad' 'pkg-mixed-sha2-bad' \ + 'pkg-md5-agree' 'pkg-md5-disagree' 'pkg-sha256-disagree'; do + touch aptarchive/${x}_1.0.dsc aptarchive/${x}_1.0.tar.gz +done + +setupaptarchive +changetowebserver +testsuccess aptget update + +testok() { + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +Download complete and in download only mode" aptget source -d "$@" + msgtest 'Files were successfully downloaded for' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz +} + +testkeep() { + touch ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Skipping already downloaded file '${1}_1.0.dsc' +Skipping already downloaded file '${1}_1.0.tar.gz' +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d "$@" + msgtest 'Files already downloaded are kept for' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz +} + +testmismatch() { + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +E: Failed to fetch http://localhost:8080/${1}_1.0.dsc Hash Sum mismatch + +E: Failed to fetch http://localhost:8080/${1}_1.0.tar.gz Hash Sum mismatch + +E: Failed to fetch some archives." aptget source -d "$@" + msgtest 'Files were not download as they have hashsum mismatches for' "$1" + testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Skipping download of file 'pkg-sha256-bad_1.0.dsc' as requested hashsum is not available for authentication +Skipping download of file 'pkg-sha256-bad_1.0.tar.gz' as requested hashsum is not available for authentication +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d "$@" -o Acquire::ForceHash=ROT26 + msgtest 'Files were not download as hash is unavailable for' "$1" + testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +Download complete and in download only mode" aptget source --allow-unauthenticated -d "$@" -o Acquire::ForceHash=ROT26 + msgtest 'Files were downloaded unauthenticated as user allowed it' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz +} + +testok pkg-md5-ok +testkeep pkg-md5-ok +testok pkg-sha256-ok +testkeep pkg-sha256-ok + +# pkg-sha256-bad has a bad SHA sum, but good MD5 sum. If apt is +# checking the best available hash (as it should), this will trigger +# a hash mismatch. +testmismatch pkg-sha256-bad +testmismatch pkg-sha256-bad +testok pkg-sha256-bad -o Acquire::ForceHash=MD5Sum + +# not having MD5 sum doesn't mean the file doesn't exist at all … +testok pkg-no-md5 +testok pkg-no-md5 -o Acquire::ForceHash=SHA256 +testequal "Reading package lists... +Building dependency tree... +Skipping download of file 'pkg-no-md5_1.0.dsc' as requested hashsum is not available for authentication +Skipping download of file 'pkg-no-md5_1.0.tar.gz' as requested hashsum is not available for authentication +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d pkg-no-md5 -o Acquire::ForceHash=MD5Sum +msgtest 'Files were not download as MD5 is not available for this package' 'pkg-no-md5' +testfailure --nomsg test -e pkg-no-md5_1.0.dsc -a -e pkg-no-md5_1.0.tar.gz + +# deal with cases in which we haven't for all files the same checksum type +# mostly pathologic as this shouldn't happen, but just to be sure +testok pkg-mixed-ok +testequal 'Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (tar) +Get:2 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) +E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc Hash Sum mismatch + +E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha1-bad +msgtest 'Only tar file is downloaded as the dsc has hashsum mismatch' 'pkg-mixed-sha1-bad' +testsuccess --nomsg test ! -e pkg-mixed-sha1-bad_1.0.dsc -a -e pkg-mixed-sha1-bad_1.0.tar.gz +testequal 'Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) +Get:2 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (dsc) +E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz Hash Sum mismatch + +E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha2-bad +msgtest 'Only dsc file is downloaded as the tar has hashsum mismatch' 'pkg-mixed-sha2-bad' +testsuccess --nomsg test -e pkg-mixed-sha2-bad_1.0.dsc -a ! -e pkg-mixed-sha2-bad_1.0.tar.gz + +# it gets even more pathologic: multiple entries for one file, some even disagreeing! +testok pkg-md5-agree +testequal 'Reading package lists... +Building dependency tree... +E: Error parsing checksum in Files of source package pkg-md5-disagree' aptget source -d pkg-md5-disagree +testequal 'Reading package lists... +Building dependency tree... +E: Error parsing checksum in Checksums-SHA256 of source package pkg-sha256-disagree' aptget source -d pkg-sha256-disagree -- cgit v1.2.3-70-g09d2 From 081c9d442a6d39fb9bc419fe3ce697cc791cb844 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 21:38:53 +0100 Subject: various small additional tests and testcases Usually they don't provide a lot in terms of what they test, but they help in covering many lines from strictly anecdotal commands (stats, moo) and error messages, so that stuff which really needs to be tested, but isn't is better visible in coverage reports. Git-Dch: Ignore --- cmdline/apt-cache.cc | 4 +- test/integration/framework | 2 +- test/integration/test-00-commands-have-help | 11 ++ test/integration/test-apt-cache | 124 +++++++++++++++++++++ test/integration/test-apt-cli-search | 3 + test/integration/test-apt-cli-show | 11 ++ test/integration/test-apt-cli-update | 7 ++ test/integration/test-apt-config | 36 ++++++ test/integration/test-apt-helper | 12 +- test/integration/test-apt-mark | 34 ++++-- test/integration/test-apt-update-stale | 6 +- .../test-external-dependency-solver-protocol | 32 +++++- 12 files changed, 263 insertions(+), 19 deletions(-) create mode 100755 test/integration/test-apt-cache create mode 100755 test/integration/test-apt-config (limited to 'cmdline') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 342ad1858..a5024c581 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -116,7 +116,7 @@ static bool ShowUnMet(pkgCache::VerIterator const &V, bool const Important) continue; // Skip conflicts and replaces - if (End.IsNegative() == true) + if (End.IsNegative() == true || End->Type == pkgCache::Dep::Replaces) continue; // Verify the or group @@ -133,7 +133,7 @@ static bool ShowUnMet(pkgCache::VerIterator const &V, bool const Important) break; } delete [] VList; - + if (Start == End) break; ++Start; diff --git a/test/integration/framework b/test/integration/framework index d28085ee4..c9f62c141 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -300,7 +300,7 @@ getarchitecture() { } getarchitectures() { - echo "$(aptconfig dump | grep APT::Architecture | cut -d'"' -f 2 | sed '/^$/ d' | sort | uniq | tr '\n' ' ')" + aptconfig dump --no-empty --format '%v%n' APT::Architecture APT::Architectures | sort -u | tr '\n' ' ' } getarchitecturesfromcommalist() { diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help index ebf8b8cfa..bbd1475eb 100755 --- a/test/integration/test-00-commands-have-help +++ b/test/integration/test-00-commands-have-help @@ -50,3 +50,14 @@ done for CMD in 'apt-dump-solver' 'apt-internal-solver'; do checkoptions "$(echo "$CMD" | tr -d '-')" done + +# in times of need, we all look for super cow to save the day +testsuccess aptget moo +testsuccess aptget moo -q=2 +testsuccess aptget moo moo +testsuccess aptget moo moo -q=2 +testsuccess aptget moo moo --color +testsuccess aptget moo moo moo +testsuccess aptget moo moo moo -q=2 +testsuccess aptget moo moo moo moo +testsuccess aptget moo moo moo moo -q=2 diff --git a/test/integration/test-apt-cache b/test/integration/test-apt-cache new file mode 100755 index 000000000..f47c0e08b --- /dev/null +++ b/test/integration/test-apt-cache @@ -0,0 +1,124 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' 'i386' + +DESCR='Some description + That has multiple lines' +insertpackage 'unstable' 'fancy' 'all' '1' +insertpackage 'unstable,installed' 'foo' 'all' '1' 'Depends: bar +Conflicts: foobar +Recommends: cool (>= 2) | cooler (<< 5)' "$DESCR" +insertpackage 'unstable' 'bar' 'all' '1' 'Depends: bar +Breaks: foo (<< 1) +Replaces: foo (<< 1)' "$DESCR" + +setupaptarchive + +# dpkg is installed by our framework +testdpkginstalled 'dpkg' +testempty aptcache unmet dpkg + +# FIXME: Find some usecase for unmet as it seems kinda useless/broken +#testsuccess aptcache unmet +#testsuccess aptcache unmet foo + +# not too useful to test, but makes coverage green… +testsuccess aptcache stats +cp rootdir/tmp/testsuccess.output stats.output +testsuccess test -s stats.output +testsuccess aptcache xvcg foo +cp rootdir/tmp/testsuccess.output xvcg.output +testsuccess test -s xvcg.output +testsuccess aptcache dotty foo +cp rootdir/tmp/testsuccess.output dotty.output +testsuccess test -s dotty.output +# for this, even the sourcecode says it is useless (expect debugging) +testsuccess aptcache dump +cp rootdir/tmp/testsuccess.output dump.output +testsuccess test -s dump.output + +testequal 'dpkg +bar +fancy +foo' aptcache pkgnames +testequal 'bar' aptcache pkgnames bar +testequal 'fancy +foo' aptcache pkgnames f + +testequal " foo | 1 | file:$(readlink -f .)/aptarchive/ unstable/main amd64 Packages" aptcache madison foo + +### depends + +testequal 'foo + Depends: bar + |Recommends: + Recommends: + Conflicts: + Conflicts: ' aptcache depends foo +testequal 'foo + Depends: bar + Recommends: + Conflicts: + Conflicts: ' aptcache depends foo -o APT::Cache::ShowOnlyFirstOr=1 +testequal 'foo + Depends: bar + |Recommends: (>= 2) + Recommends: (<< 5) + Conflicts: + Conflicts: ' aptcache depends foo -o APT::Cache::ShowVersion=1 +testequal 'foo + Depends: bar + Conflicts: + Conflicts: ' aptcache depends foo --no-recommends +testequal 'foo + Depends: bar' aptcache depends foo --important +testequal 'foo + Conflicts: + Conflicts: ' aptcache depends foo --important --no-depends --conflicts +testequal 'foo + Depends: bar + |Recommends: + Recommends: + Conflicts: + Conflicts: +bar + Depends: bar + Breaks: foo + Breaks: + Replaces: foo + Replaces: + + + + +' aptcache depends foo --recurse +testequal 'foo + Depends: bar +bar + Depends: bar + Replaces: foo + Replaces: +' aptcache depends foo --recurse --important --replaces + +## rdpends + +testequal 'foo +Reverse Depends: + bar + bar' aptcache rdepends foo +testequal 'foo +Reverse Depends: + Replaces: bar + Breaks: bar' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 +testequal 'foo +Reverse Depends: + Replaces: bar (<< 1) + Breaks: bar (<< 1)' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 -o APT::Cache::ShowVersion=1 +testequal 'foo +Reverse Depends: + Breaks: bar (<< 1)' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 -o APT::Cache::ShowVersion=1 --important --breaks diff --git a/test/integration/test-apt-cli-search b/test/integration/test-apt-cli-search index 8f009d57c..1a28ba4da 100755 --- a/test/integration/test-apt-cli-search +++ b/test/integration/test-apt-cli-search @@ -25,6 +25,9 @@ setupaptarchive APTARCHIVE=$(readlink -f ./aptarchive) +testequal 'E: You must give at least one search pattern' aptcache search +testequal 'E: You must give at least one search pattern' apt search + # with OP progress testequal "Sorting... Full Text Search... diff --git a/test/integration/test-apt-cli-show b/test/integration/test-apt-cli-show index 4c8e134d6..930e591e0 100755 --- a/test/integration/test-apt-cli-show +++ b/test/integration/test-apt-cli-show @@ -36,3 +36,14 @@ APT-Sources: file:$APTARCHIVE/ unstable/main i386 Packages Description: Some description That has multiple lines " apt show foo + +# this is the default, but disabled by the testcases +testsuccess apt show foo -o Apt::Cmd::Disable-Script-Warning=0 +cp rootdir/tmp/testsuccess.output aptshow.output +testsuccess grep '^WARNING: ' aptshow.output + +if [ "$(id -u)" != '0' ]; then + testsuccess apt install foo -s -o APT::Get::Show-User-Simulation-Note=1 + cp rootdir/tmp/testsuccess.output aptshow.output + testsuccess grep '^NOTE: ' aptshow.output +fi diff --git a/test/integration/test-apt-cli-update b/test/integration/test-apt-cli-update index 987bb9adb..83cc94b93 100755 --- a/test/integration/test-apt-cli-update +++ b/test/integration/test-apt-cli-update @@ -8,10 +8,17 @@ setupenvironment configarchitecture "i386" insertpackage 'unstable' 'foo' 'all' '2.0' +cp rootdir/var/lib/dpkg/status dpkg.status insertinstalledpackage 'foo' 'all' '1.0' setupaptarchive --no-update APTARCHIVE=$(readlink -f ./aptarchive) +testequal 'E: The update command takes no arguments' apt update -q arguments + testequal "1 package can be upgraded. Run 'apt list --upgradable' to see it." apt update -q + +cp dpkg.status rootdir/var/lib/dpkg/status +insertinstalledpackage 'foo' 'all' '2.0' +testequal 'All packages are up to date.' apt update -q diff --git a/test/integration/test-apt-config b/test/integration/test-apt-config new file mode 100755 index 000000000..2f2ff9d38 --- /dev/null +++ b/test/integration/test-apt-config @@ -0,0 +1,36 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' + +testsuccess aptconfig dump +testequal 'APT::Architecture "amd64";' aptconfig dump APT::Architecture +testempty aptconfig dump config::which::does::not::exist + +testequal 'APT::Architectures ""; +APT::Architectures:: "amd64";' aptconfig dump APT::Architectures +testequal 'APT::Architectures:: "amd64";' aptconfig dump --no-empty APT::Architectures +testequal 'amd64' aptconfig dump --no-empty --format='%v%n' APT::Architectures + +testempty aptconfig shell +testequal 'E: Arguments not in pairs' aptconfig shell APT::Architecture +testempty aptconfig shell APT::Architecture ARCH # incorrect order +testequal "ARCH='amd64'" aptconfig shell ARCH APT::Architecture + +ROOTDIR="$(readlink -f rootdir)" +testequal "CONFIG='apt.conf'" aptconfig shell CONFIG Dir::Etc::main +testequal "CONFIG='${ROOTDIR}/etc/apt/apt.conf'" aptconfig shell CONFIG Dir::Etc::main/f +testequal "CONFIG='etc/apt/'" aptconfig shell CONFIG Dir::Etc +testequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/ # old style +testequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/d + +testempty aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles +export DEB_BUILD_PROFILES='nodoc stage1' +testequal 'nodoc +stage1' aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles +unset DEB_BUILD_PROFILES +testempty aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index 06e497ff7..3c1d393a6 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -53,7 +53,7 @@ echo "http://some-proxy" EOF chmod 755 apt-proxy-detect echo "Acquire::http::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect - + testequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com @@ -64,13 +64,15 @@ echo "https://https-proxy" EOF chmod 755 apt-proxy-detect echo "Acquire::https::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect - - testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com - - + testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com } test_apt_helper_download test_apt_helper_detect_proxy +# test failure modes +testequal 'E: Invalid operation download' apthelper download +testequal 'E: Must specify at least one pair url/filename' apthelper download-file +testequal 'E: Must specify at least one pair url/filename' apthelper download-file http://example.org/ +testequal 'E: Need one URL as argument' apthelper auto-detect-proxy diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark index 69e0f933d..0f62a12b4 100755 --- a/test/integration/test-apt-mark +++ b/test/integration/test-apt-mark @@ -20,21 +20,31 @@ testdpkginstalled dpkg testnoautopkg() { testempty aptmark showauto + testempty aptcache showauto testequal 'bar dpkg foo' aptmark showmanual testequal 'bar foo' aptmark showmanual bar foo uninstalled } -testmarkonpkgasauto() { - testsuccess aptmark $1 foo +testfooisauto() { testequal 'foo' aptmark showauto + testequal 'foo' aptcache showauto testequal 'foo' aptmark showauto foo + testequal 'foo' aptcache showauto foo testequal 'bar dpkg' aptmark showmanual testequal 'bar' aptmark showmanual bar +} +testmarkonpkgasauto() { + testsuccess $1 $2 foo + testfooisauto + testsuccess $1 $2 foo + testfooisauto - testsuccess aptmark $2 foo + testsuccess $1 $3 foo + testnoautopkg + testsuccess $1 $3 foo testnoautopkg } @@ -42,8 +52,9 @@ testequal 'E: No packages found' aptmark auto testequal 'E: No packages found' aptmark manual testnoautopkg -testmarkonpkgasauto 'auto' 'manual' -testmarkonpkgasauto 'markauto' 'unmarkauto' +testmarkonpkgasauto 'aptmark' 'auto' 'manual' +testmarkonpkgasauto 'aptmark' 'markauto' 'unmarkauto' +testmarkonpkgasauto 'aptget' 'markauto' 'unmarkauto' testnoholdpkg() { testempty aptmark showhold @@ -51,10 +62,19 @@ testnoholdpkg() { testempty aptmark showhold dpkg testempty aptmark showholds dpkg } -testmarkonepkgashold() { - testsuccess aptmark hold $1 +testpkgonhold() { testequal "$1" aptmark showhold testequal "$1" aptmark showholds + testequal "$1" aptmark showhold $1 + testequal "$1" aptmark showholds $1 +} +testmarkonepkgashold() { + testsuccess aptmark hold $1 + testpkgonhold $1 + testsuccess aptmark hold $1 + testpkgonhold $1 + testsuccess aptmark unhold $1 + testnoholdpkg testsuccess aptmark unhold $1 testnoholdpkg } diff --git a/test/integration/test-apt-update-stale b/test/integration/test-apt-update-stale index 52f94591f..277aa5b09 100755 --- a/test/integration/test-apt-update-stale +++ b/test/integration/test-apt-update-stale @@ -14,9 +14,11 @@ configarchitecture "i386" insertpackage 'unstable' 'foo' 'all' '1.0' -setupaptarchive +setupaptarchive --no-update changetowebserver -aptget update -qq + +echo "Acquire::Languages \"none\";" > rootdir/etc/apt/apt.conf.d/00nolanguages +testsuccess aptget update listcurrentlistsdirectory > lists.before # insert new version diff --git a/test/integration/test-external-dependency-solver-protocol b/test/integration/test-external-dependency-solver-protocol index 07d2441b6..fd68578c5 100755 --- a/test/integration/test-external-dependency-solver-protocol +++ b/test/integration/test-external-dependency-solver-protocol @@ -12,7 +12,10 @@ insertinstalledpackage 'stuff' 'all' '1' insertpackage 'unstable' 'cool' 'all' '2' 'Multi-Arch: foreign' insertpackage 'unstable' 'stuff' 'all' '2' 'Multi-Arch: foreign' insertpackage 'unstable' 'coolstuff' 'i386,amd64' '2' 'Depends: cool, stuff' -insertpackage 'unstable' 'awesome' 'all' '2' 'Multi-Arch: foreign' +insertpackage 'unstable' 'awesome' 'all' '2' 'Multi-Arch: foreign +Conflicts: badstuff' +insertpackage 'unstable' 'badstuff' 'all' '2' 'Multi-Arch: foreign +Conflicts: awesome' insertpackage 'unstable' 'awesomecoolstuff' 'i386' '2' 'Depends: coolstuff, awesome' insertpackage 'experimental' 'cool' 'all' '3' 'Multi-Arch: foreign' @@ -44,6 +47,14 @@ The following NEW packages will be installed: Inst coolstuff (3 experimental [amd64]) Conf coolstuff (3 experimental [amd64])' aptget install --solver apt coolstuff -s +testequal 'Reading package lists... +Building dependency tree... +Execute external solver... +The following packages will be REMOVED: + cool* +0 upgraded, 0 newly installed, 1 to remove and 1 not upgraded. +Purg cool [1]' aptget purge --solver apt cool -s + testsuccess aptget install awesomecoolstuff:i386 -s testsuccess aptget install --solver apt awesomecoolstuff:i386 -s @@ -57,9 +68,13 @@ testsuccess aptget dist-upgrade -s --solver apt testsuccess aptget upgrade -s testsuccess aptget upgrade -s --solver apt +testfailure aptget install awesome badstuff -s +testfailure aptget install awesome badstuff -s --solver apt +testsuccess grep 'ERR_UNSOLVABLE' rootdir/tmp/testfailure.output + configarchitecture 'armel' msgtest 'Test direct calling is okay for' 'apt-internal-solver' -cat /tmp/dump.edsp | aptinternalsolver > solver.result 2>&1 || true +cat /tmp/dump.edsp | aptinternalsolver -q=0 > solver.result 2>&1 || true if [ "$(tail -n2 solver.result | head -n1 )" = "Message: Done" ]; then msgpass else @@ -69,3 +84,16 @@ fi rm -f /tmp/dump.edsp testfailure aptget install --solver apt awesomecoolstuff:i386 -s + +testsuccess aptinternalsolver scenario +testequal 'Package: stuff +Source: stuff +Architecture: all +Version: 1 +Installed: yes +APT-ID: 2 +Priority: optional +Section: other +APT-Pin: 100 +APT-Candidate: yes +' aptinternalsolver scenario stuff -- cgit v1.2.3-70-g09d2 From 1d838084dc775c0a4184edb4f3b9138903ac27fb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 18 Nov 2014 13:41:18 +0100 Subject: use dpkg --merge-avail only if needed in apt-mark Only "recent" versions of dpkg support stdin for merge instead of a file, so as a quick fix we delay calling it until we really need it which fixes most of the problem already. Checking for a specific dpkg version here is deemed too much work, just like using a temporary file here and depends a too high requirement for this minor usecase. After all, it didn't work at all before, so we break nobody here and can fix it if someone complains (with a patch). --- cmdline/apt-mark.cc | 100 ++++++++++++++++++++++------------------- test/integration/test-apt-mark | 9 ++++ 2 files changed, 64 insertions(+), 45 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 860982f9e..8974397c9 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -274,65 +274,75 @@ static bool DoHold(CommandLine &CmdL) return true; } - Args.erase(Args.begin() + BaseArgs, Args.end()); - Args.push_back("--merge-avail"); - Args.push_back("-"); - Args.push_back(NULL); - - int external[2] = {-1, -1}; - if (pipe(external) != 0) - return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); - - pid_t dpkgMergeAvail = ExecFork(); - if (dpkgMergeAvail == 0) - { - close(external[1]); - std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); - if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) - _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); - dup2(external[0], STDIN_FILENO); - int const nullfd = open("/dev/null", O_RDONLY); - dup2(nullfd, STDOUT_FILENO); - execvp(Args[0], (char**) &Args[0]); - _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); - _exit(2); - } - - FILE* dpkg = fdopen(external[1], "w"); + APT::PackageList keepoffset; for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (Pkg->CurrentVer != 0) continue; - char const * Arch; - if (Pkg->VersionList != 0) - Arch = Pkg.VersionList().Arch(); - else - Arch = Pkg.Arch(); - fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" - "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + keepoffset.insert(Pkg); } - fclose(dpkg); - if (dpkgMergeAvail > 0) + if (keepoffset.empty() == false) { - int Status = 0; - while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + Args.erase(Args.begin() + BaseArgs, Args.end()); + Args.push_back("--merge-avail"); + // FIXME: supported only since 1.17.7 in dpkg + Args.push_back("-"); + Args.push_back(NULL); + + int external[2] = {-1, -1}; + if (pipe(external) != 0) + return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); + + pid_t dpkgMergeAvail = ExecFork(); + if (dpkgMergeAvail == 0) { - if (errno == EINTR) - continue; - _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); - break; + close(external[1]); + std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); + if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) + _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); + dup2(external[0], STDIN_FILENO); + int const nullfd = open("/dev/null", O_RDONLY); + dup2(nullfd, STDOUT_FILENO); + execvp(Args[0], (char**) &Args[0]); + _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); + _exit(2); + } + + FILE* dpkg = fdopen(external[1], "w"); + for (APT::PackageList::iterator Pkg = keepoffset.begin(); Pkg != keepoffset.end(); ++Pkg) + { + char const * Arch; + if (Pkg->VersionList != 0) + Arch = Pkg.VersionList().Arch(); + else + Arch = Pkg.Arch(); + fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" + "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + } + fclose(dpkg); + keepoffset.clear(); + + if (dpkgMergeAvail > 0) + { + int Status = 0; + while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + { + if (errno == EINTR) + continue; + _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); + break; + } + if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) + return _error->Error(_("Executing dpkg failed. Are you root?")); } - if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) - return _error->Error(_("Executing dpkg failed. Are you root?")); } Args.erase(Args.begin() + BaseArgs, Args.end()); Args.push_back("--set-selections"); Args.push_back(NULL); - external[0] = -1; - external[1] = -1; + int external[2] = {-1, -1}; if (pipe(external) != 0) return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --set-selections"); @@ -349,7 +359,7 @@ static bool DoHold(CommandLine &CmdL) _exit(2); } - dpkg = fdopen(external[1], "w"); + FILE* dpkg = fdopen(external[1], "w"); for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (dpkgMultiArch == false) diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark index 0f62a12b4..5a3ae4b2f 100755 --- a/test/integration/test-apt-mark +++ b/test/integration/test-apt-mark @@ -85,6 +85,15 @@ testequal 'E: No packages found' aptmark unhold testnoholdpkg testmarkonepkgashold 'foo' testmarkonepkgashold 'bar' + +msgtest 'dpkg supports --merge-avail via' 'stdin' +if dpkg --merge-avail - < /dev/null >/dev/null 2>&1; then + msgpass +else + msgskip 'dpkg version too old' + exit 0 +fi + testmarkonepkgashold 'uninstalled' testmarkonepkgashold 'uninstalled-native' -- cgit v1.2.3-70-g09d2 From 69d8b8537af1dd52db5f2e0e785bdce3e52fdf8d Mon Sep 17 00:00:00 2001 From: James McCoy Date: Fri, 28 Nov 2014 14:21:06 +0100 Subject: support long keyids in "apt-key del" instead of ignoring them apt-key given a long keyid reports just "OK" all the time, but doesn't delete the mentioned key as it doesn't find the key. Note: In debian/experimental this was closed with 29f1b977100aeb6d6ebd38923eeb7a623e264ffe which just added the testcase as the rewrite of apt-key had fixed this as well. Closes: 754436 --- cmdline/apt-key.in | 2 +- test/integration/test-apt-key | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 0774cf4b7..b4e071000 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -180,7 +180,7 @@ update() { remove_key_from_keyring() { local GPG="$GPG_CMD --keyring $1" # check if the key is in this keyring: the key id is in the 5 column at the end - if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+$2:"; then + if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then return fi if [ ! -w "$1" ]; then diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index 68b3f9710..47230cb55 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -105,3 +105,9 @@ testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ + +msgtest 'Test key removal with' '8 byte key ID' +cleanplate +cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg +testsuccess --nomsg aptkey --fakeroot del 5A90D141DBAC8DAE +testempty aptkey list -- cgit v1.2.3-70-g09d2 From ed793a19ec00b83254029509bc516e3ba911c75a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 29 Nov 2014 17:59:52 +0100 Subject: dispose http(s) 416 error page as non-content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real webservers (like apache) actually send an error page with a 416 response, but our client didn't expect it leaving the page on the socket to be parsed as response for the next request (http) or as file content (https), which isn't what we want at all… Symptom is a "Bad header line" as html usually doesn't parse that well to an http-header. This manifests itself e.g. if we have a complete file (or larger) in partial/ which isn't discarded by If-Range as the server doesn't support it (or it is just newer, think: mirror rotation). It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a, which removed the filesize - 1 trick, but this had its own problems… To properly test this our webserver gains the ability to reply with transfer-encoding: chunked as most real webservers will use it to send the dynamically generated error pages. Closes: 768797 --- cmdline/apt-helper.cc | 31 +++-- methods/http.cc | 2 + methods/https.cc | 15 ++- methods/server.cc | 26 ++-- methods/server.h | 5 +- test/integration/framework | 4 +- test/integration/test-partial-file-support | 62 ++++++++- test/interactive-helper/aptwebserver.cc | 193 ++++++++++++++++++----------- 8 files changed, 233 insertions(+), 105 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index 1b832f165..a05ae90a2 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -51,22 +51,33 @@ static bool DoDownloadFile(CommandLine &CmdL) AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); pkgAcquire Fetcher(&Stat); - std::string download_uri = CmdL.FileList[1]; - std::string targetfile = CmdL.FileList[2]; - std::string hash; - if (CmdL.FileSize() > 3) - hash = CmdL.FileList[3]; - // we use download_uri as descr and targetfile as short-descr - new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, - "dest-dir-ignored", targetfile); + size_t fileind = 0; + std::vector targetfiles; + while (fileind + 2 <= CmdL.FileSize()) + { + std::string download_uri = CmdL.FileList[fileind + 1]; + std::string targetfile = CmdL.FileList[fileind + 2]; + std::string hash; + if (CmdL.FileSize() > fileind + 3) + hash = CmdL.FileList[fileind + 3]; + // we use download_uri as descr and targetfile as short-descr + new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, + "dest-dir-ignored", targetfile); + targetfiles.push_back(targetfile); + fileind += 3; + } // Disable drop-privs if "_apt" can not write to the target dir CheckDropPrivsMustBeDisabled(Fetcher); bool Failed = false; - if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true || - FileExists(targetfile) == false) + if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true) return _error->Error(_("Download Failed")); + if (targetfiles.empty() == false) + for (std::vector::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f) + if (FileExists(*f) == false) + return _error->Error(_("Download Failed")); + return true; } diff --git a/methods/http.cc b/methods/http.cc index a5de13511..ad1347d36 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -444,6 +444,8 @@ bool HttpServerState::RunData(FileFd * const File) loss of the connection means we are done */ if (Encoding == Closes) In.Limit(-1); + else if (JunkSize != 0) + In.Limit(JunkSize); else In.Limit(Size - StartPos); diff --git a/methods/https.cc b/methods/https.cc index 366148e19..23b3a10d4 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -69,6 +69,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { me->Server->Result = 200; me->Server->StartPos = me->Server->Size; + // the actual size is not important for https as curl will deal with it + // by itself and e.g. doesn't bother us with transport-encoding… + me->Server->JunkSize = std::numeric_limits::max(); } else me->Server->StartPos = 0; @@ -86,19 +89,25 @@ size_t HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) { HttpsMethod *me = (HttpsMethod *)userp; + size_t buffer_size = size * nmemb; + // we don't need to count the junk here, just drop anything we get as + // we don't always know how long it would be, e.g. in chunked encoding. + if (me->Server->JunkSize != 0) + return buffer_size; if (me->Res.Size == 0) me->URIStart(me->Res); - if(me->File->Write(buffer, size*nmemb) != true) + if(me->File->Write(buffer, buffer_size) != true) return false; if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) { me->SetFailReason("MaximumSizeExceeded"); - return _error->Error("Writing more data than expected (%llu > %llu)", + _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->MaximumSize); + return 0; } - return size*nmemb; + return buffer_size; } int diff --git a/methods/server.cc b/methods/server.cc index c4689ff12..9b3d39cf2 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, Minor = 0; Result = 0; Size = 0; + JunkSize = 0; StartPos = 0; Encoding = Closes; HaveContent = false; @@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line) Encoding = Stream; HaveContent = true; - // The length is already set from the Content-Range header - if (StartPos != 0) - return true; + unsigned long long * SizePtr = &Size; + if (Result == 416) + SizePtr = &JunkSize; - Size = strtoull(Val.c_str(), NULL, 10); - if (Size >= std::numeric_limits::max()) + *SizePtr = strtoull(Val.c_str(), NULL, 10); + if (*SizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (Size == 0) + else if (*SizePtr == 0) HaveContent = false; return true; } @@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line) // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) - { - StartPos = 1; // ignore Content-Length, it would override Size - HaveContent = false; - } + ; // we got the expected filesize which is all we wanted else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); if ((unsigned long long)StartPos > Size) @@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res) if ((unsigned long long)SBuf.st_size == Server->Size) { // the file is completely downloaded, but was not moved + if (Server->HaveContent == true) + { + // Send to error page to dev/null + FileFd DevNull("/dev/null",FileFd::WriteExists); + Server->RunData(&DevNull); + } + Server->HaveContent = false; Server->StartPos = Server->Size; Server->Result = 200; - Server->HaveContent = false; } else if (unlink(Queue->DestFile.c_str()) == 0) { diff --git a/methods/server.h b/methods/server.h index 7d5198478..b974ec89a 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,7 +34,8 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; + unsigned long long Size; // size of the usable content (aka: the file) + unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; time_t Date; bool HaveContent; @@ -73,7 +74,7 @@ struct ServerState RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; diff --git a/test/integration/framework b/test/integration/framework index 930ab9367..f7f69f5d5 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1119,8 +1119,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom } downloadfile() { - local PROTO="$(echo "$1" | cut -d':' -f 1 )" - apthelper -o Debug::Acquire::${PROTO}=1 \ + local PROTO="${1%%:*}" + apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \ download-file "$1" "$2" 2>&1 || true # only if the file exists the download was successful if [ -r "$2" ]; then diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support index 98b2f242a..b6b305d25 100755 --- a/test/integration/test-partial-file-support +++ b/test/integration/test-partial-file-support @@ -24,13 +24,25 @@ testdownloadfile() { else msgpass fi - cat "$DOWNLOADLOG" | while read field hash; do + sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log + testsuccess test -s receivedhashes.log + local HASHES_OK=0 + local HASHES_BAD=0 + while read field hash; do local EXPECTED case "$field" in 'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";; + 'Checksum-FileSize-Hash:') + #filesize is too weak to check for != + if [ "$4" = '=' ]; then + EXPECTED="$(stat -c '%s' "$TESTFILE")" + else + continue + fi + ;; *) continue;; esac if [ "$4" = '=' ]; then @@ -40,15 +52,41 @@ testdownloadfile() { fi if [ "$EXPECTED" "$4" "$hash" ]; then msgpass + HASHES_OK=$((HASHES_OK+1)); else - cat >&2 "$DOWNLOADLOG" msgfail "expected: $EXPECTED ; got: $hash" + HASHES_BAD=$((HASHES_BAD+1)); fi - done + done < receivedhashes.log + msgtest 'At least one good hash and no bad ones' + if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then + cat >&2 "$DOWNLOADLOG" + msgfail + else + msgpass + fi } TESTFILE='aptarchive/testfile' cp -a ${TESTDIR}/framework $TESTFILE +cp -a ${TESTDIR}/framework "${TESTFILE}2" + +followuprequest() { + local DOWN='./downloaded/testfile' + + copysource $TESTFILE 1M $DOWN + testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' + testwebserverlaststatuscode '416' "$DOWNLOADLOG" + + copysource $TESTFILE 1M $DOWN + copysource "${TESTFILE}2" 20 "${DOWN}2" + msgtest 'Testing download of files with' 'completely downloaded file + partial file' + testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \ + download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2" + testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output' + testsuccess diff -u "$TESTFILE" "${DOWN}" + testsuccess diff -u "${DOWN}" "${DOWN}2" +} testrun() { webserverconfig 'aptwebserver::support::range' 'true' @@ -66,9 +104,11 @@ testrun() { testdownloadfile 'invalid partial data' "${1}/testfile" "$DOWN" '!=' testwebserverlaststatuscode '206' "$DOWNLOADLOG" - copysource $TESTFILE 1M $DOWN - testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' - testwebserverlaststatuscode '416' "$DOWNLOADLOG" + webserverconfig 'aptwebserver::closeOnError' 'false' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'true' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'false' copysource /dev/zero 1M $DOWN testdownloadfile 'too-big partial file' "${1}/testfile" "$DOWN" '=' @@ -86,8 +126,18 @@ testrun() { testwebserverlaststatuscode '200' "$DOWNLOADLOG" } +msgmsg 'http: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'http://localhost:8080' +msgmsg 'http: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'http://localhost:8080' changetohttpswebserver +msgmsg 'https: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'https://localhost:4433' +msgmsg 'https: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'https://localhost:4433' diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 00004a524..ca6f88c58 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -19,6 +19,8 @@ #include #include #include + +#include #include #include #include @@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode) /*{{{*/ return NULL; } /*}}}*/ +static bool chunkedTransferEncoding(std::list const &headers) { + if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end()) + return true; + if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true) + return true; + return false; +} static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/ { - std::ostringstream contentlength; - contentlength << "Content-Length: " << data.FileSize(); - headers.push_back(contentlength.str()); - + if (chunkedTransferEncoding(headers) == false) + { + std::ostringstream contentlength; + contentlength << "Content-Length: " << data.FileSize(); + headers.push_back(contentlength.str()); + } std::string lastmodified("Last-Modified: "); lastmodified.append(TimeRFC1123(data.ModificationTime())); headers.push_back(lastmodified); @@ -92,9 +103,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/ /*}}}*/ static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/ { - std::ostringstream contentlength; - contentlength << "Content-Length: " << data.size(); - headers.push_back(contentlength.str()); + if (chunkedTransferEncoding(headers) == false) + { + std::ostringstream contentlength; + contentlength << "Content-Length: " << data.size(); + headers.push_back(contentlength.str()); + } } /*}}}*/ static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/ @@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl; bool Success = true; for (std::list::const_iterator h = headers.begin(); @@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list const &headers, FileFd &data)/*{{{*/ { bool Success = true; + bool const chunked = chunkedTransferEncoding(headers); char buffer[500]; unsigned long long actual = 0; while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true) { if (actual == 0) break; - Success &= FileFd::Write(client, buffer, actual); + + if (chunked == true) + { + std::string size; + strprintf(size, "%llX\r\n", actual); + Success &= FileFd::Write(client, size.c_str(), size.size()); + Success &= FileFd::Write(client, buffer, actual); + Success &= FileFd::Write(client, "\r\n", strlen("\r\n")); + } + else + Success &= FileFd::Write(client, buffer, actual); + } + if (chunked == true) + { + char const * const finish = "0\r\n\r\n"; + Success &= FileFd::Write(client, finish, strlen(finish)); } if (Success == false) - std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl; + std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl; return Success; } /*}}}*/ -static bool sendData(int const client, std::string const &data) /*{{{*/ +static bool sendData(int const client, std::list const &headers, std::string const &data)/*{{{*/ { - if (FileFd::Write(client, data.c_str(), data.size()) == false) + if (chunkedTransferEncoding(headers) == true) + { + unsigned long long const ullsize = data.length(); + std::string size; + strprintf(size, "%llX\r\n", ullsize); + char const * const finish = "\r\n0\r\n\r\n"; + if (FileFd::Write(client, size.c_str(), size.length()) == false || + FileFd::Write(client, data.c_str(), ullsize) == false || + FileFd::Write(client, finish, strlen(finish)) == false) + { + std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl; + return false; + } + } + else if (FileFd::Write(client, data.c_str(), data.size()) == false) { std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl; return false; @@ -157,33 +204,38 @@ static bool sendData(int const client, std::string const &data) /*{{{*/ } /*}}}*/ static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/ - bool content, std::string const &error = "", std::list headers = std::list()) + bool const content, std::string const &error, std::list &headers) { std::string response(""); response.append(httpcodeToStr(httpcode)).append(""); response.append("

").append(httpcodeToStr(httpcode)).append("

"); if (httpcode != 200) - { - if (error.empty() == false) - response.append("

Error: ").append(error).append("

"); - response.append("This error is a result of the request: 
");
-   }
+      response.append("
Error: ");
+   else
+      response.append("
Success: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("
This error is a result of the request: 
");
    else
-   {
-      if (error.empty() == false)
-	 response.append("
Success: ").append(error).append("
");
       response.append("The successfully executed operation was requested by: 
");
-   }
    response.append(request).append("
");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -220,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -262,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list &headers)
 {
-   std::list headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -310,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string ¶ms, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -332,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -344,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -359,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -370,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
 
@@ -382,9 +433,9 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       if (authConf.empty() != auth.empty())
       {
 	 if (auth.empty())
-	    sendError(client, 407, request, sendContent, "Proxy requires authentication");
+	    sendError(client, 407, request, sendContent, "Proxy requires authentication", headers);
 	 else
-	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it");
+	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it", headers);
 	return false;
       }
       if (authConf.empty() == false)
@@ -395,7 +446,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 	    auth.erase(0, strlen(basic));
 	    if (auth != authConf)
 	    {
-	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match");
+	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match", headers);
 	       return false;
 	    }
 	 }
@@ -410,7 +461,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos && APT::String::Startswith(filename, "/_config/") == false)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -421,9 +472,9 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       if (authConf.empty() != auth.empty())
       {
 	 if (auth.empty())
-	    sendError(client, 401, request, sendContent, "Server requires authentication");
+	    sendError(client, 401, request, sendContent, "Server requires authentication", headers);
 	 else
-	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it");
+	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it", headers);
 	 return false;
       }
       if (authConf.empty() == false)
@@ -434,13 +485,12 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 	    auth.erase(0, strlen(basic));
 	    if (auth != authConf)
 	    {
-	       sendError(client, 401, request, sendContent, "Authentication doesn't match");
+	       sendError(client, 401, request, sendContent, "Authentication doesn't match", headers);
 	       return false;
 	    }
 	 }
 	 else
 	 {
-	    std::list headers;
 	    headers.push_back("WWW-Authenticate: Basic");
 	    sendError(client, 401, request, sendContent, "Unsupported Authentication Scheme", headers);
 	    return false;
@@ -463,7 +513,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -499,7 +550,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector parts, std::list &headers)
 {
    size_t const pcount = parts.size();
    for (size_t i = 0; i < pcount; ++i)
@@ -507,40 +559,38 @@ static bool handleOnTheFlyReconfiguration(int const client, std::string const &r
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -549,18 +599,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -569,7 +623,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -601,7 +655,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -620,7 +674,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -676,17 +730,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -695,22 +748,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From 92e8c1ff287ab829de825e00cdf94744e699ff97 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 29 Nov 2014 17:59:52 +0100
Subject: dispose http(s) 416 error page as non-content
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Real webservers (like apache) actually send an error page with a 416
response, but our client didn't expect it leaving the page on the socket
to be parsed as response for the next request (http) or as file content
(https), which isn't what we want at all… Symptom is a "Bad header line"
as html usually doesn't parse that well to an http-header.

This manifests itself e.g. if we have a complete file (or larger) in
partial/ which isn't discarded by If-Range as the server doesn't support
it (or it is just newer, think: mirror rotation).
It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a,
which removed the filesize - 1 trick, but this had its own problems…

To properly test this our webserver gains the ability to reply with
transfer-encoding: chunked as most real webservers will use it to send
the dynamically generated error pages.

(The tests and their binary helpers had to be slightly modified to
apply, but the patch to fix the issue itself is unchanged.)

Closes: 768797
---
 cmdline/apt-helper.cc                      |  35 ++++--
 methods/http.cc                            |   2 +
 methods/https.cc                           |  12 +-
 methods/server.cc                          |  26 +++--
 methods/server.h                           |   5 +-
 test/integration/framework                 |   6 +-
 test/integration/test-apt-helper           |  24 ++--
 test/integration/test-partial-file-support |  62 +++++++++-
 test/interactive-helper/aptwebserver.cc    | 181 ++++++++++++++++++-----------
 9 files changed, 241 insertions(+), 112 deletions(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index dd43ea1bc..63f70983c 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,23 +48,34 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
-
    pkgAcquire Fetcher;
    AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
    Fetcher.Setup(&Stat);
-   std::string download_uri = CmdL.FileList[1];
-   std::string targetfile = CmdL.FileList[2];
-   std::string hash;
-   if (CmdL.FileSize() > 3)
-      hash = CmdL.FileList[3];
-   // we use download_uri as descr and targetfile as short-descr
-   new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, 
-                  "dest-dir-ignored", targetfile);
-   Fetcher.Run();
+
+   size_t fileind = 0;
+   std::vector targetfiles;
+   while (fileind + 2 <= CmdL.FileSize())
+   {
+      std::string download_uri = CmdL.FileList[fileind + 1];
+      std::string targetfile = CmdL.FileList[fileind + 2];
+      std::string hash;
+      if (CmdL.FileSize() > fileind + 3)
+	 hash = CmdL.FileList[fileind + 3];
+      // we use download_uri as descr and targetfile as short-descr
+      new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile,
+	    "dest-dir-ignored", targetfile);
+      targetfiles.push_back(targetfile);
+      fileind += 3;
+   }
+
    bool Failed = false;
-   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true ||
-	 FileExists(targetfile) == false)
+   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true)
       return _error->Error(_("Download Failed"));
+   if (targetfiles.empty() == false)
+      for (std::vector::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f)
+	 if (FileExists(*f) == false)
+	    return _error->Error(_("Download Failed"));
+
    return true;
 }
 
diff --git a/methods/http.cc b/methods/http.cc
index f2a4a4db6..1b996db98 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -440,6 +440,8 @@ bool HttpServerState::RunData(FileFd * const File)
          loss of the connection means we are done */
       if (Encoding == Closes)
 	 In.Limit(-1);
+      else if (JunkSize != 0)
+	 In.Limit(JunkSize);
       else
 	 In.Limit(Size - StartPos);
       
diff --git a/methods/https.cc b/methods/https.cc
index 0499af0c5..65a744e2a 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -59,6 +59,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       {
          me->Server->Result = 200;
 	 me->Server->StartPos = me->Server->Size;
+	 // the actual size is not important for https as curl will deal with it
+	 // by itself and e.g. doesn't bother us with transport-encoding…
+	 me->Server->JunkSize = std::numeric_limits::max();
       }
       else
 	 me->Server->StartPos = 0;
@@ -76,13 +79,18 @@ size_t
 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    HttpsMethod *me = (HttpsMethod *)userp;
+   size_t buffer_size = size * nmemb;
+   // we don't need to count the junk here, just drop anything we get as
+   // we don't always know how long it would be, e.g. in chunked encoding.
+   if (me->Server->JunkSize != 0)
+      return buffer_size;
 
    if (me->Res.Size == 0)
       me->URIStart(me->Res);
-   if(me->File->Write(buffer, size*nmemb) != true)
+   if(me->File->Write(buffer, buffer_size) != true)
       return false;
 
-   return size*nmemb;
+   return buffer_size;
 }
 
 int
diff --git a/methods/server.cc b/methods/server.cc
index 92d94e638..cb0341d5f 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
    Minor = 0; 
    Result = 0; 
    Size = 0; 
+   JunkSize = 0;
    StartPos = 0;
    Encoding = Closes;
    HaveContent = false;
@@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line)
 	 Encoding = Stream;
       HaveContent = true;
 
-      // The length is already set from the Content-Range header
-      if (StartPos != 0)
-	 return true;
+      unsigned long long * SizePtr = &Size;
+      if (Result == 416)
+	 SizePtr = &JunkSize;
 
-      Size = strtoull(Val.c_str(), NULL, 10);
-      if (Size >= std::numeric_limits::max())
+      *SizePtr = strtoull(Val.c_str(), NULL, 10);
+      if (*SizePtr >= std::numeric_limits::max())
 	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
-      else if (Size == 0)
+      else if (*SizePtr == 0)
 	 HaveContent = false;
       return true;
    }
@@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line)
 
       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
-      {
-	 StartPos = 1; // ignore Content-Length, it would override Size
-	 HaveContent = false;
-      }
+	 ; // we got the expected filesize which is all we wanted
       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
@@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
 	 if ((unsigned long long)SBuf.st_size == Server->Size)
 	 {
 	    // the file is completely downloaded, but was not moved
+	    if (Server->HaveContent == true)
+	    {
+	       // Send to error page to dev/null
+	       FileFd DevNull("/dev/null",FileFd::WriteExists);
+	       Server->RunData(&DevNull);
+	    }
+	    Server->HaveContent = false;
 	    Server->StartPos = Server->Size;
 	    Server->Result = 200;
-	    Server->HaveContent = false;
 	 }
 	 else if (unlink(Queue->DestFile.c_str()) == 0)
 	 {
diff --git a/methods/server.h b/methods/server.h
index f5e68d902..1b81e3549 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,7 +34,8 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
-   unsigned long long Size;
+   unsigned long long Size; // size of the usable content (aka: the file)
+   unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
    time_t Date;
    bool HaveContent;
@@ -71,7 +72,7 @@ struct ServerState
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
-   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
+   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
 		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true;};
    virtual bool WriteResponse(std::string const &Data) = 0;
diff --git a/test/integration/framework b/test/integration/framework
index df1942ff9..ac482a7a0 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1064,8 +1064,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 }
 
 downloadfile() {
-	local PROTO="$(echo "$1" | cut -d':' -f 1 )"
-	apthelper -o Debug::Acquire::${PROTO}=1 \
+	local PROTO="${1%%:*}"
+	apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
 		download-file "$1" "$2" 2>&1 || true
 	# only if the file exists the download was successful
 	if [ -e "$2" ]; then
@@ -1221,7 +1221,7 @@ testsuccess() {
 		msgtest 'Test for successful execution of' "$*"
 	fi
 	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
-	if $@ >${OUTPUT} 2>&1; then
+	if "$@" >${OUTPUT} 2>&1; then
 		msgpass
 	else
 		echo >&2
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index c749224ca..31e471677 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -10,34 +10,36 @@ configarchitecture "i386"
 changetohttpswebserver
 
 test_apt_helper_download() {
-    echo "foo" > aptarchive/foo
+    echo 'foo' > aptarchive/foo
+    echo 'bar' > aptarchive/foo2
 
     msgtest 'apt-file download-file md5sum'
-    apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00
     testfileequal foo2 'foo'
 
     msgtest 'apt-file download-file sha1'
-    apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
     testfileequal foo1 'foo'
 
     msgtest 'apt-file download-file sha256'
-    apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c
     testfileequal foo3 'foo'
 
     msgtest 'apt-file download-file no-hash'
-    apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo4
     testfileequal foo4 'foo'
     
     msgtest 'apt-file download-file wrong hash'
-    if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then
-        msgpass
-    else
-        msgfail
-    fi
-    testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
+    testfailure --nomsg apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc
+    testfileequal rootdir/tmp/testfailure.output 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
 
 E: Download Failed'
     testfileequal foo5.FAILED 'foo'
+
+    msgtest 'apt-file download-file md5sum sha1'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo6 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 http://localhost:8080/foo2 foo7 SHA1:e242ed3bffccdf271b7fbaf34ed72d089537b42f
+    testfileequal foo6 'foo'
+    testfileequal foo7 'bar'
 }
 
 test_apt_helper_detect_proxy() {
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 5ab326def..160d451b6 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -24,13 +24,25 @@ testdownloadfile() {
 	else
 		msgpass
 	fi
-	cat "$DOWNLOADLOG" | while read field hash; do
+	sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log
+	testsuccess test -s receivedhashes.log
+	local HASHES_OK=0
+	local HASHES_BAD=0
+	while read field hash; do
 		local EXPECTED
 		case "$field" in
 		'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";;
+		'Checksum-FileSize-Hash:')
+			#filesize is too weak to check for !=
+			if [ "$4" = '=' ]; then
+				EXPECTED="$(stat -c '%s' "$TESTFILE")"
+			else
+				continue
+			fi
+			;;
 		*) continue;;
 		esac
 		if [ "$4" = '=' ]; then
@@ -40,15 +52,41 @@ testdownloadfile() {
 		fi
 		if [ "$EXPECTED" "$4" "$hash" ]; then
 			msgpass
+			HASHES_OK=$((HASHES_OK+1));
 		else
-			cat >&2 "$DOWNLOADLOG"
 			msgfail "expected: $EXPECTED ; got: $hash"
+			HASHES_BAD=$((HASHES_BAD+1));
 		fi
-	done
+	done < receivedhashes.log
+	msgtest 'At least one good hash and no bad ones'
+	if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then
+		cat >&2 "$DOWNLOADLOG"
+		msgfail
+	else
+		msgpass
+	fi
 }
 
 TESTFILE='aptarchive/testfile'
 cp -a ${TESTDIR}/framework $TESTFILE
+cp -a ${TESTDIR}/framework "${TESTFILE}2"
+
+followuprequest() {
+	local DOWN='./testfile'
+
+	copysource $TESTFILE 1M $DOWN
+	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
+	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+
+	copysource $TESTFILE 1M $DOWN
+	copysource "${TESTFILE}2" 20 "${DOWN}2"
+	msgtest 'Testing download of files with' 'completely downloaded file + partial file'
+	testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \
+		download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2"
+	testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output'
+	testsuccess diff -u "$TESTFILE" "${DOWN}"
+	testsuccess diff -u "${DOWN}" "${DOWN}2"
+}
 
 testrun() {
 	webserverconfig 'aptwebserver::support::range' 'true'
@@ -65,9 +103,11 @@ testrun() {
 	testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
 	testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 
-	copysource $TESTFILE 1M ./testfile
-	testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '='
-	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'true'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
 
 	copysource /dev/zero 1M ./testfile
 	testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='
@@ -85,8 +125,18 @@ testrun() {
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 
+msgmsg 'http: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'http://localhost:8080'
+msgmsg 'http: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'http://localhost:8080'
 
 changetohttpswebserver
 
+msgmsg 'https: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'https://localhost:4433'
+msgmsg 'https: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'https://localhost:4433'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 34476e1af..cd52da692 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -19,6 +19,8 @@
 #include 
 #include 
 #include 
+
+#include 
 #include 
 #include 
 #include 
@@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode)		/*{{{*/
    return NULL;
 }
 									/*}}}*/
+static bool chunkedTransferEncoding(std::list const &headers) {
+   if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end())
+      return true;
+   if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true)
+      return true;
+   return false;
+}
 static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.FileSize();
-   headers.push_back(contentlength.str());
-
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.FileSize();
+      headers.push_back(contentlength.str());
+   }
    std::string lastmodified("Last-Modified: ");
    lastmodified.append(TimeRFC1123(data.ModificationTime()));
    headers.push_back(lastmodified);
@@ -92,9 +103,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
 									/*}}}*/
 static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.size();
-   headers.push_back(contentlength.str());
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.size();
+      headers.push_back(contentlength.str());
+   }
 }
 									/*}}}*/
 static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/
@@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl;
    bool Success = true;
    for (std::list::const_iterator h = headers.begin();
@@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list const &headers, FileFd &data)/*{{{*/
 {
    bool Success = true;
+   bool const chunked = chunkedTransferEncoding(headers);
    char buffer[500];
    unsigned long long actual = 0;
    while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true)
    {
       if (actual == 0)
 	 break;
-      Success &= FileFd::Write(client, buffer, actual);
+
+      if (chunked == true)
+      {
+	 std::string size;
+	 strprintf(size, "%llX\r\n", actual);
+	 Success &= FileFd::Write(client, size.c_str(), size.size());
+	 Success &= FileFd::Write(client, buffer, actual);
+	 Success &= FileFd::Write(client, "\r\n", strlen("\r\n"));
+      }
+      else
+	 Success &= FileFd::Write(client, buffer, actual);
+   }
+   if (chunked == true)
+   {
+      char const * const finish = "0\r\n\r\n";
+      Success &= FileFd::Write(client, finish, strlen(finish));
    }
    if (Success == false)
-      std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl;
+      std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl;
    return Success;
 }
 									/*}}}*/
-static bool sendData(int const client, std::string const &data)		/*{{{*/
+static bool sendData(int const client, std::list const &headers, std::string const &data)/*{{{*/
 {
-   if (FileFd::Write(client, data.c_str(), data.size()) == false)
+   if (chunkedTransferEncoding(headers) == true)
+   {
+      unsigned long long const ullsize = data.length();
+      std::string size;
+      strprintf(size, "%llX\r\n", ullsize);
+      char const * const finish = "\r\n0\r\n\r\n";
+      if (FileFd::Write(client, size.c_str(), size.length()) == false ||
+	    FileFd::Write(client, data.c_str(), ullsize) == false ||
+	    FileFd::Write(client, finish, strlen(finish)) == false)
+      {
+	 std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl;
+	 return false;
+      }
+   }
+   else if (FileFd::Write(client, data.c_str(), data.size()) == false)
    {
       std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl;
       return false;
@@ -157,34 +204,38 @@ static bool sendData(int const client, std::string const &data)		/*{{{*/
 }
 									/*}}}*/
 static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   std::list headers;
    std::string response("");
    response.append(httpcodeToStr(httpcode)).append("");
    response.append("
").append(httpcodeToStr(httpcode)).append("
");
    if (httpcode != 200)
-   {
-      if (error.empty() == false)
-	 response.append("
Error: ").append(error).append("
");
-      response.append("This error is a result of the request: 
");
-   }
+      response.append("
Error: ");
+   else
+      response.append("
Success: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("
This error is a result of the request: 
");
    else
-   {
-      if (error.empty() == false)
-	 response.append("
Success: ").append(error).append("
");
       response.append("The successfully executed operation was requested by: 
");
-   }
    response.append(request).append("
");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -221,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -263,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list &headers)
 {
-   std::list headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -311,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string ¶ms, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -333,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -345,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -360,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -371,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
       // strip the host from the request to make it an absolute path
@@ -379,7 +429,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -398,7 +448,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -434,46 +485,45 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector const &parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector parts, std::list &headers)
 {
    size_t const pcount = parts.size();
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -482,18 +532,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -502,7 +556,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -534,7 +588,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -553,7 +607,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -609,17 +663,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -628,22 +681,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From 3bf4c93ac241e270c374d1a49be11fb4d673abc1 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 9 Mar 2015 16:00:23 +0100
Subject: (error) Same iterator is used with different containers

cppcheck reports this error, its not really a problem for us as the API
can actually deal with it via implicit conversion, but being explicit
can't hurt and the less reported errors the better.

Git-Dch: Ignore
---
 cmdline/apt-mark.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc
index 8974397c9..47777009b 100644
--- a/cmdline/apt-mark.cc
+++ b/cmdline/apt-mark.cc
@@ -279,7 +279,7 @@ static bool DoHold(CommandLine &CmdL)
    {
       if (Pkg->CurrentVer != 0)
 	 continue;
-      keepoffset.insert(Pkg);
+      keepoffset.insert(*Pkg);
    }
 
    if (keepoffset.empty() == false)
-- 
cgit v1.2.3-70-g09d2


From 249aec3b7397662a678ea0014f94392085477b09 Mon Sep 17 00:00:00 2001
From: Jérémy Bobbio 
Date: Tue, 10 Mar 2015 10:09:44 +0100
Subject: stop displaying time of build in online help
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As part of the “reproducible builds” effort [1], we have noticed that
apt could not be built reproducibly.

One issue is that it uses the __DATE__ and __TIME__ macros of the C
preprocessor to display the time of build in the online help. We believe
this information not to be really useful to users as they can always
look at the package data and metadata to figure it out.

The attached patch simply removes this information. All
non-documentation packages can then be built reproducibly with our
current experimental framework.

[David: changed the string slightly to be untranslateable as well]

Closes: 774342
---
 cmdline/apt-cache.cc                        | 5 ++---
 cmdline/apt-cdrom.cc                        | 4 ++--
 cmdline/apt-config.cc                       | 5 ++---
 cmdline/apt-dump-solver.cc                  | 3 +--
 cmdline/apt-extracttemplates.cc             | 3 +--
 cmdline/apt-get.cc                          | 9 ++++-----
 cmdline/apt-helper.cc                       | 3 +--
 cmdline/apt-internal-solver.cc              | 3 +--
 cmdline/apt-mark.cc                         | 3 +--
 cmdline/apt-sortpkgs.cc                     | 3 +--
 cmdline/apt.cc                              | 3 +--
 ftparchive/apt-ftparchive.cc                | 3 +--
 test/integration/test-00-commands-have-help | 2 +-
 13 files changed, 19 insertions(+), 30 deletions(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index a5024c581..e2cf7e8b7 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -1824,9 +1824,8 @@ static bool GenCaches(CommandLine &)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
-   
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
      return true;
 
diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc
index 327039e00..d95c169cd 100644
--- a/cmdline/apt-cdrom.cc
+++ b/cmdline/apt-cdrom.cc
@@ -205,8 +205,8 @@ static bool DoIdent(CommandLine &)
 // ShowHelp - Show the help screen					/*{{{*/
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
       return true;
    
diff --git a/cmdline/apt-config.cc b/cmdline/apt-config.cc
index e0b8a624e..4479b84a7 100644
--- a/cmdline/apt-config.cc
+++ b/cmdline/apt-config.cc
@@ -80,11 +80,10 @@ static bool DoDump(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
-   
+
    cout <<
     _("Usage: apt-config [options] command\n"
       "\n"
diff --git a/cmdline/apt-dump-solver.cc b/cmdline/apt-dump-solver.cc
index f765234c5..4729eac55 100644
--- a/cmdline/apt-dump-solver.cc
+++ b/cmdline/apt-dump-solver.cc
@@ -21,9 +21,8 @@
 // ---------------------------------------------------------------------
 /* */
 static bool ShowHelp() {
-
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 	std::cout <<
-		PACKAGE " " PACKAGE_VERSION " for " COMMON_ARCH " compiled on " __DATE__ " " __TIME__ << std::endl <<
 		"Usage: apt-dump-resolver\n"
 		"\n"
 		"apt-dump-resolver is a dummy solver who just dumps its input to the\n"
diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc
index 5211ee027..3e4f89286 100644
--- a/cmdline/apt-extracttemplates.cc
+++ b/cmdline/apt-extracttemplates.cc
@@ -219,8 +219,7 @@ bool DebFile::ParseInfo()
 /* */
 static bool ShowHelp(CommandLine &)
 {
-	ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
 	if (_config->FindB("version") == true)
 		return true;
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index 33bbdd861..a0d78257c 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1607,13 +1607,12 @@ static bool DoChangelog(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
-	    
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
    {
       cout << _("Supported modules:") << endl;
-      
+
       for (unsigned I = 0; I != pkgVersioningSystem::GlobalListLen; I++)
       {
 	 pkgVersioningSystem *VS = pkgVersioningSystem::GlobalList[I];
@@ -1622,7 +1621,7 @@ static bool ShowHelp(CommandLine &)
 	 else
 	    cout << ' ';
 	 cout << "Ver: " << VS->Label << endl;
-	 
+
 	 /* Print out all the packaging systems that will work with 
 	    this VS */
 	 for (unsigned J = 0; J != pkgSystem::GlobalListLen; J++)
diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index a05ae90a2..65db857ad 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -83,8 +83,7 @@ static bool DoDownloadFile(CommandLine &CmdL)
 
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(std::cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    if (_config->FindB("version") == true)
      return true;
diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc
index 4fabeb02f..939061b93 100644
--- a/cmdline/apt-internal-solver.cc
+++ b/cmdline/apt-internal-solver.cc
@@ -44,8 +44,7 @@
 // ---------------------------------------------------------------------
 /* */
 static bool ShowHelp(CommandLine &) {
-	ioprintf(std::cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-		 COMMON_ARCH,__DATE__,__TIME__);
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
 	std::cout <<
 		_("Usage: apt-internal-solver\n"
diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc
index 47777009b..de1c80309 100644
--- a/cmdline/apt-mark.cc
+++ b/cmdline/apt-mark.cc
@@ -443,8 +443,7 @@ static bool ShowHold(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    cout <<
     _("Usage: apt-mark [options] {auto|manual} pkg1 [pkg2 ...]\n"
diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc
index 9b66ad4db..971900e4f 100644
--- a/cmdline/apt-sortpkgs.cc
+++ b/cmdline/apt-sortpkgs.cc
@@ -146,8 +146,7 @@ static bool DoIt(string InFile)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
    
diff --git a/cmdline/apt.cc b/cmdline/apt.cc
index 056cd213f..2f7eddb61 100644
--- a/cmdline/apt.cc
+++ b/cmdline/apt.cc
@@ -39,8 +39,7 @@
 
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(c1out,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(c1out, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    // FIXME: generate from CommandLine
    c1out << 
diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index adf1b6d73..69b936dff 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -616,8 +616,7 @@ static void LoadBinDir(vector &PkgList,Configuration &Setup)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
 
diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help
index bbd1475eb..f2317dbdf 100755
--- a/test/integration/test-00-commands-have-help
+++ b/test/integration/test-00-commands-have-help
@@ -13,7 +13,7 @@ configarchitecture 'amd64'
 export LD_BIND_NOW=1
 
 checkversionmessage() {
-	testsuccess grep '^apt .* compiled on ' ${1}-help.output
+	testsuccess grep '^apt .* (' ${1}-help.output
 }
 
 checkhelpmessage() {
-- 
cgit v1.2.3-70-g09d2


From d5cf8851753dde4f45bfd3b48fcdf34247a8752a Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 7 Apr 2015 22:34:34 +0200
Subject: keyids in "apt-key del" should be case-insensitive

gnupg is case-insensitive about keyids, so back then apt-key called it
directly any keyid was accepted, but now that we work more with the
keyid ourself we regressed to require uppercase keyids by accident.

This is also inconsistent with other apt-key commands which still use
gnupg directly. A single case-insensitive grep and we are fine again.

Closes: 781696
---
 cmdline/apt-key.in            | 2 +-
 test/integration/test-apt-key | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in
index b4e071000..1da311d35 100644
--- a/cmdline/apt-key.in
+++ b/cmdline/apt-key.in
@@ -180,7 +180,7 @@ update() {
 remove_key_from_keyring() {
     local GPG="$GPG_CMD --keyring $1"
     # check if the key is in this keyring: the key id is in the 5 column at the end
-    if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then
+    if ! $GPG --with-colons --list-keys 2>&1 | grep -iq "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then
 	return
     fi
     if [ ! -w "$1" ]; then
diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key
index 47230cb55..b4f823ef1 100755
--- a/test/integration/test-apt-key
+++ b/test/integration/test-apt-key
@@ -111,3 +111,9 @@ cleanplate
 cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 testsuccess --nomsg aptkey --fakeroot del 5A90D141DBAC8DAE
 testempty aptkey list
+
+msgtest 'Test key removal with' 'lowercase key ID' #keylength somewher between 8byte and short
+cleanplate
+cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
+testsuccess --nomsg aptkey --fakeroot del d141dbac8dae
+testempty aptkey list
-- 
cgit v1.2.3-70-g09d2


From dfad5beea77d75983f6ff8a1b8296b74dd48203e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sun, 15 Mar 2015 22:34:54 +0100
Subject: add a simple unit test for acquire progress

This isn't testing much of the 'complex' parts,
but its better than nothing for now.

Git-Dch: Ignore
---
 apt-private/acqprogress.cc      |  53 ++++++-------
 apt-private/acqprogress.h       |   4 +-
 apt-private/private-install.cc  |   2 +-
 apt-private/private-update.cc   |   4 +-
 cmdline/apt-get.cc              |   6 +-
 cmdline/apt-helper.cc           |   2 +-
 test/libapt/acqprogress_test.cc | 170 ++++++++++++++++++++++++++++++++++++++++
 7 files changed, 205 insertions(+), 36 deletions(-)
 create mode 100644 test/libapt/acqprogress_test.cc

(limited to 'cmdline')

diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index e834d7d6a..0c606e48e 100644
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -1,10 +1,9 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acqprogress.cc,v 1.24 2003/04/27 01:56:48 doogie Exp $
 /* ######################################################################
 
-   Acquire Progress - Command line progress meter 
-   
+   Acquire Progress - Command line progress meter
+
    ##################################################################### */
 									/*}}}*/
 // Include files							/*{{{*/
@@ -29,13 +28,11 @@
 #include 
 									/*}}}*/
 
-using namespace std;
-
 // AcqTextStatus::AcqTextStatus - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-AcqTextStatus::AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet) :
-    pkgAcquireStatus(), ScreenWidth(ScreenWidth), LastLineLength(0), ID(0), Quiet(Quiet)
+AcqTextStatus::AcqTextStatus(std::ostream &out, unsigned int &ScreenWidth,unsigned int const Quiet) :
+    pkgAcquireStatus(), out(out), ScreenWidth(ScreenWidth), LastLineLength(0), ID(0), Quiet(Quiet)
 {
    // testcases use it to disable pulses without disabling other user messages
    if (Quiet == 0 && _config->FindB("quiet::NoUpdate", false) == true)
@@ -62,8 +59,8 @@ void AcqTextStatus::IMSHit(pkgAcquire::ItemDesc &Itm)
 
    clearLastLine();
 
-   cout << _("Hit ") << Itm.Description;
-   cout << endl;
+   out << _("Hit ") << Itm.Description;
+   out << std::endl;
    Update = true;
 }
 									/*}}}*/
@@ -83,10 +80,10 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
 
    clearLastLine();
 
-   cout << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
+   out << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
    if (Itm.Owner->FileSize != 0)
-      cout << " [" << SizeToStr(Itm.Owner->FileSize) << "B]";
-   cout << endl;
+      out << " [" << SizeToStr(Itm.Owner->FileSize) << "B]";
+   out << std::endl;
 }
 									/*}}}*/
 // AcqTextStatus::Done - Completed a download				/*{{{*/
@@ -113,15 +110,15 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
 
    if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
    {
-      cout << _("Ign ") << Itm.Description << endl;
+      out << _("Ign ") << Itm.Description << std::endl;
       if (Itm.Owner->ErrorText.empty() == false &&
 	    _config->FindB("Acquire::Progress::Ignore::ShowErrorText", false) == true)
-	 cout << "  " << Itm.Owner->ErrorText << endl;
+	 out << "  " << Itm.Owner->ErrorText << std::endl;
    }
    else
    {
-      cout << _("Err ") << Itm.Description << endl;
-      cout << "  " << Itm.Owner->ErrorText << endl;
+      out << _("Err ") << Itm.Description << std::endl;
+      out << "  " << Itm.Owner->ErrorText << std::endl;
    }
 
    Update = true;
@@ -143,7 +140,7 @@ void AcqTextStatus::Stop()
       return;
 
    if (FetchedBytes != 0 && _error->PendingError() == false)
-      ioprintf(cout,_("Fetched %sB in %s (%sB/s)\n"),
+      ioprintf(out,_("Fetched %sB in %s (%sB/s)\n"),
 	       SizeToStr(FetchedBytes).c_str(),
 	       TimeToStr(ElapsedTime).c_str(),
 	       SizeToStr(CurrentCPS).c_str());
@@ -152,7 +149,7 @@ void AcqTextStatus::Stop()
 // AcqTextStatus::Pulse - Regular event pulse				/*{{{*/
 // ---------------------------------------------------------------------
 /* This draws the current progress. Each line has an overall percent
-   meter and a per active item status meter along with an overall 
+   meter and a per active item status meter along with an overall
    bandwidth and ETA indicator. */
 bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 {
@@ -250,14 +247,14 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 
    // Draw the current status
    if (_config->FindB("Apt::Color", false) == true)
-      cout << _config->Find("APT::Color::Yellow");
+      out << _config->Find("APT::Color::Yellow");
    if (LastLineLength > Line.length())
       clearLastLine();
    else
-      cout << '\r';
-   cout << Line << flush;
+      out << '\r';
+   out << Line << std::flush;
    if (_config->FindB("Apt::Color", false) == true)
-      cout << _config->Find("APT::Color::Neutral") << flush;
+      out << _config->Find("APT::Color::Neutral") << std::flush;
 
    LastLineLength = Line.length();
    Update = false;
@@ -268,7 +265,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 // AcqTextStatus::MediaChange - Media need to be swapped		/*{{{*/
 // ---------------------------------------------------------------------
 /* Prompt for a media swap */
-bool AcqTextStatus::MediaChange(string Media,string Drive)
+bool AcqTextStatus::MediaChange(std::string Media, std::string Drive)
 {
    // If we do not output on a terminal and one of the options to avoid user
    // interaction is given, we assume that no user is present who could react
@@ -281,7 +278,7 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
       return false;
 
    clearLastLine();
-   ioprintf(cout,_("Media change: please insert the disc labeled\n"
+   ioprintf(out,_("Media change: please insert the disc labeled\n"
 		   " '%s'\n"
 		   "in the drive '%s' and press enter\n"),
 	    Media.c_str(),Drive.c_str());
@@ -301,16 +298,16 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
 }
 									/*}}}*/
 void AcqTextStatus::clearLastLine() {					/*{{{*/
-   if (Quiet > 0)
+   if (Quiet > 0 || LastLineLength == 0)
       return;
 
    // do not try to clear more than the (now smaller) screen
    if (LastLineLength > ScreenWidth)
       LastLineLength = ScreenWidth;
 
-   std::cout << '\r';
+   out << '\r';
    for (size_t i = 0; i < LastLineLength; ++i)
-      std::cout << ' ';
-   std::cout << '\r' << std::flush;
+      out << ' ';
+   out << '\r' << std::flush;
 }
 									/*}}}*/
diff --git a/apt-private/acqprogress.h b/apt-private/acqprogress.h
index ab8170126..7cf990c65 100644
--- a/apt-private/acqprogress.h
+++ b/apt-private/acqprogress.h
@@ -13,9 +13,11 @@
 #include 
 
 #include 
+#include 
 
 class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
 {
+   std::ostream &out;
    unsigned int &ScreenWidth;
    size_t LastLineLength;
    unsigned long ID;
@@ -35,7 +37,7 @@ class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
 
    bool Pulse(pkgAcquire *Owner);
 
-   AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet);
+   AcqTextStatus(std::ostream &out, unsigned int &ScreenWidth,unsigned int const Quiet);
 };
 
 #endif
diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc
index 79a22edf4..acc6d42c2 100644
--- a/apt-private/private-install.cc
+++ b/apt-private/private-install.cc
@@ -116,7 +116,7 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
       return false;
 
    // Create the download object
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
    if (_config->FindB("APT::Get::Print-URIs", false) == true)
    {
diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc
index df77ac33a..73a82e988 100644
--- a/apt-private/private-update.cc
+++ b/apt-private/private-update.cc
@@ -38,8 +38,8 @@ bool DoUpdate(CommandLine &CmdL)
    pkgSourceList *List = Cache.GetSourceList();
 
    // Create the progress
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
-      
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
+
    // Just print out the uris an exit if the --print-uris flag was used
    if (_config->FindB("APT::Get::Print-URIs") == true)
    {
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index a0d78257c..1a4f4c5a8 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -624,7 +624,7 @@ static bool DoDownload(CommandLine &CmdL)
    if (verset.empty() == true)
       return false;
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet", 0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    pkgRecords Recs(Cache);
@@ -729,7 +729,7 @@ static bool DoSource(CommandLine &CmdL)
       return false;
 
    // Create the download object
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    SPtrArray Dsc = new DscFile[CmdL.FileSize()];
@@ -1550,7 +1550,7 @@ static bool DoChangelog(CommandLine &CmdL)
       return Success;
    }
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    Fetcher.SetLog(&Stat);
 
    bool const downOnly = _config->FindB("APT::Get::Download-Only", false);
diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index 65db857ad..a97fc903f 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,7 +48,7 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    size_t fileind = 0;
diff --git a/test/libapt/acqprogress_test.cc b/test/libapt/acqprogress_test.cc
new file mode 100644
index 000000000..288e05aca
--- /dev/null
+++ b/test/libapt/acqprogress_test.cc
@@ -0,0 +1,170 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+class TestItem: public pkgAcquire::Item
+{
+public:
+   TestItem(pkgAcquire * const Acq) : pkgAcquire::Item(Acq, "", NULL) {}
+
+   virtual std::string DescURI() { return ""; }
+
+};
+
+TEST(AcqProgress, IMSHit)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire::ItemDesc hit;
+   hit.URI = "http://example.org/file";
+   hit.Description = "Example File from example.org";
+   hit.ShortDesc = "Example File";
+   hit.Owner = NULL;
+
+   EXPECT_EQ("", out.str());
+   Stat.IMSHit(hit);
+   EXPECT_EQ("Hit Example File from example.org\n", out.str());
+   Stat.IMSHit(hit);
+   EXPECT_EQ("Hit Example File from example.org\n"
+	     "Hit Example File from example.org\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Hit Example File from example.org\n"
+	     "Hit Example File from example.org\n", out.str());
+}
+TEST(AcqProgress, FetchNoFileSize)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org\n", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org\n"
+	     "Get:2 Example File from example.org\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Get:1 Example File from example.org\n"
+	     "Get:2 Example File from example.org\n", out.str());
+}
+TEST(AcqProgress, FetchFileSize)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n", out.str());
+   fetchO.FileSize = 42;
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n"
+	     "Get:2 Example File from example.org [42 B]\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n"
+	     "Get:2 Example File from example.org [42 B]\n", out.str());
+}
+TEST(AcqProgress, Fail)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetchO.Status = pkgAcquire::Item::StatIdle;
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fail(fetch);
+   EXPECT_EQ("", out.str());
+   fetchO.Status = pkgAcquire::Item::StatDone;
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n", out.str());
+   fetchO.Status = pkgAcquire::Item::StatError;
+   fetchO.ErrorText = "An error test!";
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n", out.str());
+   _config->Set("Acquire::Progress::Ignore::ShowErrorText", true);
+   fetchO.Status = pkgAcquire::Item::StatDone;
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n"
+	     "Ign Example File from example.org\n"
+	     "  An error test!\n", out.str());
+   _config->Set("Acquire::Progress::Ignore::ShowErrorText", true);
+   Stat.Stop();
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n"
+	     "Ign Example File from example.org\n"
+	     "  An error test!\n", out.str());
+}
+TEST(AcqProgress, Pulse)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   _config->Set("APT::Sandbox::User", ""); // ensure we aren't sandboxing
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetchO.Status = pkgAcquire::Item::StatFetching;
+   fetch.Owner = &fetchO;
+
+   // make screen smaller and bigger again while running
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]", out.str());
+   width = 8;
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]"
+	     "\r        "
+	     "\r0% [Work", out.str());
+   width = 80;
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]"
+	     "\r        "
+	     "\r0% [Work"
+	     "\r0% [Working]", out.str());
+}
-- 
cgit v1.2.3-70-g09d2


From 353c135e45d3b76dbecc1ba1b2bd9266601181ee Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 9 May 2015 15:10:55 +0200
Subject: drop incorrect parameter implicitely converted to bool
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The helper expects to be told if it should generate messages, not where
these messages should be printed – as it isn't printing such messages,
but puts them in _error. apt-get uses in other methods a helper
specialisation which does also print stuff to a stream through, so this
is likely a copy&paste error.

Git-Dch: Ignore
---
 cmdline/apt-get.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index 1a4f4c5a8..c1f78523c 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -617,7 +617,7 @@ static bool DoDownload(CommandLine &CmdL)
    if (Cache.ReadOnlyOpen() == false)
       return false;
 
-   APT::CacheSetHelper helper(c0out);
+   APT::CacheSetHelper helper;
    APT::VersionSet verset = APT::VersionSet::FromCommandLine(Cache,
 		CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper);
 
@@ -1534,7 +1534,7 @@ static bool DoChangelog(CommandLine &CmdL)
    if (Cache.ReadOnlyOpen() == false)
       return false;
    
-   APT::CacheSetHelper helper(c0out);
+   APT::CacheSetHelper helper;
    APT::VersionList verset = APT::VersionList::FromCommandLine(Cache,
 		CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper);
    if (verset.empty() == true)
-- 
cgit v1.2.3-70-g09d2


From 88593886a42025d51d76051da5929b044e42efee Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 11 May 2015 15:08:08 +0200
Subject: rewrite all TFRewrite instances to use the new pkgTagSection::Write
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

While it is mostly busywork to rewrite all instances it actually fixes
bugs as the data storage used by the new method is std::string rather
than a char*, the later mostly created by c_str() from a std::string
which the caller has to ensure keeps in scope – something apt-ftparchive
actually didn't ensure and relied on copy-on-write behavior instead
which c++11 forbids and hence the new default gcc abi doesn't use it.
---
 apt-pkg/depcache.cc                       |  50 +++---
 apt-pkg/indexcopy.cc                      | 125 ++++++--------
 apt-pkg/indexcopy.h                       |   7 +-
 apt-private/private-show.cc               |  46 ++---
 cmdline/apt-cache.cc                      |  21 ++-
 cmdline/apt-sortpkgs.cc                   |  14 +-
 ftparchive/apt-ftparchive.cc              |  51 +++---
 ftparchive/contents.cc                    |  34 ++--
 ftparchive/contents.h                     |   7 +-
 ftparchive/multicompress.cc               |  19 +--
 ftparchive/multicompress.h                |   2 +-
 ftparchive/writer.cc                      | 204 +++++++++++-----------
 ftparchive/writer.h                       |  45 +++--
 test/integration/test-apt-cli-show        |   4 +-
 test/libapt/indexcopytosourcelist_test.cc |   2 +-
 test/libapt/tagsection_test.cc            | 270 ++++++++++++++++++++++++++++++
 16 files changed, 559 insertions(+), 342 deletions(-)
 create mode 100644 test/libapt/tagsection_test.cc

(limited to 'cmdline')

diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc
index 73c5bb320..b73c336db 100644
--- a/apt-pkg/depcache.cc
+++ b/apt-pkg/depcache.cc
@@ -33,7 +33,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 
 #include 
@@ -252,17 +251,14 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
       return _error->Error(_("Failed to open StateFile %s"),
 			   state.c_str());
 
-   FILE *OutFile;
-   string const outfile = state + ".tmp";
-   if((OutFile = fopen(outfile.c_str(),"w")) == NULL)
-      return _error->Error(_("Failed to write temporary StateFile %s"),
-			   outfile.c_str());
+   FileFd OutFile(state, FileFd::ReadWrite | FileFd::Atomic);
+   if (OutFile.IsOpen() == false || OutFile.Failed() == true)
+      return _error->Error(_("Failed to write temporary StateFile %s"), state.c_str());
 
    // first merge with the existing sections
    pkgTagFile tagfile(&StateFile);
    pkgTagSection section;
    std::set pkgs_seen;
-   const char *nullreorderlist[] = {0};
    while(tagfile.Step(section)) {
 	 string const pkgname = section.FindS("Package");
 	 string pkgarch = section.FindS("Architecture");
@@ -271,7 +267,7 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	 // Silently ignore unknown packages and packages with no actual
 	 // version.
 	 pkgCache::PkgIterator pkg = Cache->FindPkg(pkgname, pkgarch);
-	 if(pkg.end() || pkg.VersionList().end()) 
+	 if(pkg.end() || pkg.VersionList().end())
 	    continue;
 	 StateCache const &P = PkgState[pkg->ID];
 	 bool newAuto = (P.Flags & Flag::Auto);
@@ -292,21 +288,17 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	 if(_config->FindB("Debug::pkgAutoRemove",false))
 	    std::clog << "Update existing AutoInstall info: " 
 		      << pkg.FullName() << std::endl;
-	 TFRewriteData rewrite[3];
-	 rewrite[0].Tag = "Architecture";
-	 rewrite[0].Rewrite = pkg.Arch();
-	 rewrite[0].NewTag = 0;
-	 rewrite[1].Tag = "Auto-Installed";
-	 rewrite[1].Rewrite = newAuto ? "1" : "0";
-	 rewrite[1].NewTag = 0;
-	 rewrite[2].Tag = 0;
-	 TFRewrite(OutFile, section, nullreorderlist, rewrite);
-	 fprintf(OutFile,"\n");
+
+	 std::vector rewrite;
+	 rewrite.push_back(pkgTagSection::Tag::Rewrite("Architecture", pkg.Arch()));
+	 rewrite.push_back(pkgTagSection::Tag::Rewrite("Auto-Installed", newAuto ? "1" : "0"));
+	 section.Write(OutFile, NULL, rewrite);
+	 if (OutFile.Write("\n", 1) == false)
+	    return false;
 	 pkgs_seen.insert(pkg.FullName());
    }
-   
+
    // then write the ones we have not seen yet
-   std::ostringstream ostr;
    for(pkgCache::PkgIterator pkg=Cache->PkgBegin(); !pkg.end(); ++pkg) {
       StateCache const &P = PkgState[pkg->ID];
       if(P.Flags & Flag::Auto) {
@@ -325,19 +317,17 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	    continue;
 	 if(debug_autoremove)
 	    std::clog << "Writing new AutoInstall: " << pkg.FullName() << std::endl;
-	 ostr.str(string(""));
-	 ostr << "Package: " << pkg.Name()
-	      << "\nArchitecture: " << pkgarch
-	      << "\nAuto-Installed: 1\n\n";
-	 fprintf(OutFile,"%s",ostr.str().c_str());
+	 std::string stanza = "Package: ";
+	 stanza.append(pkg.Name())
+	      .append("\nArchitecture: ").append(pkgarch)
+	      .append("\nAuto-Installed: 1\n\n");
+	 if (OutFile.Write(stanza.c_str(), stanza.length()) == false)
+	    return false;
       }
    }
-   fclose(OutFile);
-
-   // move the outfile over the real file and set permissions
-   rename(outfile.c_str(), state.c_str());
+   if (OutFile.Close() == false)
+      return false;
    chmod(state.c_str(), 0644);
-
    return true;
 }
 									/*}}}*/
diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc
index 144c508be..461aa4217 100644
--- a/apt-pkg/indexcopy.cc
+++ b/apt-pkg/indexcopy.cc
@@ -108,10 +108,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
       }
       if (_error->PendingError() == true)
 	 return false;
-      FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
-      if (TargetFl == 0)
-	 return _error->Errno("fdopen","Failed to reopen fd");
-      
+
       // Setup the progress meter
       if(Progress)
 	 Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
@@ -132,14 +129,11 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	 string File;
 	 unsigned long long Size;
 	 if (GetFile(File,Size) == false)
-	 {
-	    fclose(TargetFl);
 	    return false;
-	 }
-	 
+
 	 if (Chop != 0)
 	    File = OrigPath + ChopDirs(File,Chop);
-	 
+
 	 // See if the file exists
 	 if (NoStat == false || Hits < 10)
 	 {
@@ -157,10 +151,10 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	       if (Chop != 0)
 		  File = OrigPath + ChopDirs(File,Chop);
 	    }
-	    
+
 	    // Get the size
 	    struct stat Buf;
-	    if (stat((CDROM + Prefix + File).c_str(),&Buf) != 0 || 
+	    if (stat((CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 		Buf.st_size == 0)
 	    {
 	       bool Mangled = false;
@@ -173,7 +167,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 		  File.replace(Start,End-Start,"binary-all");
 		  Mangled = true;
 	       }
-	       
+
 	       if (Mangled == false ||
 		   stat((CDROM + Prefix + File).c_str(),&Buf) != 0)
 	       {
@@ -181,9 +175,9 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 		     clog << "Missed(2): " << OrigFile << endl;
 		  NotFound++;
 		  continue;
-	       }	       
-	    }	    
-	    			    	    
+	       }
+	    }
+
 	    // Size match
 	    if ((unsigned long long)Buf.st_size != Size)
 	    {
@@ -193,21 +187,17 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	       continue;
 	    }
 	 }
-	 
+
 	 Packages++;
 	 Hits++;
-	 
-	 if (RewriteEntry(TargetFl,File) == false)
-	 {
-	    fclose(TargetFl);
+
+	 if (RewriteEntry(Target, File) == false)
 	    return false;
-	 }
       }
-      fclose(TargetFl);
 
       if (Debug == true)
 	 cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
-	 
+
       if (_config->FindB("APT::CDROM::NoAct",false) == false)
       {
 	 // Move out of the partial directory
@@ -218,39 +208,38 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	    return _error->Errno("rename","Failed to rename");
 	 ChangeOwnerAndPermissionOfFile("CopyPackages", FinalF.c_str(), "root", "root", 0644);
       }
-	 
+
       /* Mangle the source to be in the proper notation with
-       	 prefix dist [component] */ 
+	 prefix dist [component] */
       *I = string(*I,Prefix.length());
       ConvertToSourceList(CDROM,*I);
       *I = Prefix + ' ' + *I;
-      
+
       CurrentSize += FileSize;
-   }   
+   }
    if(Progress)
       Progress->Done();
-   
+
    // Some stats
    if(log) {
       stringstream msg;
       if(NotFound == 0 && WrongSize == 0)
 	 ioprintf(msg, _("Wrote %i records.\n"), Packages);
       else if (NotFound != 0 && WrongSize == 0)
-	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 		  Packages, NotFound);
       else if (NotFound == 0 && WrongSize != 0)
-	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 		  Packages, WrongSize);
       if (NotFound != 0 && WrongSize != 0)
 	 ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
    }
-   
+
    if (Packages == 0)
       _error->Warning("No valid records were found.");
 
    if (NotFound + WrongSize > 10)
       _error->Warning("A lot of entries were discarded, something may be wrong.\n");
-   
 
    return true;
 }
@@ -267,10 +256,10 @@ string IndexCopy::ChopDirs(string Path,unsigned int Depth)
       Depth--;
    }
    while (I != string::npos && Depth != 0);
-   
+
    if (I == string::npos)
       return string();
-   
+
    return string(Path,I+1);
 }
 									/*}}}*/
@@ -433,17 +422,15 @@ bool PackageCopy::GetFile(string &File,unsigned long long &Size)
 }
 									/*}}}*/
 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool PackageCopy::RewriteEntry(FILE *Target,string File)
+bool PackageCopy::RewriteEntry(FileFd &Target,string const &File)
 {
-   TFRewriteData Changes[] = {{ "Filename", File.c_str(), NULL },
-                              { NULL, NULL, NULL }};
-   
-   if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
+   string const Dir(File,0,File.rfind('/'));
+   std::vector Changes;
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", File));
+
+   if (Section->Write(Target, TFRewritePackageOrder, Changes) == false)
       return false;
-   fputc('\n',Target);
-   return true;
+   return Target.Write("\n", 1);
 }
 									/*}}}*/
 // SourceCopy::GetFile - Get the file information from the section	/*{{{*/
@@ -478,23 +465,18 @@ bool SourceCopy::GetFile(string &File,unsigned long long &Size)
 }
 									/*}}}*/
 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool SourceCopy::RewriteEntry(FILE *Target,string File)
+bool SourceCopy::RewriteEntry(FileFd &Target, std::string const &File)
 {
-   string Dir(File,0,File.rfind('/'));
-   TFRewriteData Changes[] = {{ "Directory", Dir.c_str(), NULL },
-                              { NULL, NULL, NULL }};
-   
-   if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
+   string const Dir(File,0,File.rfind('/'));
+   std::vector Changes;
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Dir));
+
+   if (Section->Write(Target, TFRewriteSourceOrder, Changes) == false)
       return false;
-   fputc('\n',Target);
-   return true;
+   return Target.Write("\n", 1);
 }
 									/*}}}*/
-// SigVerify::Verify - Verify a files md5sum against its metaindex     	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// SigVerify::Verify - Verify a files md5sum against its metaindex	/*{{{*/
 bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex)
 {
    const indexRecords::checkSum *Record = MetaIndex->Lookup(file);
@@ -702,7 +684,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
       pkgTagFile Parser(&Pkg);
       if (_error->PendingError() == true)
 	 return false;
-      
+
       // Open the output file
       char S[400];
       snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(),
@@ -719,10 +701,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
       }
       if (_error->PendingError() == true)
 	 return false;
-      FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
-      if (TargetFl == 0)
-	 return _error->Errno("fdopen","Failed to reopen fd");
-      
+
       // Setup the progress meter
       if(Progress)
 	 Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
@@ -740,20 +719,16 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
 	 if(Progress)
 	    Progress->Progress(Parser.Offset());
 
-	 const char *Start;
-	 const char *Stop;
-	 Section.GetSection(Start,Stop);
-	 fwrite(Start,Stop-Start, 1, TargetFl);
-	 fputc('\n',TargetFl);
+	 if (Section.Write(Target) == false || Target.Write("\n", 1) == false)
+	    return false;
 
 	 Packages++;
 	 Hits++;
       }
-      fclose(TargetFl);
 
       if (Debug == true)
 	 cout << " Processed by using Prefix '" << Prefix << "' and chop " << endl;
-	 
+
       if (_config->FindB("APT::CDROM::NoAct",false) == false)
       {
 	 // Move out of the partial directory
@@ -764,34 +739,32 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
 	    return _error->Errno("rename","Failed to rename");
 	 ChangeOwnerAndPermissionOfFile("CopyTranslations", FinalF.c_str(), "root", "root", 0644);
       }
-      
-      
+
       CurrentSize += FileSize;
-   }   
+   }
    if(Progress)
       Progress->Done();
-   
+
    // Some stats
    if(log) {
       stringstream msg;
       if(NotFound == 0 && WrongSize == 0)
 	 ioprintf(msg, _("Wrote %i records.\n"), Packages);
       else if (NotFound != 0 && WrongSize == 0)
-	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 		  Packages, NotFound);
       else if (NotFound == 0 && WrongSize != 0)
-	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 		  Packages, WrongSize);
       if (NotFound != 0 && WrongSize != 0)
 	 ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
    }
-   
+
    if (Packages == 0)
       _error->Warning("No valid records were found.");
 
    if (NotFound + WrongSize > 10)
       _error->Warning("A lot of entries were discarded, something may be wrong.\n");
-   
 
    return true;
 }
diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h
index 701beb075..729b0c8cb 100644
--- a/apt-pkg/indexcopy.h
+++ b/apt-pkg/indexcopy.h
@@ -28,6 +28,7 @@ using std::vector;
 class pkgTagSection;
 class indexRecords;
 class pkgCdromStatus;
+class FileFd;
 
 class IndexCopy								/*{{{*/
 {
@@ -45,7 +46,7 @@ class IndexCopy								/*{{{*/
    void ConvertToSourceList(std::string CD,std::string &Path);
    bool GrabFirst(std::string Path,std::string &To,unsigned int Depth);
    virtual bool GetFile(std::string &Filename,unsigned long long &Size) = 0;
-   virtual bool RewriteEntry(FILE *Target,std::string File) = 0;
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File) = 0;
    virtual const char *GetFileName() = 0;
    virtual const char *Type() = 0;
    
@@ -61,7 +62,7 @@ class PackageCopy : public IndexCopy					/*{{{*/
    protected:
    
    virtual bool GetFile(std::string &Filename,unsigned long long &Size);
-   virtual bool RewriteEntry(FILE *Target,std::string File);
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File);
    virtual const char *GetFileName() {return "Packages";};
    virtual const char *Type() {return "Package";};
    
@@ -72,7 +73,7 @@ class SourceCopy : public IndexCopy					/*{{{*/
    protected:
    
    virtual bool GetFile(std::string &Filename,unsigned long long &Size);
-   virtual bool RewriteEntry(FILE *Target,std::string File);
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File);
    virtual const char *GetFileName() {return "Sources";};
    virtual const char *Type() {return "Source";};
    
diff --git a/apt-private/private-show.cc b/apt-private/private-show.cc
index 289f035a6..790bc0092 100644
--- a/apt-private/private-show.cc
+++ b/apt-private/private-show.cc
@@ -97,28 +97,30 @@ static bool DisplayRecord(pkgCacheFile &CacheFile, pkgCache::VerIterator V,
       manual_installed = 0;
 
    // FIXME: add verbose that does not do the removal of the tags?
-   TFRewriteData RW[] = {
-      // delete, apt-cache show has this info and most users do not care
-      {"MD5sum", NULL, NULL},
-      {"SHA1", NULL, NULL},
-      {"SHA256", NULL, NULL},
-      {"Filename", NULL, NULL},
-      {"Multi-Arch", NULL, NULL},
-      {"Architecture", NULL, NULL},
-      {"Conffiles", NULL, NULL},
-      // we use the translated description
-      {"Description", NULL, NULL},
-      {"Description-md5", NULL, NULL},
-      // improve
-      {"Installed-Size", installed_size.c_str(), NULL},
-      {"Size", package_size.c_str(), "Download-Size"},
-      // add
-      {"APT-Manual-Installed", manual_installed, NULL},
-      {"APT-Sources", source_index_file.c_str(), NULL},
-      {NULL, NULL, NULL}
-   };
-
-   if(TFRewrite(stdout, Tags, NULL, RW) == false)
+   std::vector RW;
+   // delete, apt-cache show has this info and most users do not care
+   RW.push_back(pkgTagSection::Tag::Remove("MD5sum"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA1"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA256"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA512"));
+   RW.push_back(pkgTagSection::Tag::Remove("Filename"));
+   RW.push_back(pkgTagSection::Tag::Remove("Multi-Arch"));
+   RW.push_back(pkgTagSection::Tag::Remove("Architecture"));
+   RW.push_back(pkgTagSection::Tag::Remove("Conffiles"));
+   // we use the translated description
+   RW.push_back(pkgTagSection::Tag::Remove("Description"));
+   RW.push_back(pkgTagSection::Tag::Remove("Description-md5"));
+   // improve
+   RW.push_back(pkgTagSection::Tag::Rewrite("Installed-Size", installed_size));
+   RW.push_back(pkgTagSection::Tag::Remove("Size"));
+   RW.push_back(pkgTagSection::Tag::Rewrite("Download-Size", package_size));
+   // add
+   RW.push_back(pkgTagSection::Tag::Rewrite("APT-Manual-Installed", manual_installed));
+   RW.push_back(pkgTagSection::Tag::Rewrite("APT-Sources", source_index_file));
+
+   FileFd stdoutfd;
+   if (stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false) == false ||
+	 Tags.Write(stdoutfd, TFRewritePackageOrder, RW) == false || stdoutfd.Close() == false)
       return _error->Error("Internal Error, Unable to parse a package record");
 
    // write the description
diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index e2cf7e8b7..690b03bcc 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -580,6 +580,12 @@ static bool DumpAvail(CommandLine &)
    
    LocalitySort(VFList,Count,sizeof(*VFList));
 
+   std::vector RW;
+   RW.push_back(pkgTagSection::Tag::Remove("Status"));
+   RW.push_back(pkgTagSection::Tag::Remove("Config-Version"));
+   FileFd stdoutfd;
+   stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
+
    // Iterate over all the package files and write them out.
    char *Buffer = new char[Cache->HeaderP->MaxVerFileSize+10];
    for (pkgCache::VerFile **J = VFList; *J != 0;)
@@ -620,35 +626,32 @@ static bool DumpAvail(CommandLine &)
 	 if (PkgF.Read(Buffer,VF.Size + Jitter) == false)
 	    break;
 	 Buffer[VF.Size + Jitter] = '\n';
-	 
+
 	 // See above..
 	 if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
 	 {
 	    pkgTagSection Tags;
-	    TFRewriteData RW[] = {{"Status", NULL, NULL},{"Config-Version", NULL, NULL},{NULL, NULL, NULL}};
-	    const char *Zero = 0;
 	    if (Tags.Scan(Buffer+Jitter,VF.Size+1) == false ||
-		TFRewrite(stdout,Tags,&Zero,RW) == false)
+		Tags.Write(stdoutfd, NULL, RW) == false ||
+		stdoutfd.Write("\n", 1) == false)
 	    {
 	       _error->Error("Internal Error, Unable to parse a package record");
 	       break;
 	    }
-	    fputc('\n',stdout);
 	 }
 	 else
 	 {
-	    if (fwrite(Buffer+Jitter,VF.Size+1,1,stdout) != 1)
+	    if (stdoutfd.Write(Buffer + Jitter, VF.Size + 1) == false)
 	       break;
 	 }
-	 
+
 	 Pos = VF.Offset + VF.Size;
       }
 
-      fflush(stdout);
       if (_error->PendingError() == true)
          break;
    }
-   
+
    delete [] Buffer;
    delete [] VFList;
    return !_error->PendingError();
diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc
index 971900e4f..12ef8dda0 100644
--- a/cmdline/apt-sortpkgs.cc
+++ b/cmdline/apt-sortpkgs.cc
@@ -108,8 +108,10 @@ static bool DoIt(string InFile)
    const char **Order = TFRewritePackageOrder;
    if (Source == true)
       Order = TFRewriteSourceOrder;
-   
+
    // Emit
+   FileFd stdoutfd;
+   stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
    unsigned char *Buffer = new unsigned char[Largest+1];
    for (vector::iterator I = List.begin(); I != List.end(); ++I)
    {
@@ -119,8 +121,8 @@ static bool DoIt(string InFile)
 	 delete [] Buffer;
 	 return false;
       }
-      
-      Buffer[I->Length] = '\n';      
+
+      Buffer[I->Length] = '\n';
       if (Section.Scan((char *)Buffer,I->Length+1) == false)
       {
 	 delete [] Buffer;
@@ -128,15 +130,13 @@ static bool DoIt(string InFile)
       }
 
       // Sort the section
-      if (TFRewrite(stdout,Section,Order,0) == false)
+      if (Section.Write(stdoutfd, Order) == false || stdoutfd.Write("\n", 1) == false)
       {
 	 delete [] Buffer;
 	 return _error->Error("Internal error, failed to sort fields");
       }
-      
-      fputc('\n',stdout);      
    }
-   
+
    delete [] Buffer;
    return true;
 }
diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index 69b936dff..62108f7ca 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -178,7 +178,9 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    PkgDone = true;
    
    // Create a package writer object.
-   PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
+   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
+		      PkgCompress,Permissions);
+   PackagesWriter Packages(&Comp.Input, flCombine(CacheDir,BinCacheDB),
 			   flCombine(OverrideDir,BinOverride),
 			   flCombine(OverrideDir,ExtraOverride),
 			   Arch);
@@ -197,10 +199,6 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
    Packages.DeLinkLimit = DeLinkLimit;
 
-   // Create a compressor object
-   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
-		      PkgCompress,Permissions);
-   Packages.Output = Comp.Input;
    if (_error->PendingError() == true)
       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
    
@@ -272,7 +270,9 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    SrcDone = true;
    
    // Create a package writer object.
-   SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
+   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
+		      SrcCompress,Permissions);
+   SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 			 flCombine(OverrideDir,BinOverride),
 			 flCombine(OverrideDir,SrcOverride),
 			 flCombine(OverrideDir,SrcExtraOverride));
@@ -287,11 +287,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 
    Sources.DeLinkLimit = DeLinkLimit;
    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
-   
-   // Create a compressor object
-   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
-		      SrcCompress,Permissions);
-   Sources.Output = Comp.Input;
+
    if (_error->PendingError() == true)
       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 
@@ -365,16 +361,15 @@ bool PackageMap::GenContents(Configuration &Setup,
    gettimeofday(&StartTime,0);   
    
    // Create a package writer object.
-   ContentsWriter Contents("", Arch);
+   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
+		      CntCompress,Permissions);
+   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
+   ContentsWriter Contents(&Comp.Input, "", Arch);
    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
       return false;
 
-   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
-		      CntCompress,Permissions);
-   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
-   Contents.Output = Comp.Input;
    if (_error->PendingError() == true)
       return false;
 
@@ -384,7 +379,7 @@ bool PackageMap::GenContents(Configuration &Setup,
       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
       if (_error->PendingError() == true)
 	 return false;
-      
+
       unsigned long long Size = Head.Size();
       unsigned char Buf[4096];
       while (Size != 0)
@@ -392,17 +387,17 @@ bool PackageMap::GenContents(Configuration &Setup,
 	 unsigned long long ToRead = Size;
 	 if (Size > sizeof(Buf))
 	    ToRead = sizeof(Buf);
-	 
+
 	 if (Head.Read(Buf,ToRead) == false)
 	    return false;
-	 
-	 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
+
+	 if (Comp.Input.Write(Buf, ToRead) == false)
 	    return _error->Errno("fwrite",_("Error writing header to contents file"));
-	 
+
 	 Size -= ToRead;
-      }            
-   }  
-      
+      }
+   }
+
    /* Go over all the package file records and parse all the package
       files associated with this contents file into one great big honking
       memory structure, then dump the sorted version */
@@ -676,7 +671,7 @@ static bool SimpleGenPackages(CommandLine &CmdL)
       Override = CmdL.FileList[2];
    
    // Create a package writer object.
-   PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
+   PackagesWriter Packages(NULL, _config->Find("APT::FTPArchive::DB"),
 			   Override, "", _config->Find("APT::FTPArchive::Architecture"));
    if (_error->PendingError() == true)
       return false;
@@ -704,7 +699,7 @@ static bool SimpleGenContents(CommandLine &CmdL)
       return ShowHelp(CmdL);
    
    // Create a package writer object.
-   ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
+   ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
    if (_error->PendingError() == true)
       return false;
    
@@ -737,7 +732,7 @@ static bool SimpleGenSources(CommandLine &CmdL)
 			     SOverride.c_str());
        
    // Create a package writer object.
-   SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
+   SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
    if (_error->PendingError() == true)
       return false;
    
@@ -764,7 +759,7 @@ static bool SimpleGenRelease(CommandLine &CmdL)
 
    string Dir = CmdL.FileList[1];
 
-   ReleaseWriter Release("");
+   ReleaseWriter Release(NULL, "");
    Release.DirStrip = Dir;
 
    if (_error->PendingError() == true)
diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc
index 8c4181eda..145f3910e 100644
--- a/ftparchive/contents.cc
+++ b/ftparchive/contents.cc
@@ -38,6 +38,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -238,19 +239,19 @@ void GenContents::Add(const char *Dir,const char *Package)
 // GenContents::WriteSpace - Write a given number of white space chars	/*{{{*/
 // ---------------------------------------------------------------------
 /* We mod 8 it and write tabs where possible. */
-void GenContents::WriteSpace(FILE *Out,unsigned int Current,unsigned int Target)
+void GenContents::WriteSpace(std::string &out, size_t Current, size_t Target)
 {
    if (Target <= Current)
       Target = Current + 1;
-   
+
    /* Now we write tabs so long as the next tab stop would not pass
       the target */
    for (; (Current/8 + 1)*8 < Target; Current = (Current/8 + 1)*8)
-      fputc('\t',Out);
+      out.append("\t");
 
    // Fill the last bit with spaces
    for (; Current < Target; Current++)
-      fputc(' ',Out);
+      out.append(" ");
 }
 									/*}}}*/
 // GenContents::Print - Display the tree				/*{{{*/
@@ -259,13 +260,13 @@ void GenContents::WriteSpace(FILE *Out,unsigned int Current,unsigned int Target)
    calls itself and runs over each section of the tree printing out
    the pathname and the hit packages. We use Buf to build the pathname
    summed over all the directory parents of this node. */
-void GenContents::Print(FILE *Out)
+void GenContents::Print(FileFd &Out)
 {
    char Buffer[1024];
    Buffer[0] = 0;
    DoPrint(Out,&Root,Buffer);
 }
-void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf)
+void GenContents::DoPrint(FileFd &Out,GenContents::Node *Top, char *Buf)
 {
    if (Top == 0)
       return;
@@ -278,26 +279,27 @@ void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf)
    if (Top->Path != 0)
    {
       strcat(Buf,Top->Path);
-      
+
       // Do not show the item if it is a directory with dups
       if (Top->Path[strlen(Top->Path)-1] != '/' /*|| Top->Dups == 0*/)
       {
-	 fputs(Buf,Out);
-	 WriteSpace(Out,strlen(Buf),60);
+	 std::string out = Buf;
+	 WriteSpace(out, out.length(), 60);
 	 for (Node *I = Top; I != 0; I = I->Dups)
 	 {
 	    if (I != Top)
-	       fputc(',',Out);
-	    fputs(I->Package,Out);
+	       out.append(",");
+	    out.append(I->Package);
 	 }
-         fputc('\n',Out);
-      }      
-   }   
-   
+         out.append("\n");
+	 Out.Write(out.c_str(), out.length());
+      }
+   }
+
    // Go along the directory link
    DoPrint(Out,Top->DirDown,Buf);
    *OldEnd = 0;
-   
+
    // Go right
    DoPrint(Out,Top->BTreeRight,Buf);  
 }
diff --git a/ftparchive/contents.h b/ftparchive/contents.h
index f58e3278e..953d0d54b 100644
--- a/ftparchive/contents.h
+++ b/ftparchive/contents.h
@@ -17,6 +17,7 @@
 #include 
 
 class debDebFile;
+class FileFd;
 
 class GenContents
 {
@@ -54,14 +55,14 @@ class GenContents
    unsigned long NodeLeft;
    
    Node *Grab(Node *Top,const char *Name,const char *Package);
-   void WriteSpace(FILE *Out,unsigned int Current,unsigned int Target);
-   void DoPrint(FILE *Out,Node *Top, char *Buf);
+   void WriteSpace(std::string &out, size_t Current, size_t Target);
+   void DoPrint(FileFd &Out,Node *Top, char *Buf);
    
    public:
    
    char *Mystrdup(const char *From);
    void Add(const char *Dir,const char *Package);   
-   void Print(FILE *Out);
+   void Print(FileFd &Out);
 
    GenContents() : BlockList(0), StrPool(0), StrLeft(0), 
                    NodePool(0), NodeLeft(0) {};
diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc
index f35d5304a..08a3cff5a 100644
--- a/ftparchive/multicompress.cc
+++ b/ftparchive/multicompress.cc
@@ -46,7 +46,6 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress,
 {
    Outputs = 0;
    Outputter = -1;
-   Input = 0;
    UpdateMTime = 0;
 
    /* Parse the compression string, a space separated lists of compresison
@@ -187,12 +186,11 @@ bool MultiCompress::Start()
    };
 
    close(Pipe[0]);
-   Input = fdopen(Pipe[1],"w");
-   if (Input == 0)
-      return _error->Errno("fdopen",_("Failed to create FILE*"));
-   
+   if (Input.OpenDescriptor(Pipe[1], FileFd::WriteOnly, true) == false)
+      return false;
+
    if (Outputter == -1)
-      return _error->Errno("fork",_("Failed to fork"));   
+      return _error->Errno("fork",_("Failed to fork"));
    return true;
 }
 									/*}}}*/
@@ -201,11 +199,10 @@ bool MultiCompress::Start()
 /* */
 bool MultiCompress::Die()
 {
-   if (Input == 0)
+   if (Input.IsOpen() == false)
       return true;
-   
-   fclose(Input);
-   Input = 0;
+
+   Input.Close();
    bool Res = ExecWait(Outputter,_("Compress child"),false);
    Outputter = -1;
    return Res;
@@ -217,7 +214,7 @@ bool MultiCompress::Die()
 bool MultiCompress::Finalize(unsigned long long &OutSize)
 {
    OutSize = 0;
-   if (Input == 0 || Die() == false)
+   if (Input.IsOpen() == false || Die() == false)
       return false;
    
    time_t Now;
diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h
index ddd1815a3..161716b86 100644
--- a/ftparchive/multicompress.h
+++ b/ftparchive/multicompress.h
@@ -48,7 +48,7 @@ class MultiCompress
    public:
    
    // The FD to write to for compression.
-   FILE *Input;
+   FileFd Input;
    unsigned long UpdateMTime;
    
    bool Finalize(unsigned long long &OutSize);
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index 855e0ef79..7cf7e6efc 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -51,20 +51,6 @@
 using namespace std;
 FTWScanner *FTWScanner::Owner;
 
-// SetTFRewriteData - Helper for setting rewrite lists			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-static inline TFRewriteData SetTFRewriteData(const char *tag,
-			     const char *rewrite,
-			     const char *newtag = 0)
-{
-   TFRewriteData tfrd;
-   tfrd.Tag = tag;
-   tfrd.Rewrite = rewrite;
-   tfrd.NewTag = newtag;
-   return tfrd;
-}
-									/*}}}*/
 // ConfigToDoHashes - which hashes to generate				/*{{{*/
 static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
 {
@@ -85,8 +71,15 @@ static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
 // FTWScanner::FTWScanner - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-FTWScanner::FTWScanner(string const &Arch): Arch(Arch), DoHashes(~0)
+FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch): Arch(Arch), DoHashes(~0)
 {
+   if (GivenOutput == NULL)
+   {
+      Output = new FileFd;
+      Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
+   }
+   else
+      Output = GivenOutput;
    ErrorPrinted = false;
    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
    ConfigToDoHashes(DoHashes, "APT::FTPArchive");
@@ -331,11 +324,10 @@ bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 // PackagesWriter::PackagesWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
+PackagesWriter::PackagesWriter(FileFd * const GivenOutput, string const &DB,string const &Overrides,string const &ExtOverrides,
 			       string const &Arch) :
-   FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
+   FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 {
-   Output = stdout;
    SetExts(".deb .udeb");
    DeLinkLimit = 0;
 
@@ -461,32 +453,32 @@ bool PackagesWriter::DoPackage(string FileName)
    }
 
    // This lists all the changes to the fields we are going to make.
-   std::vector Changes;
+   std::vector Changes;
 
    std::string Size;
    strprintf(Size, "%llu", (unsigned long long) FileSize);
-   Changes.push_back(SetTFRewriteData("Size", Size.c_str()));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Size", Size));
 
    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
    {
       if (hs->HashType() == "MD5Sum")
-	 Changes.push_back(SetTFRewriteData("MD5sum", hs->HashValue().c_str()));
+	 Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue()));
       else if (hs->HashType() == "Checksum-FileSize")
 	 continue;
       else
-	 Changes.push_back(SetTFRewriteData(hs->HashType().c_str(), hs->HashValue().c_str()));
+	 Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue()));
    }
-   Changes.push_back(SetTFRewriteData("Filename", NewFileName.c_str()));
-   Changes.push_back(SetTFRewriteData("Priority", OverItem->Priority.c_str()));
-   Changes.push_back(SetTFRewriteData("Status", 0));
-   Changes.push_back(SetTFRewriteData("Optional", 0));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority));
+   Changes.push_back(pkgTagSection::Tag::Remove("Status"));
+   Changes.push_back(pkgTagSection::Tag::Remove("Optional"));
 
    string DescriptionMd5;
    if (LongDescription == false) {
       MD5Summation descmd5;
       descmd5.Add(desc.c_str());
       DescriptionMd5 = descmd5.Result().Value();
-      Changes.push_back(SetTFRewriteData("Description-md5", DescriptionMd5.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5));
       if (TransWriter != NULL)
 	 TransWriter->DoPackage(Package, desc, DescriptionMd5);
    }
@@ -505,7 +497,7 @@ bool PackagesWriter::DoPackage(string FileName)
    }
 
    if (NewMaint.empty() == false)
-      Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint));
 
    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
@@ -517,19 +509,17 @@ bool PackagesWriter::DoPackage(string FileName)
    {
       if (Tags.FindS("Suggests").empty() == false)
 	 OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
-      Changes.push_back(SetTFRewriteData("Suggests", OptionalStr.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Suggests", OptionalStr));
    }
 
    for (map::const_iterator I = OverItem->FieldOverride.begin();
         I != OverItem->FieldOverride.end(); ++I)
-      Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
-
-   Changes.push_back(SetTFRewriteData( 0, 0));
+      Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 
    // Rewrite and store the fields.
-   if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes.data()) == false)
+   if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false ||
+	 Output->Write("\n", 1) == false)
       return false;
-   fprintf(Output,"\n");
 
    return Db.Finish();
 }
@@ -539,14 +529,13 @@ bool PackagesWriter::DoPackage(string FileName)
 // ---------------------------------------------------------------------
 /* Create a Translation-Master file for this Packages file */
 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
-					mode_t const &Permissions) : Output(NULL),
-							RefCounter(0)
+					mode_t const &Permissions) : RefCounter(0)
 {
    if (File.empty() == true)
       return;
 
    Comp = new MultiCompress(File, TransCompress, Permissions);
-   Output = Comp->Input;
+   Output = &Comp->Input;
 }
 									/*}}}*/
 // TranslationWriter::DoPackage - Process a single package		/*{{{*/
@@ -565,8 +554,10 @@ bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
    if (Included.find(Record) != Included.end())
       return true;
 
-   fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
+   std::string out;
+   strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 	   Pkg.c_str(), MD5.c_str(), Desc.c_str());
+   Output->Write(out.c_str(), out.length());
 
    Included.insert(Record);
    return true;
@@ -587,11 +578,10 @@ TranslationWriter::~TranslationWriter()
 // SourcesWriter::SourcesWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
+SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides,
 			     string const &ExtOverrides) :
-   Db(DB), Stats(Db.Stats)
+   FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats)
 {
-   Output = stdout;
    AddPattern("*.dsc");
    DeLinkLimit = 0;
    Buffer = 0;
@@ -620,15 +610,16 @@ SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string c
 }
 									/*}}}*/
 // SourcesWriter::DoPackage - Process a single package			/*{{{*/
-static std::ostream& addDscHash(std::ostream &out, unsigned int const DoHashes,
+static std::string getDscHash(unsigned int const DoHashes,
       Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, char const * const FieldName,
       HashString const * const Hash, unsigned long long Size, std::string FileName)
 {
    if ((DoHashes & DoIt) != DoIt || Tags.Exists(FieldName) == false || Hash == NULL)
-      return out;
+      return "";
+   std::ostringstream out;
    out << "\n " << Hash->HashValue() << " " << Size << " " << FileName
       << "\n " << Tags.FindS(FieldName);
-   return out;
+   return out.str();
 }
 bool SourcesWriter::DoPackage(string FileName)
 {
@@ -721,16 +712,10 @@ bool SourcesWriter::DoPackage(string FileName)
 
    // Add the dsc to the files hash list
    string const strippedName = flNotDir(FileName);
-   std::ostringstream ostreamFiles;
-   addDscHash(ostreamFiles, DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
-   string const Files = ostreamFiles.str();
-
-   std::ostringstream ostreamSha1;
-   addDscHash(ostreamSha1, DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
-   std::ostringstream ostreamSha256;
-   addDscHash(ostreamSha256, DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
-   std::ostringstream ostreamSha512;
-   addDscHash(ostreamSha512, DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
+   std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
+   std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
+   std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
+   std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
 
    // Strip the DirStrip prefix from the FileName and add the PathPrefix
    string NewFileName;
@@ -752,7 +737,7 @@ bool SourcesWriter::DoPackage(string FileName)
    char *RealPath = NULL;
    for (;isspace(*C); C++);
    while (*C != 0)
-   {   
+   {
       // Parse each of the elements
       if (ParseQuoteWord(C,ParseJnk) == false ||
 	  ParseQuoteWord(C,ParseJnk) == false ||
@@ -782,21 +767,21 @@ bool SourcesWriter::DoPackage(string FileName)
 	    if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
 	       continue;
 	    char const * fieldname;
-	    std::ostream * out;
+	    std::string * out;
 	    if (hs->HashType() == "SHA1")
 	    {
 	       fieldname = "Checksums-Sha1";
-	       out = &ostreamSha1;
+	       out = &ChecksumsSha1;
 	    }
 	    else if (hs->HashType() == "SHA256")
 	    {
 	       fieldname = "Checksums-Sha256";
-	       out = &ostreamSha256;
+	       out = &ChecksumsSha256;
 	    }
 	    else if (hs->HashType() == "SHA512")
 	    {
 	       fieldname = "Checksums-Sha512";
-	       out = &ostreamSha512;
+	       out = &ChecksumsSha512;
 	    }
 	    else
 	    {
@@ -805,10 +790,12 @@ bool SourcesWriter::DoPackage(string FileName)
 	    }
 	    if (Tags.Exists(fieldname) == true)
 	       continue;
-	    (*out) << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
+	    std::ostringstream streamout;
+	    streamout << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
+	    out->append(streamout.str());
 	 }
 
-	 // write back the GetFileInfo() stats data 
+	 // write back the GetFileInfo() stats data
 	 Db.Finish();
       }
 
@@ -829,54 +816,48 @@ bool SourcesWriter::DoPackage(string FileName)
    if (Directory.length() > 2)
       Directory.erase(Directory.end()-1);
 
-   string const ChecksumsSha1 = ostreamSha1.str();
-   string const ChecksumsSha256 = ostreamSha256.str();
-   string const ChecksumsSha512 = ostreamSha512.str();
-
    // This lists all the changes to the fields we are going to make.
    // (5 hardcoded + checksums + maintainer + end marker)
-   std::vector Changes;
+   std::vector Changes;
 
-   Changes.push_back(SetTFRewriteData("Source", 0));
-   Changes.push_back(SetTFRewriteData("Package",Package.c_str()));
+   Changes.push_back(pkgTagSection::Tag::Remove("Source"));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package));
    if (Files.empty() == false)
-      Changes.push_back(SetTFRewriteData("Files",Files.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files));
    if (ChecksumsSha1.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha1",ChecksumsSha1.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1));
    if (ChecksumsSha256.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha256",ChecksumsSha256.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256));
    if (ChecksumsSha512.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha512",ChecksumsSha512.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512));
    if (Directory != "./")
-      Changes.push_back(SetTFRewriteData("Directory",Directory.c_str()));
-   Changes.push_back(SetTFRewriteData("Priority",BestPrio.c_str()));
-   Changes.push_back(SetTFRewriteData("Status",0));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio));
+   Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 
    // Rewrite the maintainer field if necessary
    bool MaintFailed;
-   string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
+   string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"), MaintFailed);
    if (MaintFailed == true)
    {
       if (NoOverride == false)
       {
-	 NewLine(1);	 
+	 NewLine(1);
 	 ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 	       Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
-      }      
+      }
    }
    if (NewMaint.empty() == false)
-      Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
-   
-   for (map::const_iterator I = SOverItem->FieldOverride.begin(); 
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str()));
+
+   for (map::const_iterator I = SOverItem->FieldOverride.begin();
         I != SOverItem->FieldOverride.end(); ++I)
-      Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 
-   Changes.push_back(SetTFRewriteData(0, 0));
-      
    // Rewrite and store the fields.
-   if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes.data()) == false)
+   if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false ||
+	 Output->Write("\n", 1) == false)
       return false;
-   fprintf(Output,"\n");
 
    Stats.Packages++;
    
@@ -887,12 +868,11 @@ bool SourcesWriter::DoPackage(string FileName)
 // ContentsWriter::ContentsWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
-		    FTWScanner(Arch), Db(DB), Stats(Db.Stats)
+ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB, string const &Arch) :
+		    FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats)
 
 {
    SetExts(".deb");
-   Output = stdout;
 }
 									/*}}}*/
 // ContentsWriter::DoPackage - Process a single package			/*{{{*/
@@ -974,7 +954,7 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres
 // ReleaseWriter::ReleaseWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-ReleaseWriter::ReleaseWriter(string const &/*DB*/)
+ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput)
 {
    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
    {
@@ -996,7 +976,6 @@ ReleaseWriter::ReleaseWriter(string const &/*DB*/)
    }
    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 
-   Output = stdout;
    time_t const now = time(NULL);
 
    setlocale(LC_TIME, "C");
@@ -1040,7 +1019,8 @@ ReleaseWriter::ReleaseWriter(string const &/*DB*/)
       if (Value == "")
          continue;
 
-      fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
+      std::string const out = I->first + ": " + Value + "\n";
+      Output->Write(out.c_str(), out.length());
    }
 
    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
@@ -1087,29 +1067,35 @@ bool ReleaseWriter::DoPackage(string FileName)
 									/*}}}*/
 // ReleaseWriter::Finish - Output the checksums				/*{{{*/
 // ---------------------------------------------------------------------
-static void printChecksumTypeRecord(FILE * const Output, char const * const Type, map const &CheckSums)
+static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map const &CheckSums)
 {
-      fprintf(Output, "%s:\n", Type);
-      for(map::const_iterator I = CheckSums.begin();
-	  I != CheckSums.end(); ++I)
-      {
-	 HashString const * const hs = I->second.Hashes.find(Type);
-	 if (hs == NULL)
-	    continue;
-	 fprintf(Output, " %s %16llu %s\n",
-		 hs->HashValue().c_str(),
-		 (*I).second.size,
-		 (*I).first.c_str());
-      }
+   {
+      std::string out;
+      strprintf(out, "%s:\n", Type);
+      Output.Write(out.c_str(), out.length());
+   }
+   for(map::const_iterator I = CheckSums.begin();
+	 I != CheckSums.end(); ++I)
+   {
+      HashString const * const hs = I->second.Hashes.find(Type);
+      if (hs == NULL)
+	 continue;
+      std::string out;
+      strprintf(out, " %s %16llu %s\n",
+	    hs->HashValue().c_str(),
+	    (*I).second.size,
+	    (*I).first.c_str());
+      Output.Write(out.c_str(), out.length());
+   }
 }
 void ReleaseWriter::Finish()
 {
    if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
-      printChecksumTypeRecord(Output, "MD5Sum", CheckSums);
+      printChecksumTypeRecord(*Output, "MD5Sum", CheckSums);
    if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
-      printChecksumTypeRecord(Output, "SHA1", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA1", CheckSums);
    if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
-      printChecksumTypeRecord(Output, "SHA256", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA256", CheckSums);
    if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
-      printChecksumTypeRecord(Output, "SHA512", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA512", CheckSums);
 }
diff --git a/ftparchive/writer.h b/ftparchive/writer.h
index 226996475..0ba60db5e 100644
--- a/ftparchive/writer.h
+++ b/ftparchive/writer.h
@@ -42,10 +42,10 @@ class FTWScanner
    string Arch;
    const char *OriginalPath;
    bool ErrorPrinted;
-   
+
    // Stuff for the delinker
    bool NoLinkAct;
-   
+
    static FTWScanner *Owner;
    static int ScannerFTW(const char *File,const struct stat *sb,int Flag);
    static int ScannerFile(const char *File, bool const &ReadLink);
@@ -59,10 +59,11 @@ class FTWScanner
       {
 	 c1out << endl;
 	 ErrorPrinted = true;
-      }	 
+      }
    }
-   
+
    public:
+   FileFd *Output;
    unsigned int DoHashes;
 
    unsigned long DeLinkLimit;
@@ -76,8 +77,8 @@ class FTWScanner
    void AddPattern(char const *Pattern) { Patterns.push_back(Pattern); };
    void AddPatterns(std::vector const &patterns) { Patterns.insert(Patterns.end(), patterns.begin(), patterns.end()); };
    bool SetExts(string const &Vals);
-      
-   FTWScanner(string const &Arch = string());
+
+   FTWScanner(FileFd * const Output, string const &Arch = string());
    virtual ~FTWScanner() {};
 };
 
@@ -86,9 +87,9 @@ class MultiCompress;
 class TranslationWriter
 {
    MultiCompress *Comp;
-   FILE *Output;
    std::set Included;
    unsigned short RefCounter;
+   FileFd *Output;
 
    public:
    void IncreaseRefCounter() { ++RefCounter; };
@@ -97,7 +98,7 @@ class TranslationWriter
    bool DoPackage(string const &Pkg, string const &Desc, string const &MD5);
 
    TranslationWriter(string const &File, string const &TransCompress, mode_t const &Permissions);
-   TranslationWriter() : Comp(NULL), Output(NULL), RefCounter(0) {};
+   TranslationWriter() : Comp(NULL), RefCounter(0) {};
    ~TranslationWriter();
 };
 
@@ -105,7 +106,7 @@ class PackagesWriter : public FTWScanner
 {
    Override Over;
    CacheDB Db;
-      
+
    public:
 
    // Some flags
@@ -117,7 +118,6 @@ class PackagesWriter : public FTWScanner
    // General options
    string PathPrefix;
    string DirStrip;
-   FILE *Output;
    struct CacheDB::Stats &Stats;
    TranslationWriter *TransWriter;
 
@@ -126,7 +126,7 @@ class PackagesWriter : public FTWScanner
       {return Over.ReadExtraOverride(File);};
    virtual bool DoPackage(string FileName);
 
-   PackagesWriter(string const &DB,
+   PackagesWriter(FileFd * const Output, string const &DB,
                   string const &Overrides,
                   string const &ExtOverrides = "",
 		  string const &Arch = "");
@@ -136,25 +136,24 @@ class PackagesWriter : public FTWScanner
 class ContentsWriter : public FTWScanner
 {
    CacheDB Db;
-   
+
    GenContents Gen;
-   
+
    public:
 
    // General options
-   FILE *Output;
    struct CacheDB::Stats &Stats;
    string Prefix;
-   
+
    bool DoPackage(string FileName,string Package);
    virtual bool DoPackage(string FileName) 
              {return DoPackage(FileName,string());};
    bool ReadFromPkgs(string const &PkgFile,string const &PkgCompress);
 
-   void Finish() {Gen.Print(Output);};
+   void Finish() {Gen.Print(*Output);};
    inline bool ReadyDB(string const &DB) {return Db.ReadyDB(DB);};
-   
-   ContentsWriter(string const &DB, string const &Arch = string());
+
+   ContentsWriter(FileFd * const Output, string const &DB, string const &Arch = string());
    virtual ~ContentsWriter() {};
 };
 
@@ -165,21 +164,20 @@ class SourcesWriter : public FTWScanner
    Override SOver;
    char *Buffer;
    unsigned long long BufSize;
-   
+
    public:
 
    bool NoOverride;
    bool DoAlwaysStat;
-   
+
    // General options
    string PathPrefix;
    string DirStrip;
-   FILE *Output;
    struct CacheDB::Stats &Stats;
 
    virtual bool DoPackage(string FileName);
 
-   SourcesWriter(string const &DB,string const &BOverrides,string const &SOverrides,
+   SourcesWriter(FileFd * const Output, string const &DB,string const &BOverrides,string const &SOverrides,
 		 string const &ExtOverrides=string());
    virtual ~SourcesWriter() {free(Buffer);};
 };
@@ -187,11 +185,10 @@ class SourcesWriter : public FTWScanner
 class ReleaseWriter : public FTWScanner
 {
 public:
-   ReleaseWriter(string const &DB);
+   ReleaseWriter(FileFd * const Output, string const &DB);
    virtual bool DoPackage(string FileName);
    void Finish();
 
-   FILE *Output;
    // General options
    string PathPrefix;
    string DirStrip;
diff --git a/test/integration/test-apt-cli-show b/test/integration/test-apt-cli-show
index 5604620fd..43072cf03 100755
--- a/test/integration/test-apt-cli-show
+++ b/test/integration/test-apt-cli-show
@@ -25,11 +25,11 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 # note that we do not display Description-md5 with the "apt" cmd
 # and also show some additional fields that are calculated
 testsuccessequal "Package: foo
+Version: 1.0
 Priority: optional
 Section: other
-Installed-Size: 43.0 kB
 Maintainer: Joe Sixpack 
-Version: 1.0
+Installed-Size: 43.0 kB
 Download-Size: unknown
 APT-Manual-Installed: yes
 APT-Sources: file:$APTARCHIVE/ unstable/main i386 Packages
diff --git a/test/libapt/indexcopytosourcelist_test.cc b/test/libapt/indexcopytosourcelist_test.cc
index bec87601f..1b0427564 100644
--- a/test/libapt/indexcopytosourcelist_test.cc
+++ b/test/libapt/indexcopytosourcelist_test.cc
@@ -16,7 +16,7 @@ class NoCopy : public IndexCopy {
 	 return Path;
       }
       bool GetFile(std::string &/*Filename*/, unsigned long long &/*Size*/) { return false; }
-      bool RewriteEntry(FILE * /*Target*/, std::string /*File*/) { return false; }
+      bool RewriteEntry(FileFd & /*Target*/, std::string const &/*File*/) { return false; }
       const char *GetFileName() { return NULL; }
       const char *Type() { return NULL; }
 
diff --git a/test/libapt/tagsection_test.cc b/test/libapt/tagsection_test.cc
new file mode 100644
index 000000000..f250177af
--- /dev/null
+++ b/test/libapt/tagsection_test.cc
@@ -0,0 +1,270 @@
+#include 
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+
+#include "file-helpers.h"
+
+std::string packageValue = "aaaa";
+std::string typoValue = "aa\n"
+   " .\n"
+   " cc";
+std::string typoRawValue = "\n " + typoValue;
+std::string overrideValue = "1";
+/*
+   std::cerr << "FILECONTENT: »";
+   char buffer[3000];
+   while (fd.ReadLine(buffer, sizeof(buffer)))
+      std::cerr << buffer;
+   std::cerr << "«" << std::endl;;
+*/
+
+void setupTestcaseStart(FileFd &fd, pkgTagSection §ion, std::string &content)
+{
+   createTemporaryFile("writesection", fd, NULL, NULL);
+   content = "Package: " + packageValue + "\n"
+      "TypoA:\n " + typoValue + "\n"
+      "Override: " + overrideValue + "\n"
+      "Override-Backup: " + overrideValue + "\n"
+      "\n";
+   EXPECT_TRUE(section.Scan(content.c_str(), content.length(), true));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(typoRawValue, section.FindRawS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodified)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   EXPECT_TRUE(section.Write(fd));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrder)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrderReversed)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Override", "TypoA", "Package", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrderNotAll)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRename)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rename("TypoA", "TypoB"));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoB"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRemove)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Remove("TypoA"));
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", ""));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_FALSE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(2, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRewrite)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", "42"));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(42, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRename)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rename("TypoA", "TypoB"));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoB"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRemove)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Remove("TypoA"));
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", ""));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_FALSE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(2, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRewrite)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", "42"));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(42, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
-- 
cgit v1.2.3-70-g09d2


From 8881b11eacd735148d087c8c0f53827cb537b582 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Thu, 11 Jun 2015 16:40:45 +0200
Subject: implement 'apt-get files' to access index targets

Downloading additional files is only half the job. We still need a way
to allow external tools to know where the files are they requested for
download given that we don't want them to choose their own location.

'apt-get files' is our answer to this showing by default in a deb822
format information about each IndexTarget with the potential to filter
the records based on lines and an option to change the output format.

The command serves also as an example on how to get to this information
via libapt.
---
 apt-pkg/deb/debmetaindex.cc                        | 17 +++--
 apt-pkg/indexfile.cc                               | 18 ++++-
 apt-pkg/indexfile.h                                |  3 +
 apt-private/private-cmndline.cc                    |  4 +
 cmdline/apt-get.cc                                 | 87 ++++++++++++++++++++++
 doc/acquire-additional-files.txt                   | 87 +++++++++++++++++++++-
 doc/apt-get.8.xml                                  | 15 ++++
 test/integration/test-apt-acquire-additional-files |  2 +
 8 files changed, 223 insertions(+), 10 deletions(-)

(limited to 'cmdline')

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index c05e7cdae..8e4c2be2d 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -136,11 +136,14 @@ void foreachTarget(std::string const URI, std::string const Dist,
 	       std::map Options;
 	       Options.insert(std::make_pair("SITE", Site));
 	       Options.insert(std::make_pair("RELEASE", Release));
-	       Options.insert(std::make_pair("COMPONENT", (*I)->Section));
-	       Options.insert(std::make_pair("LANGUAGE", *l));
+	       if (MetaKey.find("$(COMPONENT)") != std::string::npos)
+		  Options.insert(std::make_pair("COMPONENT", (*I)->Section));
+	       if (MetaKey.find("$(LANGUAGE)") != std::string::npos)
+		  Options.insert(std::make_pair("LANGUAGE", *l));
 	       Options.insert(std::make_pair("ARCHITECTURE", "source"));
 	       Options.insert(std::make_pair("BASE_URI", baseURI));
 	       Options.insert(std::make_pair("REPO_URI", URI));
+	       Options.insert(std::make_pair("TARGET_OF", "deb-src"));
 	       Options.insert(std::make_pair("CREATED_BY", *T));
 	       Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options);
 
@@ -183,11 +186,15 @@ void foreachTarget(std::string const URI, std::string const Dist,
 	       std::map Options;
 	       Options.insert(std::make_pair("SITE", Site));
 	       Options.insert(std::make_pair("RELEASE", Release));
-	       Options.insert(std::make_pair("COMPONENT", (*I)->Section));
-	       Options.insert(std::make_pair("LANGUAGE", *l));
-	       Options.insert(std::make_pair("ARCHITECTURE", a->first));
+	       if (MetaKey.find("$(COMPONENT)") != std::string::npos)
+		  Options.insert(std::make_pair("COMPONENT", (*I)->Section));
+	       if (MetaKey.find("$(LANGUAGE)") != std::string::npos)
+		  Options.insert(std::make_pair("LANGUAGE", *l));
+	       if (MetaKey.find("$(ARCHITECTURE)") != std::string::npos)
+		  Options.insert(std::make_pair("ARCHITECTURE", a->first));
 	       Options.insert(std::make_pair("BASE_URI", baseURI));
 	       Options.insert(std::make_pair("REPO_URI", URI));
+	       Options.insert(std::make_pair("TARGET_OF", "deb"));
 	       Options.insert(std::make_pair("CREATED_BY", *T));
 	       Call(MetaKey, ShortDesc, LongDesc, IsOptional, Options);
 
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 72d35ddcc..33fb48e35 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -133,8 +133,10 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(ARCHITECTURE);
       APT_CASE(BASE_URI);
       APT_CASE(REPO_URI);
+      APT_CASE(TARGET_OF);
       APT_CASE(CREATED_BY);
 #undef APT_CASE
+      case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
    }
    std::map::const_iterator const M = Options.find(Key);
    if (M == Options.end())
@@ -142,6 +144,20 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
    return M->second;
 }
 									/*}}}*/
+std::string IndexTarget::Format(std::string format) const		/*{{{*/
+{
+   for (std::map::const_iterator O = Options.begin(); O != Options.end(); ++O)
+   {
+      format = SubstVar(format, std::string("$(") + O->first + ")", O->second);
+   }
+   format = SubstVar(format, "$(METAKEY)", MetaKey);
+   format = SubstVar(format, "$(SHORTDESC)", ShortDesc);
+   format = SubstVar(format, "$(DESCRIPTION)", Description);
+   format = SubstVar(format, "$(URI)", URI);
+   format = SubstVar(format, "$(FILENAME)", Option(IndexTarget::FILENAME));
+   return format;
+}
+									/*}}}*/
 
 pkgIndexTargetFile::pkgIndexTargetFile(IndexTarget const &Target, bool const Trusted) :/*{{{*/
    pkgIndexFile(Trusted), Target(Target)
@@ -162,7 +178,7 @@ std::string pkgIndexTargetFile::Describe(bool Short) const		/*{{{*/
 									/*}}}*/
 std::string pkgIndexTargetFile::IndexFileName() const			/*{{{*/
 {
-   std::string const s =_config->FindDir("Dir::State::lists") + URItoFileName(Target.URI);
+   std::string const s = Target.Option(IndexTarget::FILENAME);
    if (FileExists(s))
       return s;
 
diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index c38cf0bf0..220c415ac 100644
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -79,8 +79,11 @@ class IndexTarget							/*{{{*/
       BASE_URI,
       REPO_URI,
       CREATED_BY,
+      TARGET_OF,
+      FILENAME,
    };
    std::string Option(OptionKeys const Key) const;
+   std::string Format(std::string format) const;
 };
 									/*}}}*/
 
diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc
index 41aab81f6..458051bf2 100644
--- a/apt-private/private-cmndline.cc
+++ b/apt-private/private-cmndline.cc
@@ -163,6 +163,10 @@ static bool addArgumentsAPTGet(std::vector &Args, char const
       // once sbuild is fixed, this option can be removed
       addArg('f', "fix-broken", "APT::Get::Fix-Broken", 0);
    }
+   else if (CmdMatches("files"))
+   {
+      addArg(0,"format","APT::Get::Files::Format", CommandLine::HasArg);
+   }
    else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
 	    CmdMatches("markauto", "unmarkauto")) // deprecated commands
       ;
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index c1f78523c..0fff2db58 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -86,6 +86,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -1602,6 +1603,91 @@ static bool DoChangelog(CommandLine &CmdL)
    return true;
 }
 									/*}}}*/
+// DoFiles - Lists all IndexTargets					/*{{{*/
+static std::string format_key(std::string key)
+{
+   // deb822 is case-insensitive, but the human eye prefers candy
+   std::transform(key.begin(), key.end(), key.begin(), ::tolower);
+   key[0] = ::toupper(key[0]);
+   size_t found = key.find("_uri");
+   if (found != std::string::npos)
+      key.replace(found, 4, "-URI");
+   while ((found = key.find('_')) != std::string::npos)
+   {
+      key[found] = '-';
+      key[found + 1] = ::toupper(key[found + 1]);
+   }
+   return key;
+}
+static bool DoFiles(CommandLine &CmdL)
+{
+   pkgCacheFile CacheFile;
+   pkgSourceList *SrcList = CacheFile.GetSourceList();
+
+   if (SrcList == NULL)
+      return false;
+
+   std::string const Format = _config->Find("APT::Get::Files::Format");
+   bool Filtered = CmdL.FileSize() > 1;
+   for (pkgSourceList::const_iterator S = SrcList->begin(); S != SrcList->end(); ++S)
+   {
+      std::vector const targets = (*S)->GetIndexTargets();
+      std::map AddOptions;
+      AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+
+      for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
+      {
+	 std::ostringstream stanza;
+	 if (Filtered || Format.empty())
+	 {
+	    stanza << "MetaKey: " << T->MetaKey << "\n"
+	       << "ShortDesc: " << T->ShortDesc << "\n"
+	       << "Description: " << T->Description << "\n"
+	       << "URI: " << T->URI << "\n"
+	       << "Filename: " << T->Option(IndexTarget::FILENAME) << "\n"
+	       << "Optional: " << (T->IsOptional ? "yes" : "no") << "\n";
+	    for (std::map::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
+	       stanza << format_key(O->first) << ": " << O->second << "\n";
+	    for (std::map::const_iterator O = T->Options.begin(); O != T->Options.end(); ++O)
+	       stanza << format_key(O->first) << ": " << O->second << "\n";
+	    stanza << "\n";
+
+	    if (Filtered)
+	    {
+	       // that is a bit crude, but good enough for now
+	       bool found = true;
+	       std::string haystack = std::string("\n") + stanza.str() + "\n";
+	       std::transform(haystack.begin(), haystack.end(), haystack.begin(), ::tolower);
+	       size_t const filesize = CmdL.FileSize() - 1;
+	       for (size_t i = 0; i != filesize; ++i)
+	       {
+		  std::string needle = std::string("\n") + CmdL.FileList[i + 1] + "\n";
+		  std::transform(needle.begin(), needle.end(), needle.begin(), ::tolower);
+		  if (haystack.find(needle) != std::string::npos)
+		     continue;
+		  found = false;
+		  break;
+	       }
+	       if (found == false)
+		  continue;
+	    }
+	 }
+
+	 if (Format.empty())
+	    cout << stanza.str();
+	 else
+	 {
+	    std::string out = T->Format(Format);
+	    for (std::map::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
+	       out = SubstVar(out, std::string("$(") + O->first + ")", O->second);
+	    cout << out << std::endl;
+	 }
+      }
+   }
+
+   return true;
+}
+									/*}}}*/
 // ShowHelp - Show a help screen					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -1716,6 +1802,7 @@ int main(int argc,const char *argv[])					/*{{{*/
 				   {"source",&DoSource},
                                    {"download",&DoDownload},
                                    {"changelog",&DoChangelog},
+				   {"files",&DoFiles},
 				   {"moo",&DoMoo},
 				   {"help",&ShowHelp},
                                    {0,0}};
diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index ab833440b..a47dac2d4 100644
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -47,11 +47,13 @@ All files which should be downloaded (nicknamed 'Targets') are mentioned
 below the APT::Acquire::Targets scope. 'deb' is here the type of the
 sources.list entry the file should be acquired for. The only other
 supported value is hence 'deb-src'. Beware: You can't specify multiple
-types here and you can't download the same MetaKey form multiple types!
+types here and you can't download the same (evaluated) MetaKey from
+multiple types!
 
 After the type you can pick any valid and unique string which preferable
 refers to the file it downloads (In the example we picked 'Packages').
-This string is never shown or used.
+This string is used as identifier for the target class and accessible as
+'Created-By' e.g. in the "apt-get files" output as detailed below.
 
 All targets have three main properties you can define:
 * MetaKey: The identifier of the file to be downloaded as used in the
@@ -123,11 +125,12 @@ by the acquire system. The following variables are known; note that
 unknown variables have no default value nor are they touched: They are
 printed literally.
 
-* $(SITE): An identifier of the site we access, e.g.
-  "http://example.org/".
+* $(SITE): An identifier of the site we access as seen in sources.list,
+  e.g. "http://example.org/debian" or "file:/path/to/a/repository".
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
   "stretch".  Note that flat-style repositories do not have a archive-
   or codename per-se, so the value might very well be just "/" or so.
+  Again, as seen in the sources.list.
 * $(COMPONENT): as given in the sources.list, e.g. "main", "non-free" or
   "universe".  Note that flat-style repositories again do not really
   have a meaningful value here.
@@ -137,3 +140,79 @@ printed literally.
   APT::Architectures (potentially modified by sources.list options),
   e.g. "amd64", "i386" or "armel" for the 'deb' type. In type 'deb-src'
   this variable has the value "source".
+
+Note that while more variables might exist in the implementation, these
+are to be considered undefined and their usage strongly discouraged. If
+you have a need for another variable contact us instead.
+
+# Accessing files
+
+Do NOT hardcode specific file locations, names or compression types in
+your application! You will notice that the configuration options give
+you no choice over where the downloaded files will be stored. This is by
+design so multiple applications can download and use the same file
+rather than each and every one of them potentially downloads and uses
+its own copy somewhere on disk.
+
+"apt-get files" can be used to get the location as well as other
+information about all files downloaded (aka: you will see Packages,
+Sources and Translation-* files here as well). Provide a line of the
+default output format as parameter to filter out all entries which do
+not have such a line. With --format, you can further more define your
+own output style. The variables are what you see in the output, just all
+uppercase and wrapped in $(), as in the configuration file.
+
+To get all the filenames of all Translation-en files you can e.g. call:
+	apt-get files --format '$(FILENAME)' "Created-By: Translations" "Language: en"
+
+Accessing this information via libapt is done by reading the
+sources.lists (pkgSourceList), iterating over the metaIndex objects this
+creates and calling GetIndexTargets() on them. See the sourcecode of
+"apt-get files" for a complete example.
+
+Remarks on the available fields:
+* MetaKey, ShortDesc, Description, Site, Release: as defined
+  by the configuration and described further above.
+* Created-By: configuration entity responsible for this target
+* Target-Of: type of the sources.list entry
+* URI, Repo-URI: avoid using. Contains potentially username/password.
+  Prefer 'Site', especially for display.
+* Filename: The mentioned file doesn't need to exist, e.g. because the
+  file wasn't downloaded yet – or it does exist compressed. libapt users
+  are encouraged to use FileFd to open such files as it can handle
+  decompression transparently.
+* Trusted: As of the last 'apt-get update' call denoting if e.g. apt-get
+  would print the unauthenticated warning while installing packages
+  mentioned in this file (example assumes Packages file) [Not really
+  a property of the target itself, but of the metaIndex].
+* Optional: Decodes the option of the same name from the configuration.
+  Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
+* Language, Architecture, Component: as defined further above, but with
+  the catch that they might be missing if they don't effect the target
+  (aka: They weren't used while evaluating the MetaKey template).
+
+Again, additional fields might be visible in certain implementation, but
+you should avoid using them and instead talk to us about a portable
+implementation.
+
+# Multiple application requiring the same files
+
+It is highly encouraged that applications talk to each other and to us
+about which files they require. It is usually best to have a common
+package ship the configuration needed to get the files, but specific
+needs might require specific solutions. Again: talk to us.
+
+# Acquiring files not mentioned in the Release file
+
+You can't. This is by design as these files couldn't be verified to not
+be modified in transit, corrupted by the download process or simple if
+they are present at all on the server, which would require apt to probe
+for them. APT did this in the past for legacy reasons, we do not intend
+to go back to these dark times.
+
+This is also why you can't request files from a different server. It
+would have the additional problem that this server might not even be
+accessible (e.g. proxy settings) or that local sources (file:/, cdrom:/)
+start requesting online files…
+
+In other words: We would be opening Pandora's box.
diff --git a/doc/apt-get.8.xml b/doc/apt-get.8.xml
index a372a0d30..da077afa7 100644
--- a/doc/apt-get.8.xml
+++ b/doc/apt-get.8.xml
@@ -243,6 +243,21 @@
        
      
 
+     
+	Displays by default a deb822 formatted listing of
+	      information about all data files apt-get
+		 update would download. Supports a
+	       option to modify the output format as
+	      well as accepts lines of the default output to filter the records
+	      by. The command is mainly used as an interface for external tools
+	      working with APT to get information as well as filenames for
+	      downloaded files so they can use them as well instead of
+	      downloading them again on their own. Detailed documentation is
+	      omitted here and can instead be found in the source tree in
+	      doc/acquire-additional-files.txt.
+	   
+	
+     
 
 
    
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 4a6845f40..971cfac2e 100755
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -48,6 +48,7 @@ Get:1 http://localhost:8080 unstable/main amd64 Contents [$(stat -c%s aptarchive
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64' 'aptarchive/dists/unstable/main/Contents-amd64'
 
 # no automatic uncompress based on the name please,
@@ -72,6 +73,7 @@ Get:1 http://localhost:8080 unstable/main amd64 Contents.gz [$(stat -c%s aptarch
 Reading package lists..." aptget update
 
 testequal 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' find rootdir/var/lib/apt/lists -name '*Contents*'
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
 testsuccess cmp 'rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz' 'aptarchive/dists/unstable/main/Contents-amd64.gz'
 
 rm -f rootdir/etc/apt/apt.conf.d/content-target.conf
-- 
cgit v1.2.3-70-g09d2


From b07aeb1a6e24825e534167a737043441e871de9f Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 12 Jun 2015 02:08:53 +0200
Subject: store Release files data in the Cache

We used to read the Release file for each Packages file and store the
data in the PackageFile struct even through potentially many Packages
(and Translation-*) files could use the same data. The point of the
exercise isn't the duplicated data through. Having the Release files as
first-class citizens in the Cache allows us to properly track their
state as well as allows us to use the information also for files which
aren't in the cache, but where we know to which Release file they
belong (Sources are an example for this).

This modifies the pkgCache structs, especially the PackagesFile struct
which depending on how libapt users access the data in these structs can
mean huge breakage or no visible change. As a single data point:
aptitude seems to be fine with this. Even if there is breakage it is
trivial to fix in a backportable way while avoiding breakage for
everyone would be a huge pain for us.

Note that not all PackageFile structs have a corresponding ReleaseFile.
In particular the dpkg/status file as well as *.deb files have not. As
these have only a Archive property need, the Component property takes
over this duty and the ReleaseFile remains zero. This is also the reason
why it isn't needed nor particularily recommended to change from
PackagesFile to ReleaseFile blindly. Sticking with the earlier is
usually the better option.
---
 apt-pkg/acquire-item.cc              |   6 +-
 apt-pkg/cacheiterators.h             |  46 ++++++-
 apt-pkg/clean.cc                     |   2 +-
 apt-pkg/deb/debindexfile.cc          |  42 ++-----
 apt-pkg/deb/debindexfile.h           |   1 -
 apt-pkg/deb/deblistparser.cc         |  37 ------
 apt-pkg/deb/deblistparser.h          |   6 +-
 apt-pkg/deb/debmetaindex.cc          | 124 ++++++++++++++++++-
 apt-pkg/deb/debmetaindex.h           |   6 +
 apt-pkg/deb/dpkgpm.cc                |   4 +-
 apt-pkg/depcache.cc                  |   6 +-
 apt-pkg/edsp.cc                      |   2 +-
 apt-pkg/edsp/edspindexfile.cc        |   5 +-
 apt-pkg/edsp/edsplistparser.cc       |   4 +-
 apt-pkg/edsp/edsplistparser.h        |   4 +-
 apt-pkg/metaindex.cc                 |  17 +++
 apt-pkg/metaindex.h                  |   6 +
 apt-pkg/pkgcache.cc                  |  77 ++++++++----
 apt-pkg/pkgcache.h                   |  75 ++++++++++--
 apt-pkg/pkgcachegen.cc               | 226 +++++++++++++++++++++++++++--------
 apt-pkg/pkgcachegen.h                |  10 +-
 apt-pkg/policy.cc                    |  15 ++-
 apt-pkg/versionmatch.cc              |  45 +++----
 apt-pkg/versionmatch.h               |   4 +-
 cmdline/apt-cache.cc                 |  11 +-
 test/integration/framework           |   9 +-
 test/integration/test-policy-pinning |   2 +-
 27 files changed, 573 insertions(+), 219 deletions(-)

(limited to 'cmdline')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index d4191f00e..d6e9ccbe0 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -2568,7 +2568,7 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire * const Owner,pkgSourceList * const Sour
    // Skip not source sources, they do not have file fields.
    for (; Vf.end() == false; ++Vf)
    {
-      if ((Vf.File()->Flags & pkgCache::Flag::NotSource) != 0)
+      if (Vf.File().Flagged(pkgCache::Flag::NotSource))
 	 continue;
       break;
    }
@@ -2636,14 +2636,14 @@ bool pkgAcqArchive::QueueNext()
    {
       pkgCache::PkgFileIterator const PkgF = Vf.File();
       // Ignore not source sources
-      if ((PkgF->Flags & pkgCache::Flag::NotSource) != 0)
+      if (PkgF.Flagged(pkgCache::Flag::NotSource))
 	 continue;
 
       // Try to cross match against the source list
       pkgIndexFile *Index;
       if (Sources->FindIndex(PkgF, Index) == false)
 	    continue;
-      LocalSource = (PkgF->Flags & pkgCache::Flag::LocalSource) == pkgCache::Flag::LocalSource;
+      LocalSource = PkgF.Flagged(pkgCache::Flag::LocalSource);
 
       // only try to get a trusted package from another source if that source
       // is also trusted
diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h
index fe798799c..301da6fc4 100644
--- a/apt-pkg/cacheiterators.h
+++ b/apt-pkg/cacheiterators.h
@@ -367,27 +367,61 @@ class pkgCache::PrvIterator : public Iterator {
 	}
 };
 									/*}}}*/
-// Package file								/*{{{*/
-class pkgCache::PkgFileIterator : public Iterator {
+// Release file								/*{{{*/
+class pkgCache::RlsFileIterator : public Iterator {
 	protected:
-	inline PackageFile* OwnerPointer() const {
-		return (Owner != 0) ? Owner->PkgFileP : 0;
+	inline ReleaseFile* OwnerPointer() const {
+		return (Owner != 0) ? Owner->RlsFileP : 0;
 	}
 
 	public:
 	// Iteration
-	void operator ++(int) {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile;}
+	void operator ++(int) {if (S != Owner->RlsFileP) S = Owner->RlsFileP + S->NextFile;}
 	inline void operator ++() {operator ++(0);}
 
 	// Accessors
 	inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;}
 	inline const char *Archive() const {return S->Archive == 0?0:Owner->StrP + S->Archive;}
-	inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;}
 	inline const char *Version() const {return S->Version == 0?0:Owner->StrP + S->Version;}
 	inline const char *Origin() const {return S->Origin == 0?0:Owner->StrP + S->Origin;}
 	inline const char *Codename() const {return S->Codename ==0?0:Owner->StrP + S->Codename;}
 	inline const char *Label() const {return S->Label == 0?0:Owner->StrP + S->Label;}
 	inline const char *Site() const {return S->Site == 0?0:Owner->StrP + S->Site;}
+	inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return (S->Flags & flag) == flag; }
+
+	bool IsOk();
+	std::string RelStr();
+
+	// Constructors
+	inline RlsFileIterator() : Iterator() {}
+	inline RlsFileIterator(pkgCache &Owner) : Iterator(Owner, Owner.RlsFileP) {}
+	inline RlsFileIterator(pkgCache &Owner,ReleaseFile *Trg) : Iterator(Owner, Trg) {}
+};
+									/*}}}*/
+// Package file								/*{{{*/
+class pkgCache::PkgFileIterator : public Iterator {
+	protected:
+	inline PackageFile* OwnerPointer() const {
+		return (Owner != 0) ? Owner->PkgFileP : 0;
+	}
+
+	public:
+	// Iteration
+	void operator ++(int) {if (S != Owner->PkgFileP) S = Owner->PkgFileP + S->NextFile;}
+	inline void operator ++() {operator ++(0);}
+
+	// Accessors
+	inline const char *FileName() const {return S->FileName == 0?0:Owner->StrP + S->FileName;}
+	inline pkgCache::RlsFileIterator ReleaseFile() const {return RlsFileIterator(*Owner, Owner->RlsFileP + S->Release);}
+	inline const char *Archive() const {return S->Release == 0 ? Component() : ReleaseFile().Archive();}
+	inline const char *Version() const {return S->Release == 0 ? NULL : ReleaseFile().Version();}
+	inline const char *Origin() const {return S->Release == 0 ? NULL : ReleaseFile().Origin();}
+	inline const char *Codename() const {return S->Release == 0 ? NULL : ReleaseFile().Codename();}
+	inline const char *Label() const {return S->Release == 0 ? NULL : ReleaseFile().Label();}
+	inline const char *Site() const {return S->Release == 0 ? NULL : ReleaseFile().Site();}
+	inline bool Flagged(pkgCache::Flag::ReleaseFileFlags const flag) const {return S->Release== 0 ? false : ReleaseFile().Flagged(flag);}
+	inline bool Flagged(pkgCache::Flag::PkgFFlags const flag) const {return (S->Flags & flag) == flag;}
+	inline const char *Component() const {return S->Component == 0?0:Owner->StrP + S->Component;}
 	inline const char *Architecture() const {return S->Architecture == 0?0:Owner->StrP + S->Architecture;}
 	inline const char *IndexType() const {return S->IndexType == 0?0:Owner->StrP + S->IndexType;}
 
diff --git a/apt-pkg/clean.cc b/apt-pkg/clean.cc
index 6edce5b6d..0fca60ba9 100644
--- a/apt-pkg/clean.cc
+++ b/apt-pkg/clean.cc
@@ -106,7 +106,7 @@ bool pkgArchiveCleaner::Go(std::string Dir,pkgCache &Cache)
 		 J.end() == false; ++J)
 	    {
 	       if (CleanInstalled == true &&
-		   (J.File()->Flags & pkgCache::Flag::NotSource) != 0)
+		   J.File().Flagged(pkgCache::Flag::NotSource))
 		  continue;
 	       IsFetchable = true;
 	       break;
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index 7aad65c0e..3a79cbc58 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -128,7 +128,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    if (Dist.empty())
       Dist = "/";
    ::URI Tmp(URI);
-   if (Gen.SelectFile(PackageFile,Tmp.Host,*this) == false)
+   if (Gen.SelectFile(PackageFile, *this, Target.Option(IndexTarget::COMPONENT)) == false)
       return _error->Error("Problem with SelectFile %s",PackageFile.c_str());
 
    // Store the IMS information
@@ -136,31 +136,10 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
    pkgCacheGenerator::Dynamic DynFile(File);
    File->Size = Pkg.FileSize();
    File->mtime = Pkg.ModificationTime();
-   
+
    if (Gen.MergeList(Parser) == false)
       return _error->Error("Problem with MergeList %s",PackageFile.c_str());
 
-   // Check the release file
-   string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease");
-   bool releaseExists = false;
-   if (FileExists(ReleaseFile) == true)
-      releaseExists = true;
-   else
-      ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release");
-
-   if (releaseExists == true || FileExists(ReleaseFile) == true)
-   {
-      FileFd Rel;
-      // Beware: The 'Release' file might be clearsigned in case the
-      // signature for an 'InRelease' file couldn't be checked
-      if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false)
-	 return false;
-
-      if (_error->PendingError() == true)
-	 return false;
-      Parser.LoadReleaseInfo(File, Rel, Target.Option(IndexTarget::COMPONENT));
-   }
-   
    return true;
 }
 									/*}}}*/
@@ -221,17 +200,17 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
      debTranslationsParser TransParser(&Trans);
      if (_error->PendingError() == true)
        return false;
-     
+
      if (Prog != NULL)
 	Prog->SubProgress(0, Target.Description);
-     if (Gen.SelectFile(TranslationFile,string(),*this) == false)
+     if (Gen.SelectFile(TranslationFile, *this, Target.Option(IndexTarget::COMPONENT), pkgCache::Flag::NotSource) == false)
        return _error->Error("Problem with SelectFile %s",TranslationFile.c_str());
 
      // Store the IMS information
      pkgCache::PkgFileIterator TransFile = Gen.GetCurFile();
      TransFile->Size = Trans.FileSize();
      TransFile->mtime = Trans.ModificationTime();
-   
+
      if (Gen.MergeList(TransParser) == false)
        return _error->Error("Problem with MergeList %s",TranslationFile.c_str());
    }
@@ -305,18 +284,17 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 
    if (Prog != NULL)
       Prog->SubProgress(0,File);
-   if (Gen.SelectFile(File,string(),*this,pkgCache::Flag::NotSource) == false)
+   if (Gen.SelectFile(File, *this, "now", pkgCache::Flag::NotSource) == false)
       return _error->Error("Problem with SelectFile %s",File.c_str());
 
    // Store the IMS information
    pkgCache::PkgFileIterator CFile = Gen.GetCurFile();
+   pkgCacheGenerator::Dynamic DynFile(CFile);
    CFile->Size = Pkg.FileSize();
    CFile->mtime = Pkg.ModificationTime();
-   map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "now");
-   CFile->Archive = storage;
-   
+
    if (Gen.MergeList(Parser) == false)
-      return _error->Error("Problem with MergeList %s",File.c_str());   
+      return _error->Error("Problem with MergeList %s",File.c_str());
    return true;
 }
 									/*}}}*/
@@ -431,7 +409,7 @@ bool debDebPkgFileIndex::Merge(pkgCacheGenerator& Gen, OpProgress* Prog) const
 
    // and give it to the list parser
    debDebFileParser Parser(DebControl, DebFile);
-   if(Gen.SelectFile(DebFile, "local", *this, pkgCache::Flag::LocalSource) == false)
+   if(Gen.SelectFile(DebFile, *this, "now", pkgCache::Flag::LocalSource) == false)
       return _error->Error("Problem with SelectFile %s", DebFile.c_str());
 
    pkgCache::PkgFileIterator File = Gen.GetCurFile();
diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h
index dd3f212b2..6b8c78e5a 100644
--- a/apt-pkg/deb/debindexfile.h
+++ b/apt-pkg/deb/debindexfile.h
@@ -45,7 +45,6 @@ class APT_HIDDEN debStatusIndex : public pkgIndexFile
    virtual bool HasPackages() const {return true;};
    virtual unsigned long Size() const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
-   bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog, unsigned long const Flag) const;
    virtual pkgCache::PkgFileIterator FindInCache(pkgCache &Cache) const;
 
    debStatusIndex(std::string File);
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index b80b57bc4..c5e77b0ff 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -951,43 +951,6 @@ bool debListParser::Step()
    return false;
 }
 									/*}}}*/
-// ListParser::LoadReleaseInfo - Load the release information		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,
-				    FileFd &File, string component)
-{
-   // apt-secure does no longer download individual (per-section) Release
-   // file. to provide Component pinning we use the section name now
-   map_stringitem_t const storage = StoreString(pkgCacheGenerator::MIXED, component);
-   FileI->Component = storage;
-
-   pkgTagFile TagFile(&File, File.Size());
-   pkgTagSection Section;
-   if (_error->PendingError() == true || TagFile.Step(Section) == false)
-      return false;
-
-   std::string data;
-   #define APT_INRELEASE(TYPE, TAG, STORE) \
-   data = Section.FindS(TAG); \
-   if (data.empty() == false) \
-   { \
-      map_stringitem_t const storage = StoreString(pkgCacheGenerator::TYPE, data); \
-      STORE = storage; \
-   }
-   APT_INRELEASE(MIXED, "Suite", FileI->Archive)
-   APT_INRELEASE(MIXED, "Component", FileI->Component)
-   APT_INRELEASE(VERSIONNUMBER, "Version", FileI->Version)
-   APT_INRELEASE(MIXED, "Origin", FileI->Origin)
-   APT_INRELEASE(MIXED, "Codename", FileI->Codename)
-   APT_INRELEASE(MIXED, "Label", FileI->Label)
-   #undef APT_INRELEASE
-   Section.FindFlag("NotAutomatic", FileI->Flags, pkgCache::Flag::NotAutomatic);
-   Section.FindFlag("ButAutomaticUpgrades", FileI->Flags, pkgCache::Flag::ButAutomaticUpgrades);
-
-   return !_error->PendingError();
-}
-									/*}}}*/
 // ListParser::GetPrio - Convert the priority from a string		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h
index 6279d8399..420d5ff08 100644
--- a/apt-pkg/deb/deblistparser.h
+++ b/apt-pkg/deb/deblistparser.h
@@ -80,9 +80,9 @@ class APT_HIDDEN debListParser : public pkgCacheGenerator::ListParser
    virtual map_filesize_t Size() {return Section.size();};
 
    virtual bool Step();
-   
-   bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File,
-			std::string section);
+
+   bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File,
+			std::string const §ion);
 
    APT_PUBLIC static const char *ParseDepends(const char *Start,const char *Stop,
 	 std::string &Package,std::string &Ver,unsigned int &Op);
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 8e4c2be2d..994b95849 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -1,5 +1,6 @@
 #include 
 
+#include 
 #include 
 #include 
 #include 
@@ -10,10 +11,12 @@
 #include 
 #include 
 #include 
-#include 
 #include 
+#include 
+#include 
+#include 
+#include 
 
-#include 
 #include 
 #include 
 #include 
@@ -21,6 +24,11 @@
 #include 
 #include 
 
+#include 
+#include 
+#include 
+#include 
+
 using namespace std;
 
 string debReleaseIndex::MetaIndexInfo(const char *Type) const
@@ -37,6 +45,10 @@ string debReleaseIndex::MetaIndexInfo(const char *Type) const
    Info += Type;
    return Info;
 }
+std::string debReleaseIndex::Describe() const
+{
+   return MetaIndexInfo("Release");
+}
 
 string debReleaseIndex::MetaIndexFile(const char *Type) const
 {
@@ -339,6 +351,114 @@ debReleaseIndex::debSectionEntry::debSectionEntry (string const &Section,
 		bool const &IsSrc): Section(Section), IsSrc(IsSrc)
 {}
 
+static bool ReleaseFileName(debReleaseIndex const * const That, std::string &ReleaseFile)
+{
+   ReleaseFile = That->MetaIndexFile("InRelease");
+   bool releaseExists = false;
+   if (FileExists(ReleaseFile) == true)
+      releaseExists = true;
+   else
+   {
+      ReleaseFile = That->MetaIndexFile("Release");
+      if (FileExists(ReleaseFile))
+	 releaseExists = true;
+   }
+   return releaseExists;
+}
+
+bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/*{{{*/
+{
+   std::string ReleaseFile;
+   bool const releaseExists = ReleaseFileName(this, ReleaseFile);
+
+   ::URI Tmp(URI);
+   if (Gen.SelectReleaseFile(ReleaseFile, Tmp.Host) == false)
+      return _error->Error("Problem with SelectReleaseFile %s", ReleaseFile.c_str());
+
+   if (releaseExists == false)
+      return true;
+
+   FileFd Rel;
+   // Beware: The 'Release' file might be clearsigned in case the
+   // signature for an 'InRelease' file couldn't be checked
+   if (OpenMaybeClearSignedFile(ReleaseFile, Rel) == false)
+      return false;
+   if (_error->PendingError() == true)
+      return false;
+
+   // Store the IMS information
+   pkgCache::RlsFileIterator File = Gen.GetCurRlsFile();
+   pkgCacheGenerator::Dynamic DynFile(File);
+   // Rel can't be used as this is potentially a temporary file
+   struct stat Buf;
+   if (stat(ReleaseFile.c_str(), &Buf) != 0)
+      return _error->Errno("fstat", "Unable to stat file %s", ReleaseFile.c_str());
+   File->Size = Buf.st_size;
+   File->mtime = Buf.st_mtime;
+
+   pkgTagFile TagFile(&Rel, Rel.Size());
+   pkgTagSection Section;
+   if (_error->PendingError() == true || TagFile.Step(Section) == false)
+      return false;
+
+   std::string data;
+   #define APT_INRELEASE(TYPE, TAG, STORE) \
+   data = Section.FindS(TAG); \
+   if (data.empty() == false) \
+   { \
+      map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::TYPE, data); \
+      STORE = storage; \
+   }
+   APT_INRELEASE(MIXED, "Suite", File->Archive)
+   APT_INRELEASE(VERSIONNUMBER, "Version", File->Version)
+   APT_INRELEASE(MIXED, "Origin", File->Origin)
+   APT_INRELEASE(MIXED, "Codename", File->Codename)
+   APT_INRELEASE(MIXED, "Label", File->Label)
+   #undef APT_INRELEASE
+   Section.FindFlag("NotAutomatic", File->Flags, pkgCache::Flag::NotAutomatic);
+   Section.FindFlag("ButAutomaticUpgrades", File->Flags, pkgCache::Flag::ButAutomaticUpgrades);
+
+   return !_error->PendingError();
+}
+									/*}}}*/
+// ReleaseIndex::FindInCache - Find this index				/*{{{*/
+pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
+{
+   std::string ReleaseFile;
+   bool const releaseExists = ReleaseFileName(this, ReleaseFile);
+
+   pkgCache::RlsFileIterator File = Cache.RlsFileBegin();
+   for (; File.end() == false; ++File)
+   {
+       if (File->FileName == 0 || ReleaseFile != File.FileName())
+	 continue;
+
+       // empty means the file does not exist by "design"
+       if (releaseExists == false && File->Size == 0)
+	  return File;
+
+      struct stat St;
+      if (stat(File.FileName(),&St) != 0)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "ReleaseIndex::FindInCache - stat failed on " << File.FileName() << std::endl;
+	 return pkgCache::RlsFileIterator(Cache);
+      }
+      if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "ReleaseIndex::FindInCache - size (" << St.st_size << " <> " << File->Size
+			<< ") or mtime (" << St.st_mtime << " <> " << File->mtime
+			<< ") doesn't match for " << File.FileName() << std::endl;
+	 return pkgCache::RlsFileIterator(Cache);
+      }
+      return File;
+   }
+
+   return File;
+}
+									/*}}}*/
+
 class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type
 {
    protected:
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 0b1b08432..7e9942710 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -20,6 +20,8 @@ class pkgAcquire;
 class pkgIndexFile;
 class debDebPkgFileIndex;
 class IndexTarget;
+class pkgCacheGenerator;
+class OpProgress;
 
 class APT_HIDDEN debReleaseIndex : public metaIndex {
    public:
@@ -48,6 +50,10 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const;
    virtual std::vector GetIndexTargets() const;
 
+   virtual std::string Describe() const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
+
    std::string MetaIndexInfo(const char *Type) const;
    std::string MetaIndexFile(const char *Types) const;
    std::string MetaIndexURI(const char *Type) const;
diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index a7a66c75d..6ee939edd 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -205,8 +205,10 @@ pkgCache::VerIterator FindNowVersion(const pkgCache::PkgIterator &Pkg)
    for (Ver = Pkg.VersionList(); Ver.end() == false; ++Ver)
       for (pkgCache::VerFileIterator Vf = Ver.FileList(); Vf.end() == false; ++Vf)
 	 for (pkgCache::PkgFileIterator F = Vf.File(); F.end() == false; ++F)
-	    if (F->Archive != 0 && strcmp(F.Archive(), "now") == 0)
+	 {
+	    if (F.Archive() != 0 && strcmp(F.Archive(), "now") == 0)
 	       return Ver;
+	 }
    return Ver;
 }
 									/*}}}*/
diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc
index b73c336db..36e1ac9ec 100644
--- a/apt-pkg/depcache.cc
+++ b/apt-pkg/depcache.cc
@@ -1685,13 +1685,13 @@ pkgCache::VerIterator pkgDepCache::Policy::GetCandidateVer(PkgIterator const &Pk
       
       for (VerFileIterator J = I.FileList(); J.end() == false; ++J)
       {
-	 if ((J.File()->Flags & Flag::NotSource) != 0)
+	 if (J.File().Flagged(Flag::NotSource))
 	    continue;
 
 	 /* Stash the highest version of a not-automatic source, we use it
 	    if there is nothing better */
-	 if ((J.File()->Flags & Flag::NotAutomatic) != 0 ||
-	     (J.File()->Flags & Flag::ButAutomaticUpgrades) != 0)
+	 if (J.File().Flagged(Flag::NotAutomatic) ||
+	     J.File().Flagged(Flag::ButAutomaticUpgrades))
 	 {
 	    if (Last.end() == true)
 	       Last = I;
diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc
index 3c6a7e30f..41cc2cdfe 100644
--- a/apt-pkg/edsp.cc
+++ b/apt-pkg/edsp.cc
@@ -129,7 +129,7 @@ void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgI
       signed short const p = Cache.GetPolicy().GetPriority(File);
       if (Pin < p)
 	 Pin = p;
-      if ((File->Flags & pkgCache::Flag::NotSource) != pkgCache::Flag::NotSource) {
+      if (File.Flagged(pkgCache::Flag::NotSource) == false) {
 	 string Release = File.RelStr();
 	 if (!Release.empty())
 	    Releases.insert(Release);
diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc
index d00536362..43dd44a79 100644
--- a/apt-pkg/edsp/edspindexfile.cc
+++ b/apt-pkg/edsp/edspindexfile.cc
@@ -49,15 +49,14 @@ bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 
    if (Prog != NULL)
       Prog->SubProgress(0,File);
-   if (Gen.SelectFile(File,std::string(),*this) == false)
+   if (Gen.SelectFile(File, *this, "edsp") == false)
       return _error->Error("Problem with SelectFile %s",File.c_str());
 
    // Store the IMS information
    pkgCache::PkgFileIterator CFile = Gen.GetCurFile();
+   pkgCacheGenerator::Dynamic DynFile(CFile);
    CFile->Size = Pkg.FileSize();
    CFile->mtime = Pkg.ModificationTime();
-   map_stringitem_t const storage = Gen.StoreString(pkgCacheGenerator::MIXED, "edsp::scenario");
-   CFile->Archive = storage;
 
    if (Gen.MergeList(Parser) == false)
       return _error->Error("Problem with MergeList %s",File.c_str());
diff --git a/apt-pkg/edsp/edsplistparser.cc b/apt-pkg/edsp/edsplistparser.cc
index 212dc7840..d62abe709 100644
--- a/apt-pkg/edsp/edsplistparser.cc
+++ b/apt-pkg/edsp/edsplistparser.cc
@@ -86,8 +86,8 @@ bool edspListParser::ParseStatus(pkgCache::PkgIterator &Pkg,
 }
 									/*}}}*/
 // ListParser::LoadReleaseInfo - Load the release information		/*{{{*/
-APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::PkgFileIterator & /*FileI*/,
-				    FileFd & /*File*/, std::string /*component*/)
+APT_CONST bool edspListParser::LoadReleaseInfo(pkgCache::RlsFileIterator & /*FileI*/,
+				    FileFd & /*File*/, std::string const &/*component*/)
 {
    return true;
 }
diff --git a/apt-pkg/edsp/edsplistparser.h b/apt-pkg/edsp/edsplistparser.h
index 86cd77606..abe2ef139 100644
--- a/apt-pkg/edsp/edsplistparser.h
+++ b/apt-pkg/edsp/edsplistparser.h
@@ -34,8 +34,8 @@ class APT_HIDDEN edspListParser : public debListParser
    virtual MD5SumValue Description_md5();
    virtual unsigned short VersionHash();
 
-   bool LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,FileFd &File,
-			std::string section);
+   bool LoadReleaseInfo(pkgCache::RlsFileIterator &FileI,FileFd &File,
+			std::string const §ion);
 
    edspListParser(FileFd *File, std::string const &Arch = "");
 
diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc
index 31a8ec009..35ab6c53b 100644
--- a/apt-pkg/metaindex.cc
+++ b/apt-pkg/metaindex.cc
@@ -1,4 +1,5 @@
 // Include Files                                                       /*{{{*/
+#include 
 #include 
 #include 
 
@@ -22,6 +23,22 @@ std::string metaIndex::LocalFileName() const
 }
 #endif
 
+std::string metaIndex::Describe() const
+{
+   return "Release";
+}
+
+pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache) const
+{
+   return pkgCache::RlsFileIterator(Cache);
+}
+
+bool metaIndex::Merge(pkgCacheGenerator &Gen,OpProgress *) const
+{
+   return Gen.SelectReleaseFile("", "");
+}
+
+
 metaIndex::metaIndex(std::string const &URI, std::string const &Dist,
       char const * const Type)
 : Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(false)
diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h
index 4e5de8be0..20c879a89 100644
--- a/apt-pkg/metaindex.h
+++ b/apt-pkg/metaindex.h
@@ -23,6 +23,8 @@ using std::string;
 
 class pkgAcquire;
 class IndexTarget;
+class pkgCacheGenerator;
+class OpProgress;
 
 class metaIndex
 {
@@ -51,6 +53,10 @@ class metaIndex
    virtual std::vector *GetIndexFiles() = 0;
    virtual bool IsTrusted() const = 0;
 
+   virtual std::string Describe() const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
+
    metaIndex(std::string const &URI, std::string const &Dist,
              char const * const Type);
    virtual ~metaIndex();
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index 864ae0f60..9fe382108 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -61,6 +61,7 @@ pkgCache::Header::Header()
    HeaderSz = sizeof(pkgCache::Header);
    GroupSz = sizeof(pkgCache::Group);
    PackageSz = sizeof(pkgCache::Package);
+   ReleaseFileSz = sizeof(pkgCache::ReleaseFile);
    PackageFileSz = sizeof(pkgCache::PackageFile);
    VersionSz = sizeof(pkgCache::Version);
    DescriptionSz = sizeof(pkgCache::Description);
@@ -74,6 +75,7 @@ pkgCache::Header::Header()
    VersionCount = 0;
    DescriptionCount = 0;
    DependsCount = 0;
+   ReleaseFileCount = 0;
    PackageFileCount = 0;
    VerFileCount = 0;
    DescFileCount = 0;
@@ -82,6 +84,7 @@ pkgCache::Header::Header()
    MaxDescFileSize = 0;
    
    FileList = 0;
+   RlsFileList = 0;
 #if APT_PKG_ABI < 413
    APT_IGNORE_DEPRECATED(StringList = 0;)
 #endif
@@ -102,6 +105,7 @@ bool pkgCache::Header::CheckSizes(Header &Against) const
    if (HeaderSz == Against.HeaderSz &&
        GroupSz == Against.GroupSz &&
        PackageSz == Against.PackageSz &&
+       ReleaseFileSz == Against.ReleaseFileSz &&
        PackageFileSz == Against.PackageFileSz &&
        VersionSz == Against.VersionSz &&
        DescriptionSz == Against.DescriptionSz &&
@@ -140,6 +144,7 @@ bool pkgCache::ReMap(bool const &Errorchecks)
    PkgP = (Package *)Map.Data();
    VerFileP = (VerFile *)Map.Data();
    DescFileP = (DescFile *)Map.Data();
+   RlsFileP = (ReleaseFile *)Map.Data();
    PkgFileP = (PackageFile *)Map.Data();
    VerP = (Version *)Map.Data();
    DescP = (Description *)Map.Data();
@@ -814,7 +819,7 @@ APT_PURE bool pkgCache::VerIterator::Downloadable() const
 {
    VerFileIterator Files = FileList();
    for (; Files.end() == false; ++Files)
-      if ((Files.File()->Flags & pkgCache::Flag::NotSource) != pkgCache::Flag::NotSource)
+      if (Files.File().Flagged(pkgCache::Flag::NotSource) == false)
 	 return true;
    return false;
 }
@@ -828,7 +833,7 @@ APT_PURE bool pkgCache::VerIterator::Automatic() const
    VerFileIterator Files = FileList();
    for (; Files.end() == false; ++Files)
       // Do not check ButAutomaticUpgrades here as it is kind of automatic…
-      if ((Files.File()->Flags & pkgCache::Flag::NotAutomatic) != pkgCache::Flag::NotAutomatic)
+      if (Files.File().Flagged(pkgCache::Flag::NotAutomatic) == false)
 	 return true;
    return false;
 }
@@ -861,27 +866,27 @@ string pkgCache::VerIterator::RelStr() const
    for (pkgCache::VerFileIterator I = this->FileList(); I.end() == false; ++I)
    {
       // Do not print 'not source' entries'
-      pkgCache::PkgFileIterator File = I.File();
-      if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
+      pkgCache::PkgFileIterator const File = I.File();
+      if (File.Flagged(pkgCache::Flag::NotSource))
 	 continue;
 
       // See if we have already printed this out..
       bool Seen = false;
       for (pkgCache::VerFileIterator J = this->FileList(); I != J; ++J)
       {
-	 pkgCache::PkgFileIterator File2 = J.File();
-	 if (File2->Label == 0 || File->Label == 0)
+	 pkgCache::PkgFileIterator const File2 = J.File();
+	 if (File2.Label() == 0 || File.Label() == 0)
 	    continue;
 
 	 if (strcmp(File.Label(),File2.Label()) != 0)
 	    continue;
 	 
-	 if (File2->Version == File->Version)
+	 if (File2.Version() == File.Version())
 	 {
 	    Seen = true;
 	    break;
 	 }
-	 if (File2->Version == 0 || File->Version == 0)
+	 if (File2.Version() == 0 || File.Version() == 0)
 	    break;
 	 if (strcmp(File.Version(),File2.Version()) == 0)
 	    Seen = true;
@@ -895,12 +900,12 @@ string pkgCache::VerIterator::RelStr() const
       else
 	 First = false;
       
-      if (File->Label != 0)
+      if (File.Label() != 0)
 	 Res = Res + File.Label() + ':';
 
-      if (File->Archive != 0)
+      if (File.Archive() != 0)
       {
-	 if (File->Version == 0)
+	 if (File.Version() == 0)
 	    Res += File.Archive();
 	 else
 	    Res = Res + File.Version() + '/' +  File.Archive();
@@ -908,7 +913,7 @@ string pkgCache::VerIterator::RelStr() const
       else
       {
 	 // No release file, print the host name that this came from
-	 if (File->Site == 0 || File.Site()[0] == 0)
+	 if (File.Site() == 0 || File.Site()[0] == 0)
 	    Res += "localhost";
 	 else
 	    Res += File.Site();
@@ -931,12 +936,12 @@ const char * pkgCache::VerIterator::MultiArchType() const
    return "none";
 }
 									/*}}}*/
-// PkgFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
+// RlsFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
 // ---------------------------------------------------------------------
 /* This stats the file and compares its stats with the ones that were
-   stored during generation. Date checks should probably also be 
+   stored during generation. Date checks should probably also be
    included here. */
-bool pkgCache::PkgFileIterator::IsOk()
+bool pkgCache::RlsFileIterator::IsOk()
 {
    struct stat Buf;
    if (stat(FileName(),&Buf) != 0)
@@ -948,10 +953,8 @@ bool pkgCache::PkgFileIterator::IsOk()
    return true;
 }
 									/*}}}*/
-// PkgFileIterator::RelStr - Return the release string			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-string pkgCache::PkgFileIterator::RelStr()
+// RlsFileIterator::RelStr - Return the release string			/*{{{*/
+string pkgCache::RlsFileIterator::RelStr()
 {
    string Res;
    if (Version() != 0)
@@ -964,8 +967,40 @@ string pkgCache::PkgFileIterator::RelStr()
       Res = Res + (Res.empty() == true?"n=":",n=")  + Codename();
    if (Label() != 0)
       Res = Res + (Res.empty() == true?"l=":",l=")  + Label();
-   if (Component() != 0)
-      Res = Res + (Res.empty() == true?"c=":",c=")  + Component();
+   return Res;
+}
+									/*}}}*/
+// PkgFileIterator::IsOk - Checks if the cache is in sync with the file	/*{{{*/
+// ---------------------------------------------------------------------
+/* This stats the file and compares its stats with the ones that were
+   stored during generation. Date checks should probably also be
+   included here. */
+bool pkgCache::PkgFileIterator::IsOk()
+{
+   struct stat Buf;
+   if (stat(FileName(),&Buf) != 0)
+      return false;
+
+   if (Buf.st_size != (signed)S->Size || Buf.st_mtime != S->mtime)
+      return false;
+
+   return true;
+}
+									/*}}}*/
+string pkgCache::PkgFileIterator::RelStr()				/*{{{*/
+{
+   std::string Res;
+   if (ReleaseFile() == 0)
+   {
+      if (Component() != 0)
+	 Res = Res + (Res.empty() == true?"a=":",a=")  + Component();
+   }
+   else
+   {
+      Res = ReleaseFile().RelStr();
+      if (Component() != 0)
+	 Res = Res + (Res.empty() == true?"c=":",c=")  + Component();
+   }
    if (Architecture() != 0)
       Res = Res + (Res.empty() == true?"b=":",b=")  + Architecture();
    return Res;
diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h
index b4d56611a..696b3b94d 100644
--- a/apt-pkg/pkgcache.h
+++ b/apt-pkg/pkgcache.h
@@ -133,6 +133,7 @@ class pkgCache								/*{{{*/
    struct Header;
    struct Group;
    struct Package;
+   struct ReleaseFile;
    struct PackageFile;
    struct Version;
    struct Description;
@@ -150,6 +151,7 @@ class pkgCache								/*{{{*/
    class DescIterator;
    class DepIterator;
    class PrvIterator;
+   class RlsFileIterator;
    class PkgFileIterator;
    class VerFileIterator;
    class DescFileIterator;
@@ -192,9 +194,11 @@ class pkgCache								/*{{{*/
       enum PkgFlags {Auto=(1<<0),Essential=(1<<3),Important=(1<<4)};
       enum PkgFFlags {
 	 NotSource=(1<<0), /*!< packages can't be fetched from here, e.g. dpkg/status file */
-	 NotAutomatic=(1<<1), /*!< archive has a default pin of 1 */
-	 ButAutomaticUpgrades=(1<<2), /*!< (together with the previous) archive has a default pin of 100 */
-	 LocalSource=(1<<3), /*!< local sources can't and will not be verified by hashes */
+	 LocalSource=(1<<1), /*!< local sources can't and will not be verified by hashes */
+      };
+      enum ReleaseFileFlags {
+	 NotAutomatic=(1<<0), /*!< archive has a default pin of 1 */
+	 ButAutomaticUpgrades=(1<<1), /*!< (together with the previous) archive has a default pin of 100 */
       };
    };
    
@@ -215,6 +219,7 @@ class pkgCache								/*{{{*/
    Package *PkgP;
    VerFile *VerFileP;
    DescFile *DescFileP;
+   ReleaseFile *RlsFileP;
    PackageFile *PkgFileP;
    Version *VerP;
    Description *DescP;
@@ -247,6 +252,8 @@ class pkgCache								/*{{{*/
    inline PkgIterator PkgEnd();
    inline PkgFileIterator FileBegin();
    inline PkgFileIterator FileEnd();
+   inline RlsFileIterator RlsFileBegin();
+   inline RlsFileIterator RlsFileEnd();
 
    inline bool MultiArchCache() const { return MultiArchEnabled; }
    inline char const * NativeArch();
@@ -296,6 +303,7 @@ struct pkgCache::Header
    unsigned short HeaderSz;
    unsigned short GroupSz;
    unsigned short PackageSz;
+   unsigned short ReleaseFileSz;
    unsigned short PackageFileSz;
    unsigned short VersionSz;
    unsigned short DescriptionSz;
@@ -314,6 +322,7 @@ struct pkgCache::Header
    map_id_t VersionCount;
    map_id_t DescriptionCount;
    map_id_t DependsCount;
+   map_fileid_t ReleaseFileCount;
    map_fileid_t PackageFileCount;
    map_fileid_t VerFileCount;
    map_fileid_t DescFileCount;
@@ -324,6 +333,9 @@ struct pkgCache::Header
        The PackageFile structures are singly linked lists that represent
        all package files that have been merged into the cache. */
    map_pointer_t FileList;
+   /** \brief index of the first ReleaseFile structure */
+   map_pointer_t RlsFileList;
+
 #if APT_PKG_ABI < 413
    APT_DEPRECATED map_pointer_t StringList;
 #endif
@@ -482,15 +494,14 @@ struct pkgCache::Package
    unsigned long Flags;
 };
 									/*}}}*/
-// Package File structure						/*{{{*/
-/** \brief stores information about the files used to generate the cache
+// Release File structure						/*{{{*/
+/** \brief stores information about the release files used to generate the cache
 
-    Package files are referenced by Version structures to be able to know
-    after the generation still from which Packages file includes this Version
-    as we need this information later on e.g. for pinning. */
-struct pkgCache::PackageFile
+    PackageFiles reference ReleaseFiles as we need to keep record of which
+    version belongs to which release e.g. for pinning. */
+struct pkgCache::ReleaseFile
 {
-   /** \brief physical disk file that this PackageFile represents */
+   /** \brief physical disk file that this ReleaseFile represents */
    map_stringitem_t FileName;
    /** \brief the release information
 
@@ -498,13 +509,47 @@ struct pkgCache::PackageFile
        release information means. */
    map_stringitem_t Archive;
    map_stringitem_t Codename;
-   map_stringitem_t Component;
    map_stringitem_t Version;
    map_stringitem_t Origin;
    map_stringitem_t Label;
-   map_stringitem_t Architecture;
    /** \brief The site the index file was fetched from */
    map_stringitem_t Site;
+
+   /** \brief Size of the file
+
+       Used together with the modification time as a
+       simple check to ensure that the Packages
+       file has not been altered since Cache generation. */
+   map_filesize_t Size;
+   /** \brief Modification time for the file */
+   time_t mtime;
+
+   /** @TODO document PackageFile::Flags */
+   unsigned long Flags;
+
+   // Linked list
+   /** \brief Link to the next ReleaseFile in the Cache */
+   map_pointer_t NextFile;
+   /** \brief unique sequel ID */
+   should_be_map_fileid_t ID;
+};
+									/*}}}*/
+// Package File structure						/*{{{*/
+/** \brief stores information about the files used to generate the cache
+
+    Package files are referenced by Version structures to be able to know
+    after the generation still from which Packages file includes this Version
+    as we need this information later on e.g. for pinning. */
+struct pkgCache::PackageFile
+{
+   /** \brief physical disk file that this PackageFile represents */
+   map_stringitem_t FileName;
+   /** \brief the release information */
+   map_pointer_t Release;
+
+   map_stringitem_t Component;
+   map_stringitem_t Architecture;
+
    /** \brief indicates what sort of index file this is
 
        @TODO enumerate at least the possible indexes */
@@ -744,6 +789,11 @@ inline pkgCache::PkgFileIterator pkgCache::FileBegin()
        {return PkgFileIterator(*this,PkgFileP + HeaderP->FileList);}
 inline pkgCache::PkgFileIterator pkgCache::FileEnd()
        {return PkgFileIterator(*this,PkgFileP);}
+inline pkgCache::RlsFileIterator pkgCache::RlsFileBegin()
+       {return RlsFileIterator(*this,RlsFileP + HeaderP->RlsFileList);}
+inline pkgCache::RlsFileIterator pkgCache::RlsFileEnd()
+       {return RlsFileIterator(*this,RlsFileP);}
+
 
 // Oh I wish for Real Name Space Support
 class pkgCache::Namespace						/*{{{*/
@@ -755,6 +805,7 @@ class pkgCache::Namespace						/*{{{*/
    typedef pkgCache::DescIterator DescIterator;
    typedef pkgCache::DepIterator DepIterator;
    typedef pkgCache::PrvIterator PrvIterator;
+   typedef pkgCache::RlsFileIterator RlsFileIterator;
    typedef pkgCache::PkgFileIterator PkgFileIterator;
    typedef pkgCache::VerFileIterator VerFileIterator;
    typedef pkgCache::Version Version;
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index ba454f057..2d0e3960d 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -54,10 +54,8 @@ using std::string;
 /* We set the dirty flag and make sure that is written to the disk */
 pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) :
 		    Map(*pMap), Cache(pMap,false), Progress(Prog),
-		    FoundFileDeps(0)
+		     CurrentRlsFile(NULL), CurrentFile(NULL), FoundFileDeps(0)
 {
-   CurrentFile = 0;
-
    if (_error->PendingError() == true)
       return;
 
@@ -145,6 +143,7 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
    Cache.ReMap(false);
 
    CurrentFile += (pkgCache::PackageFile const * const) newMap - (pkgCache::PackageFile const * const) oldMap;
+   CurrentRlsFile += (pkgCache::ReleaseFile const * const) newMap - (pkgCache::ReleaseFile const * const) oldMap;
 
    for (std::vector::const_iterator i = Dynamic::toReMap.begin();
 	i != Dynamic::toReMap.end(); ++i)
@@ -167,6 +166,9 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM
    for (std::vector::const_iterator i = Dynamic::toReMap.begin();
 	i != Dynamic::toReMap.end(); ++i)
       (*i)->ReMap(oldMap, newMap);
+   for (std::vector::const_iterator i = Dynamic::toReMap.begin();
+	i != Dynamic::toReMap.end(); ++i)
+      (*i)->ReMap(oldMap, newMap);
 }									/*}}}*/
 // CacheGenerator::WriteStringInMap					/*{{{*/
 map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String,
@@ -1072,13 +1074,47 @@ bool pkgCacheGenerator::ListParser::SameVersion(unsigned short const Hash,/*{{{*
    return Hash == Ver->Hash;
 }
 									/*}}}*/
+// CacheGenerator::SelectReleaseFile - Select the current release file the indexes belong to	/*{{{*/
+bool pkgCacheGenerator::SelectReleaseFile(const string &File,const string &Site,
+				   unsigned long Flags)
+{
+   if (File.empty() && Site.empty())
+   {
+      CurrentRlsFile = NULL;
+      return true;
+   }
+
+   // Get some space for the structure
+   map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentRlsFile));
+   if (unlikely(idxFile == 0))
+      return false;
+   CurrentRlsFile = Cache.RlsFileP + idxFile;
+
+   // Fill it in
+   map_stringitem_t const idxFileName = WriteStringInMap(File);
+   map_stringitem_t const idxSite = StoreString(MIXED, Site);
+   if (unlikely(idxFileName == 0 || idxSite == 0))
+      return false;
+   CurrentRlsFile->FileName = idxFileName;
+   CurrentRlsFile->Site = idxSite;
+   CurrentRlsFile->NextFile = Cache.HeaderP->RlsFileList;
+   CurrentRlsFile->Flags = Flags;
+   CurrentRlsFile->ID = Cache.HeaderP->ReleaseFileCount;
+   RlsFileName = File;
+   Cache.HeaderP->RlsFileList = CurrentRlsFile - Cache.RlsFileP;
+   Cache.HeaderP->ReleaseFileCount++;
+
+   return true;
+}
+									/*}}}*/
 // CacheGenerator::SelectFile - Select the current file being parsed	/*{{{*/
 // ---------------------------------------------------------------------
 /* This is used to select which file is to be associated with all newly
    added versions. The caller is responsible for setting the IMS fields. */
-bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
-				   const pkgIndexFile &Index,
-				   unsigned long Flags)
+bool pkgCacheGenerator::SelectFile(std::string const &File,
+				   pkgIndexFile const &Index,
+				   std::string const &Component,
+				   unsigned long const Flags)
 {
    // Get some space for the structure
    map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile));
@@ -1088,18 +1124,24 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site,
 
    // Fill it in
    map_stringitem_t const idxFileName = WriteStringInMap(File);
-   map_stringitem_t const idxSite = StoreString(MIXED, Site);
-   if (unlikely(idxFileName == 0 || idxSite == 0))
+   if (unlikely(idxFileName == 0))
       return false;
    CurrentFile->FileName = idxFileName;
-   CurrentFile->Site = idxSite;
    CurrentFile->NextFile = Cache.HeaderP->FileList;
-   CurrentFile->Flags = Flags;
    CurrentFile->ID = Cache.HeaderP->PackageFileCount;
    map_stringitem_t const idxIndexType = StoreString(MIXED, Index.GetType()->Label);
    if (unlikely(idxIndexType == 0))
       return false;
    CurrentFile->IndexType = idxIndexType;
+   map_stringitem_t const component = StoreString(pkgCacheGenerator::MIXED, Component);
+   if (unlikely(component == 0))
+      return false;
+   CurrentFile->Component = component;
+   CurrentFile->Flags = Flags;
+   if (CurrentRlsFile != NULL)
+      CurrentFile->Release = CurrentRlsFile - Cache.RlsFileP;
+   else
+      CurrentFile->Release = 0;
    PkgFileName = File;
    Cache.HeaderP->FileList = CurrentFile - Cache.PkgFileP;
    Cache.HeaderP->PackageFileCount++;
@@ -1174,35 +1216,59 @@ static bool CheckValidity(const string &CacheFile,
       _error->Discard();
       return false;
    }
-   
+
+   SPtrArray RlsVisited = new bool[Cache.HeaderP->ReleaseFileCount];
+   memset(RlsVisited,0,sizeof(*RlsVisited)*Cache.HeaderP->ReleaseFileCount);
+   std::vector Files;
+   for (pkgSourceList::const_iterator i = List.begin(); i != List.end(); ++i)
+   {
+      if (Debug == true)
+	 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
+      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache);
+      if (RlsFile.end() == true)
+      {
+	 if (Debug == true)
+	    std::clog << "FindInCache returned end-Pointer" << std::endl;
+	 return false;
+      }
+
+      RlsVisited[RlsFile->ID] = true;
+      if (Debug == true)
+	 std::clog << "with ID " << RlsFile->ID << " is valid" << std::endl;
+
+      std::vector  *Indexes = (*i)->GetIndexFiles();
+      for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
+	 if ((*j)->HasPackages())
+	    Files.push_back (*j);
+   }
+   for (unsigned I = 0; I != Cache.HeaderP->ReleaseFileCount; ++I)
+      if (RlsVisited[I] == false)
+      {
+	 if (Debug == true)
+	    std::clog << "RlsFile with ID" << I << " wasn't visited" << std::endl;
+	 return false;
+      }
+
+   for (; Start != End; ++Start)
+      Files.push_back(*Start);
+
    /* Now we check every index file, see if it is in the cache,
       verify the IMS data and check that it is on the disk too.. */
    SPtrArray Visited = new bool[Cache.HeaderP->PackageFileCount];
    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
-   for (; Start != End; ++Start)
+   for (std::vector::const_reverse_iterator PkgFile = Files.rbegin(); PkgFile != Files.rend(); ++PkgFile)
    {
       if (Debug == true)
-	 std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
-      if ((*Start)->HasPackages() == false)
-      {
-         if (Debug == true)
-	    std::clog << "Has NO packages" << std::endl;
-	 continue;
-      }
-    
-      if ((*Start)->Exists() == false)
+	 std::clog << "Checking PkgFile " << (*PkgFile)->Describe() << ": ";
+      if ((*PkgFile)->Exists() == false)
       {
-#if 0 // mvo: we no longer give a message here (Default Sources spec)
-	 _error->WarningE("stat",_("Couldn't stat source package list %s"),
-			  (*Start)->Describe().c_str());
-#endif
          if (Debug == true)
 	    std::clog << "file doesn't exist" << std::endl;
 	 continue;
       }
 
       // FindInCache is also expected to do an IMS check.
-      pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
+      pkgCache::PkgFileIterator File = (*PkgFile)->FindInCache(Cache);
       if (File.end() == true)
       {
 	 if (Debug == true)
@@ -1214,15 +1280,15 @@ static bool CheckValidity(const string &CacheFile,
       if (Debug == true)
 	 std::clog << "with ID " << File->ID << " is valid" << std::endl;
    }
-   
+
    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
       if (Visited[I] == false)
       {
 	 if (Debug == true)
-	    std::clog << "File with ID" << I << " wasn't visited" << std::endl;
+	    std::clog << "PkgFile with ID" << I << " wasn't visited" << std::endl;
 	 return false;
       }
-   
+
    if (_error->PendingError() == true)
    {
       if (Debug == true)
@@ -1243,9 +1309,20 @@ static bool CheckValidity(const string &CacheFile,
 // ---------------------------------------------------------------------
 /* Size is kind of an abstract notion that is only used for the progress
    meter */
-static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
+static map_filesize_t ComputeSize(pkgSourceList const * const List, FileIterator Start,FileIterator End)
 {
    map_filesize_t TotalSize = 0;
+   if (List !=  NULL)
+   {
+      for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
+      {
+	 std::vector  *Indexes = (*i)->GetIndexFiles();
+	 for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j)
+	    if ((*j)->HasPackages() == true)
+	       TotalSize += (*j)->Size();
+      }
+   }
+
    for (; Start < End; ++Start)
    {
       if ((*Start)->HasPackages() == false)
@@ -1261,11 +1338,63 @@ static map_filesize_t ComputeSize(FileIterator Start,FileIterator End)
 static bool BuildCache(pkgCacheGenerator &Gen,
 		       OpProgress *Progress,
 		       map_filesize_t &CurrentSize,map_filesize_t TotalSize,
+		       pkgSourceList const * const List,
 		       FileIterator Start, FileIterator End)
 {
+   std::vector Files;
+   bool const HasFileDeps = Gen.HasFileDeps();
+
+   if (List !=  NULL)
+   {
+      for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
+      {
+	 if ((*i)->FindInCache(Gen.GetCache()).end() == false)
+	 {
+	    _error->Warning("Duplicate sources.list entry %s",
+		  (*i)->Describe().c_str());
+	    continue;
+	 }
+
+	 if ((*i)->Merge(Gen, Progress) == false)
+	    return false;
+
+	 std::vector  *Indexes = (*i)->GetIndexFiles();
+	 for (std::vector::const_iterator I = Indexes->begin(); I != Indexes->end(); ++I)
+	 {
+	    if (HasFileDeps)
+	       Files.push_back(*I);
+
+	    if ((*I)->HasPackages() == false)
+	       continue;
+
+	    if ((*I)->Exists() == false)
+	       continue;
+
+	    if ((*I)->FindInCache(Gen.GetCache()).end() == false)
+	    {
+	       _error->Warning("Duplicate sources.list entry %s",
+		     (*I)->Describe().c_str());
+	       continue;
+	    }
+
+	    map_filesize_t Size = (*I)->Size();
+	    if (Progress != NULL)
+	       Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists"));
+	    CurrentSize += Size;
+
+	    if ((*I)->Merge(Gen,Progress) == false)
+	       return false;
+	 }
+      }
+   }
+
+   Gen.SelectReleaseFile("", "");
    FileIterator I;
    for (I = Start; I != End; ++I)
    {
+      if (HasFileDeps)
+	 Files.push_back(*I);
+
       if ((*I)->HasPackages() == false)
 	 continue;
       
@@ -1288,13 +1417,13 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 	 return false;
    }   
 
-   if (Gen.HasFileDeps() == true)
+   if (HasFileDeps == true)
    {
       if (Progress != NULL)
 	 Progress->Done();
-      TotalSize = ComputeSize(Start, End);
+      TotalSize = ComputeSize(List, Start, End);
       CurrentSize = 0;
-      for (I = Start; I != End; ++I)
+      for (std::vector::const_iterator I = Files.begin(); I != Files.end(); ++I)
       {
 	 map_filesize_t Size = (*I)->Size();
 	 if (Progress != NULL)
@@ -1339,6 +1468,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
 
    std::vector Files;
+   /*
    for (std::vector::const_iterator i = List.begin();
         i != List.end();
         ++i)
@@ -1349,8 +1479,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 	   ++j)
          Files.push_back (*j);
    }
-   
-   map_filesize_t const EndOfSource = Files.size();
+*/
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
@@ -1442,8 +1571,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
    // Lets try the source cache.
    map_filesize_t CurrentSize = 0;
    map_filesize_t TotalSize = 0;
-   if (CheckValidity(SrcCacheFile, List, Files.begin(),
-		     Files.begin()+EndOfSource) == true)
+   if (CheckValidity(SrcCacheFile, List, Files.end(),
+		     Files.end()) == true)
    {
       if (Debug == true)
 	 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
@@ -1455,28 +1584,28 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
 				SCacheF.Size()) == false)
 	 return false;
 
-      TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
+      TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
 
       // Build the status cache
       pkgCacheGenerator Gen(Map.Get(),Progress);
       if (_error->PendingError() == true)
 	 return false;
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin()+EndOfSource,Files.end()) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
+		     Files.begin(),Files.end()) == false)
 	 return false;
    }
    else
    {
       if (Debug == true)
 	 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
-      TotalSize = ComputeSize(Files.begin(),Files.end());
+      TotalSize = ComputeSize(&List, Files.begin(),Files.end());
       
       // Build the source cache
       pkgCacheGenerator Gen(Map.Get(),Progress);
       if (_error->PendingError() == true)
 	 return false;
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin(),Files.begin()+EndOfSource) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, &List,
+		     Files.end(),Files.end()) == false)
 	 return false;
       
       // Write it back
@@ -1503,8 +1632,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress
       }
       
       // Build the status cache
-      if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		     Files.begin()+EndOfSource,Files.end()) == false)
+      if (BuildCache(Gen, Progress, CurrentSize, TotalSize, NULL,
+		     Files.begin(), Files.end()) == false)
 	 return false;
    }
    if (Debug == true)
@@ -1536,7 +1665,6 @@ APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **Ou
 bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap)
 {
    std::vector Files;
-   map_filesize_t EndOfSource = Files.size();
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
@@ -1544,7 +1672,7 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    map_filesize_t CurrentSize = 0;
    map_filesize_t TotalSize = 0;
    
-   TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
+   TotalSize = ComputeSize(NULL, Files.begin(), Files.end());
    
    // Build the status cache
    if (Progress != NULL)
@@ -1552,8 +1680,8 @@ bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **O
    pkgCacheGenerator Gen(Map.Get(),Progress);
    if (_error->PendingError() == true)
       return false;
-   if (BuildCache(Gen,Progress,CurrentSize,TotalSize,
-		  Files.begin()+EndOfSource,Files.end()) == false)
+   if (BuildCache(Gen,Progress,CurrentSize,TotalSize, NULL,
+		  Files.begin(), Files.end()) == false)
       return false;
 
    if (_error->PendingError() == true)
diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h
index c4ace713d..2795d09d6 100644
--- a/apt-pkg/pkgcachegen.h
+++ b/apt-pkg/pkgcachegen.h
@@ -69,7 +69,9 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    DynamicMMap ⤅
    pkgCache Cache;
    OpProgress *Progress;
-   
+
+   std::string RlsFileName;
+   pkgCache::ReleaseFile *CurrentRlsFile;
    std::string PkgFileName;
    pkgCache::PackageFile *CurrentFile;
 
@@ -100,12 +102,14 @@ class APT_HIDDEN pkgCacheGenerator					/*{{{*/
    inline map_stringitem_t StoreString(enum StringType const type, const std::string &S) {return StoreString(type, S.c_str(),S.length());};
 
    void DropProgress() {Progress = 0;};
-   bool SelectFile(const std::string &File,const std::string &Site,pkgIndexFile const &Index,
-		   unsigned long Flags = 0);
+   bool SelectFile(const std::string &File,pkgIndexFile const &Index, std::string const &Component, unsigned long Flags = 0);
+   bool SelectReleaseFile(const std::string &File, const std::string &Site, unsigned long Flags = 0);
    bool MergeList(ListParser &List,pkgCache::VerIterator *Ver = 0);
    inline pkgCache &GetCache() {return Cache;};
    inline pkgCache::PkgFileIterator GetCurFile() 
          {return pkgCache::PkgFileIterator(Cache,CurrentFile);};
+   inline pkgCache::RlsFileIterator GetCurRlsFile() 
+         {return pkgCache::RlsFileIterator(Cache,CurrentRlsFile);};
 
    bool HasFileDeps() {return FoundFileDeps;};
    bool MergeFileProvides(ListParser &List);
diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index 00693ce54..bd40ad2d9 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -63,9 +63,9 @@ pkgPolicy::pkgPolicy(pkgCache *Owner) : Pins(0), PFPriority(0), Cache(Owner)
       pkgVersionMatch vm("", pkgVersionMatch::None);
       for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
       {
-	 if ((F->Archive != 0 && vm.ExpressionMatches(DefRel, F.Archive()) == true) ||
-	     (F->Codename != 0 && vm.ExpressionMatches(DefRel, F.Codename()) == true) ||
-	     (F->Version != 0 && vm.ExpressionMatches(DefRel, F.Version()) == true) ||
+	 if (vm.ExpressionMatches(DefRel, F.Archive()) ||
+	     vm.ExpressionMatches(DefRel, F.Codename()) ||
+	     vm.ExpressionMatches(DefRel, F.Version()) ||
 	     (DefRel.length() > 2 && DefRel[1] == '='))
 	    found = true;
       }
@@ -86,11 +86,11 @@ bool pkgPolicy::InitDefaults()
    for (pkgCache::PkgFileIterator I = Cache->FileBegin(); I != Cache->FileEnd(); ++I)
    {
       PFPriority[I->ID] = 500;
-      if ((I->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
+      if (I.Flagged(pkgCache::Flag::NotSource))
 	 PFPriority[I->ID] = 100;
-      else if ((I->Flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades)
+      else if (I.Flagged(pkgCache::Flag::ButAutomaticUpgrades))
 	 PFPriority[I->ID] = 100;
-      else if ((I->Flags & pkgCache::Flag::NotAutomatic) == pkgCache::Flag::NotAutomatic)
+      else if (I.Flagged(pkgCache::Flag::NotAutomatic))
 	 PFPriority[I->ID] = 1;
    }
 
@@ -170,8 +170,7 @@ pkgCache::VerIterator pkgPolicy::GetCandidateVer(pkgCache::PkgIterator const &Pk
 	    then it is not a candidate for installation, ever. This weeds
 	    out bogus entries that may be due to config-file states, or
 	    other. */
-	 if ((VF.File()->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource &&
-	     instVer == false)
+	 if (VF.File().Flagged(pkgCache::Flag::NotSource) && instVer == false)
 	    continue;
 
 	 signed Prio = PFPriority[VF.File()->ID];
diff --git a/apt-pkg/versionmatch.cc b/apt-pkg/versionmatch.cc
index 284098bdf..86c1b7d4a 100644
--- a/apt-pkg/versionmatch.cc
+++ b/apt-pkg/versionmatch.cc
@@ -137,7 +137,10 @@ pkgVersionMatch::pkgVersionMatch(string Data,MatchType Type) : Type(Type)
 // ---------------------------------------------------------------------
 /* */
 bool pkgVersionMatch::MatchVer(const char *A,string B,bool Prefix)
-{   
+{
+   if (A == NULL)
+      return false;
+
    const char *Ab = A;
    const char *Ae = Ab + strlen(A);
    
@@ -178,13 +181,16 @@ pkgCache::VerIterator pkgVersionMatch::Find(pkgCache::PkgIterator Pkg)
    // This will be Ended by now.
    return Ver;
 }
+									/*}}}*/
 
 #ifndef FNM_CASEFOLD
 #define FNM_CASEFOLD 0
 #endif
 
-bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string)
+bool pkgVersionMatch::ExpressionMatches(const char *pattern, const char *string)/*{{{*/
 {
+   if (pattern == NULL || string == NULL)
+      return false;
    if (pattern[0] == '/') {
       size_t length = strlen(pattern);
       if (pattern[length - 1] == '/') {
@@ -230,38 +236,30 @@ bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File)
 	 return false;
 
       if (RelVerStr.empty() == false)
-	 if (File->Version == 0 ||
-	     (MatchVer(File.Version(),RelVerStr,RelVerPrefixMatch) == false &&
-	      ExpressionMatches(RelVerStr, File.Version()) == false))
+	 if (MatchVer(File.Version(),RelVerStr,RelVerPrefixMatch) == false &&
+	       ExpressionMatches(RelVerStr, File.Version()) == false)
 	    return false;
       if (RelOrigin.empty() == false)
-	 if (File->Origin == 0 || !ExpressionMatches(RelOrigin,File.Origin()))
+	 if (!ExpressionMatches(RelOrigin,File.Origin()))
 	    return false;
       if (RelArchive.empty() == false)
-	 if (File->Archive == 0 ||
-	     !ExpressionMatches(RelArchive,File.Archive()))
+	 if (!ExpressionMatches(RelArchive,File.Archive()))
             return false;
       if (RelCodename.empty() == false)
-	 if (File->Codename == 0 ||
-	     !ExpressionMatches(RelCodename,File.Codename()))
+	 if (!ExpressionMatches(RelCodename,File.Codename()))
             return false;
       if (RelRelease.empty() == false)
-	 if ((File->Archive == 0 ||
-	     !ExpressionMatches(RelRelease,File.Archive())) &&
-             (File->Codename == 0 ||
-	      !ExpressionMatches(RelRelease,File.Codename())))
+	 if (!ExpressionMatches(RelRelease,File.Archive()) &&
+             !ExpressionMatches(RelRelease,File.Codename()))
 	       return false;
       if (RelLabel.empty() == false)
-	 if (File->Label == 0 ||
-	     !ExpressionMatches(RelLabel,File.Label()))
+	 if (!ExpressionMatches(RelLabel,File.Label()))
 	    return false;
       if (RelComponent.empty() == false)
-	 if (File->Component == 0 ||
-	     !ExpressionMatches(RelComponent,File.Component()))
+	 if (!ExpressionMatches(RelComponent,File.Component()))
 	    return false;
       if (RelArchitecture.empty() == false)
-	 if (File->Architecture == 0 ||
-	     !ExpressionMatches(RelArchitecture,File.Architecture()))
+	 if (!ExpressionMatches(RelArchitecture,File.Architecture()))
 	    return false;
       return true;
    }
@@ -269,11 +267,14 @@ bool pkgVersionMatch::FileMatch(pkgCache::PkgFileIterator File)
    if (Type == Origin)
    {
       if (OrSite.empty() == false) {
-	 if (File->Site == 0)
+	 if (File.Site() == NULL)
 	    return false;
       } else // so we are talking about file:// or status file
-	 if (strcmp(File.Site(),"") == 0 && File->Archive != 0 && strcmp(File.Archive(),"now") == 0) // skip the status file
+      {
+	 pkgCache::RlsFileIterator const RlsFile = File.ReleaseFile();
+	 if (strcmp(File.Site(),"") == 0 && RlsFile->Archive != 0 && strcmp(RlsFile.Archive(),"now") == 0) // skip the status file
 	    return false;
+      }
       return (ExpressionMatches(OrSite, File.Site())); /* both strings match */
    }
 
diff --git a/apt-pkg/versionmatch.h b/apt-pkg/versionmatch.h
index 4c8f704c8..bb950b9c7 100644
--- a/apt-pkg/versionmatch.h
+++ b/apt-pkg/versionmatch.h
@@ -70,8 +70,8 @@ class pkgVersionMatch
    enum MatchType {None = 0,Version,Release,Origin} Type;
 
    bool MatchVer(const char *A,std::string B,bool Prefix) APT_PURE;
-   bool ExpressionMatches(const char *pattern, const char *string);
-   bool ExpressionMatches(const std::string& pattern, const char *string);
+   static bool ExpressionMatches(const char *pattern, const char *string);
+   static bool ExpressionMatches(const std::string& pattern, const char *string);
    bool FileMatch(pkgCache::PkgFileIterator File);
    pkgCache::VerIterator Find(pkgCache::PkgIterator Pkg);
 
diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index 690b03bcc..3b8ebc96d 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -413,17 +413,21 @@ static bool Stats(CommandLine &)
 	    stritems.insert(Prv->ProvideVersion);
       }
    }
-   for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
+   for (pkgCache::RlsFileIterator F = Cache->RlsFileBegin(); F != Cache->RlsFileEnd(); ++F)
    {
       stritems.insert(F->FileName);
       stritems.insert(F->Archive);
       stritems.insert(F->Codename);
-      stritems.insert(F->Component);
       stritems.insert(F->Version);
       stritems.insert(F->Origin);
       stritems.insert(F->Label);
-      stritems.insert(F->Architecture);
       stritems.insert(F->Site);
+   }
+   for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F)
+   {
+      stritems.insert(F->FileName);
+      stritems.insert(F->Architecture);
+      stritems.insert(F->Component);
       stritems.insert(F->IndexType);
    }
    unsigned long Size = 0;
@@ -446,6 +450,7 @@ static bool Stats(CommandLine &)
       APT_CACHESIZE(VersionCount, VersionSz) +
       APT_CACHESIZE(DescriptionCount, DescriptionSz) +
       APT_CACHESIZE(DependsCount, DependencySz) +
+      APT_CACHESIZE(ReleaseFileCount, ReleaseFileSz) +
       APT_CACHESIZE(PackageFileCount, PackageFileSz) +
       APT_CACHESIZE(VerFileCount, VerFileSz) +
       APT_CACHESIZE(DescFileCount, DescFileSz) +
diff --git a/test/integration/framework b/test/integration/framework
index 110758e82..7b03c09ef 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -137,7 +137,14 @@ dpkgcheckbuilddeps() {
 	command dpkg-checkbuilddeps --admindir=${TMPWORKINGDIRECTORY}/rootdir/var/lib/dpkg "$@"
 }
 gdb() {
-	local CMD="$1"
+	local CMD
+	case "$1" in
+	aptget) CMD="apt-get";;
+	aptcache) CMD="apt-cache";;
+	aptmark) CMD="apt-mark";;
+	apthelper) CMD="apt-helper";;
+	*) CMD="$1";;
+	esac
 	shift
 	runapt command gdb --quiet -ex run "${BUILDDIRECTORY}/$CMD" --args "${BUILDDIRECTORY}/$CMD" "$@"
 }
diff --git a/test/integration/test-policy-pinning b/test/integration/test-policy-pinning
index 9cd54264b..9f7f457ae 100755
--- a/test/integration/test-policy-pinning
+++ b/test/integration/test-policy-pinning
@@ -22,7 +22,7 @@ testequalpolicy() {
      release a=now
  $(echo "$AP" | awk '{ printf("%3s\n",$0) }') file:${APTARCHIVE}  Packages
      release c=
-Pinned packages:" aptcache policy $*
+Pinned packages:" aptcache policy "$@"
 }
 
 testglobalpolicy() {
-- 
cgit v1.2.3-70-g09d2


From 3fd89e62e985c89b1f9a545ab72c20987b756aff Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 12 Jun 2015 12:06:29 +0200
Subject: implement default apt-get file --release-info mode

Selecting targets based on the Release they belong to isn't to
unrealistic. In fact, it is assumed to be the most used case so it is
made the default especially as this allows to bundle another thing we
have to be careful with: Filenames and only showing targets we have
acquired.

Closes: 752702
---
 apt-pkg/deb/debmetaindex.cc                        |  4 +-
 apt-pkg/deb/debmetaindex.h                         |  2 +-
 apt-pkg/indexfile.cc                               | 12 ++++
 apt-pkg/indexfile.h                                |  1 +
 apt-pkg/metaindex.cc                               |  2 +-
 apt-pkg/metaindex.h                                |  2 +-
 apt-pkg/pkgcachegen.cc                             |  4 +-
 apt-private/private-cmndline.cc                    |  1 +
 cmdline/apt-get.cc                                 | 26 ++++++++-
 doc/acquire-additional-files.txt                   | 65 +++++++++++++---------
 test/integration/test-apt-acquire-additional-files |  6 ++
 11 files changed, 88 insertions(+), 37 deletions(-)

(limited to 'cmdline')

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 994b95849..b328fcea8 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -422,7 +422,7 @@ bool debReleaseIndex::Merge(pkgCacheGenerator &Gen,OpProgress * /*Prog*/) const/
 }
 									/*}}}*/
 // ReleaseIndex::FindInCache - Find this index				/*{{{*/
-pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
+pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache, bool const ModifyCheck) const
 {
    std::string ReleaseFile;
    bool const releaseExists = ReleaseFileName(this, ReleaseFile);
@@ -434,7 +434,7 @@ pkgCache::RlsFileIterator debReleaseIndex::FindInCache(pkgCache &Cache) const
 	 continue;
 
        // empty means the file does not exist by "design"
-       if (releaseExists == false && File->Size == 0)
+       if (ModifyCheck == false || (releaseExists == false && File->Size == 0))
 	  return File;
 
       struct stat St;
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 7e9942710..b448ecc53 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -51,7 +51,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex {
    virtual std::vector GetIndexTargets() const;
 
    virtual std::string Describe() const;
-   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    std::string MetaIndexInfo(const char *Type) const;
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index 33fb48e35..605bbeb47 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -137,6 +137,18 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(CREATED_BY);
 #undef APT_CASE
       case FILENAME: return _config->FindDir("Dir::State::lists") + URItoFileName(URI);
+      case EXISTING_FILENAME:
+	 std::string const filename = Option(FILENAME);
+	 std::vector const types = APT::Configuration::getCompressionTypes();
+	 for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t)
+	 {
+	    if (t->empty())
+	       continue;
+	    std::string const file = (*t == "uncompressed") ? filename : (filename + "." + *t);
+	    if (FileExists(file))
+	       return file;
+	 }
+	 return "";
    }
    std::map::const_iterator const M = Options.find(Key);
    if (M == Options.end())
diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index 220c415ac..042e5c2f7 100644
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -81,6 +81,7 @@ class IndexTarget							/*{{{*/
       CREATED_BY,
       TARGET_OF,
       FILENAME,
+      EXISTING_FILENAME,
    };
    std::string Option(OptionKeys const Key) const;
    std::string Format(std::string format) const;
diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc
index 35ab6c53b..3c1b696bd 100644
--- a/apt-pkg/metaindex.cc
+++ b/apt-pkg/metaindex.cc
@@ -28,7 +28,7 @@ std::string metaIndex::Describe() const
    return "Release";
 }
 
-pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache) const
+pkgCache::RlsFileIterator metaIndex::FindInCache(pkgCache &Cache, bool const) const
 {
    return pkgCache::RlsFileIterator(Cache);
 }
diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h
index 20c879a89..e1810fb27 100644
--- a/apt-pkg/metaindex.h
+++ b/apt-pkg/metaindex.h
@@ -54,7 +54,7 @@ class metaIndex
    virtual bool IsTrusted() const = 0;
 
    virtual std::string Describe() const;
-   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache) const;
+   virtual pkgCache::RlsFileIterator FindInCache(pkgCache &Cache, bool const ModifyCheck) const;
    virtual bool Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const;
 
    metaIndex(std::string const &URI, std::string const &Dist,
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 2d0e3960d..9db4ef41f 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -1224,7 +1224,7 @@ static bool CheckValidity(const string &CacheFile,
    {
       if (Debug == true)
 	 std::clog << "Checking RlsFile " << (*i)->Describe() << ": ";
-      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache);
+      pkgCache::RlsFileIterator const RlsFile = (*i)->FindInCache(Cache, true);
       if (RlsFile.end() == true)
       {
 	 if (Debug == true)
@@ -1348,7 +1348,7 @@ static bool BuildCache(pkgCacheGenerator &Gen,
    {
       for (pkgSourceList::const_iterator i = List->begin(); i != List->end(); ++i)
       {
-	 if ((*i)->FindInCache(Gen.GetCache()).end() == false)
+	 if ((*i)->FindInCache(Gen.GetCache(), false).end() == false)
 	 {
 	    _error->Warning("Duplicate sources.list entry %s",
 		  (*i)->Describe().c_str());
diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc
index 458051bf2..11e88b1e7 100644
--- a/apt-private/private-cmndline.cc
+++ b/apt-private/private-cmndline.cc
@@ -166,6 +166,7 @@ static bool addArgumentsAPTGet(std::vector &Args, char const
    else if (CmdMatches("files"))
    {
       addArg(0,"format","APT::Get::Files::Format", CommandLine::HasArg);
+      addArg(0,"release-info","APT::Get::Files::ReleaseInfo", 0);
    }
    else if (CmdMatches("clean", "autoclean", "check", "download", "changelog") ||
 	    CmdMatches("markauto", "unmarkauto")) // deprecated commands
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index 0fff2db58..a4cd3c377 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1628,15 +1628,34 @@ static bool DoFiles(CommandLine &CmdL)
       return false;
 
    std::string const Format = _config->Find("APT::Get::Files::Format");
+   bool const ReleaseInfo = _config->FindB("APT::Get::Files::ReleaseInfo", true);
    bool Filtered = CmdL.FileSize() > 1;
    for (pkgSourceList::const_iterator S = SrcList->begin(); S != SrcList->end(); ++S)
    {
       std::vector const targets = (*S)->GetIndexTargets();
       std::map AddOptions;
-      AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+      if (ReleaseInfo)
+      {
+	 AddOptions.insert(std::make_pair("TRUSTED", ((*S)->IsTrusted() ? "yes" : "no")));
+	 pkgCache &Cache = *CacheFile.GetPkgCache();
+	 pkgCache::RlsFileIterator const RlsFile = (*S)->FindInCache(Cache, false);
+	 if (RlsFile.end())
+	    continue;
+#define APT_RELEASE(X,Y) if (RlsFile.Y() != NULL) AddOptions.insert(std::make_pair(X, RlsFile.Y()))
+	 APT_RELEASE("CODENAME", Codename);
+	 APT_RELEASE("SUITE", Archive);
+	 APT_RELEASE("VERSION", Version);
+	 APT_RELEASE("ORIGIN", Origin);
+	 APT_RELEASE("LABEL", Label);
+#undef APT_RELEASE
+      }
 
       for (std::vector::const_iterator T = targets.begin(); T != targets.end(); ++T)
       {
+	 std::string filename = T->Option(ReleaseInfo ? IndexTarget::EXISTING_FILENAME : IndexTarget::FILENAME);
+	 if (filename.empty())
+	    continue;
+
 	 std::ostringstream stanza;
 	 if (Filtered || Format.empty())
 	 {
@@ -1644,7 +1663,7 @@ static bool DoFiles(CommandLine &CmdL)
 	       << "ShortDesc: " << T->ShortDesc << "\n"
 	       << "Description: " << T->Description << "\n"
 	       << "URI: " << T->URI << "\n"
-	       << "Filename: " << T->Option(IndexTarget::FILENAME) << "\n"
+	       << "Filename: " << filename << "\n"
 	       << "Optional: " << (T->IsOptional ? "yes" : "no") << "\n";
 	    for (std::map::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
 	       stanza << format_key(O->first) << ": " << O->second << "\n";
@@ -1677,7 +1696,8 @@ static bool DoFiles(CommandLine &CmdL)
 	    cout << stanza.str();
 	 else
 	 {
-	    std::string out = T->Format(Format);
+	    std::string out = SubstVar(Format, "$(FILENAME)", filename);
+	    out = T->Format(out);
 	    for (std::map::const_iterator O = AddOptions.begin(); O != AddOptions.end(); ++O)
 	       out = SubstVar(out, std::string("$(") + O->first + ")", O->second);
 	    cout << out << std::endl;
diff --git a/doc/acquire-additional-files.txt b/doc/acquire-additional-files.txt
index a47dac2d4..f9a16318d 100644
--- a/doc/acquire-additional-files.txt
+++ b/doc/acquire-additional-files.txt
@@ -18,12 +18,6 @@ might want to use and would therefore need to be downloaded as well
 This file describes the configuration scheme such a frontend can use to
 instruct the Acquire system to download those additional files.
 
-Note that you can't download files from other sources. It must be files
-in the same repository and below the Release file. The Release file must
-also contain hashes for the file itself as well as for the compressed
-file if wanted, otherwise a download isn't even tried!
-
-
 # The Configuration Stanza
 
 The Acquire system uses the same configuration settings to implement the
@@ -81,15 +75,17 @@ Additional optional properties:
 * Optional: The default value is 'true' and should be kept at this
   value.  If enabled the acquire system will skip the download if the
   file isn't mentioned in the Release file. Otherwise this is treated as
-  a hard error and the update process fails.
+  a hard error and the update process fails. Note that failures while
+  downloading (e.g. 404 or hash verification errors) are failures,
+  regardless of this setting.
 
 
-Note that the acquire system will automatically choose to download
-a compressed file if it is available and uncompress it for you, just as
-it will also use pdiff patching if provided by the repository and
-enabled by the user. You only have to ensure that the Release file
-contains the information about the compressed files/pdiffs to make this
-happen. NO properties have to be set to enable this.
+The acquire system will automatically choose to download a compressed
+file if it is available and uncompress it for you, just as it will also
+use pdiff patching if provided by the repository and enabled by the
+user. You only have to ensure that the Release file contains the
+information about the compressed files/pdiffs to make this happen.
+NO properties have to be set to enable this.
 
 # More examples
 
@@ -123,10 +119,12 @@ APT::Acquire::Targets {
 As seen in the examples, properties can contain placeholders filled in
 by the acquire system. The following variables are known; note that
 unknown variables have no default value nor are they touched: They are
-printed literally.
+printed as-is.
 
 * $(SITE): An identifier of the site we access as seen in sources.list,
-  e.g. "http://example.org/debian" or "file:/path/to/a/repository".
+  e.g. "http://example.org/debian" or "file:/path/to/a/repository". You
+  can't use this field in {,flat}MetaKey, it is for description proposes
+  only.
 * $(RELEASE): This is usually an archive- or codename, e.g. "stable" or
   "stretch".  Note that flat-style repositories do not have a archive-
   or codename per-se, so the value might very well be just "/" or so.
@@ -143,7 +141,7 @@ printed literally.
 
 Note that while more variables might exist in the implementation, these
 are to be considered undefined and their usage strongly discouraged. If
-you have a need for another variable contact us instead.
+you have a need for other variables contact us.
 
 # Accessing files
 
@@ -170,29 +168,42 @@ sources.lists (pkgSourceList), iterating over the metaIndex objects this
 creates and calling GetIndexTargets() on them. See the sourcecode of
 "apt-get files" for a complete example.
 
-Remarks on the available fields:
+Note that by default targets are not listed if they weren't downloaded.
+If you want to see all targets, you can use the --no-release-info, which
+also removes the Codename, Suite, Version, Origin, Label and Trusted
+fields from the output as these also display data which needs to be
+downloaded first and could hence be inaccurate [on the pro-side: This
+mode is faster as it doesn't require a valid binary cache to operate].
+The most notable difference perhaps is in the Filename field through: By
+default it indicates an existing file, potentially compressed (Hint:
+libapt users can use FileFd to open compressed files transparently). In
+the --no-release-info mode the indicated file doesn't need to exist and
+it will always refer to an uncompressed file, even if the index would be
+(or is) stored compressed.
+
+Remarks on fields only available in (default) --release-info mode:
+* Trusted: Denotes with a 'yes' or 'no' if the data in this file is
+  authenticated by a trustchain rooted in a trusted gpg key. You should
+  be careful with untrusted data and warn the user if you use it.
+* Codename, Suite, Version, Origin and Label are fields from the Release
+  file, are only present if they are present in the Release file and
+  contain the same data.
+
+Remarks on other available fields:
 * MetaKey, ShortDesc, Description, Site, Release: as defined
   by the configuration and described further above.
 * Created-By: configuration entity responsible for this target
 * Target-Of: type of the sources.list entry
 * URI, Repo-URI: avoid using. Contains potentially username/password.
   Prefer 'Site', especially for display.
-* Filename: The mentioned file doesn't need to exist, e.g. because the
-  file wasn't downloaded yet – or it does exist compressed. libapt users
-  are encouraged to use FileFd to open such files as it can handle
-  decompression transparently.
-* Trusted: As of the last 'apt-get update' call denoting if e.g. apt-get
-  would print the unauthenticated warning while installing packages
-  mentioned in this file (example assumes Packages file) [Not really
-  a property of the target itself, but of the metaIndex].
 * Optional: Decodes the option of the same name from the configuration.
   Note that it is using 'yes' and 'no' instead of 'true' and 'false'.
 * Language, Architecture, Component: as defined further above, but with
   the catch that they might be missing if they don't effect the target
   (aka: They weren't used while evaluating the MetaKey template).
 
-Again, additional fields might be visible in certain implementation, but
-you should avoid using them and instead talk to us about a portable
+Again, additional fields might be visible in certain implementations,
+but you should avoid using them and instead talk to us about a portable
 implementation.
 
 # Multiple application requiring the same files
diff --git a/test/integration/test-apt-acquire-additional-files b/test/integration/test-apt-acquire-additional-files
index 971cfac2e..ffb9f3135 100755
--- a/test/integration/test-apt-acquire-additional-files
+++ b/test/integration/test-apt-acquire-additional-files
@@ -37,6 +37,12 @@ APT::Acquire::Targets::deb::Contents {
 };
 EOF
 
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64)" aptget files --no-release-info --format '$(FILENAME)' 'Created-By: Contents'
+testempty aptget files --format '$(FILENAME)' 'Created-By: Contents'
+# lets fake the existence of a compressed Contents file
+touch ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz
+testequal "$(readlink -f ./rootdir/var/lib/apt/lists/localhost:8080_dists_unstable_main_Contents-amd64.gz)" aptget files --format '$(FILENAME)' 'Created-By: Contents'
+
 testequal "'http://localhost:8080/dists/unstable/InRelease' localhost:8080_dists_unstable_InRelease 0 
 'http://localhost:8080/dists/unstable/main/source/Sources.bz2' localhost:8080_dists_unstable_main_source_Sources 0 
 'http://localhost:8080/dists/unstable/main/binary-amd64/Packages.bz2' localhost:8080_dists_unstable_main_binary-amd64_Packages 0 
-- 
cgit v1.2.3-70-g09d2


From d2cb5b153fb13d587b1ff632cab34ce0c403326e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 12 Jun 2015 15:48:00 +0200
Subject: hide Translation-* in 'apt-cache policy' output
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Translation-* files are internally handled as PackageFiles which isn't
super nice, but giving them their own struct is a bit overkill so let it
be for the moment. They always appeared in the policy output because of
this through and now that they are properly linked to a ReleaseFile they
even display all the pinning information on them, but they don't contain
any packages which could be pinned… No problem, but useless and
potentially confusing output.

Adding a 'NoPackages' flag which can be set on those files and be used
in applications seems like a simple way to fix this display issue.
---
 apt-pkg/deb/debindexfile.cc | 2 +-
 apt-pkg/pkgcache.h          | 1 +
 cmdline/apt-cache.cc        | 2 ++
 3 files changed, 4 insertions(+), 1 deletion(-)

(limited to 'cmdline')

diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index f089cda81..944cbe0bf 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -203,7 +203,7 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
 
      if (Prog != NULL)
 	Prog->SubProgress(0, Target.Description);
-     if (Gen.SelectFile(TranslationFile, *this, "", Target.Option(IndexTarget::COMPONENT), pkgCache::Flag::NotSource) == false)
+     if (Gen.SelectFile(TranslationFile, *this, "", Target.Option(IndexTarget::COMPONENT), pkgCache::Flag::NotSource | pkgCache::Flag::NoPackages) == false)
        return _error->Error("Problem with SelectFile %s",TranslationFile.c_str());
 
      // Store the IMS information
diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h
index 696b3b94d..3cc85f1e8 100644
--- a/apt-pkg/pkgcache.h
+++ b/apt-pkg/pkgcache.h
@@ -195,6 +195,7 @@ class pkgCache								/*{{{*/
       enum PkgFFlags {
 	 NotSource=(1<<0), /*!< packages can't be fetched from here, e.g. dpkg/status file */
 	 LocalSource=(1<<1), /*!< local sources can't and will not be verified by hashes */
+	 NoPackages=(1<<2), /*!< the file includes no package records itself, but additions like Translations */
       };
       enum ReleaseFileFlags {
 	 NotAutomatic=(1<<0), /*!< archive has a default pin of 1 */
diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index 3b8ebc96d..c2f6dbd5c 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -1638,6 +1638,8 @@ static bool Policy(CommandLine &CmdL)
       cout << _("Package files:") << endl;   
       for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F.end() == false; ++F)
       {
+	 if (F.Flagged(pkgCache::Flag::NoPackages))
+	    continue;
 	 // Locate the associated index files so we can derive a description
 	 pkgIndexFile *Indx;
 	 if (SrcList->FindIndex(F,Indx) == false &&
-- 
cgit v1.2.3-70-g09d2


From d56e2917f27a722b54685de13aeb1bb7592fc61b Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 13 Jun 2015 11:13:45 +0200
Subject: provide a public interface for acquiring changelogs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Provided is a specialized acquire item which given a version can figure
out the correct URI to try by itself and if not provides an error
message alongside with static methods to get just the URI it would try
to download if it should just be displayed or similar such.

The URI is constructed as follows:
Release files can provide an URI template in the "Changelogs" field,
otherwise we lookup a configuration item based on the "Label" or
"Origin" of the Release file to get a (hopefully known) default value
for now. This template should contain the string CHANGEPATH which is
replaced with the information about the version we want the changelog
for (e.g.  main/a/apt/apt_1.1). This middleway was choosen as this path
part was consistent over the three known implementations (+1 defunct),
while the rest of the URI varies widely between them.

The benefit of this construct is that it is now easy to get changelogs
for Debian packages on Ubuntu and vice versa – even at the moment where
the Changelogs field is present nowhere. Strictly better than what
apt-get had before as it would even fail to get changelogs from
security… Now it will notice that security identifies as Origin: Debian
and pick this setting (assuming again that no Changelogs field exists).
If on the other hand security would ship its changelogs in a different
location we could set it via the Label option overruling Origin.

Closes: 687147, 739854, 784027, 787190
---
 apt-pkg/acquire-item.cc                            | 211 +++++++++++++++++++++
 apt-pkg/acquire-item.h                             | 114 +++++++++++
 apt-pkg/init.cc                                    |   4 +
 cmdline/apt-get.cc                                 | 204 ++++----------------
 doc/apt-get.8.xml                                  |  16 +-
 doc/apt.conf.5.xml                                 |  27 +++
 doc/examples/configure-index                       |  11 +-
 test/integration/framework                         |  24 ++-
 test/integration/test-apt-get-changelog            | 104 +++++++---
 .../test-bug-722207-print-uris-even-if-very-quiet  |   3 +-
 test/integration/test-bug-738785-switch-protocol   |   7 +-
 vendor/ubuntu/apt.conf-01-vendor-ubuntu            |   6 -
 12 files changed, 507 insertions(+), 224 deletions(-)

(limited to 'cmdline')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index d6e9ccbe0..4bf4e62f8 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -31,6 +31,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -2836,6 +2837,216 @@ std::string pkgAcqArchive::ShortDesc() const				/*{{{*/
 }
 									/*}}}*/
 
+// AcqChangelog::pkgAcqChangelog - Constructors				/*{{{*/
+pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
+      std::string const &DestDir, std::string const &DestFilename) :
+   pkgAcquire::Item(Owner), d(NULL), SrcName(Ver.SourcePkgName()), SrcVersion(Ver.SourceVerStr())
+{
+   Desc.URI = URI(Ver);
+   Init(DestDir, DestFilename);
+}
+// some parameters are char* here as they come likely from char* interfaces – which can also return NULL
+pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &RlsFile,
+      char const * const Component, char const * const SrcName, char const * const SrcVersion,
+      const string &DestDir, const string &DestFilename) :
+   pkgAcquire::Item(Owner), d(NULL), SrcName(SrcName), SrcVersion(SrcVersion)
+{
+   Desc.URI = URI(RlsFile, Component, SrcName, SrcVersion);
+   Init(DestDir, DestFilename);
+}
+pkgAcqChangelog::pkgAcqChangelog(pkgAcquire * const Owner,
+      std::string const &URI, char const * const SrcName, char const * const SrcVersion,
+      const string &DestDir, const string &DestFilename) :
+   pkgAcquire::Item(Owner), d(NULL), SrcName(SrcName), SrcVersion(SrcVersion)
+{
+   Desc.URI = URI;
+   Init(DestDir, DestFilename);
+}
+void pkgAcqChangelog::Init(std::string const &DestDir, std::string const &DestFilename)
+{
+   if (Desc.URI.empty())
+   {
+      Status = StatError;
+      // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
+      strprintf(ErrorText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
+      // Let the error message print something sensible rather than "Failed to fetch /"
+      if (DestFilename.empty())
+	 DestFile = SrcName + ".changelog";
+      else
+	 DestFile = DestFilename;
+      Desc.URI = "changelog:/" + DestFile;
+      return;
+   }
+
+   if (DestDir.empty())
+   {
+      std::string const systemTemp = GetTempDir();
+      char tmpname[100];
+      snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", systemTemp.c_str());
+      if (NULL == mkdtemp(tmpname))
+      {
+	 _error->Errno("mkdtemp", "mkdtemp failed in changelog acquire of %s %s", SrcName.c_str(), SrcVersion.c_str());
+	 Status = StatError;
+	 return;
+      }
+      DestFile = TemporaryDirectory = tmpname;
+   }
+   else
+      DestFile = DestDir;
+
+   if (DestFilename.empty())
+      DestFile = flCombine(DestFile, SrcName + ".changelog");
+   else
+      DestFile = flCombine(DestFile, DestFilename);
+
+   Desc.ShortDesc = "Changelog";
+   strprintf(Desc.Description, "%s %s %s Changelog", URI::SiteOnly(Desc.URI).c_str(), SrcName.c_str(), SrcVersion.c_str());
+   Desc.Owner = this;
+   QueueURI(Desc);
+
+   if (Status == StatDone) // this happens if we queue the same changelog two times
+   {
+      Complete = true;
+      for (pkgAcquire::UriIterator I = Owner->UriBegin(); I != Owner->UriEnd(); ++I)
+	 if (I->URI == Desc.URI)
+	    if (DestFile != I->Owner->DestFile)
+	       if (symlink(I->Owner->DestFile.c_str(), DestFile.c_str()) != 0)
+	       {
+		  ; // ignore error, there isn't anthing we could do to handle the edgecase of an edgecase
+	       }
+   }
+}
+									/*}}}*/
+std::string pkgAcqChangelog::URI(pkgCache::VerIterator const &Ver)	/*{{{*/
+{
+   char const * const SrcName = Ver.SourcePkgName();
+   char const * const SrcVersion = Ver.SourceVerStr();
+   pkgCache::PkgFileIterator PkgFile;
+   // find the first source for this version which promises a changelog
+   for (pkgCache::VerFileIterator VF = Ver.FileList(); VF.end() == false; ++VF)
+   {
+      pkgCache::PkgFileIterator const PF = VF.File();
+      if (PF.Flagged(pkgCache::Flag::NotSource) || PF->Release == 0)
+	 continue;
+      PkgFile = PF;
+      pkgCache::RlsFileIterator const RF = PF.ReleaseFile();
+      std::string const uri = URI(RF, PF.Component(), SrcName, SrcVersion);
+      if (uri.empty())
+	 continue;
+      return uri;
+   }
+   return "";
+}
+std::string pkgAcqChangelog::URITemplate(pkgCache::RlsFileIterator const &Rls)
+{
+   if (Rls.end() == true || (Rls->Label == 0 && Rls->Origin == 0))
+      return "";
+   std::string const serverConfig = "Acquire::Changelogs::URI";
+   std::string server;
+#define APT_EMPTY_SERVER \
+   if (server.empty() == false) \
+   { \
+      if (server != "no") \
+	 return server; \
+      return ""; \
+   }
+#define APT_CHECK_SERVER(X, Y) \
+   if (Rls->X != 0) \
+   { \
+      std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls.X(); \
+      server = _config->Find(specialServerConfig); \
+      APT_EMPTY_SERVER \
+   }
+   // this way e.g. Debian-Security can fallback to Debian
+   APT_CHECK_SERVER(Label, "Override::")
+   APT_CHECK_SERVER(Origin, "Override::")
+
+   if (RealFileExists(Rls.FileName()))
+   {
+      _error->PushToStack();
+      FileFd rf;
+      /* This can be costly. A caller wanting to get millions of URIs might
+	 want to do this on its own once and use Override settings.
+	 We don't do this here as Origin/Label are not as unique as they
+	 should be so this could produce request order-dependent anomalies */
+      if (OpenMaybeClearSignedFile(Rls.FileName(), rf) == true)
+      {
+	 pkgTagFile TagFile(&rf, rf.Size());
+	 pkgTagSection Section;
+	 if (TagFile.Step(Section) == true)
+	    server = Section.FindS("Changelogs");
+      }
+      _error->RevertToStack();
+      APT_EMPTY_SERVER
+   }
+
+   APT_CHECK_SERVER(Label, "")
+   APT_CHECK_SERVER(Origin, "")
+#undef APT_CHECK_SERVER
+#undef APT_EMPTY_SERVER
+   return "";
+}
+std::string pkgAcqChangelog::URI(pkgCache::RlsFileIterator const &Rls,
+	 char const * const Component, char const * const SrcName,
+	 char const * const SrcVersion)
+{
+   return URI(URITemplate(Rls), Component, SrcName, SrcVersion);
+}
+std::string pkgAcqChangelog::URI(std::string const &Template,
+	 char const * const Component, char const * const SrcName,
+	 char const * const SrcVersion)
+{
+   if (Template.find("CHANGEPATH") == std::string::npos)
+      return "";
+
+   // the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. main/a/apt/1.1 or contrib/liba/libapt/2.0
+   std::string Src = SrcName;
+   std::string path = APT::String::Startswith(SrcName, "lib") ? Src.substr(0, 4) : Src.substr(0,1);
+   path.append("/").append(Src).append("/");
+   path.append(Src).append("_").append(StripEpoch(SrcVersion));
+   // we omit component for releases without one (= flat-style repositories)
+   if (Component != NULL && strlen(Component) != 0)
+      path = std::string(Component) + "/" + path;
+
+   return SubstVar(Template, "CHANGEPATH", path);
+}
+									/*}}}*/
+// AcqChangelog::Failed - Failure handler				/*{{{*/
+void pkgAcqChangelog::Failed(string const &Message, pkgAcquire::MethodConfig const * const Cnf)
+{
+   Item::Failed(Message,Cnf);
+
+   std::string errText;
+   // TRANSLATOR: %s=%s is sourcename=sourceversion, e.g. apt=1.1
+   strprintf(errText, _("Changelog unavailable for %s=%s"), SrcName.c_str(), SrcVersion.c_str());
+
+   // Error is probably something techy like 404 Not Found
+   if (ErrorText.empty())
+      ErrorText = errText;
+   else
+      ErrorText = errText + " (" + ErrorText + ")";
+   return;
+}
+									/*}}}*/
+// AcqChangelog::Done - Item downloaded OK				/*{{{*/
+void pkgAcqChangelog::Done(string const &Message,HashStringList const &CalcHashes,
+		      pkgAcquire::MethodConfig const * const Cnf)
+{
+   Item::Done(Message,CalcHashes,Cnf);
+
+   Complete = true;
+}
+									/*}}}*/
+pkgAcqChangelog::~pkgAcqChangelog()					/*{{{*/
+{
+   if (TemporaryDirectory.empty() == false)
+   {
+      unlink(DestFile.c_str());
+      rmdir(TemporaryDirectory.c_str());
+   }
+}
+									/*}}}*/
+
 // AcqFile::pkgAcqFile - Constructor					/*{{{*/
 pkgAcqFile::pkgAcqFile(pkgAcquire * const Owner,string const &URI, HashStringList const &Hashes,
 		       unsigned long long const Size,string const &Dsc,string const &ShortDesc,
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index b6d569737..606fd4173 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -1031,6 +1031,120 @@ class pkgAcqArchive : public pkgAcquire::Item
 		 std::string &StoreFilename);
 };
 									/*}}}*/
+/** \brief Retrieve the changelog for the given version			{{{
+ *
+ *  Downloads the changelog to a temporary file it will also remove again
+ *  while it is deconstructed or downloads it to a named location.
+ */
+class pkgAcqChangelog : public pkgAcquire::Item
+{
+   void *d;
+   std::string TemporaryDirectory;
+   std::string const SrcName;
+   std::string const SrcVersion;
+
+   public:
+   // we will never have hashes for changelogs.
+   // If you need verified ones, download the deb and extract the changelog.
+   virtual HashStringList GetExpectedHashes() const { return HashStringList(); }
+   virtual bool HashesRequired() const { return false; }
+
+   // Specialized action members
+   virtual void Failed(std::string const &Message,pkgAcquire::MethodConfig const * const Cnf);
+   virtual void Done(std::string const &Message, HashStringList const &CalcHashes,
+		     pkgAcquire::MethodConfig const * const Cnf);
+   virtual std::string DescURI() const {return Desc.URI;};
+
+   /** returns the URI to the changelog of this version
+    *
+    * @param Ver is the version to get the changelog for
+    * @return the URI which will be used to acquire the changelog
+    */
+   static std::string URI(pkgCache::VerIterator const &Ver);
+
+   /** returns the URI to the changelog of this version
+    *
+    *  \param Rls is the Release file the package comes from
+    *  \param Component in which the package resides, can be empty
+    *  \param SrcName is the source package name
+    *  \param SrcVersion is the source package version
+    * @return the URI which will be used to acquire the changelog
+    */
+   static std::string URI(pkgCache::RlsFileIterator const &Rls,
+	 char const * const Component, char const * const SrcName,
+	 char const * const SrcVersion);
+
+   /** returns the URI to the changelog of this version
+    *
+    *  \param Template URI where CHANGEPATH has to be filled in
+    *  \param Component in which the package resides, can be empty
+    *  \param SrcName is the source package name
+    *  \param SrcVersion is the source package version
+    * @return the URI which will be used to acquire the changelog
+    */
+   static std::string URI(std::string const &Template,
+	 char const * const Component, char const * const SrcName,
+	 char const * const SrcVersion);
+
+   /** returns the URI template for this release file
+    *
+    *  \param Rls is a Release file
+    * @return the URI template to use for this release file
+    */
+   static std::string URITemplate(pkgCache::RlsFileIterator const &Rls);
+
+   /** \brief Create a new pkgAcqChangelog object.
+    *
+    *  \param Owner The pkgAcquire object with which this object is
+    *  associated.
+    *  \param Ver is the version to get the changelog for
+    *  \param DestDir The directory the file should be downloaded into.
+    *  Will be an autocreated (and cleaned up) temporary directory if not set.
+    *  \param DestFilename The filename the file should have in #DestDir
+    *  Defaults to sourcepackagename.changelog if not set.
+    */
+   pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::VerIterator const &Ver,
+	 std::string const &DestDir="", std::string const &DestFilename="");
+
+   /** \brief Create a new pkgAcqChangelog object.
+    *
+    *  \param Owner The pkgAcquire object with which this object is
+    *  associated.
+    *  \param Rls is the Release file the package comes from
+    *  \param Component in which the package resides, can be empty
+    *  \param SrcName is the source package name
+    *  \param SrcVersion is the source package version
+    *  \param DestDir The directory the file should be downloaded into.
+    *  Will be an autocreated (and cleaned up) temporary directory if not set.
+    *  \param DestFilename The filename the file should have in #DestDir
+    *  Defaults to sourcepackagename.changelog if not set.
+    */
+   pkgAcqChangelog(pkgAcquire * const Owner, pkgCache::RlsFileIterator const &Rls,
+	 char const * const Component, char const * const SrcName, char const * const SrcVersion,
+	 std::string const &DestDir="", std::string const &DestFilename="");
+
+   /** \brief Create a new pkgAcqChangelog object.
+    *
+    *  \param Owner The pkgAcquire object with which this object is
+    *  associated.
+    *  \param URI is to be used to get the changelog
+    *  \param SrcName is the source package name
+    *  \param SrcVersion is the source package version
+    *  \param DestDir The directory the file should be downloaded into.
+    *  Will be an autocreated (and cleaned up) temporary directory if not set.
+    *  \param DestFilename The filename the file should have in #DestDir
+    *  Defaults to sourcepackagename.changelog if not set.
+    */
+   pkgAcqChangelog(pkgAcquire * const Owner, std::string const &URI,
+	 char const * const SrcName, char const * const SrcVersion,
+	 std::string const &DestDir="", std::string const &DestFilename="");
+
+   virtual ~pkgAcqChangelog();
+
+private:
+   APT_HIDDEN void Init(std::string const &DestDir, std::string const &DestFilename);
+};
+									/*}}}*/
 /** \brief Retrieve an arbitrary file to the current directory.		{{{
  *
  *  The file is retrieved even if it is accessed via a URL type that
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index e2239a906..96966b249 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -119,6 +119,10 @@ bool pkgInitConfig(Configuration &Cnf)
    Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::flatDescription", "$(SITE) $(RELEASE) Sources");
    Cnf.CndSet("APT::Acquire::Targets::deb-src::Sources::Optional", false);
 
+   Cnf.CndSet("Acquire::Changelogs::URI::Origin::Debian", "http://metadata.ftp-master.debian.org/changelogs/CHANGEPATH_changelog");
+   Cnf.CndSet("Acquire::Changelogs::URI::Origin::Ubuntu", "http://changelogs.ubuntu.com/changelogs/pool/CHANGEPATH/changelog");
+   Cnf.CndSet("Acquire::Changelogs::URI::Origin::Ultimedia", "http://packages.ultimediaos.com/changelogs/pool/CHANGEPATH/changelog.txt");
+
    bool Res = true;
 
    // Read an alternate config file
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index a4cd3c377..184b51d23 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1410,196 +1410,70 @@ static bool DoBuildDep(CommandLine &CmdL)
    return true;
 }
 									/*}}}*/
-// GetChangelogPath - return a path pointing to a changelog file or dir /*{{{*/
-// ---------------------------------------------------------------------
-/* This returns a "path" string for the changelog url construction.
- * Please note that its not complete, it either needs a "/changelog"
- * appended (for the packages.debian.org/changelogs site) or a
- * ".changelog" (for third party sites that store the changelog in the
- * pool/ next to the deb itself)
- * Example return: "pool/main/a/apt/apt_0.8.8ubuntu3" 
- */
-static string GetChangelogPath(CacheFile &Cache,
-                        pkgCache::VerIterator Ver)
-{
-   pkgRecords Recs(Cache);
-   pkgRecords::Parser &rec=Recs.Lookup(Ver.FileList());
-   string path = flNotFile(rec.FileName());
-#if APT_PKG_ABI >= 413
-   path.append(Ver.SourcePkgName());
-   path.append("_");
-   path.append(StripEpoch(Ver.SourceVerStr()));
-#else
-   string srcpkg = rec.SourcePkg().empty() ? Ver.ParentPkg().Name() : rec.SourcePkg();
-   string ver = Ver.VerStr();
-   // if there is a source version it always wins
-   if (rec.SourceVer() != "")
-      ver = rec.SourceVer();
-   path += srcpkg + "_" + StripEpoch(ver);
-#endif
-   return path;
-}
-									/*}}}*/
-// GuessThirdPartyChangelogUri - return url 			        /*{{{*/
-// ---------------------------------------------------------------------
-/* Contruct a changelog file path for third party sites that do not use
- * packages.debian.org/changelogs
- * This simply uses the ArchiveURI() of the source pkg and looks for
- * a .changelog file there, Example for "mediabuntu":
- * apt-get changelog mplayer-doc:
- *  http://packages.medibuntu.org/pool/non-free/m/mplayer/mplayer_1.0~rc4~try1.dsfg1-1ubuntu1+medibuntu1.changelog
- */
-static bool GuessThirdPartyChangelogUri(CacheFile &Cache, 
-                                 pkgCache::VerIterator Ver,
-                                 string &out_uri)
-{
-   // get the binary deb server path
-   pkgCache::VerFileIterator Vf = Ver.FileList();
-   if (Vf.end() == true)
-      return false;
-   pkgCache::PkgFileIterator F = Vf.File();
-   pkgIndexFile *index;
-   pkgSourceList *SrcList = Cache.GetSourceList();
-   if(SrcList->FindIndex(F, index) == false)
-      return false;
-
-   // get archive uri for the binary deb
-   string path_without_dot_changelog = GetChangelogPath(Cache, Ver);
-   out_uri = index->ArchiveURI(path_without_dot_changelog + ".changelog");
-
-   // now strip away the filename and add srcpkg_srcver.changelog
-   return true;
-}
-									/*}}}*/
-// DownloadChangelog - Download the changelog 			        /*{{{*/
-// ---------------------------------------------------------------------
-static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, 
-                       pkgCache::VerIterator Ver, string targetfile)
-/* Download a changelog file for the given package version to
- * targetfile. This will first try the server from Apt::Changelogs::Server
- * (http://packages.debian.org/changelogs by default) and if that gives
- * a 404 tries to get it from the archive directly (see 
- * GuessThirdPartyChangelogUri for details how)
- */
-{
-   // make the server root configurable
-   string const server = _config->Find("Apt::Changelogs::Server",
-                          "http://packages.debian.org/changelogs");
-   string const path = GetChangelogPath(CacheFile, Ver);
-   string changelog_uri;
-   if (APT::String::Endswith(server, "/") == true)
-      strprintf(changelog_uri, "%s%s/changelog", server.c_str(), path.c_str());
-   else
-      strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str());
-   if (_config->FindB("APT::Get::Print-URIs", false) == true)
-   {
-      std::cout << '\'' << changelog_uri << '\'' << std::endl;
-      return true;
-   }
-   pkgCache::PkgIterator const Pkg = Ver.ParentPkg();
-
-   string descr;
-   strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), changelog_uri.c_str());
-   // queue it
-   pkgAcquire::Item const * itm = new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile);
-
-   // Disable drop-privs if "_apt" can not write to the target dir
-   CheckDropPrivsMustBeDisabled(Fetcher);
-
-   // try downloading it, if that fails, try third-party-changelogs location
-   // FIXME: Fetcher.Run() is "Continue" even if I get a 404?!?
-   Fetcher.Run();
-   if (itm->Status != pkgAcquire::Item::StatDone)
-   {
-      string third_party_uri;
-      if (GuessThirdPartyChangelogUri(CacheFile, Ver, third_party_uri))
-      {
-         strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), third_party_uri.c_str());
-         itm = new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile);
-         Fetcher.Run();
-      }
-   }
-
-   if (itm->Status == pkgAcquire::Item::StatDone)
-      return true;
-
-   // error
-   return _error->Error("changelog download failed");
-}
-									/*}}}*/
 // DoChangelog - Get changelog from the command line			/*{{{*/
-// ---------------------------------------------------------------------
 static bool DoChangelog(CommandLine &CmdL)
 {
    CacheFile Cache;
    if (Cache.ReadOnlyOpen() == false)
       return false;
-   
+
    APT::CacheSetHelper helper;
    APT::VersionList verset = APT::VersionList::FromCommandLine(Cache,
 		CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper);
    if (verset.empty() == true)
       return false;
    pkgAcquire Fetcher;
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
+   Fetcher.SetLog(&Stat);
 
-   if (_config->FindB("APT::Get::Print-URIs", false) == true)
+   bool const downOnly = _config->FindB("APT::Get::Download-Only", false);
+   bool const printOnly = _config->FindB("APT::Get::Print-URIs", false);
+
+   for (APT::VersionList::const_iterator Ver = verset.begin();
+        Ver != verset.end();
+        ++Ver)
    {
-      bool Success = true;
-      for (APT::VersionList::const_iterator Ver = verset.begin();
-	   Ver != verset.end(); ++Ver)
-	 Success &= DownloadChangelog(Cache, Fetcher, Ver, "");
-      return Success;
+      if (printOnly)
+	 new pkgAcqChangelog(&Fetcher, Ver, "/dev/null");
+      else if (downOnly)
+	 new pkgAcqChangelog(&Fetcher, Ver, ".");
+      else
+	 new pkgAcqChangelog(&Fetcher, Ver);
    }
 
-   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
-   Fetcher.SetLog(&Stat);
+   if (printOnly == false)
+   {
+      // Disable drop-privs if "_apt" can not write to the target dir
+      CheckDropPrivsMustBeDisabled(Fetcher);
+      if (_error->PendingError() == true)
+	 return false;
 
-   bool const downOnly = _config->FindB("APT::Get::Download-Only", false);
+      bool Failed = false;
+      if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true)
+	 return false;
+   }
 
-   char tmpname[100];
-   const char* tmpdir = NULL;
-   if (downOnly == false)
+   if (downOnly == false || printOnly == true)
    {
-      std::string systemTemp = GetTempDir();
-      snprintf(tmpname, sizeof(tmpname), "%s/apt-changelog-XXXXXX", 
-               systemTemp.c_str());
-      tmpdir = mkdtemp(tmpname);
-      if (tmpdir == NULL)
-	 return _error->Errno("mkdtemp", "mkdtemp failed");
-
-      std::string const SandboxUser = _config->Find("APT::Sandbox::User");
-      if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it
+      bool Failed = false;
+      for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin(); I != Fetcher.ItemsEnd(); ++I)
       {
-	 struct passwd const * const pw = getpwnam(SandboxUser.c_str());
-	 struct group const * const gr = getgrnam("root");
-	 if (pw != NULL && gr != NULL)
+	 if (printOnly)
 	 {
-	    // chown the tmp dir directory we use to the sandbox user
-	    if(chown(tmpdir, pw->pw_uid, gr->gr_gid) != 0)
-	       _error->WarningE("DoChangelog", "chown to %s:%s of directory %s failed", SandboxUser.c_str(), "root", tmpdir);
+	    if ((*I)->ErrorText.empty() == false)
+	    {
+	       Failed = true;
+	       _error->Error("%s", (*I)->ErrorText.c_str());
+	    }
+	    else
+	       cout << '\'' << (*I)->DescURI() << "' " << flNotDir((*I)->DestFile)  << std::endl;
 	 }
+	 else
+	    DisplayFileInPager((*I)->DestFile);
       }
+      return Failed == false;
    }
 
-   for (APT::VersionList::const_iterator Ver = verset.begin(); 
-        Ver != verset.end(); 
-        ++Ver) 
-   {
-      string changelogfile;
-      if (downOnly == false)
-	 changelogfile.append(tmpname).append("/changelog");
-      else
-	 changelogfile.append(Ver.ParentPkg().Name()).append(".changelog");
-      if (DownloadChangelog(Cache, Fetcher, Ver, changelogfile) && downOnly == false)
-      {
-         DisplayFileInPager(changelogfile);
-         // cleanup temp file
-	 unlink(changelogfile.c_str());
-      }
-   }
-   // clenaup tmp dir
-   if (tmpdir != NULL)
-      rmdir(tmpdir);
    return true;
 }
 									/*}}}*/
diff --git a/doc/apt-get.8.xml b/doc/apt-get.8.xml
index da077afa7..5b6788ed4 100644
--- a/doc/apt-get.8.xml
+++ b/doc/apt-get.8.xml
@@ -230,16 +230,12 @@
      
 
      
-       changelog downloads a package changelog and displays
-           it through sensible-pager. The server name and base
-           directory is defined in the APT::Changelogs::Server
-	   variable (e.g. packages.debian.org/changelogs for
-	   Debian or changelogs.ubuntu.com/changelogs for
-           Ubuntu).
-           By default it displays the changelog for the version that is
-           installed.  However, you can specify the same options as for
-           the  command.
-         
+	changelog tries to download the
+	      changelog of a package and displays it through
+	      sensible-pager.  By default it
+	      displays the changelog for the version that is installed.
+	      However, you can specify the same options as for the
+	       command.
        
      
 
diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml
index efe986ea8..7d5f7e9b3 100644
--- a/doc/apt.conf.5.xml
+++ b/doc/apt.conf.5.xml
@@ -618,6 +618,33 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
          
      
 
+      scope
+	
+	  Acquiring changelogs can only be done if an URI is known from where to get them.
+	  Preferable the Release file indicates this in a 'Changelogs' field. If this isn't
+	  available the Label/Origin field of the Release file is used to check if a
+	  Acquire::Changelogs::URI::Label::LABEL or
+	  Acquire::Changelogs::URI::Origin::ORIGIN option
+	  exists and if so this value is taken. The value in the Release file can be overridden
+	  with Acquire::Changelogs::URI::Override::Label::LABEL
+	  or Acquire::Changelogs::URI::Override::Origin::ORIGIN.
+
+	  The value should be a normal URI to a text file, expect that package specific data is
+	  replaced with the placeholder CHANGEPATH. The
+	  value for it is: 1. if the package is from a component (e.g. main)
+	  this is the first part otherwise it is omitted, 2. the first letter of source package name,
+	  expect if the source package name starts with 'lib' in which case it will
+	  be the first four letters. 3. The complete source package name. 4. the complete name again and
+	  5. the source version.
+	  The first (if present), second, third and fourth part are separated by a slash ('/')
+	  and between the fourth and fifth part is an underscore ('_').
+
+	  The special value 'no' is available for this option indicating that
+	  this source can't be used to acquire changelog files from. Another source will be tried
+	  if available in this case.
+	
+    
+
    
  
 
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index ef1ae056d..1339335fa 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -117,14 +117,6 @@ APT
   // does a ExecFork)
   Keep-Fds {};
 
-  Changelogs
-  {
-    // server the provides the changelogs, the code will assume
-    // the changlogs are in the pool/ under a srcpkg_ver directory 
-    // with the name "changelog"
-    Server "http://packages.debian.org/changelogs";
-  }:
-
   // control parameters for cron jobs by /etc/cron.daily/apt
   Periodic
   {
@@ -305,6 +297,9 @@ Acquire
      "none";
      "fr";
   };
+
+  // Location of the changelogs with the placeholder CHANGEPATH (e.g. "main/a/apt/apt_1.1")
+  Changelogs::URI::Origin::Debian "http://metadata.ftp-master.debian.org/changelogs/CHANGEPATH_changelog";
 };
 
 // Directory layout
diff --git a/test/integration/framework b/test/integration/framework
index 7b03c09ef..1b99929e2 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -520,6 +520,12 @@ Package: $NAME" > debian/control
 
 buildsimplenativepackage() {
 	local NAME="$1"
+	local NM
+	if [ "$(echo "$NAME" | cut -c 1-3)" = 'lib' ]; then
+		NM="$(echo "$NAME" | cut -c 1-4)"
+	else
+		NM="$(echo "$NAME" | cut -c 1)"
+	fi
 	local ARCH="$2"
 	local VERSION="$3"
 	local RELEASE="${4:-unstable}"
@@ -600,15 +606,15 @@ Package: $NAME" >> ${BUILDDIR}/debian/control
 		(cd ${BUILDDIR}; dpkg-gencontrol -DArchitecture=$arch)
 		(cd ${BUILDDIR}/debian/tmp; md5sum $(find usr/ -type f) > DEBIAN/md5sums)
 		local LOG="${BUILDDIR}/../${NAME}_${VERSION}_${arch}.dpkg-deb.log"
-		# ensure the right permissions as dpkg-deb ensists
+		# ensure the right permissions as dpkg-deb insists
 		chmod 755 ${BUILDDIR}/debian/tmp/DEBIAN
 		testsuccess --nomsg dpkg-deb -Z${COMPRESS_TYPE} --build ${BUILDDIR}/debian/tmp ${BUILDDIR}/..
 		echo "pool/${NAME}_${VERSION}_${arch}.deb" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.pkglist
 	done
 
-	mkdir -p ${BUILDDIR}/../${NAME}_${VERSION}
-	cp ${BUILDDIR}/debian/changelog ${BUILDDIR}/../${NAME}_${VERSION}/
-	cp ${BUILDDIR}/debian/changelog ${BUILDDIR}/../${NAME}_${VERSION}.changelog
+	local CHANGEPATH="${BUILDDIR}/../${DISTSECTION}/${NM}/${NAME}/${NAME}_${VERSION}"
+	mkdir -p $CHANGEPATH
+	cp ${BUILDDIR}/debian/changelog $CHANGEPATH
 	rm -rf "${BUILDDIR}"
 	msgdone "info"
 }
@@ -876,6 +882,7 @@ getcodenamefromsuite() {
 }
 getreleaseversionfromsuite() { true; }
 getlabelfromsuite() { true; }
+getoriginfromsuite() { true; }
 
 generatereleasefiles() {
 	# $1 is the Date header and $2 is the ValidUntil header to be set
@@ -887,16 +894,21 @@ generatereleasefiles() {
 			local CODENAME="$(getcodenamefromsuite $SUITE)"
 			local VERSION="$(getreleaseversionfromsuite $SUITE)"
 			local LABEL="$(getlabelfromsuite $SUITE)"
+			local ORIGIN="$(getoriginfromsuite $SUITE)"
 			if [ -n "$VERSION" ]; then
 				VERSION="-o APT::FTPArchive::Release::Version=${VERSION}"
 			fi
 			if [ -n "$LABEL" ]; then
 				LABEL="-o APT::FTPArchive::Release::Label=${LABEL}"
 			fi
+			if [ -n "$ORIGIN" ]; then
+				ORIGIN="-o APT::FTPArchive::Release::Origin=${ORIGIN}"
+			fi
 			aptftparchive -qq release $dir \
 				-o APT::FTPArchive::Release::Suite="${SUITE}" \
 				-o APT::FTPArchive::Release::Codename="${CODENAME}" \
 				${LABEL} \
+				${ORIGIN} \
 				${VERSION} \
 					| sed -e '/0 Release$/ d' > $dir/Release # remove the self reference
 			if [ "$SUITE" = "experimental" -o "$SUITE" = "experimental2" ]; then
@@ -1450,9 +1462,9 @@ testfilestats() {
 		msgpass
 	else
 		echo >&2
-		ls -ld >&2 "$1"
+		ls -ld >&2 "$1" || true
 		echo -n >&2 "stat(1) reports for $2: "
-		stat --format "$2" "$1"
+		stat --format "$2" "$1" || true
 		msgfail
 	fi
 }
diff --git a/test/integration/test-apt-get-changelog b/test/integration/test-apt-get-changelog
index 7e81c71b6..5fa8543b9 100755
--- a/test/integration/test-apt-get-changelog
+++ b/test/integration/test-apt-get-changelog
@@ -5,44 +5,98 @@ TESTDIR=$(readlink -f $(dirname $0))
 . $TESTDIR/framework
 
 setupenvironment
-configarchitecture "i386"
+configarchitecture 'native'
 
-buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
+buildsimplenativepackage 'foo' 'all' '1.0' 'stable'
+buildsimplenativepackage 'libbar' 'all' '1.0' 'stable'
+
+getlabelfromsuite() { echo 'Testcases'; }
+getoriginfromsuite() { echo 'Debian'; }
 
 setupaptarchive --no-update
 changetowebserver
 testsuccess aptget update
 
-# simulate normal user with non-existent root-owned directories
-rm -rf rootdir/var/cache/apt/archives/
-mkdir rootdir/var/cache/apt/archives/
-addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/cache/apt/archives || true;"
-chmod -R -w rootdir/var/cache/apt/archives
+testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/f/foo/foo_1.0_changelog' foo.changelog
+'http://metadata.ftp-master.debian.org/changelogs/main/libb/libbar/libbar_1.0_changelog' libbar.changelog" aptget changelog foo libbar --print-uris
+
+releasechanger() {
+	# modifying the Release files in lists… bad stuff. Good that this is only a test…
+	sed -i "s#^${1}: .*#${1}: ${2}#" $(find rootdir/var/lib/apt/lists -name '*Release')
+	rm -f rootdir/var/cache/apt/*.bin
+}
+releasechanger 'Origin' 'Ubuntu'
+testsuccessequal "'http://changelogs.ubuntu.com/changelogs/pool/main/f/foo/foo_1.0/changelog' foo.changelog
+'http://changelogs.ubuntu.com/changelogs/pool/main/libb/libbar/libbar_1.0/changelog' libbar.changelog" aptget changelog foo libbar --print-uris
+
+releasechanger 'Label' 'Debian'
+testsuccessequal "'http://changelogs.ubuntu.com/changelogs/pool/main/f/foo/foo_1.0/changelog' foo.changelog
+'http://changelogs.ubuntu.com/changelogs/pool/main/libb/libbar/libbar_1.0/changelog' libbar.changelog" aptget changelog foo libbar --print-uris
+
+testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0.changelog' foo.changelog
+'http://localhost:8080/main/libb/libbar/libbar_1.0.changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Label::Debian='http://localhost:8080/CHANGEPATH.changelog'
+
+sed -i '/^Origin: / a\
+Changelogs: http://example.org/CHANGEPATH-changelog' $(find rootdir/var/lib/apt/lists -name '*Release')
+rm -f rootdir/var/cache/apt/*.bin
 
-echo 'Apt::Changelogs::Server "http://localhost:8080/";' > rootdir/etc/apt/apt.conf.d/changelog.conf
+testsuccessequal "'http://example.org/main/f/foo/foo_1.0-changelog' foo.changelog
+'http://example.org/main/libb/libbar/libbar_1.0-changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Label::Debian='http://localhost:8080/CHANGEPATH.changelog'
 
-testsuccessequal "'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
+testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0.changelog' foo.changelog
+'http://localhost:8080/main/libb/libbar/libbar_1.0.changelog' libbar.changelog" aptget changelog foo libbar --print-uris -o Acquire::Changelogs::URI::Override::Label::Debian='http://localhost:8080/CHANGEPATH.changelog'
 
-testsuccessequal "'http://localhost:8080/pool/apt_1.0/changelog'
-'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt apt --print-uris
+releasechanger 'Changelogs' 'no'
+testequal 'E: Failed to fetch changelog:/foo.changelog  Changelog unavailable for foo=1.0
+' aptget changelog foo -qq -d
+
+sed -i '/^Changelogs: / d' $(find rootdir/var/lib/apt/lists -name '*Release')
+releasechanger 'Label' 'Testcases'
+
+echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/CHANGEPATH/change.txt";' > rootdir/etc/apt/apt.conf.d/changelog.conf
+testsuccessequal "'http://localhost:8080/main/f/foo/foo_1.0/change.txt' foo.changelog
+'http://localhost:8080/main/libb/libbar/libbar_1.0/change.txt' libbar.changelog" aptget changelog foo libbar --print-uris
+
+echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/pool/CHANGEPATH/changelog";' > rootdir/etc/apt/apt.conf.d/changelog.conf
+testsuccessequal "'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog" aptget changelog foo --print-uris
 
 cd downloaded
 
-testsuccess aptget changelog apt -qq
-testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/apt_1.0/changelog)"
+testsuccess aptget changelog foo -qq
+testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+
+testsuccess aptget changelog foo libbar -qq
+testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)
+$(cat ../aptarchive/pool/main/libb/libbar/libbar_1.0/changelog)"
+
+testsuccess aptget changelog foo -d
+testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
+testfileequal 'foo.changelog' "$(cat  ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+rm -f foo.changelog
 
-testsuccess aptget changelog apt -d
-testfileequal 'apt.changelog' "$(cat  ../aptarchive/pool/apt_1.0/changelog)"
-testfilestats 'apt.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
-rm -f apt.changelog ../aptarchive/pool/apt_1.0/changelog
+testsuccess aptget changelog libbar foo -d
+testfilestats 'libbar.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
+testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
+testfileequal 'libbar.changelog' "$(cat  ../aptarchive/pool/main/libb/libbar/libbar_1.0/changelog)"
+testfileequal 'foo.changelog' "$(cat  ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+rm -f libbar.changelog foo.changelog
 
-testequal "$(cat ../aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \
-	-qq -o APT::Changelogs::Server='http://not-on-the-main-server:8080/'
+# as such bogus, but can happen with multiple binaries from the same source
+testsuccessequal "'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog
+'http://localhost:8080/pool/main/f/foo/foo_1.0/changelog' foo.changelog" aptget changelog foo foo --print-uris
+testsuccess aptget changelog foo foo -qq
+testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)
+$(cat ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+testsuccess aptget changelog foo foo -d
+testfilestats 'foo.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
+testfileequal 'foo.changelog' "$(cat  ../aptarchive/pool/main/f/foo/foo_1.0/changelog)"
+rm -f foo.changelog
 
-testsuccess aptget changelog apt -d
-testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0.changelog)"
-testfilestats 'apt.changelog' '%U:%G:%a' '=' "${TEST_DEFAULT_USER}:${TEST_DEFAULT_GROUP}:644"
-rm -f apt.changelog ../aptarchive/pool/apt_1.0.changelog
+# no CHANGEPATH in the URI
+testequal 'E: Failed to fetch changelog:/foo.changelog  Changelog unavailable for foo=1.0
+' aptget changelog foo -qq -d -o Acquire::Changelogs::URI::Label::Testcases='http://localhost:8080/change.txt'
+testfailure test -e foo.changelog
 
-testequal 'E: changelog download failed' aptget changelog apt -qq -d -o APT::Changelogs::Server='http://not-on-the-main-server:8080/'
-testfailure test -e apt.changelog
+testequal 'E: Failed to fetch http://localhost:8080/does/not/exist/main/f/foo/foo_1.0/change.txt  Changelog unavailable for foo=1.0 (404  Not Found)
+' aptget changelog foo -qq -d -o Acquire::Changelogs::URI::Label::Testcases='http://localhost:8080/does/not/exist/CHANGEPATH/change.txt'
+testfailure test -e foo.changelog
diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
index 2cad929cc..e51d72ccd 100755
--- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet
+++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet
@@ -12,6 +12,7 @@ insertpackage 'unstable' 'apt' 'all' '2'
 insertsource 'unstable' 'apt' 'all' '2'
 insertsource 'unstable' 'apt2' 'all' '1'
 
+getoriginfromsuite() { echo 'Debian'; }
 setupaptarchive
 
 APTARCHIVE=$(readlink -f ./aptarchive)
@@ -22,7 +23,7 @@ testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.d
 testsuccessequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris
 testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris
-testsuccessequal "'http://packages.debian.org/changelogs/pool/main/apt/apt_2/changelog'" aptget changelog apt -qq --print-uris
+testsuccessequal "'http://metadata.ftp-master.debian.org/changelogs/main/a/apt/apt_2_changelog' apt.changelog" aptget changelog apt -qq --print-uris
 
 testsuccessequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e
diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol
index f6336ffe3..539080200 100755
--- a/test/integration/test-bug-738785-switch-protocol
+++ b/test/integration/test-bug-738785-switch-protocol
@@ -10,6 +10,7 @@ configarchitecture "i386"
 buildsimplenativepackage 'apt' 'all' '1.0' 'stable'
 
 # setup http redirecting to https
+getlabelfromsuite() { echo 'Testcases'; }
 setupaptarchive --no-update
 changetowebserver -o 'aptwebserver::redirect::replace::/redirectme/=https://localhost:4433/' \
 	-o 'aptwebserver::redirect::replace::/downgrademe/=http://localhost:8080/' \
@@ -20,10 +21,10 @@ sed -i -e 's#:4433/#:8080/redirectme#' -e 's# https:# http:#' rootdir/etc/apt/so
 testsuccess aptget update -o Debug::Acquire::http=1 -o Debug::Acquire::https=1 -o Debug::pkgAcquire::Worker=1
 
 msgtest 'Test that the webserver does not answer' 'http requests'
-downloadfile 'http://localhost:8080/pool/apt_1.0/changelog' changelog >/dev/null 2>&1 && msgfail || msgpass
+downloadfile 'http://localhost:8080/pool/main/a/apt/apt_1.0/changelog' changelog >/dev/null 2>&1 && msgfail || msgpass
 
-echo 'Apt::Changelogs::Server "http://localhost:8080/redirectme";' > rootdir/etc/apt/apt.conf.d/changelog.conf
-testsuccessequal "'http://localhost:8080/redirectme/pool/apt_1.0/changelog'" aptget changelog apt --print-uris
+echo 'Acquire::Changelogs::URI::Label::Testcases "http://localhost:8080/redirectme/pool/CHANGEPATH/changelog";' > rootdir/etc/apt/apt.conf.d/changelog.conf
+testsuccessequal "'http://localhost:8080/redirectme/pool/main/a/apt/apt_1.0/changelog' apt.changelog" aptget changelog apt --print-uris
 
 cd downloaded
 testsuccess aptget changelog apt -d
diff --git a/vendor/ubuntu/apt.conf-01-vendor-ubuntu b/vendor/ubuntu/apt.conf-01-vendor-ubuntu
index c4092ff44..e69de29bb 100644
--- a/vendor/ubuntu/apt.conf-01-vendor-ubuntu
+++ b/vendor/ubuntu/apt.conf-01-vendor-ubuntu
@@ -1,6 +0,0 @@
-// Server information for apt-changelog
-APT {
-  Changelogs {
-    Server "http://changelogs.ubuntu.com/changelogs";
-  };
-};
-- 
cgit v1.2.3-70-g09d2