From cb6a2b3eaca4353d7f490fb360b98c08d64a2d8c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 14 Jan 2013 07:09:51 +0100 Subject: first iteration that adds support for checksums-{sha512,sha256} --- cmdline/apt-get.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 1bb981b20..68e7f829c 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -2555,15 +2555,12 @@ bool DoSource(CommandLine &CmdL) if(queued.find(Last->Index().ArchiveURI(I->Path)) != queued.end()) continue; queued.insert(Last->Index().ArchiveURI(I->Path)); - + // check if we have a file with that md5 sum already localy - if(!I->MD5Hash.empty() && FileExists(flNotDir(I->Path))) + if(!I->Hash.empty() && FileExists(flNotDir(I->Path))) { - FileFd Fd(flNotDir(I->Path), FileFd::ReadOnly); - MD5Summation sum; - sum.AddFD(Fd.Fd(), Fd.Size()); - Fd.Close(); - if((string)sum.Result() == I->MD5Hash) + HashString hash_string = HashString(I->Hash); + if(hash_string.VerifyFile(I->Path)) { ioprintf(c1out,_("Skipping already downloaded file '%s'\n"), flNotDir(I->Path).c_str()); @@ -2572,7 +2569,7 @@ bool DoSource(CommandLine &CmdL) } new pkgAcqFile(&Fetcher,Last->Index().ArchiveURI(I->Path), - I->MD5Hash,I->Size, + I->Hash,I->Size, Last->Index().SourceInfo(*Last,*I),Src); } } -- cgit v1.2.3-70-g09d2 From 64680d3bf12db6599e93f950546e380e0bd5ddf4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 14 Jan 2013 07:59:52 +0100 Subject: fix skipping of already downloaded files and add some FIXMEs --- apt-pkg/deb/debsrcrecords.cc | 13 +++++++++---- cmdline/apt-get.cc | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc index 45cc0ae82..c620c2298 100644 --- a/apt-pkg/deb/debsrcrecords.cc +++ b/apt-pkg/deb/debsrcrecords.cc @@ -116,16 +116,21 @@ bool debSrcRecordParser::Files(std::vector &List) { List.erase(List.begin(),List.end()); + // FIXME: build string dynamically from + // Hashes::SupportedHashes const char *hash_field[] = { "Checksums-Sha512", "Checksums-Sha256", "Checksums-Sha1", "Files", // historic name NULL, }; - const char *hash_type[] = { "Sha512", - "Sha256", - "Sha1", - "md5sum", + + // FIXME: use string from Hashes::SupportedHashes + // FIXME2: this is case senstivie + const char *hash_type[] = { "SHA512", + "SHA256", + "SHA1", + "MD5Sum", NULL, }; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 68e7f829c..a2a19cd11 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -2560,7 +2560,7 @@ bool DoSource(CommandLine &CmdL) if(!I->Hash.empty() && FileExists(flNotDir(I->Path))) { HashString hash_string = HashString(I->Hash); - if(hash_string.VerifyFile(I->Path)) + if(hash_string.VerifyFile(flNotDir(I->Path))) { ioprintf(c1out,_("Skipping already downloaded file '%s'\n"), flNotDir(I->Path).c_str()); -- cgit v1.2.3-70-g09d2 From 2a49601f69e08f06fb2727d869d420daacdd09d5 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 23 Apr 2013 08:16:21 +0200 Subject: merge patch from Daniel Hartwig to Show a error message if {,dist-}upgrade is used with additional arguments (closes: #705510) --- cmdline/apt-get.cc | 6 ++++++ debian/changelog | 2 ++ 2 files changed, 8 insertions(+) (limited to 'cmdline') diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index eaa50519c..cce80ccfe 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1845,6 +1845,9 @@ bool DoAutomaticRemove(CacheFile &Cache) packages */ bool DoUpgrade(CommandLine &CmdL) { + if (CmdL.FileSize() != 1) + return _error->Error(_("The upgrade command takes no arguments")); + CacheFile Cache; if (Cache.OpenForInstall() == false || Cache.CheckDeps() == false) return false; @@ -2178,6 +2181,9 @@ bool DoMarkAuto(CommandLine &CmdL) /* Intelligent upgrader that will install and remove packages at will */ bool DoDistUpgrade(CommandLine &CmdL) { + if (CmdL.FileSize() != 1) + return _error->Error(_("The dist-upgrade command takes no arguments")); + CacheFile Cache; if (Cache.OpenForInstall() == false || Cache.CheckDeps() == false) return false; diff --git a/debian/changelog b/debian/changelog index 5b0aaec3d..0e3529006 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,8 @@ apt (0.9.8~exp1) UNRELEASED; urgency=low [ Daniel Hartwig ] * Clarify units of Acquire::http::Dl-Limit (closes: #705445) + * Show a error message if {,dist-}upgrade is used with additional + arguments (closes: #705510) [ Michael Vogt ] * lp:~mvo/apt/webserver-simulate-broken-with-fix346386: -- cgit v1.2.3-70-g09d2 From feab34c5216941ca95aae1a389238a77b662d1de Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 22 Apr 2014 17:59:09 +0200 Subject: add support for apt-get build-dep foo.dsc --- apt-pkg/deb/debsrcrecords.cc | 19 ++++++++ apt-pkg/deb/debsrcrecords.h | 7 +++ apt-pkg/tagfile.cc | 11 +++++ apt-pkg/tagfile.h | 2 + cmdline/apt-get.cc | 20 ++++++-- test/integration/test-apt-get-build-dep | 85 +++++++++++++++++++++++++++++++++ 6 files changed, 141 insertions(+), 3 deletions(-) create mode 100755 test/integration/test-apt-get-build-dep (limited to 'cmdline') diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc index b09588dd3..fc748c40f 100644 --- a/apt-pkg/deb/debsrcrecords.cc +++ b/apt-pkg/deb/debsrcrecords.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -189,3 +190,21 @@ debSrcRecordParser::~debSrcRecordParser() delete[] Buffer; } /*}}}*/ + + +debDscRecordParser::debDscRecordParser(std::string const &DscFile) + : debSrcRecordParser(DscFile, NULL) +{ + // support clear signed files + if (OpenMaybeClearSignedFile(DscFile, Fd) == false) + { + _error->Error("Failed to open %s", DscFile.c_str()); + return; + } + + // re-init to ensure the updated Fd is used + Tags.Init(&Fd); + // read the first (and only) record + Step(); + +} diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h index b65d1480b..0cd664818 100644 --- a/apt-pkg/deb/debsrcrecords.h +++ b/apt-pkg/deb/debsrcrecords.h @@ -26,6 +26,7 @@ class debSrcRecordParser : public pkgSrcRecords::Parser /** \brief dpointer placeholder (for later in case we need it) */ void *d; + protected: FileFd Fd; pkgTagFile Tags; pkgTagSection Sect; @@ -60,4 +61,10 @@ class debSrcRecordParser : public pkgSrcRecords::Parser virtual ~debSrcRecordParser(); }; +class debDscRecordParser : public debSrcRecordParser +{ + public: + debDscRecordParser(std::string const &DscFile); +}; + #endif diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 91d176e3c..009ed7d74 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -51,12 +51,23 @@ public: // --------------------------------------------------------------------- /* */ pkgTagFile::pkgTagFile(FileFd *pFd,unsigned long long Size) + : d(NULL) +{ + Init(pFd, Size); +} + +void pkgTagFile::Init(FileFd *pFd,unsigned long long Size) { /* The size is increased by 4 because if we start with the Size of the filename we need to try to read 1 char more to see an EOF faster, 1 char the end-pointer can be on and maybe 2 newlines need to be added to the end of the file -> 4 extra chars */ Size += 4; + if(d != NULL) + { + free(d->Buffer); + delete d; + } d = new pkgTagFilePrivate(pFd, Size); if (d->Fd.IsOpen() == false) diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index d5b62e76d..d1a24ba45 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -105,6 +105,8 @@ class pkgTagFile unsigned long Offset(); bool Jump(pkgTagSection &Tag,unsigned long long Offset); + void Init(FileFd *F,unsigned long long Size = 32*1024); + pkgTagFile(FileFd *F,unsigned long long Size = 32*1024); virtual ~pkgTagFile(); }; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index caf69da2a..92384931b 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -57,6 +57,9 @@ #include #include +// FIXME: direct include of deb specific header +#include + #include #include #include @@ -1053,12 +1056,23 @@ static bool DoBuildDep(CommandLine &CmdL) for (const char **I = CmdL.FileList + 1; *I != 0; I++, J++) { string Src; - pkgSrcRecords::Parser *Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); + pkgSrcRecords::Parser *Last = 0; + vector BuildDeps; + + // support local .dsc files + if (FileExists(*I) && flExtension(*I) == "dsc") + { + // FIXME: add a layer of abstraction + Last = new debDscRecordParser(*I); + Src = *I; + } else { + Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); + } if (Last == 0) return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); // Process the build-dependencies - vector BuildDeps; + // FIXME: Can't specify architecture to use for [wildcard] matching, so switch default arch temporary if (hostArch.empty() == false) { @@ -1071,7 +1085,7 @@ static bool DoBuildDep(CommandLine &CmdL) } else if (Last->BuildDepends(BuildDeps, _config->FindB("APT::Get::Arch-Only", false), StripMultiArch) == false) return _error->Error(_("Unable to get build-dependency information for %s"),Src.c_str()); - + // Also ensure that build-essential packages are present Configuration::Item const *Opts = _config->Tree("APT::Build-Essential"); if (Opts) diff --git a/test/integration/test-apt-get-build-dep b/test/integration/test-apt-get-build-dep new file mode 100755 index 000000000..6516aa705 --- /dev/null +++ b/test/integration/test-apt-get-build-dep @@ -0,0 +1,85 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +buildsimplenativepackage 'debhelper' 'i386' '7' 'stable' +buildsimplenativepackage 'build-essential' 'i386' '1' 'stable' + +setupaptarchive +cat > 2vcard_0.5-3.dsc < +Uploaders: Marcela Tiznado +Standards-Version: 3.8.0 +Build-Depends: debhelper (>= 5.0.37) +Checksums-Sha1: + b7f1ce31ec856414a3f0f1090689f91aa7456d56 9398 2vcard_0.5.orig.tar.gz + 5f9acd07ebda6ab00fa6b4fe3198c13e94090862 2036 2vcard_0.5-3.diff.gz +Checksums-Sha256: + efdc22859ac2f8f030d038dc4faa9020082ebae34212498c288968ffd45c9764 9398 2vcard_0.5.orig.tar.gz + 82673ff3456af571094066c89bcea87b25c23c87cf1d0050b731e5222563626b 2036 2vcard_0.5-3.diff.gz +Files: + f73a69c170f772f3f6e75f2d11bbb792 9398 2vcard_0.5.orig.tar.gz + 1e806d32233af87437258d86b1561f57 2036 2vcard_0.5-3.diff.gz +EOF + +testequal "Reading package lists... +Building dependency tree... +The following NEW packages will be installed: + build-essential debhelper +0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded. +Inst build-essential (1 stable [i386]) +Inst debhelper (7 stable [i386]) +Conf build-essential (1 stable [i386]) +Conf debhelper (7 stable [i386])" aptget build-dep -s 2vcard_0.5-3.dsc + +cat > 2vcard_0.5-3.dsc < +Uploaders: Marcela Tiznado +Standards-Version: 3.8.0 +Build-Depends: debhelper (>= 5.0.37) +Checksums-Sha1: + b7f1ce31ec856414a3f0f1090689f91aa7456d56 9398 2vcard_0.5.orig.tar.gz + 5f9acd07ebda6ab00fa6b4fe3198c13e94090862 2036 2vcard_0.5-3.diff.gz +Checksums-Sha256: + efdc22859ac2f8f030d038dc4faa9020082ebae34212498c288968ffd45c9764 9398 2vcard_0.5.orig.tar.gz + 82673ff3456af571094066c89bcea87b25c23c87cf1d0050b731e5222563626b 2036 2vcard_0.5-3.diff.gz +Files: + f73a69c170f772f3f6e75f2d11bbb792 9398 2vcard_0.5.orig.tar.gz + 1e806d32233af87437258d86b1561f57 2036 2vcard_0.5-3.diff.gz + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.9 (GNU/Linux) + +iEYEARECAAYFAkijKhsACgkQsrBfRdYmq7aA2gCfaOW9riTYVQMx5ajKQVAcctlC +z2UAn1oXgTai6opwhVfkxrlmJ+iRxzuc +=4eRd +-----END PGP SIGNATURE----- +EOF + +testequal "Reading package lists... +Building dependency tree... +The following NEW packages will be installed: + build-essential debhelper +0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded. +Inst build-essential (1 stable [i386]) +Inst debhelper (7 stable [i386]) +Conf build-essential (1 stable [i386]) +Conf debhelper (7 stable [i386])" aptget build-dep --simulate 2vcard_0.5-3.dsc -- cgit v1.2.3-70-g09d2 From a49e7948029d8219d7cb182fbc1b0adb587691b8 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 23 Apr 2014 13:51:48 +0200 Subject: create debIFTypeDscFile type --- apt-pkg/deb/debindexfile.cc | 52 ++++++++++++++++++++++++++++++++++++++++++++ apt-pkg/deb/debindexfile.h | 18 +++++++++++++++ apt-pkg/deb/debsrcrecords.cc | 4 ++-- apt-pkg/deb/debsrcrecords.h | 2 +- apt-pkg/indexfile.h | 1 + apt-pkg/pkgsystem.h | 6 +++-- apt-pkg/sourcelist.h | 2 +- cmdline/apt-get.cc | 20 ++++++++--------- 8 files changed, 89 insertions(+), 16 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index eee758b7a..e7e764dd4 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -667,6 +667,42 @@ APT_CONST bool debStatusIndex::Exists() const } /*}}}*/ + +// debDscFileIndex stuff +debDscFileIndex::debDscFileIndex(std::string &DscFile) + : pkgIndexFile(true), DscFile(DscFile) +{ +} + +bool debDscFileIndex::Exists() const +{ + return FileExists(DscFile); +} + +unsigned long debDscFileIndex::Size() const +{ + struct stat buf; + if(stat(DscFile.c_str(), &buf) == 0) + return buf.st_size; + return 0; +} + +// DscFileIndex::CreateSrcParser - Get a parser for the .dsc file /*{{{*/ +// --------------------------------------------------------------------- +/* */ +pkgSrcRecords::Parser *debDscFileIndex::CreateSrcParser() const +{ + if (!FileExists(DscFile)) + return NULL; + + return new debDscRecordParser(DscFile,this); +} + /*}}}*/ + + + + +// --------------------------------------------------------------------- // Index File types for Debian /*{{{*/ class debIFTypeSrc : public pkgIndexFile::Type { @@ -699,10 +735,22 @@ class debIFTypeStatus : public pkgIndexFile::Type }; debIFTypeStatus() {Label = "Debian dpkg status file";}; }; +class debIFTypeDscFile : public pkgIndexFile::Type +{ + public: + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string DscFile) const + { + return new debDscRecordParser(DscFile, NULL); + }; + debIFTypeDscFile() {Label = "dsc File Source Index";}; +}; + static debIFTypeSrc _apt_Src; static debIFTypePkg _apt_Pkg; static debIFTypeTrans _apt_Trans; static debIFTypeStatus _apt_Status; +// file based pseudo indexes +static debIFTypeDscFile _apt_DscFile; const pkgIndexFile::Type *debSourcesIndex::GetType() const { @@ -720,5 +768,9 @@ const pkgIndexFile::Type *debStatusIndex::GetType() const { return &_apt_Status; } +const pkgIndexFile::Type *debDscFileIndex::GetType() const +{ + return &_apt_DscFile; +} /*}}}*/ diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index 017c69a0a..88abfdd9d 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -164,4 +164,22 @@ class debSourcesIndex : public pkgIndexFile virtual ~debSourcesIndex() {}; }; +class debDscFileIndex : public pkgIndexFile +{ + private: + std::string DscFile; + public: + virtual const Type *GetType() const APT_CONST; + virtual pkgSrcRecords::Parser *CreateSrcParser() const; + virtual bool Exists() const; + virtual bool HasPackages() const {return false;}; + virtual unsigned long Size() const; + virtual std::string Describe(bool /*Short*/) const { + return DscFile; + }; + + debDscFileIndex(std::string &DscFile); + virtual ~debDscFileIndex() {}; +}; + #endif diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc index fc748c40f..8aad81e3c 100644 --- a/apt-pkg/deb/debsrcrecords.cc +++ b/apt-pkg/deb/debsrcrecords.cc @@ -192,8 +192,8 @@ debSrcRecordParser::~debSrcRecordParser() /*}}}*/ -debDscRecordParser::debDscRecordParser(std::string const &DscFile) - : debSrcRecordParser(DscFile, NULL) +debDscRecordParser::debDscRecordParser(std::string const &DscFile, pkgIndexFile const *Index) + : debSrcRecordParser(DscFile, Index) { // support clear signed files if (OpenMaybeClearSignedFile(DscFile, Fd) == false) diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h index 0cd664818..a0a151875 100644 --- a/apt-pkg/deb/debsrcrecords.h +++ b/apt-pkg/deb/debsrcrecords.h @@ -64,7 +64,7 @@ class debSrcRecordParser : public pkgSrcRecords::Parser class debDscRecordParser : public debSrcRecordParser { public: - debDscRecordParser(std::string const &DscFile); + debDscRecordParser(std::string const &DscFile, pkgIndexFile const *Index); }; #endif diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h index b5c9ac77e..817165f08 100644 --- a/apt-pkg/indexfile.h +++ b/apt-pkg/indexfile.h @@ -59,6 +59,7 @@ class pkgIndexFile const char *Label; virtual pkgRecords::Parser *CreatePkgParser(pkgCache::PkgFileIterator /*File*/) const {return 0;}; + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string /*File*/) const {return 0;}; Type(); virtual ~Type() {}; }; diff --git a/apt-pkg/pkgsystem.h b/apt-pkg/pkgsystem.h index 6e33c67ed..f88ffa7c8 100644 --- a/apt-pkg/pkgsystem.h +++ b/apt-pkg/pkgsystem.h @@ -85,10 +85,12 @@ class pkgSystem virtual bool AddStatusFiles(std::vector &List) = 0; virtual bool FindIndex(pkgCache::PkgFileIterator File, pkgIndexFile *&Found) const = 0; - + /* Evauluate how 'right' we are for this system based on the filesystem etc.. */ - virtual signed Score(Configuration const &/*Cnf*/) {return 0;}; + virtual signed Score(Configuration const &/*Cnf*/) { + return 0; + }; pkgSystem(); virtual ~pkgSystem() {}; diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 9df0c1d74..0ebf4e080 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -86,7 +86,7 @@ class pkgSourceList typedef std::vector::const_iterator const_iterator; - protected: + public: std::vector SrcList; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 92384931b..5418c351b 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -57,9 +57,6 @@ #include #include -// FIXME: direct include of deb specific header -#include - #include #include #include @@ -1057,22 +1054,25 @@ static bool DoBuildDep(CommandLine &CmdL) { string Src; pkgSrcRecords::Parser *Last = 0; - vector BuildDeps; - // support local .dsc files - if (FileExists(*I) && flExtension(*I) == "dsc") + // if its a local file (e.g. .dsc) use this + if (FileExists(*I)) { - // FIXME: add a layer of abstraction - Last = new debDscRecordParser(*I); - Src = *I; + // see if we can get a parser for this pkgIndexFile type + string TypeName = flExtension(*I) + " File Source Index"; + pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); + if(Type != NULL) + Last = Type->CreateSrcPkgParser(*I); } else { + // normal case, search the cache for the source file Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); } + if (Last == 0) return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); // Process the build-dependencies - + vector BuildDeps; // FIXME: Can't specify architecture to use for [wildcard] matching, so switch default arch temporary if (hostArch.empty() == false) { -- cgit v1.2.3-70-g09d2 From 77da39b95870498431fc21df65900acc5ce2f7ea Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 25 Apr 2014 09:47:50 +0200 Subject: add support for apt-get build-dep unpacked-source-dir --- apt-pkg/deb/debindexfile.cc | 14 +++++++++++ apt-pkg/deb/debindexfile.h | 5 ++++ cmdline/apt-get.cc | 11 ++++++++- test/integration/test-apt-get-build-dep | 41 +++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) (limited to 'cmdline') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index e7e764dd4..bb6884123 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -744,6 +744,15 @@ class debIFTypeDscFile : public pkgIndexFile::Type }; debIFTypeDscFile() {Label = "dsc File Source Index";}; }; +class debIFTypeDebianSourceDir : public pkgIndexFile::Type +{ + public: + virtual pkgSrcRecords::Parser *CreateSrcPkgParser(std::string SourceDir) const + { + return new debDscRecordParser(SourceDir + string("/debian/control"), NULL); + }; + debIFTypeDebianSourceDir() {Label = "debian/control File Source Index";}; +}; static debIFTypeSrc _apt_Src; static debIFTypePkg _apt_Pkg; @@ -751,6 +760,7 @@ static debIFTypeTrans _apt_Trans; static debIFTypeStatus _apt_Status; // file based pseudo indexes static debIFTypeDscFile _apt_DscFile; +static debIFTypeDebianSourceDir _apt_DebianSourceDir; const pkgIndexFile::Type *debSourcesIndex::GetType() const { @@ -772,5 +782,9 @@ const pkgIndexFile::Type *debDscFileIndex::GetType() const { return &_apt_DscFile; } +const pkgIndexFile::Type *debDebianSourceDirIndex::GetType() const +{ + return &_apt_DebianSourceDir; +} /*}}}*/ diff --git a/apt-pkg/deb/debindexfile.h b/apt-pkg/deb/debindexfile.h index 88abfdd9d..64ca558d2 100644 --- a/apt-pkg/deb/debindexfile.h +++ b/apt-pkg/deb/debindexfile.h @@ -182,4 +182,9 @@ class debDscFileIndex : public pkgIndexFile virtual ~debDscFileIndex() {}; }; +class debDebianSourceDirIndex : public debDscFileIndex +{ + virtual const Type *GetType() const APT_CONST; +}; + #endif diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 5418c351b..6d6ba5ff9 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1055,8 +1055,17 @@ static bool DoBuildDep(CommandLine &CmdL) string Src; pkgSrcRecords::Parser *Last = 0; + // a unpacked debian source tree + if (DirectoryExists(*I)) + { + // FIXME: how can we make this more elegant? + std::string TypeName = "debian/control File Source Index"; + pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); + if(Type != NULL) + Last = Type->CreateSrcPkgParser(*I); + } // if its a local file (e.g. .dsc) use this - if (FileExists(*I)) + else if (FileExists(*I)) { // see if we can get a parser for this pkgIndexFile type string TypeName = flExtension(*I) + " File Source Index"; diff --git a/test/integration/test-apt-get-build-dep b/test/integration/test-apt-get-build-dep index 6516aa705..f71beae9c 100755 --- a/test/integration/test-apt-get-build-dep +++ b/test/integration/test-apt-get-build-dep @@ -83,3 +83,44 @@ Inst build-essential (1 stable [i386]) Inst debhelper (7 stable [i386]) Conf build-essential (1 stable [i386]) Conf debhelper (7 stable [i386])" aptget build-dep --simulate 2vcard_0.5-3.dsc + + +# unpacked source dir +mkdir -p foo-1.0/debian +cat > foo-1.0/debian/control <<'EOF' +Source: apturl +Section: admin +Priority: optional +Maintainer: Michael Vogt +Build-Depends: debhelper (>= 7) +X-Python3-Version: >= 3.2 +Standards-Version: 3.9.3 + +Package: apturl-common +Architecture: any +Depends: ${python3:Depends}, + ${shlibs:Depends}, + ${misc:Depends}, + python3-apt, + python3-update-manager +Replaces: apturl (<< 0.3.6ubuntu2) +Description: install packages using the apt protocol - common data + AptUrl is a simple graphical application that takes an URL (which follows the + apt-protocol) as a command line option, parses it and carries out the + operations that the URL describes (that is, it asks the user if he wants the + indicated packages to be installed and if the answer is positive does so for + him). + . + This package contains the common data shared between the frontends. + +EOF + +testequal "Reading package lists... +Building dependency tree... +The following NEW packages will be installed: + build-essential debhelper +0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded. +Inst build-essential (1 stable [i386]) +Inst debhelper (7 stable [i386]) +Conf build-essential (1 stable [i386]) +Conf debhelper (7 stable [i386])" aptget build-dep --simulate ./foo-1.0 -- cgit v1.2.3-70-g09d2 From 1262d35895c930f3fa49d7b4182cdd7a4a841f74 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 18 Aug 2013 23:27:24 +0200 Subject: use 'best' hash for source authentication Collect all hashes we can get from the source record and put them into a HashStringList so that 'apt-get source' can use it instead of using always the MD5sum. We therefore also deprecate the MD5 struct member in favor of the list. While at it, the parsing of the Files is enhanced so that records which miss "Files" (aka MD5 checksums) are still searched for other checksums as they include just as much data, just not with a nice and catchy name. LP: 1098738 --- apt-pkg/deb/debsrcrecords.cc | 170 ++++++++------ apt-pkg/srcrecords.h | 17 +- cmdline/apt-get.cc | 19 +- test/integration/test-debsrc-hashes | 77 ------ .../test-ubuntu-bug-1098738-apt-get-source-md5sum | 260 +++++++++++++++++++++ 5 files changed, 385 insertions(+), 158 deletions(-) delete mode 100755 test/integration/test-debsrc-hashes create mode 100755 test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum (limited to 'cmdline') diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc index 7b9a828d3..bf5e56ec9 100644 --- a/apt-pkg/deb/debsrcrecords.cc +++ b/apt-pkg/deb/debsrcrecords.cc @@ -124,85 +124,113 @@ bool debSrcRecordParser::Files(std::vector &List) { List.erase(List.begin(),List.end()); - // map from the Hashsum field to the hashsum function, - // unfortunately this is not a 1:1 mapping from - // Hashes::SupporedHashes as e.g. Files is a historic name for the md5 - const std::pair SourceHashFields[] = { - std::make_pair( "Checksums-Sha512", "SHA512"), - std::make_pair( "Checksums-Sha256", "SHA256"), - std::make_pair( "Checksums-Sha1", "SHA1"), - std::make_pair( "Files", "MD5Sum"), // historic Name - }; - - for (unsigned int i=0; - i < sizeof(SourceHashFields)/sizeof(SourceHashFields[0]); - i++) - { - string Files = Sect.FindS(SourceHashFields[i].first); - if (Files.empty() == true) - continue; + // Stash the / terminated directory prefix + string Base = Sect.FindS("Directory"); + if (Base.empty() == false && Base[Base.length()-1] != '/') + Base += '/'; - // Stash the / terminated directory prefix - string Base = Sect.FindS("Directory"); - if (Base.empty() == false && Base[Base.length()-1] != '/') - Base += '/'; + std::vector const compExts = APT::Configuration::getCompressorExtensions(); - std::vector const compExts = APT::Configuration::getCompressorExtensions(); + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + // derive field from checksum type + std::string checksumField("Checksums-"); + if (strcmp(*type, "MD5Sum") == 0) + checksumField = "Files"; // historic name for MD5 checksums + else + checksumField.append(*type); + + string const Files = Sect.FindS(checksumField.c_str()); + if (Files.empty() == true) + continue; // Iterate over the entire list grabbing each triplet const char *C = Files.c_str(); while (*C != 0) - { - pkgSrcRecords::File F; - string Size; - - // Parse each of the elements - std::string RawHash; - if (ParseQuoteWord(C, RawHash) == false || - ParseQuoteWord(C, Size) == false || - ParseQuoteWord(C, F.Path) == false) - return _error->Error("Error parsing '%s' record", - SourceHashFields[i].first); - // assign full hash string - F.Hash = HashString(SourceHashFields[i].second, RawHash).toStr(); - // API compat hack - if(strcmp(SourceHashFields[i].second, "MD5Sum") == 0) - F.MD5Hash = RawHash; - - // Parse the size and append the directory - F.Size = atoi(Size.c_str()); - F.Path = Base + F.Path; - - // Try to guess what sort of file it is we are getting. - string::size_type Pos = F.Path.length()-1; - while (1) - { - string::size_type Tmp = F.Path.rfind('.',Pos); - if (Tmp == string::npos) - break; - if (F.Type == "tar") { - // source v3 has extension 'debian.tar.*' instead of 'diff.*' - if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") - F.Type = "diff"; - break; - } - F.Type = string(F.Path,Tmp+1,Pos-Tmp); - - if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || - F.Type == "tar") - { - Pos = Tmp-1; - continue; - } - - break; - } - - List.push_back(F); + { + string hash, size, path; + + // Parse each of the elements + if (ParseQuoteWord(C, hash) == false || + ParseQuoteWord(C, size) == false || + ParseQuoteWord(C, path) == false) + return _error->Error("Error parsing file record in %s of source package %s", checksumField.c_str(), Package().c_str()); + + HashString const hashString(*type, hash); + if (Base.empty() == false) + path = Base + path; + + // look if we have a record for this file already + std::vector::iterator file = List.begin(); + for (; file != List.end(); ++file) + if (file->Path == path) + break; + + // we have it already, store the new hash and be done + if (file != List.end()) + { +#if __GNUC__ >= 4 + // set for compatibility only, so warn users not us + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + if (checksumField == "Files") + file->MD5Hash = hash; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + // an error here indicates that we have two different hashes for the same file + if (file->Hashes.push_back(hashString) == false) + return _error->Error("Error parsing checksum in %s of source package %s", checksumField.c_str(), Package().c_str()); + continue; + } + + // we haven't seen this file yet + pkgSrcRecords::File F; + F.Path = path; + F.Size = strtoull(size.c_str(), NULL, 10); + F.Hashes.push_back(hashString); + +#if __GNUC__ >= 4 + // set for compatibility only, so warn users not us + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + if (checksumField == "Files") + F.MD5Hash = hash; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + + // Try to guess what sort of file it is we are getting. + string::size_type Pos = F.Path.length()-1; + while (1) + { + string::size_type Tmp = F.Path.rfind('.',Pos); + if (Tmp == string::npos) + break; + if (F.Type == "tar") { + // source v3 has extension 'debian.tar.*' instead of 'diff.*' + if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") + F.Type = "diff"; + break; + } + F.Type = string(F.Path,Tmp+1,Pos-Tmp); + + if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || + F.Type == "tar") + { + Pos = Tmp-1; + continue; + } + + break; + } + List.push_back(F); } - break; } - return (List.size() > 0); + + return true; } /*}}}*/ // SrcRecordParser::~SrcRecordParser - Destructor /*{{{*/ diff --git a/apt-pkg/srcrecords.h b/apt-pkg/srcrecords.h index 58a5e242f..69b3cfd99 100644 --- a/apt-pkg/srcrecords.h +++ b/apt-pkg/srcrecords.h @@ -14,6 +14,7 @@ #define PKGLIB_SRCRECORDS_H #include +#include #include #include @@ -29,16 +30,24 @@ class pkgSrcRecords { public: +#if __GNUC__ >= 4 + // ensure that con- & de-structor don't trigger this warning + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif // Describes a single file struct File { - std::string MD5Hash; - std::string Hash; - unsigned long Size; std::string Path; std::string Type; + unsigned long long Size; + HashStringList Hashes; + APT_DEPRECATED std::string MD5Hash; }; - +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + // Abstract parser for each source record class Parser { diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 1148dbbf3..fc6223989 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -828,19 +828,26 @@ static bool DoSource(CommandLine &CmdL) queued.insert(Last->Index().ArchiveURI(I->Path)); // check if we have a file with that md5 sum already localy - if(!I->Hash.empty() && FileExists(flNotDir(I->Path))) - { - HashString hash_string = HashString(I->Hash); - if(hash_string.VerifyFile(flNotDir(I->Path))) + std::string localFile = flNotDir(I->Path); + if (FileExists(localFile) == true) + if(I->Hashes.VerifyFile(localFile) == true) { ioprintf(c1out,_("Skipping already downloaded file '%s'\n"), - flNotDir(I->Path).c_str()); + localFile.c_str()); continue; } + + // see if we have a hash (Acquire::ForceHash is the only way to have none) + HashString const * const hs = I->Hashes.find(NULL); + if (hs == NULL && _config->FindB("APT::Get::AllowUnauthenticated",false) == false) + { + ioprintf(c1out, "Skipping download of file '%s' as requested hashsum is not available for authentication\n", + localFile.c_str()); + continue; } new pkgAcqFile(&Fetcher,Last->Index().ArchiveURI(I->Path), - I->Hash,I->Size, + hs != NULL ? hs->toStr() : "", I->Size, Last->Index().SourceInfo(*Last,*I),Src); } } diff --git a/test/integration/test-debsrc-hashes b/test/integration/test-debsrc-hashes deleted file mode 100755 index 5e917232b..000000000 --- a/test/integration/test-debsrc-hashes +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -set -e - -TESTDIR=$(readlink -f $(dirname $0)) -. $TESTDIR/framework - -setupenvironment -configarchitecture "i386" - -# pkg-sha256-bad has a bad SHA sum, but good MD5 sum. If apt is -# checking the best available hash (as it should), this will trigger -# a hash mismatch. - -cat > aptarchive/Sources < -Architecture: i386 -Files: - d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.dsc - d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.tar.gz - -Package: pkg-sha256-ok -Binary: pkg-sha256-ok -Version: 1.0 -Maintainer: Joe Sixpack -Architecture: i386 -Files: - d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.dsc - d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.tar.gz -Checksums-Sha1: - da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.dsc - da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.tar.gz -Checksums-Sha256: - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.dsc - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.tar.gz - -Package: pkg-sha256-bad -Binary: pkg-sha256-bad -Version: 1.0 -Maintainer: Joe Sixpack -Architecture: i386 -Files: - d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.dsc - d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.tar.gz -Checksums-Sha1: - da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.dsc - da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.tar.gz -Checksums-Sha256: - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 0 pkg-sha256-bad_1.0.dsc - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 0 pkg-sha256-bad_1.0.tar.gz -EOF - -# create fetchable files -for x in "pkg-md5-ok" "pkg-sha256-ok" "pkg-sha256-bad"; do - touch aptarchive/${x}_1.0.dsc - touch aptarchive/${x}_1.0.tar.gz -done - -testok() { - msgtest "Test for hash ok of" "$*" - $* 2>&1 | grep "Download complete" > /dev/null && msgpass || msgfail -} - -testmismatch() { - msgtest "Test for hash mismatch of" "$*" - $* 2>&1 | grep "Hash Sum mismatch" > /dev/null && msgpass || msgfail -} - -setupaptarchive -changetowebserver -aptget update -qq - -testok aptget source -d pkg-md5-ok -testok aptget source -d pkg-sha256-ok -testmismatch aptget source -d pkg-sha256-bad diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum new file mode 100755 index 000000000..9bdc81264 --- /dev/null +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -0,0 +1,260 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'native' + +cat > aptarchive/Sources < +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.tar.gz + +Package: pkg-sha256-ok +Binary: pkg-sha256-ok +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.tar.gz + +Package: pkg-sha256-bad +Binary: pkg-sha256-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.tar.gz +Checksums-Sha256: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-bad_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-bad_1.0.tar.gz + +Package: pkg-no-md5 +Binary: pkg-no-md5 +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.tar.gz + +Package: pkg-mixed-ok +Binary: pkg-mixed-ok +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-ok_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-ok_1.0.dsc + +Package: pkg-mixed-sha1-bad +Binary: pkg-mixed-sha1-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-mixed-sha1-bad_1.0.dsc +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-sha1-bad_1.0.tar.gz + +Package: pkg-mixed-sha2-bad +Binary: pkg-mixed-sha2-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-sha2-bad_1.0.dsc +Checksums-Sha256: + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-mixed-sha2-bad_1.0.tar.gz + +Package: pkg-md5-disagree +Binary: pkg-md5-disagree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.tar.gz + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-md5-disagree_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-md5-disagree_1.0.tar.gz + +Package: pkg-md5-agree +Binary: pkg-md5-agree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc + +Package: pkg-sha256-disagree +Binary: pkg-sha256-disagree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.tar.gz + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-disagree_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-disagree_1.0.tar.gz +EOF + +# create fetchable files +for x in 'pkg-md5-ok' 'pkg-sha256-ok' 'pkg-sha256-bad' 'pkg-no-md5' \ + 'pkg-mixed-ok' 'pkg-mixed-sha1-bad' 'pkg-mixed-sha2-bad' \ + 'pkg-md5-agree' 'pkg-md5-disagree' 'pkg-sha256-disagree'; do + touch aptarchive/${x}_1.0.dsc aptarchive/${x}_1.0.tar.gz +done + +setupaptarchive +changetowebserver +testsuccess aptget update + +testok() { + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +Download complete and in download only mode" aptget source -d "$@" + msgtest 'Files were successfully downloaded for' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz +} + +testkeep() { + touch ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Skipping already downloaded file '${1}_1.0.dsc' +Skipping already downloaded file '${1}_1.0.tar.gz' +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d "$@" + msgtest 'Files already downloaded are kept for' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz +} + +testmismatch() { + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +E: Failed to fetch http://localhost:8080/${1}_1.0.dsc Hash Sum mismatch + +E: Failed to fetch http://localhost:8080/${1}_1.0.tar.gz Hash Sum mismatch + +E: Failed to fetch some archives." aptget source -d "$@" + msgtest 'Files were not download as they have hashsum mismatches for' "$1" + testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Skipping download of file 'pkg-sha256-bad_1.0.dsc' as requested hashsum is not available for authentication +Skipping download of file 'pkg-sha256-bad_1.0.tar.gz' as requested hashsum is not available for authentication +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d "$@" -o Acquire::ForceHash=ROT26 + msgtest 'Files were not download as hash is unavailable for' "$1" + testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +Download complete and in download only mode" aptget source --allow-unauthenticated -d "$@" -o Acquire::ForceHash=ROT26 + msgtest 'Files were downloaded unauthenticated as user allowed it' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz +} + +testok pkg-md5-ok +testkeep pkg-md5-ok +testok pkg-sha256-ok +testkeep pkg-sha256-ok + +# pkg-sha256-bad has a bad SHA sum, but good MD5 sum. If apt is +# checking the best available hash (as it should), this will trigger +# a hash mismatch. +testmismatch pkg-sha256-bad +testmismatch pkg-sha256-bad +testok pkg-sha256-bad -o Acquire::ForceHash=MD5Sum + +# not having MD5 sum doesn't mean the file doesn't exist at all … +testok pkg-no-md5 +testok pkg-no-md5 -o Acquire::ForceHash=SHA256 +testequal "Reading package lists... +Building dependency tree... +Skipping download of file 'pkg-no-md5_1.0.dsc' as requested hashsum is not available for authentication +Skipping download of file 'pkg-no-md5_1.0.tar.gz' as requested hashsum is not available for authentication +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d pkg-no-md5 -o Acquire::ForceHash=MD5Sum +msgtest 'Files were not download as MD5 is not available for this package' 'pkg-no-md5' +testfailure --nomsg test -e pkg-no-md5_1.0.dsc -a -e pkg-no-md5_1.0.tar.gz + +# deal with cases in which we haven't for all files the same checksum type +# mostly pathologic as this shouldn't happen, but just to be sure +testok pkg-mixed-ok +testequal 'Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (tar) +Get:2 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) +E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc Hash Sum mismatch + +E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha1-bad +msgtest 'Only tar file is downloaded as the dsc has hashsum mismatch' 'pkg-mixed-sha1-bad' +testsuccess --nomsg test ! -e pkg-mixed-sha1-bad_1.0.dsc -a -e pkg-mixed-sha1-bad_1.0.tar.gz +testequal 'Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) +Get:2 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (dsc) +E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz Hash Sum mismatch + +E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha2-bad +msgtest 'Only dsc file is downloaded as the tar has hashsum mismatch' 'pkg-mixed-sha2-bad' +testsuccess --nomsg test -e pkg-mixed-sha2-bad_1.0.dsc -a ! -e pkg-mixed-sha2-bad_1.0.tar.gz + +# it gets even more pathologic: multiple entries for one file, some even disagreeing! +testok pkg-md5-agree +testequal 'Reading package lists... +Building dependency tree... +E: Error parsing checksum in Files of source package pkg-md5-disagree' aptget source -d pkg-md5-disagree +testequal 'Reading package lists... +Building dependency tree... +E: Error parsing checksum in Checksums-SHA256 of source package pkg-sha256-disagree' aptget source -d pkg-sha256-disagree -- cgit v1.2.3-70-g09d2 From b3501edb7091ca3aa6c2d6d96dc667b8161dd2b9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 19 Aug 2013 00:00:23 +0200 Subject: use HashStringList in the acquire system It is not very extensible to have the supported Hashes hardcoded everywhere and especially if it is part of virtual method names. It is also possible that a method does not support the 'best' hash (yet), so we might end up not being able to verify a file even though we have a common subset of supported hashes. And those are just two of the cases in which it is handy to have a more dynamic selection. The downside is that this is a MAJOR API break, but the HashStringList has a string constructor for compatibility, so with a bit of luck the few frontends playing with the acquire system directly are okay. --- apt-pkg/acquire-item.cc | 202 ++++++++++----------- apt-pkg/acquire-item.h | 146 +++++++-------- apt-pkg/acquire-method.cc | 39 ++-- apt-pkg/acquire-method.h | 9 +- apt-pkg/acquire-worker.cc | 37 ++-- apt-pkg/contrib/hashes.cc | 97 +++++++--- apt-pkg/contrib/hashes.h | 84 +++++++-- apt-pkg/deb/debmetaindex.cc | 2 +- apt-pkg/deb/debrecords.cc | 41 ++--- apt-pkg/deb/debrecords.h | 8 +- apt-pkg/indexcopy.cc | 8 +- apt-pkg/indexrecords.cc | 2 +- apt-pkg/indexrecords.h | 12 +- apt-pkg/pkgrecords.h | 26 ++- cmdline/apt-get.cc | 6 +- test/integration/test-apt-sources-deb822 | 48 ++--- .../test-bug-722207-print-uris-even-if-very-quiet | 8 +- test/integration/test-pdiff-usage | 6 - .../test-ubuntu-bug-346386-apt-get-update-paywall | 2 +- test/libapt/hashsums_test.cc | 85 ++++++++- 20 files changed, 507 insertions(+), 361 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 913764f64..1156d6f33 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -50,6 +50,19 @@ using namespace std; +static void printHashSumComparision(std::string const &URI, HashStringList const &Expected, HashStringList const &Actual) /*{{{*/ +{ + if (_config->FindB("Debug::Acquire::HashSumMismatch", false) == false) + return; + std::cerr << std::endl << URI << ":" << std::endl << " Expected Hash: " << std::endl; + for (HashStringList::const_iterator hs = Expected.begin(); hs != Expected.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; + std::cerr << " Actual Hash: " << std::endl; + for (HashStringList::const_iterator hs = Actual.begin(); hs != Actual.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; +} + /*}}}*/ + // Acquire::Item::Item - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -118,7 +131,7 @@ void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size) // Acquire::Item::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcquire::Item::Done(string Message,unsigned long long Size,string /*Hash*/, +void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringList const &/*Hash*/, pkgAcquire::MethodConfig * /*Cnf*/) { // We just downloaded something.. @@ -229,8 +242,8 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) possibly query additional files */ pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI, string const &URIDesc, string const &ShortDesc, - HashString const &ExpectedHash) - : Item(Owner), ExpectedHash(ExpectedHash) + HashStringList const &ExpectedHashes) + : Item(Owner), ExpectedHashes(ExpectedHashes) { /* XXX: Beware: Currently this class does nothing (of value) anymore ! */ Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); @@ -252,7 +265,7 @@ pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI, // AcqSubIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqSubIndex::Custom600Headers() +string pkgAcqSubIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(Desc.URI); @@ -275,7 +288,7 @@ void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/* // No good Index is provided } /*}}}*/ -void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqSubIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) @@ -297,7 +310,7 @@ void pkgAcqSubIndex::Done(string Message,unsigned long long Size,string Md5Hash, return; } - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI); @@ -343,12 +356,11 @@ bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ * the original packages file */ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, - IndexTarget const *Target, - HashString ExpectedHash, - indexRecords *MetaIndexParser) - : Item(Owner), ExpectedHash(ExpectedHash), Target(Target), + struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, + indexRecords *MetaIndexParser) + : Item(Owner), ExpectedHashes(ExpectedHashes), Target(Target), MetaIndexParser(MetaIndexParser) - { Debug = _config->FindB("Debug::pkgAcquire::Diffs",false); @@ -393,7 +405,7 @@ pkgAcqDiffIndex::pkgAcqDiffIndex(pkgAcquire *Owner, // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqDiffIndex::Custom600Headers() +string pkgAcqDiffIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI) + string(".IndexDiff"); @@ -447,7 +459,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ // list cleanup needs to know that this file as well as the already // present index is ours, so we create an empty diff to save it for us new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + ExpectedHashes, ServerSha1, available_patches); return true; } else @@ -533,12 +545,12 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ if (pdiff_merge == false) new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + ExpectedHashes, ServerSha1, available_patches); else { std::vector *diffs = new std::vector(available_patches.size()); for(size_t i = 0; i < available_patches.size(); ++i) - (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHash, + (*diffs)[i] = new pkgAcqIndexMergeDiffs(Owner, RealURI, Description, Desc.ShortDesc, ExpectedHashes, available_patches[i], diffs); } @@ -563,20 +575,20 @@ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/ std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; - new pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser); + new pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser); Complete = false; Status = StatDone; Dequeue(); } /*}}}*/ -void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqDiffIndex::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); @@ -607,10 +619,10 @@ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,string Md5Hash */ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - HashString ExpectedHash, + HashStringList const &ExpectedHashes, string ServerSha1, vector diffs) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), + : Item(Owner), RealURI(URI), ExpectedHashes(ExpectedHashes), available_patches(diffs), ServerSha1(ServerSha1) { @@ -642,7 +654,7 @@ void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, - ExpectedHash); + ExpectedHashes); Finish(); } /*}}}*/ @@ -656,7 +668,7 @@ void pkgAcqIndexDiffs::Finish(bool allDone) DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); - if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile)) + if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile)) { RenameOnError(HashSumMismatch); Dequeue(); @@ -734,13 +746,13 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ return true; } /*}}}*/ -void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size, HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqIndexDiffs::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message, Size, Hashes, Cnf); string FinalFile; FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); @@ -783,7 +795,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Has // see if there is more to download if(available_patches.empty() == false) { new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc, - ExpectedHash, ServerSha1, available_patches); + ExpectedHashes, ServerSha1, available_patches); return Finish(); } else return Finish(true); @@ -793,10 +805,10 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long long Size,string Md5Has // AcqIndexMergeDiffs::AcqIndexMergeDiffs - Constructor /*{{{*/ pkgAcqIndexMergeDiffs::pkgAcqIndexMergeDiffs(pkgAcquire *Owner, string const &URI, string const &URIDesc, - string const &ShortDesc, HashString const &ExpectedHash, + string const &ShortDesc, HashStringList const &ExpectedHashes, DiffInfo const &patch, std::vector const * const allPatches) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), + : Item(Owner), RealURI(URI), ExpectedHashes(ExpectedHashes), patch(patch),allPatches(allPatches), State(StateFetchDiff) { @@ -839,16 +851,16 @@ void pkgAcqIndexMergeDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*C State = StateErrorDiff; std::clog << "Falling back to normal index file acquire" << std::endl; new pkgAcqIndex(Owner, RealURI, Description,Desc.ShortDesc, - ExpectedHash); + ExpectedHashes); } /*}}}*/ -void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string Md5Hash, /*{{{*/ +void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cnf) { if(Debug) std::clog << "pkgAcqIndexMergeDiffs::Done(): " << Desc.URI << std::endl; - Item::Done(Message,Size,Md5Hash,Cnf); + Item::Done(Message,Size,Hashes,Cnf); string const FinalFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); @@ -884,7 +896,7 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string M else if (State == StateApplyDiff) { // see if we really got the expected file - if(!ExpectedHash.empty() && !ExpectedHash.VerifyFile(DestFile)) + if(ExpectedHashes.usable() && !ExpectedHashes.VerifyFile(DestFile)) { RenameOnError(HashSumMismatch); return; @@ -921,8 +933,8 @@ void pkgAcqIndexMergeDiffs::Done(string Message,unsigned long long Size,string M instantiated to fetch the revision file */ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc, - HashString ExpectedHash, string comprExt) - : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash), Target(0), + HashStringList const &ExpectedHashes, string comprExt) + : Item(Owner), RealURI(URI), ExpectedHashes(ExpectedHashes), Target(0), MetaIndexParser(0) { if(comprExt.empty() == true) @@ -940,14 +952,14 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, Init(URI, URIDesc, ShortDesc); } -pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, - HashString const &ExpectedHash, indexRecords *MetaIndexParser) - : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash) +pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) + : Item(Owner), RealURI(Target->URI), ExpectedHashes(ExpectedHashes) { // autoselect the compression method std::vector types = APT::Configuration::getCompressionTypes(); CompressionExtension = ""; - if (ExpectedHash.empty() == false) + if (ExpectedHashes.usable()) { for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true) @@ -1016,7 +1028,7 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqIndex::Custom600Headers() +string pkgAcqIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@ -1062,22 +1074,17 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ to the uncompressed version of the file. If this is so the file is copied into the partial directory. In all other cases the file is decompressed with a gzip uri. */ -void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, +void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,Hash,Cfg); + Item::Done(Message,Size,Hashes,Cfg); if (Decompression == true) { - if (_config->FindB("Debug::pkgAcquire::Auth", false)) - { - std::cerr << std::endl << RealURI << ": Computed Hash: " << Hash; - std::cerr << " Expected Hash: " << ExpectedHash.toStr() << std::endl; - } - - if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash) + if (ExpectedHashes.usable() && ExpectedHashes != Hashes) { RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, ExpectedHashes, Hashes); return; } @@ -1199,12 +1206,12 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, /* The Translation file is added to the queue */ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc) - : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "") + : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashStringList(), "") { } -pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, - HashString const &ExpectedHash, indexRecords *MetaIndexParser) - : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser) +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, struct IndexTarget const * const Target, + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser) + : pkgAcqIndex(Owner, Target, ExpectedHashes, MetaIndexParser) { // load the filesize indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); @@ -1214,7 +1221,7 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, /*}}}*/ // AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- -string pkgAcqIndexTrans::Custom600Headers() +string pkgAcqIndexTrans::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@ -1311,7 +1318,7 @@ pkgAcqMetaSig::~pkgAcqMetaSig() /*{{{*/ // pkgAcqMetaSig::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqMetaSig::Custom600Headers() +string pkgAcqMetaSig::Custom600Headers() const { struct stat Buf; if (stat(LastGoodSig.c_str(),&Buf) != 0) @@ -1320,10 +1327,10 @@ string pkgAcqMetaSig::Custom600Headers() return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } -void pkgAcqMetaSig::Done(string Message,unsigned long long Size,string MD5, +void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,MD5,Cfg); + Item::Done(Message, Size, Hashes, Cfg); string FileName = LookupTag(Message,"Filename"); if (FileName.empty() == true) @@ -1426,7 +1433,7 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ // pkgAcqMetaIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqMetaIndex::Custom600Headers() +string pkgAcqMetaIndex::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@ -1438,10 +1445,10 @@ string pkgAcqMetaIndex::Custom600Headers() return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); } /*}}}*/ -void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,string Hash, /*{{{*/ +void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,Hash,Cfg); + Item::Done(Message,Size,Hashes,Cfg); // MetaIndexes are done in two passes: one to download the // metaindex with an appropriate method, and a second to verify it @@ -1616,8 +1623,8 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ Target != IndexTargets->end(); ++Target) { - HashString ExpectedIndexHash; - indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); + HashStringList ExpectedIndexHashes; + const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); bool compressedAvailable = false; if (Record == NULL) { @@ -1640,14 +1647,16 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ } else { - ExpectedIndexHash = Record->Hash; + ExpectedIndexHashes = Record->Hashes; if (_config->FindB("Debug::pkgAcquire::Auth", false)) { - std::cerr << "Queueing: " << (*Target)->URI << std::endl; - std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl; + std::cerr << "Queueing: " << (*Target)->URI << std::endl + << "Expected Hash:" << std::endl; + for (HashStringList::const_iterator hs = ExpectedIndexHashes.begin(); hs != ExpectedIndexHashes.end(); ++hs) + std::cerr << "\t- " << hs->toStr() << std::endl; std::cerr << "For: " << Record->MetaKeyFilename << std::endl; } - if (verify == true && ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false) + if (verify == true && ExpectedIndexHashes.empty() == true && (*Target)->IsOptional() == false) { Status = StatAuthError; strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str()); @@ -1659,14 +1668,14 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ { if ((*Target)->IsSubIndex() == true) new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + (*Target)->ShortDesc, ExpectedIndexHashes); else if (transInRelease == false || Record != NULL || compressedAvailable == true) { if (_config->FindB("Acquire::PDiffs",true) == true && transInRelease == true && MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true) - new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); } continue; } @@ -1677,9 +1686,9 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ instead, but passing the required info to it is to much hassle */ if(_config->FindB("Acquire::PDiffs",true) == true && (verify == false || MetaIndexParser->Exists((*Target)->MetaKey + ".diff/Index") == true)) - new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + new pkgAcqDiffIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); else - new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + new pkgAcqIndex(Owner, *Target, ExpectedIndexHashes, MetaIndexParser); } } /*}}}*/ @@ -1868,7 +1877,7 @@ pkgAcqMetaClearSig::~pkgAcqMetaClearSig() /*{{{*/ // pkgAcqMetaClearSig::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- // FIXME: this can go away once the InRelease file is used widely -string pkgAcqMetaClearSig::Custom600Headers() +string pkgAcqMetaClearSig::Custom600Headers() const { string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); @@ -2002,7 +2011,6 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *Owner,pkgSourceList *Sources, checking later. */ bool pkgAcqArchive::QueueNext() { - string const ForceHash = _config->Find("Acquire::ForceHash"); for (; Vf.end() == false; ++Vf) { // Ignore not source sources @@ -2023,31 +2031,10 @@ bool pkgAcqArchive::QueueNext() pkgRecords::Parser &Parse = Recs->Lookup(Vf); if (_error->PendingError() == true) return false; - + string PkgFile = Parse.FileName(); - if (ForceHash.empty() == false) - { - if(stringcasecmp(ForceHash, "sha512") == 0) - ExpectedHash = HashString("SHA512", Parse.SHA512Hash()); - else if(stringcasecmp(ForceHash, "sha256") == 0) - ExpectedHash = HashString("SHA256", Parse.SHA256Hash()); - else if (stringcasecmp(ForceHash, "sha1") == 0) - ExpectedHash = HashString("SHA1", Parse.SHA1Hash()); - else - ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); - } - else - { - string Hash; - if ((Hash = Parse.SHA512Hash()).empty() == false) - ExpectedHash = HashString("SHA512", Hash); - else if ((Hash = Parse.SHA256Hash()).empty() == false) - ExpectedHash = HashString("SHA256", Hash); - else if ((Hash = Parse.SHA1Hash()).empty() == false) - ExpectedHash = HashString("SHA1", Hash); - else - ExpectedHash = HashString("MD5Sum", Parse.MD5Hash()); - } + ExpectedHashes = Parse.Hashes(); + if (PkgFile.empty() == true) return _error->Error(_("The package index files are corrupted. No Filename: " "field for package %s."), @@ -2134,10 +2121,10 @@ bool pkgAcqArchive::QueueNext() // AcqArchive::Done - Finished fetching /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash, +void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList const &CalcHashes, pkgAcquire::MethodConfig *Cfg) { - Item::Done(Message,Size,CalcHash,Cfg); + Item::Done(Message, Size, CalcHashes, Cfg); // Check the size if (Size != Version->Size) @@ -2145,12 +2132,12 @@ void pkgAcqArchive::Done(string Message,unsigned long long Size,string CalcHash, RenameOnError(SizeMismatch); return; } - - // Check the hash + // FIXME: could this empty() check impose *any* sort of security issue? - if(ExpectedHash.empty() == false && ExpectedHash.toStr() != CalcHash) + if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes) { RenameOnError(HashSumMismatch); + printHashSumComparision(DestFile, ExpectedHashes, CalcHashes); return; } @@ -2222,7 +2209,7 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*}}}*/ // AcqArchive::IsTrusted - Determine whether this archive comes from a trusted source /*{{{*/ // --------------------------------------------------------------------- -APT_PURE bool pkgAcqArchive::IsTrusted() +APT_PURE bool pkgAcqArchive::IsTrusted() const { return Trusted; } @@ -2241,11 +2228,11 @@ void pkgAcqArchive::Finished() // AcqFile::pkgAcqFile - Constructor /*{{{*/ // --------------------------------------------------------------------- /* The file is added to the queue */ -pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, +pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashes, unsigned long long Size,string Dsc,string ShortDesc, const string &DestDir, const string &DestFilename, bool IsIndexFile) : - Item(Owner), ExpectedHash(Hash), IsIndexFile(IsIndexFile) + Item(Owner), ExpectedHashes(Hashes), IsIndexFile(IsIndexFile) { Retries = _config->FindI("Acquire::Retries",0); @@ -2282,15 +2269,16 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI,string Hash, // AcqFile::Done - Item downloaded OK /*{{{*/ // --------------------------------------------------------------------- /* */ -void pkgAcqFile::Done(string Message,unsigned long long Size,string CalcHash, +void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList const &CalcHashes, pkgAcquire::MethodConfig *Cnf) { - Item::Done(Message,Size,CalcHash,Cnf); + Item::Done(Message,Size,CalcHashes,Cnf); // Check the hash - if(!ExpectedHash.empty() && ExpectedHash.toStr() != CalcHash) + if(ExpectedHashes.usable() && ExpectedHashes != CalcHashes) { RenameOnError(HashSumMismatch); + printHashSumComparision(DestFile, ExpectedHashes, CalcHashes); return; } @@ -2361,7 +2349,7 @@ void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf) // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ // --------------------------------------------------------------------- /* The only header we use is the last-modified header. */ -string pkgAcqFile::Custom600Headers() +string pkgAcqFile::Custom600Headers() const { if (IsIndexFile) return "\nIndex-File: true"; diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index eab5bb222..2c39be46c 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -211,12 +211,12 @@ class pkgAcquire::Item : public WeakPointable * \param Message Data from the acquire method. Use LookupTag() * to parse it. * \param Size The size of the object that was fetched. - * \param Hash The HashSum of the object that was fetched. + * \param Hashes The HashSums of the object that was fetched. * \param Cnf The method via which the object was fetched. * * \sa pkgAcqMethod */ - virtual void Done(std::string Message,unsigned long long Size,std::string Hash, + virtual void Done(std::string Message, unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); /** \brief Invoked when the worker starts to fetch this object. @@ -238,34 +238,35 @@ class pkgAcquire::Item : public WeakPointable * line, so they should (if nonempty) have a leading newline and * no trailing newline. */ - virtual std::string Custom600Headers() {return std::string();}; + virtual std::string Custom600Headers() const {return std::string();}; /** \brief A "descriptive" URI-like string. * * \return a URI that should be used to describe what is being fetched. */ - virtual std::string DescURI() = 0; + virtual std::string DescURI() const = 0; /** \brief Short item description. * * \return a brief description of the object being fetched. */ - virtual std::string ShortDesc() {return DescURI();} + virtual std::string ShortDesc() const {return DescURI();} /** \brief Invoked by the worker when the download is completely done. */ virtual void Finished() {}; - /** \brief HashSum + /** \brief HashSums * - * \return the HashSum of this object, if applicable; otherwise, an - * empty string. + * \return the HashSums of this object, if applicable; otherwise, an + * empty list. */ - virtual std::string HashSum() {return std::string();}; + virtual HashStringList HashSums() const {return HashStringList();}; + std::string HashSum() const {HashStringList const hashes = HashSums(); HashString const * const hs = hashes.find(NULL); return hs != NULL ? hs->toStr() : ""; }; /** \return the acquire process with which this item is associated. */ - pkgAcquire *GetOwner() {return Owner;}; + pkgAcquire *GetOwner() const {return Owner;}; /** \return \b true if this object is being fetched from a trusted source. */ - virtual bool IsTrusted() {return false;}; + virtual bool IsTrusted() const {return false;}; // report mirror problems /** \brief Report mirror problem @@ -338,17 +339,17 @@ class pkgAcqSubIndex : public pkgAcquire::Item /** \brief The item that is currently being downloaded. */ pkgAcquire::ItemDesc Desc; - /** \brief The Hash that this file should have after download + /** \brief The Hashes that this file should have after download */ - HashString ExpectedHash; + HashStringList ExpectedHashes; public: // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return Desc.URI;}; - virtual std::string Custom600Headers(); + virtual std::string DescURI() const {return Desc.URI;}; + virtual std::string Custom600Headers() const; virtual bool ParseIndex(std::string const &IndexFile); /** \brief Create a new pkgAcqSubIndex. @@ -361,10 +362,10 @@ class pkgAcqSubIndex : public pkgAcquire::Item * * \param ShortDesc A short description of the list file to download. * - * \param ExpectedHash The list file's MD5 signature. + * \param ExpectedHashes The list file's hashsums which are expected. */ pkgAcqSubIndex(pkgAcquire *Owner, std::string const &URI,std::string const &URIDesc, - std::string const &ShortDesc, HashString const &ExpectedHash); + std::string const &ShortDesc, HashStringList const &ExpectedHashes); }; /*}}}*/ /** \brief An item that is responsible for fetching an index file of {{{ @@ -390,10 +391,10 @@ class pkgAcqDiffIndex : public pkgAcquire::Item */ std::string RealURI; - /** \brief The Hash that the real index file should have after + /** \brief The Hashes that the real index file should have after * all patches have been applied. */ - HashString ExpectedHash; + HashStringList ExpectedHashes; /** \brief The index file which will be patched to generate the new * file. @@ -413,10 +414,10 @@ class pkgAcqDiffIndex : public pkgAcquire::Item public: // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return RealURI + "Index";}; - virtual std::string Custom600Headers(); + virtual std::string DescURI() const {return RealURI + "Index";}; + virtual std::string Custom600Headers() const; /** \brief Parse the Index file for a set of Packages diffs. * @@ -441,11 +442,11 @@ class pkgAcqDiffIndex : public pkgAcquire::Item * * \param ShortDesc A short description of the list file to download. * - * \param ExpectedHash The list file's MD5 signature. + * \param ExpectedHashes The list file's hashsums which are expected. */ pkgAcqDiffIndex(pkgAcquire *Owner, struct IndexTarget const * const Target, - HashString ExpectedHash, + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); }; /*}}}*/ @@ -479,10 +480,10 @@ class pkgAcqIndexMergeDiffs : public pkgAcquire::Item */ std::string RealURI; - /** \brief HashSum of the package index file that is being - * reconstructed. + /** \brief The Hashes that the real index file should have after + * all patches have been applied. */ - HashString ExpectedHash; + HashStringList ExpectedHashes; /** \brief description of the file being downloaded. */ std::string Description; @@ -516,10 +517,9 @@ class pkgAcqIndexMergeDiffs : public pkgAcquire::Item * outright; its arguments are ignored. */ virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, - pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return RealURI + "Index";}; + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, + pkgAcquire::MethodConfig *Cnf); + virtual std::string DescURI() const {return RealURI + "Index";}; /** \brief Create an index merge-diff item. * @@ -532,7 +532,7 @@ class pkgAcqIndexMergeDiffs : public pkgAcquire::Item * * \param ShortDesc A brief description of this item. * - * \param ExpectedHash The expected md5sum of the completely + * \param ExpectedHashes The expected md5sum of the completely * reconstructed package index file; the index file will be tested * against this value when it is entirely reconstructed. * @@ -543,7 +543,7 @@ class pkgAcqIndexMergeDiffs : public pkgAcquire::Item * check if it was the last one to complete the download step */ pkgAcqIndexMergeDiffs(pkgAcquire *Owner,std::string const &URI,std::string const &URIDesc, - std::string const &ShortDesc, HashString const &ExpectedHash, + std::string const &ShortDesc, HashStringList const &ExpectedHashes, DiffInfo const &patch, std::vector const * const allPatches); }; /*}}}*/ @@ -576,8 +576,8 @@ class pkgAcqIndexDiffs : public pkgAcquire::Item /** \brief Handle tasks that must be performed after the item * finishes downloading. * - * Dequeues the item and checks the resulting file's md5sum - * against ExpectedHash after the last patch was applied. + * Dequeues the item and checks the resulting file's hashsums + * against ExpectedHashes after the last patch was applied. * There is no need to check the md5/sha1 after a "normal" * patch because QueueNextDiff() will check the sha1 later. * @@ -603,10 +603,10 @@ class pkgAcqIndexDiffs : public pkgAcquire::Item */ std::string RealURI; - /** \brief The HashSum of the package index file that is being + /** \brief The HashSums of the package index file that is being * reconstructed. */ - HashString ExpectedHash; + HashStringList ExpectedHashes; /** A description of the file being downloaded. */ std::string Description; @@ -649,9 +649,9 @@ class pkgAcqIndexDiffs : public pkgAcquire::Item */ virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return RealURI + "Index";}; + virtual std::string DescURI() const {return RealURI + "Index";}; /** \brief Create an index diff item. * @@ -667,7 +667,7 @@ class pkgAcqIndexDiffs : public pkgAcquire::Item * * \param ShortDesc A brief description of this item. * - * \param ExpectedHash The expected md5sum of the completely + * \param ExpectedHashes The expected md5sum of the completely * reconstructed package index file; the index file will be tested * against this value when it is entirely reconstructed. * @@ -678,7 +678,7 @@ class pkgAcqIndexDiffs : public pkgAcquire::Item * that depends on it. */ pkgAcqIndexDiffs(pkgAcquire *Owner,std::string URI,std::string URIDesc, - std::string ShortDesc, HashString ExpectedHash, + std::string ShortDesc, HashStringList const &ExpectedHashes, std::string ServerSha1, std::vector diffs=std::vector()); }; @@ -723,7 +723,7 @@ class pkgAcqIndex : public pkgAcquire::Item std::string RealURI; /** \brief The expected hashsum of the decompressed index file. */ - HashString ExpectedHash; + HashStringList ExpectedHashes; /** \brief The compression-related file extensions that are being * added to the downloaded file one by one if first fails (e.g., "gz bz2"). @@ -739,11 +739,11 @@ class pkgAcqIndex : public pkgAcquire::Item // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); - virtual std::string DescURI() {return Desc.URI;}; - virtual std::string HashSum() {return ExpectedHash.toStr(); }; + virtual std::string Custom600Headers() const; + virtual std::string DescURI() const {return Desc.URI;}; + virtual HashStringList HashSums() const {return ExpectedHashes; }; /** \brief Create a pkgAcqIndex. * @@ -756,7 +756,7 @@ class pkgAcqIndex : public pkgAcquire::Item * * \param ShortDesc A brief description of this index file. * - * \param ExpectedHash The expected hashsum of this index file. + * \param ExpectedHashes The expected hashsum of this index file. * * \param compressExt The compression-related extension with which * this index file should be downloaded, or "" to autodetect @@ -765,10 +765,10 @@ class pkgAcqIndex : public pkgAcquire::Item * fallback is ".gz" or none. */ pkgAcqIndex(pkgAcquire *Owner,std::string URI,std::string URIDesc, - std::string ShortDesc, HashString ExpectedHash, + std::string ShortDesc, HashStringList const &ExpectedHashes, std::string compressExt=""); pkgAcqIndex(pkgAcquire *Owner, struct IndexTarget const * const Target, - HashString const &ExpectedHash, indexRecords *MetaIndexParser); + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); void Init(std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc); }; /*}}}*/ @@ -784,7 +784,7 @@ class pkgAcqIndexTrans : public pkgAcqIndex public: virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); + virtual std::string Custom600Headers() const; /** \brief Create a pkgAcqIndexTrans. * @@ -800,7 +800,7 @@ class pkgAcqIndexTrans : public pkgAcqIndex pkgAcqIndexTrans(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc); pkgAcqIndexTrans(pkgAcquire *Owner, struct IndexTarget const * const Target, - HashString const &ExpectedHash, indexRecords *MetaIndexParser); + HashStringList const &ExpectedHashes, indexRecords *MetaIndexParser); }; /*}}}*/ /** \brief Information about an index file. */ /*{{{*/ @@ -904,10 +904,10 @@ class pkgAcqMetaSig : public pkgAcquire::Item // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Md5Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); - virtual std::string DescURI() {return RealURI; }; + virtual std::string Custom600Headers() const; + virtual std::string DescURI() const {return RealURI; }; /** \brief Create a new pkgAcqMetaSig. */ pkgAcqMetaSig(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc, @@ -1000,10 +1000,10 @@ class pkgAcqMetaIndex : public pkgAcquire::Item // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size, std::string Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); - virtual std::string DescURI() {return RealURI; }; + virtual std::string Custom600Headers() const; + virtual std::string DescURI() const {return RealURI; }; /** \brief Create a new pkgAcqMetaIndex. */ pkgAcqMetaIndex(pkgAcquire *Owner, @@ -1036,7 +1036,7 @@ class pkgAcqMetaClearSig : public pkgAcqMetaIndex public: void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual std::string Custom600Headers(); + virtual std::string Custom600Headers() const; /** \brief Create a new pkgAcqMetaClearSig. */ pkgAcqMetaClearSig(pkgAcquire *Owner, @@ -1073,7 +1073,7 @@ class pkgAcqArchive : public pkgAcquire::Item pkgRecords *Recs; /** \brief The hashsum of this package. */ - HashString ExpectedHash; + HashStringList ExpectedHashes; /** \brief A location in which the actual filename of the package * should be stored. @@ -1101,13 +1101,13 @@ class pkgAcqArchive : public pkgAcquire::Item public: virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string Hash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return Desc.URI;}; - virtual std::string ShortDesc() {return Desc.ShortDesc;}; + virtual std::string DescURI() const {return Desc.URI;}; + virtual std::string ShortDesc() const {return Desc.ShortDesc;}; virtual void Finished(); - virtual std::string HashSum() {return ExpectedHash.toStr(); }; - virtual bool IsTrusted(); + virtual HashStringList HashSums() const {return ExpectedHashes; }; + virtual bool IsTrusted() const; /** \brief Create a new pkgAcqArchive. * @@ -1144,7 +1144,7 @@ class pkgAcqFile : public pkgAcquire::Item pkgAcquire::ItemDesc Desc; /** \brief The hashsum of the file to download, if it is known. */ - HashString ExpectedHash; + HashStringList ExpectedHashes; /** \brief How many times to retry the download, set from * Acquire::Retries. @@ -1158,11 +1158,11 @@ class pkgAcqFile : public pkgAcquire::Item // Specialized action members virtual void Failed(std::string Message,pkgAcquire::MethodConfig *Cnf); - virtual void Done(std::string Message,unsigned long long Size,std::string CalcHash, + virtual void Done(std::string Message,unsigned long long Size, HashStringList const &CalcHashes, pkgAcquire::MethodConfig *Cnf); - virtual std::string DescURI() {return Desc.URI;}; - virtual std::string HashSum() {return ExpectedHash.toStr(); }; - virtual std::string Custom600Headers(); + virtual std::string DescURI() const {return Desc.URI;}; + virtual HashStringList HashSums() const { return ExpectedHashes; }; + virtual std::string Custom600Headers() const; /** \brief Create a new pkgAcqFile object. * @@ -1171,8 +1171,8 @@ class pkgAcqFile : public pkgAcquire::Item * * \param URI The URI to download. * - * \param Hash The hashsum of the file to download, if it is known; - * otherwise "". + * \param Hashes The hashsums of the file to download, if they are known; + * otherwise empty list. * * \param Size The size of the file to download, if it is known; * otherwise 0. @@ -1195,7 +1195,7 @@ class pkgAcqFile : public pkgAcquire::Item * is the absolute name to which the file should be downloaded. */ - pkgAcqFile(pkgAcquire *Owner, std::string URI, std::string Hash, unsigned long long Size, + pkgAcqFile(pkgAcquire *Owner, std::string URI, HashStringList const &Hashes, unsigned long long Size, std::string Desc, std::string ShortDesc, const std::string &DestDir="", const std::string &DestFilename="", bool IsIndexFile=false); diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 746c553f1..58d214068 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -147,6 +147,16 @@ void pkgAcqMethod::URIStart(FetchResult &Res) // AcqMethod::URIDone - A URI is finished /*{{{*/ // --------------------------------------------------------------------- /* */ +static void printHashStringList(HashStringList const * const list) +{ + for (HashStringList::const_iterator hash = list->begin(); hash != list->end(); ++hash) + { + // very old compatibility name for MD5Sum + if (hash->HashType() == "MD5Sum") + std::cout << "MD5-Hash: " << hash->HashValue() << "\n"; + std::cout << hash->HashType() << "-Hash: " << hash->HashValue() << "\n"; + } +} void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) { if (Queue == 0) @@ -164,15 +174,8 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) if (Res.LastModified != 0) std::cout << "Last-Modified: " << TimeRFC1123(Res.LastModified) << "\n"; - if (Res.MD5Sum.empty() == false) - std::cout << "MD5-Hash: " << Res.MD5Sum << "\n" - << "MD5Sum-Hash: " << Res.MD5Sum << "\n"; - if (Res.SHA1Sum.empty() == false) - std::cout << "SHA1-Hash: " << Res.SHA1Sum << "\n"; - if (Res.SHA256Sum.empty() == false) - std::cout << "SHA256-Hash: " << Res.SHA256Sum << "\n"; - if (Res.SHA512Sum.empty() == false) - std::cout << "SHA512-Hash: " << Res.SHA512Sum << "\n"; + printHashStringList(&Res.Hashes); + if (UsedMirror.empty() == false) std::cout << "UsedMirror: " << UsedMirror << "\n"; if (Res.GPGVOutput.empty() == false) @@ -200,15 +203,8 @@ void pkgAcqMethod::URIDone(FetchResult &Res, FetchResult *Alt) if (Alt->LastModified != 0) std::cout << "Alt-Last-Modified: " << TimeRFC1123(Alt->LastModified) << "\n"; - if (Alt->MD5Sum.empty() == false) - std::cout << "Alt-MD5-Hash: " << Alt->MD5Sum << "\n"; - if (Alt->SHA1Sum.empty() == false) - std::cout << "Alt-SHA1-Hash: " << Alt->SHA1Sum << "\n"; - if (Alt->SHA256Sum.empty() == false) - std::cout << "Alt-SHA256-Hash: " << Alt->SHA256Sum << "\n"; - if (Alt->SHA512Sum.empty() == false) - std::cout << "Alt-SHA512-Hash: " << Alt->SHA512Sum << "\n"; - + printHashStringList(&Alt->Hashes); + if (Alt->IMSHit == true) std::cout << "Alt-IMS-Hit: true\n"; } @@ -442,12 +438,9 @@ pkgAcqMethod::FetchResult::FetchResult() : LastModified(0), // --------------------------------------------------------------------- /* This hides the number of hashes we are supporting from the caller. It just deals with the hash class. */ -void pkgAcqMethod::FetchResult::TakeHashes(Hashes &Hash) +void pkgAcqMethod::FetchResult::TakeHashes(class Hashes &Hash) { - MD5Sum = Hash.MD5.Result(); - SHA1Sum = Hash.SHA1.Result(); - SHA256Sum = Hash.SHA256.Result(); - SHA512Sum = Hash.SHA512.Result(); + Hashes = Hash.GetHashStringList(); } /*}}}*/ void pkgAcqMethod::Dequeue() { /*{{{*/ diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 221ccf273..8a17a5fd5 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -20,6 +20,7 @@ #ifndef PKGLIB_ACQUIRE_METHOD_H #define PKGLIB_ACQUIRE_METHOD_H +#include #include #include @@ -33,7 +34,6 @@ #include #endif -class Hashes; class pkgAcqMethod { protected: @@ -51,10 +51,7 @@ class pkgAcqMethod struct FetchResult { - std::string MD5Sum; - std::string SHA1Sum; - std::string SHA256Sum; - std::string SHA512Sum; + HashStringList Hashes; std::vector GPGVOutput; time_t LastModified; bool IMSHit; @@ -62,7 +59,7 @@ class pkgAcqMethod unsigned long long Size; unsigned long long ResumePoint; - void TakeHashes(Hashes &Hash); + void TakeHashes(class Hashes &Hash); FetchResult(); }; diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 047a655ce..ec1b2fc46 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -326,25 +326,30 @@ bool pkgAcquire::Worker::RunMessages() Owner->DestFile.c_str(), LookupTag(Message,"Size","0").c_str(),TotalSize); // see if there is a hash to verify - string RecivedHash; - HashString expectedHash(Owner->HashSum()); - if(!expectedHash.empty()) + HashStringList RecivedHashes; + HashStringList expectedHashes = Owner->HashSums(); + for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs) { - string hashTag = expectedHash.HashType()+"-Hash"; - string hashSum = LookupTag(Message, hashTag.c_str()); - if(!hashSum.empty()) - RecivedHash = expectedHash.HashType() + ":" + hashSum; - if(_config->FindB("Debug::pkgAcquire::Auth", false) == true) - { - clog << "201 URI Done: " << Owner->DescURI() << endl - << "RecivedHash: " << RecivedHash << endl - << "ExpectedHash: " << expectedHash.toStr() - << endl << endl; - } + std::string const tagname = hs->HashType() + "-Hash"; + std::string const hashsum = LookupTag(Message, tagname.c_str()); + if (hashsum.empty() == false) + RecivedHashes.push_back(HashString(hs->HashType(), hashsum)); + } + + if(_config->FindB("Debug::pkgAcquire::Auth", false) == true) + { + std::clog << "201 URI Done: " << Owner->DescURI() << endl + << "RecivedHash:" << endl; + for (HashStringList::const_iterator hs = RecivedHashes.begin(); hs != RecivedHashes.end(); ++hs) + std::clog << "\t- " << hs->toStr() << std::endl; + std::clog << "ExpectedHash:" << endl; + for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs) + std::clog << "\t- " << hs->toStr() << std::endl; + std::clog << endl; } - Owner->Done(Message, ServerSize, RecivedHash.c_str(), Config); + Owner->Done(Message, ServerSize, RecivedHashes, Config); ItemDone(); - + // Log that we are done if (Log != 0) { diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index bb11a3fca..199e395f6 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -141,11 +141,21 @@ APT_PURE bool HashString::operator!=(HashString const &other) const } /*}}}*/ +bool HashStringList::usable() const /*{{{*/ +{ + if (empty() == true) + return false; + std::string const forcedType = _config->Find("Acquire::ForceHash", ""); + if (forcedType.empty() == true) + return true; + return find(forcedType) != NULL; +} + /*}}}*/ HashString const * HashStringList::find(char const * const type) const /*{{{*/ { if (type == NULL || type[0] == '\0') { - std::string forcedType = _config->Find("Acquire::ForceHash", ""); + std::string const forcedType = _config->Find("Acquire::ForceHash", ""); if (forcedType.empty() == false) return find(forcedType.c_str()); for (char const * const * t = HashString::SupportedHashes(); *t != NULL; ++t) @@ -196,6 +206,15 @@ bool HashStringList::VerifyFile(std::string filename) const /*{{{*/ /*}}}*/ bool HashStringList::operator==(HashStringList const &other) const /*{{{*/ { + std::string const forcedType = _config->Find("Acquire::ForceHash", ""); + if (forcedType.empty() == false) + { + HashString const * const hs = other.find(forcedType); + HashString const * const ohs = other.find(forcedType); + if (hs == NULL || ohs == NULL) + return false; + return hs == ohs; + } short matches = 0; for (const_iterator hs = begin(); hs != end(); ++hs) { @@ -216,11 +235,28 @@ bool HashStringList::operator!=(HashStringList const &other) const } /*}}}*/ -// Hashes::AddFD - Add the contents of the FD /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool Hashes::AddFD(int const Fd,unsigned long long Size, bool const addMD5, - bool const addSHA1, bool const addSHA256, bool const addSHA512) +// Hashes::Add* - Add the contents of data or FD /*{{{*/ +bool Hashes::Add(const unsigned char * const Data,unsigned long long const Size, unsigned int const Hashes) +{ + bool Res = true; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + if ((Hashes & MD5SUM) == MD5SUM) + Res &= MD5.Add(Data, Size); + if ((Hashes & SHA1SUM) == SHA1SUM) + Res &= SHA1.Add(Data, Size); + if ((Hashes & SHA256SUM) == SHA256SUM) + Res &= SHA256.Add(Data, Size); + if ((Hashes & SHA512SUM) == SHA512SUM) + Res &= SHA512.Add(Data, Size); +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + return Res; +} +bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes) { unsigned char Buf[64*64]; bool const ToEOF = (Size == UntilEOF); @@ -234,19 +270,12 @@ bool Hashes::AddFD(int const Fd,unsigned long long Size, bool const addMD5, if (ToEOF && Res == 0) // EOF break; Size -= Res; - if (addMD5 == true) - MD5.Add(Buf,Res); - if (addSHA1 == true) - SHA1.Add(Buf,Res); - if (addSHA256 == true) - SHA256.Add(Buf,Res); - if (addSHA512 == true) - SHA512.Add(Buf,Res); + if (Add(Buf, Res, Hashes) == false) + return false; } return true; } -bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, bool const addMD5, - bool const addSHA1, bool const addSHA256, bool const addSHA512) +bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes) { unsigned char Buf[64*64]; bool const ToEOF = (Size == 0); @@ -265,15 +294,35 @@ bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, bool const addMD5, else if (a == 0) // EOF break; Size -= a; - if (addMD5 == true) - MD5.Add(Buf, a); - if (addSHA1 == true) - SHA1.Add(Buf, a); - if (addSHA256 == true) - SHA256.Add(Buf, a); - if (addSHA512 == true) - SHA512.Add(Buf, a); + if (Add(Buf, a, Hashes) == false) + return false; } return true; } /*}}}*/ +HashStringList Hashes::GetHashStringList() +{ + HashStringList hashes; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + hashes.push_back(HashString("MD5Sum", MD5.Result().Value())); + hashes.push_back(HashString("SHA1", SHA1.Result().Value())); + hashes.push_back(HashString("SHA256", SHA256.Result().Value())); + hashes.push_back(HashString("SHA512", SHA512.Result().Value())); +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + return hashes; +} +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" + #pragma GCC diagnostic ignored "-Wsuggest-attribute=const" +#endif +Hashes::Hashes() {} +Hashes::~Hashes() {} +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h index 490282209..caeba006d 100644 --- a/apt-pkg/contrib/hashes.h +++ b/apt-pkg/contrib/hashes.h @@ -114,6 +114,15 @@ class HashStringList */ bool empty() const { return list.empty(); } + /** has the list at least one good entry + * + * similar to #empty, but handles forced hashes. + * + * @return if no hash is forced, same result as #empty, + * if one is forced \b true if this has is available, \b false otherwise + */ + bool usable() const; + typedef std::vector::const_iterator const_iterator; /** iterator to the first element */ @@ -128,8 +137,10 @@ class HashStringList /** compare two HashStringList for similarity. * * Two lists are similar if at least one hashtype is in both lists - * and the hashsum matches. All hashes are checked, if one doesn't - * match false is returned regardless of how many matched before. + * and the hashsum matches. All hashes are checked by default, + * if one doesn't match false is returned regardless of how many + * matched before. If a hash is forced, only this hash is compared, + * all others are ignored. */ bool operator==(HashStringList const &other) const; bool operator!=(HashStringList const &other) const; @@ -152,30 +163,63 @@ class HashStringList class Hashes { + /** \brief dpointer placeholder */ + void *d; + public: + /* those will disappear in the future as it is hard to add new ones this way. + * Use Add* to build the results and get them via GetHashStringList() instead */ + APT_DEPRECATED MD5Summation MD5; + APT_DEPRECATED SHA1Summation SHA1; + APT_DEPRECATED SHA256Summation SHA256; + APT_DEPRECATED SHA512Summation SHA512; - MD5Summation MD5; - SHA1Summation SHA1; - SHA256Summation SHA256; - SHA512Summation SHA512; - static const int UntilEOF = 0; - inline bool Add(const unsigned char *Data,unsigned long long Size) + bool Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes = ~0); + inline bool Add(const char * const Data) + {return Add((unsigned char const * const)Data,strlen(Data));}; + inline bool Add(const unsigned char * const Beg,const unsigned char * const End) + {return Add(Beg,End-Beg);}; + + enum SupportedHashes { MD5SUM = (1 << 0), SHA1SUM = (1 << 1), SHA256SUM = (1 << 2), + SHA512SUM = (1 << 3) }; + bool AddFD(int const Fd,unsigned long long Size = 0, unsigned int const Hashes = ~0); + bool AddFD(FileFd &Fd,unsigned long long Size = 0, unsigned int const Hashes = ~0); + + HashStringList GetHashStringList(); + +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + Hashes(); + virtual ~Hashes(); +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + + private: + APT_HIDDEN APT_CONST inline unsigned int boolsToFlag(bool const addMD5, bool const addSHA1, bool const addSHA256, bool const addSHA512) { - return MD5.Add(Data,Size) && SHA1.Add(Data,Size) && SHA256.Add(Data,Size) && SHA512.Add(Data,Size); + unsigned int Hashes = ~0; + if (addMD5 == false) Hashes &= ~MD5SUM; + if (addSHA1 == false) Hashes &= ~SHA1SUM; + if (addSHA256 == false) Hashes &= ~SHA256SUM; + if (addSHA512 == false) Hashes &= ~SHA512SUM; + return Hashes; + } + + public: + APT_DEPRECATED bool AddFD(int const Fd, unsigned long long Size, bool const addMD5, + bool const addSHA1, bool const addSHA256, bool const addSHA512) { + return AddFD(Fd, Size, boolsToFlag(addMD5, addSHA1, addSHA256, addSHA512)); + }; + + APT_DEPRECATED bool AddFD(FileFd &Fd, unsigned long long Size, bool const addMD5, + bool const addSHA1, bool const addSHA256, bool const addSHA512) { + return AddFD(Fd, Size, boolsToFlag(addMD5, addSHA1, addSHA256, addSHA512)); }; - inline bool Add(const char *Data) {return Add((unsigned char const *)Data,strlen(Data));}; - inline bool AddFD(int const Fd,unsigned long long Size = 0) - { return AddFD(Fd, Size, true, true, true, true); }; - bool AddFD(int const Fd, unsigned long long Size, bool const addMD5, - bool const addSHA1, bool const addSHA256, bool const addSHA512); - inline bool AddFD(FileFd &Fd,unsigned long long Size = 0) - { return AddFD(Fd, Size, true, true, true, true); }; - bool AddFD(FileFd &Fd, unsigned long long Size, bool const addMD5, - bool const addSHA1, bool const addSHA256, bool const addSHA512); - inline bool Add(const unsigned char *Beg,const unsigned char *End) - {return Add(Beg,End-Beg);}; }; #endif diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index b4839ada4..607dae882 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -258,7 +258,7 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const vector *targets = ComputeIndexTargets(); for (vector ::const_iterator Target = targets->begin(); Target != targets->end(); ++Target) { new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, HashString()); + (*Target)->ShortDesc, HashStringList()); } delete targets; diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc index 6063db5a8..b824e11b4 100644 --- a/apt-pkg/deb/debrecords.cc +++ b/apt-pkg/deb/debrecords.cc @@ -73,36 +73,17 @@ string debRecordParser::Homepage() return Section.FindS("Homepage"); } /*}}}*/ -// RecordParser::MD5Hash - Return the archive hash /*{{{*/ -// --------------------------------------------------------------------- -/* */ -string debRecordParser::MD5Hash() -{ - return Section.FindS("MD5Sum"); -} - /*}}}*/ -// RecordParser::SHA1Hash - Return the archive hash /*{{{*/ -// --------------------------------------------------------------------- -/* */ -string debRecordParser::SHA1Hash() -{ - return Section.FindS("SHA1"); -} - /*}}}*/ -// RecordParser::SHA256Hash - Return the archive hash /*{{{*/ -// --------------------------------------------------------------------- -/* */ -string debRecordParser::SHA256Hash() -{ - return Section.FindS("SHA256"); -} - /*}}}*/ -// RecordParser::SHA512Hash - Return the archive hash /*{{{*/ -// --------------------------------------------------------------------- -/* */ -string debRecordParser::SHA512Hash() -{ - return Section.FindS("SHA512"); +// RecordParser::Hashes - return the available archive hashes /*{{{*/ +HashStringList debRecordParser::Hashes() const +{ + HashStringList hashes; + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + std::string const hash = Section.FindS(*type); + if (hash.empty() == false) + hashes.push_back(HashString(*type, hash)); + } + return hashes; } /*}}}*/ // RecordParser::Maintainer - Return the maintainer email /*{{{*/ diff --git a/apt-pkg/deb/debrecords.h b/apt-pkg/deb/debrecords.h index d572bc5c2..90c88fefe 100644 --- a/apt-pkg/deb/debrecords.h +++ b/apt-pkg/deb/debrecords.h @@ -42,13 +42,11 @@ class debRecordParser : public pkgRecords::Parser // These refer to the archive file for the Version virtual std::string FileName(); - virtual std::string MD5Hash(); - virtual std::string SHA1Hash(); - virtual std::string SHA256Hash(); - virtual std::string SHA512Hash(); virtual std::string SourcePkg(); virtual std::string SourceVer(); - + + virtual HashStringList Hashes() const; + // These are some general stats about the package virtual std::string Maintainer(); virtual std::string ShortDesc(); diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index 854ba1bd7..bb3b5d340 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -516,7 +516,7 @@ bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex) return false; } - if (!Record->Hash.VerifyFile(prefix+file)) + if (!Record->Hashes.VerifyFile(prefix+file)) { _error->Warning(_("Hash mismatch for: %s"),file.c_str()); return false; @@ -524,8 +524,10 @@ bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex) if(Debug == true) { - cout << "File: " << prefix+file << endl; - cout << "Expected Hash " << Record->Hash.toStr() << endl; + cout << "File: " << prefix+file << endl + << "Expected Hash " << endl; + for (HashStringList::const_iterator hs = Record->Hashes.begin(); hs != Record->Hashes.end(); ++hs) + std::cout << "\t- " << hs->toStr() << std::endl; } return true; diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc index 68ebdce08..5dcaadd76 100644 --- a/apt-pkg/indexrecords.cc +++ b/apt-pkg/indexrecords.cc @@ -105,7 +105,7 @@ bool indexRecords::Load(const string Filename) /*{{{*/ return false; indexRecords::checkSum *Sum = new indexRecords::checkSum; Sum->MetaKeyFilename = Name; - Sum->Hash = HashString(HashString::SupportedHashes()[i],Hash); + Sum->Hashes.push_back(HashString(HashString::SupportedHashes()[i],Hash)); Sum->Size = Size; Entries[Name] = Sum; } diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h index 2260a4ae1..14b03c4d5 100644 --- a/apt-pkg/indexrecords.h +++ b/apt-pkg/indexrecords.h @@ -55,11 +55,21 @@ class indexRecords virtual ~indexRecords(){}; }; +#if __GNUC__ >= 4 + // ensure that con- & de-structor don't trigger this warning + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif struct indexRecords::checkSum { std::string MetaKeyFilename; - HashString Hash; + HashStringList Hashes; unsigned long long Size; + + APT_DEPRECATED HashString Hash; }; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif #endif diff --git a/apt-pkg/pkgrecords.h b/apt-pkg/pkgrecords.h index b5237b3a0..b0b449ae1 100644 --- a/apt-pkg/pkgrecords.h +++ b/apt-pkg/pkgrecords.h @@ -18,6 +18,8 @@ #define PKGLIB_PKGRECORDS_H #include +#include +#include #include #include @@ -56,13 +58,21 @@ class pkgRecords::Parser /*{{{*/ // These refer to the archive file for the Version virtual std::string FileName() {return std::string();}; - virtual std::string MD5Hash() {return std::string();}; - virtual std::string SHA1Hash() {return std::string();}; - virtual std::string SHA256Hash() {return std::string();}; - virtual std::string SHA512Hash() {return std::string();}; virtual std::string SourcePkg() {return std::string();}; virtual std::string SourceVer() {return std::string();}; + /** return all known hashes in this record. + * + * For authentication proposes packages come with hashsums which + * this method is supposed to parse and return so that clients can + * choose the hash to be used. + */ + virtual HashStringList Hashes() const { return HashStringList(); }; + APT_DEPRECATED std::string MD5Hash() const { return GetHashFromHashes("MD5Sum"); }; + APT_DEPRECATED std::string SHA1Hash() const { return GetHashFromHashes("SHA1"); }; + APT_DEPRECATED std::string SHA256Hash() const { return GetHashFromHashes("SHA256"); }; + APT_DEPRECATED std::string SHA512Hash() const { return GetHashFromHashes("SHA512"); }; + // These are some general stats about the package virtual std::string Maintainer() {return std::string();}; virtual std::string ShortDesc() {return std::string();}; @@ -77,6 +87,14 @@ class pkgRecords::Parser /*{{{*/ virtual void GetRec(const char *&Start,const char *&Stop) {Start = Stop = 0;}; virtual ~Parser() {}; + + private: + APT_HIDDEN std::string GetHashFromHashes(char const * const type) const + { + HashStringList const hashes = Hashes(); + HashString const * const hs = hashes.find(type); + return hs != NULL ? hs->HashValue() : ""; + }; }; /*}}}*/ #endif diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index fc6223989..3388351d9 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -838,8 +838,7 @@ static bool DoSource(CommandLine &CmdL) } // see if we have a hash (Acquire::ForceHash is the only way to have none) - HashString const * const hs = I->Hashes.find(NULL); - if (hs == NULL && _config->FindB("APT::Get::AllowUnauthenticated",false) == false) + if (I->Hashes.usable() == false && _config->FindB("APT::Get::AllowUnauthenticated",false) == false) { ioprintf(c1out, "Skipping download of file '%s' as requested hashsum is not available for authentication\n", localFile.c_str()); @@ -847,8 +846,7 @@ static bool DoSource(CommandLine &CmdL) } new pkgAcqFile(&Fetcher,Last->Index().ArchiveURI(I->Path), - hs != NULL ? hs->toStr() : "", I->Size, - Last->Index().SourceInfo(*Last,*I),Src); + I->Hashes, I->Size, Last->Index().SourceInfo(*Last,*I), Src); } } diff --git a/test/integration/test-apt-sources-deb822 b/test/integration/test-apt-sources-deb822 index 5f54b7531..d8b2334ad 100755 --- a/test/integration/test-apt-sources-deb822 +++ b/test/integration/test-apt-sources-deb822 @@ -23,14 +23,14 @@ Description: summay msgtest 'Test sources.list' 'old style' echo "deb http://ftp.debian.org/debian stable main" > $SOURCES -testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris msgtest 'Test sources.list' 'simple deb822' echo "$BASE" > $SOURCES -testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris @@ -39,29 +39,29 @@ msgtest 'Test deb822 with' 'two entries' echo "$BASE" > $SOURCES echo "" >> $SOURCES echo "$BASE" | sed s/stable/unstable/ >> $SOURCES -testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 -'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 : +'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 " aptget update --print-uris # two suite entries msgtest 'Test deb822 with' 'two Suite entries' echo "$BASE" | sed -e "s/stable/stable unstable/" > $SOURCES -testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 -'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 : +'http://ftp.debian.org/debian/dists/unstable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_unstable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/unstable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_unstable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/unstable/InRelease' ftp.debian.org_debian_dists_unstable_InRelease 0 " aptget update --print-uris msgtest 'Test deb822' 'architecture option' echo "$BASE" > $SOURCES echo "Architectures: amd64 armel" >> $SOURCES -testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris @@ -85,19 +85,19 @@ testempty aptget update --print-uris # multiple URIs msgtest 'Test deb822 sources.list file which has' 'Multiple URIs work' echo "$BASE" | sed -e 's#http://ftp.debian.org/debian#http://ftp.debian.org/debian http://ftp.de.debian.org/debian#' > $SOURCES -testequal --nomsg "'http://ftp.de.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.de.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : -'http://ftp.de.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.de.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +testequal --nomsg "'http://ftp.de.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.de.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.de.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.de.debian.org_debian_dists_stable_main_i18n_Translation-en 0 'http://ftp.de.debian.org/debian/dists/stable/InRelease' ftp.de.debian.org_debian_dists_stable_InRelease 0 -'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris # multiple Type in one field msgtest 'Test deb822 sources.list file which has' 'Multiple Types work' echo "$BASE" | sed -e 's#Types: deb#Types: deb deb-src#' > $SOURCES -testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/source/Sources.bz2' ftp.debian.org_debian_dists_stable_main_source_Sources 0 : -'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 : -'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 : +testequal --nomsg "'http://ftp.debian.org/debian/dists/stable/main/source/Sources.bz2' ftp.debian.org_debian_dists_stable_main_source_Sources 0 +'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.bz2' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 +'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.bz2' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 " aptget update --print-uris # a Suite @@ -107,6 +107,6 @@ Types: deb URIs: http://emacs.naquadah.org Suites: stable/ EOF -testequal --nomsg "'http://emacs.naquadah.org/stable/Packages.bz2' emacs.naquadah.org_stable_Packages 0 : -'http://emacs.naquadah.org/stable/en.bz2' emacs.naquadah.org_stable_en 0 : +testequal --nomsg "'http://emacs.naquadah.org/stable/Packages.bz2' emacs.naquadah.org_stable_Packages 0 +'http://emacs.naquadah.org/stable/en.bz2' emacs.naquadah.org_stable_en 0 'http://emacs.naquadah.org/stable/InRelease' emacs.naquadah.org_stable_InRelease 0 " aptget update --print-uris diff --git a/test/integration/test-bug-722207-print-uris-even-if-very-quiet b/test/integration/test-bug-722207-print-uris-even-if-very-quiet index f2d95da19..9a5685703 100755 --- a/test/integration/test-bug-722207-print-uris-even-if-very-quiet +++ b/test/integration/test-bug-722207-print-uris-even-if-very-quiet @@ -16,10 +16,10 @@ setupaptarchive APTARCHIVE=$(readlink -f ./aptarchive) -testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 MD5Sum:" aptget upgrade -qq --print-uris -testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 MD5Sum:" aptget dist-upgrade -qq --print-uris -testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 MD5Sum:" aptget install apt -qq --print-uris -testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 MD5Sum:" aptget download apt -qq --print-uris +testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget upgrade -qq --print-uris +testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget dist-upgrade -qq --print-uris +testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget install apt -qq --print-uris +testequal "'file://${APTARCHIVE}/pool/main/apt/apt_2_all.deb' apt_2_all.deb 0 " aptget download apt -qq --print-uris testequal "'file://${APTARCHIVE}/apt_2.dsc' apt_2.dsc 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e 'file://${APTARCHIVE}/apt_2.tar.gz' apt_2.tar.gz 0 MD5Sum:d41d8cd98f00b204e9800998ecf8427e" aptget source apt -qq --print-uris testequal "'http://packages.debian.org/changelogs/pool/main/apt/apt_2/changelog'" aptget changelog apt -qq --print-uris diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index afe1ad443..74749d6ab 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -13,12 +13,6 @@ changetowebserver PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')" -echo '#!/bin/sh -touch merge-was-used -/usr/bin/diffindex-rred "$@"' > extrred -chmod +x extrred -echo 'Dir::Bin::rred "./extrred";' > rootdir/etc/apt/apt.conf.d/99rred - wasmergeused() { msgtest 'Test for successful execution of' "$*" local OUTPUT=$(mktemp) diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall index 1a7db0adc..8e50843f3 100755 --- a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall +++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall @@ -22,7 +22,7 @@ Config-Item: Acquire::http::DependOnSTDIN=0 600 Acquire URI URI: http://localhost:8080/holygrail Filename: knights-talking -' | LD_LIBRARY_PATH=${BUILDDIRECTORY} ${METHODSDIR}/http >/dev/null 2>&1 && msgpass || msgfail +' | runapt ${METHODSDIR}/http >/dev/null 2>&1 && msgpass || msgfail testfileequal knights-talking 'ni ni ni' ensure_n_canary_strings_in_dir() { diff --git a/test/libapt/hashsums_test.cc b/test/libapt/hashsums_test.cc index ac7d41582..2159996ff 100644 --- a/test/libapt/hashsums_test.cc +++ b/test/libapt/hashsums_test.cc @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -166,20 +167,26 @@ TEST(HashSumsTest, FileBased) { Hashes hashes; hashes.AddFD(fd.Fd()); - EXPECT_EQ(md5.Value(), hashes.MD5.Result().Value()); - EXPECT_EQ(sha1.Value(), hashes.SHA1.Result().Value()); - EXPECT_EQ(sha256.Value(), hashes.SHA256.Result().Value()); - EXPECT_EQ(sha512.Value(), hashes.SHA512.Result().Value()); + HashStringList list = hashes.GetHashStringList(); + EXPECT_FALSE(list.empty()); + EXPECT_EQ(4, list.size()); + EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue()); + EXPECT_EQ(sha1.Value(), list.find("SHA1")->HashValue()); + EXPECT_EQ(sha256.Value(), list.find("SHA256")->HashValue()); + EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue()); } unsigned long sz = fd.FileSize(); fd.Seek(0); { Hashes hashes; hashes.AddFD(fd.Fd(), sz); - EXPECT_EQ(md5.Value(), hashes.MD5.Result().Value()); - EXPECT_EQ(sha1.Value(), hashes.SHA1.Result().Value()); - EXPECT_EQ(sha256.Value(), hashes.SHA256.Result().Value()); - EXPECT_EQ(sha512.Value(), hashes.SHA512.Result().Value()); + HashStringList list = hashes.GetHashStringList(); + EXPECT_FALSE(list.empty()); + EXPECT_EQ(4, list.size()); + EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue()); + EXPECT_EQ(sha1.Value(), list.find("SHA1")->HashValue()); + EXPECT_EQ(sha256.Value(), list.find("SHA256")->HashValue()); + EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue()); } fd.Seek(0); { @@ -260,3 +267,65 @@ TEST(HashSumsTest, FileBased) EXPECT_FALSE(similar == hashes); EXPECT_TRUE(similar != hashes); } +TEST(HashSumsTest, HashStringList) +{ + _config->Clear("Acquire::ForceHash"); + + HashStringList list; + EXPECT_TRUE(list.empty()); + EXPECT_FALSE(list.usable()); + EXPECT_EQ(0, list.size()); + EXPECT_EQ(NULL, list.find(NULL)); + EXPECT_EQ(NULL, list.find("")); + EXPECT_EQ(NULL, list.find("MD5Sum")); + + HashStringList list2; + EXPECT_FALSE(list == list2); + EXPECT_TRUE(list != list2); + + Hashes hashes; + hashes.Add("The quick brown fox jumps over the lazy dog"); + list = hashes.GetHashStringList(); + EXPECT_FALSE(list.empty()); + EXPECT_TRUE(list.usable()); + EXPECT_EQ(4, list.size()); + EXPECT_TRUE(NULL != list.find(NULL)); + EXPECT_TRUE(NULL != list.find("")); + EXPECT_TRUE(NULL != list.find("MD5Sum")); + EXPECT_TRUE(NULL == list.find("ROT26")); + + _config->Set("Acquire::ForceHash", "MD5Sum"); + EXPECT_FALSE(list.empty()); + EXPECT_TRUE(list.usable()); + EXPECT_EQ(4, list.size()); + EXPECT_TRUE(NULL != list.find(NULL)); + EXPECT_TRUE(NULL != list.find("")); + EXPECT_TRUE(NULL != list.find("MD5Sum")); + EXPECT_TRUE(NULL == list.find("ROT26")); + + _config->Set("Acquire::ForceHash", "ROT26"); + EXPECT_FALSE(list.empty()); + EXPECT_FALSE(list.usable()); + EXPECT_EQ(4, list.size()); + EXPECT_TRUE(NULL == list.find(NULL)); + EXPECT_TRUE(NULL == list.find("")); + EXPECT_TRUE(NULL != list.find("MD5Sum")); + EXPECT_TRUE(NULL == list.find("ROT26")); + + _config->Clear("Acquire::ForceHash"); + + list2.push_back(*list.find("MD5Sum")); + EXPECT_TRUE(list == list2); + EXPECT_FALSE(list != list2); + + // introduce a mismatch to the list + list2.push_back(HashString("SHA1", "cacecbd74968bc90ea3342767e6b94f46ddbcafc")); + EXPECT_FALSE(list == list2); + EXPECT_TRUE(list != list2); + + _config->Set("Acquire::ForceHash", "MD5Sum"); + EXPECT_TRUE(list == list2); + EXPECT_FALSE(list != list2); + + _config->Clear("Acquire::ForceHash"); +} -- cgit v1.2.3-70-g09d2 From 8710a36a01c0cb1648926792c2ad05185535558e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 10 May 2014 11:24:44 +0200 Subject: improve pkgTagSection scanning and parsing Removes the 256 fields limit, deals consistently with spaces littered all over the place and is even a tiny bit faster than before. Even comes with a bunch of new tests to validate these claims. --- apt-pkg/tagfile.cc | 209 ++++++++++++++--------- apt-pkg/tagfile.h | 63 ++++--- cmdline/apt-extracttemplates.cc | 13 +- test/integration/test-apt-ftparchive-src-cachedb | 5 + test/libapt/tagfile_test.cc | 179 +++++++++++++++++++ 5 files changed, 365 insertions(+), 104 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 009ed7d74..52f4da2d5 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -47,6 +47,22 @@ public: unsigned long long Size; }; +static unsigned long AlphaHash(const char *Text, size_t Length) /*{{{*/ +{ + /* This very simple hash function for the last 8 letters gives + very good performance on the debian package files */ + if (Length > 8) + { + Text += (Length - 8); + Length = 8; + } + unsigned long Res = 0; + for (size_t i = 0; i < Length; ++i) + Res = ((unsigned long)(Text[i]) & 0xDF) ^ (Res << 1); + return Res & 0xFF; +} + /*}}}*/ + // TagFile::pkgTagFile - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -139,18 +155,23 @@ bool pkgTagFile::Resize(unsigned long long const newSize) */ bool pkgTagFile::Step(pkgTagSection &Tag) { - while (Tag.Scan(d->Start,d->End - d->Start) == false) + if(Tag.Scan(d->Start,d->End - d->Start) == false) { - if (Fill() == false) - return false; - - if(Tag.Scan(d->Start,d->End - d->Start)) - break; + do + { + if (Fill() == false) + return false; + + if(Tag.Scan(d->Start,d->End - d->Start, false)) + break; + + if (Resize() == false) + return _error->Error(_("Unable to parse package file %s (1)"), + d->Fd.Name().c_str()); - if (Resize() == false) - return _error->Error(_("Unable to parse package file %s (1)"), - d->Fd.Name().c_str()); + } while (Tag.Scan(d->Start,d->End - d->Start, false) == false); } + d->Start += Tag.size(); d->iOffset += Tag.size(); @@ -244,7 +265,7 @@ bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset) if (Fill() == false) return false; - if (Tag.Scan(d->Start, d->End - d->Start) == false) + if (Tag.Scan(d->Start, d->End - d->Start, false) == false) return _error->Error(_("Unable to parse package file %s (2)"),d->Fd.Name().c_str()); return true; @@ -254,27 +275,46 @@ bool pkgTagFile::Jump(pkgTagSection &Tag,unsigned long long Offset) // --------------------------------------------------------------------- /* */ pkgTagSection::pkgTagSection() - : Section(0), TagCount(0), d(NULL), Stop(0) + : Section(0), d(NULL), Stop(0) { - memset(&Indexes, 0, sizeof(Indexes)); - memset(&AlphaIndexes, 0, sizeof(AlphaIndexes)); + memset(&LookupTable, 0, sizeof(LookupTable)); } /*}}}*/ // TagSection::Scan - Scan for the end of the header information /*{{{*/ -// --------------------------------------------------------------------- -/* This looks for the first double new line in the data stream. - It also indexes the tags in the section. */ -bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength) +bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength, bool const Restart) { + Section = Start; const char *End = Start + MaxLength; - Stop = Section = Start; - memset(AlphaIndexes,0,sizeof(AlphaIndexes)); + + if (Restart == false && Tags.empty() == false) + { + Stop = Section + Tags.back().StartTag; + if (End <= Stop) + return false; + Stop = (const char *)memchr(Stop,'\n',End - Stop); + if (Stop == NULL) + return false; + ++Stop; + } + else + { + Stop = Section; + if (Tags.empty() == false) + { + memset(&LookupTable, 0, sizeof(LookupTable)); + Tags.clear(); + } + Tags.reserve(0x100); + } + size_t TagCount = Tags.size(); if (Stop == 0) return false; - TagCount = 0; - while (TagCount+1 < sizeof(Indexes)/sizeof(Indexes[0]) && Stop < End) + TagData lastTagData(0); + lastTagData.EndTag = 0; + unsigned long lastTagHash = 0; + while (Stop < End) { TrimRecord(true,End); @@ -286,12 +326,39 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength) // Start a new index and add it to the hash if (isspace(Stop[0]) == 0) { - Indexes[TagCount++] = Stop - Section; - AlphaIndexes[AlphaHash(Stop,End)] = TagCount; + // store the last found tag + if (lastTagData.EndTag != 0) + { + if (LookupTable[lastTagHash] != 0) + lastTagData.NextInBucket = LookupTable[lastTagHash]; + LookupTable[lastTagHash] = TagCount; + Tags.push_back(lastTagData); + } + + ++TagCount; + lastTagData = TagData(Stop - Section); + // find the colon separating tag and value + char const * Colon = (char const *) memchr(Stop, ':', End - Stop); + if (Colon == NULL) + return false; + // find the end of the tag (which might or might not be the colon) + char const * EndTag = Colon; + --EndTag; + for (; EndTag > Stop && isspace(*EndTag) != 0; --EndTag) + ; + ++EndTag; + lastTagData.EndTag = EndTag - Section; + lastTagHash = AlphaHash(Stop, EndTag - Stop); + // find the beginning of the value + Stop = Colon + 1; + for (; isspace(*Stop) != 0; ++Stop); + if (Stop >= End) + return false; + lastTagData.StartValue = Stop - Section; } Stop = (const char *)memchr(Stop,'\n',End - Stop); - + if (Stop == 0) return false; @@ -302,7 +369,16 @@ bool pkgTagSection::Scan(const char *Start,unsigned long MaxLength) // Double newline marks the end of the record if (Stop+1 < End && Stop[1] == '\n') { - Indexes[TagCount] = Stop - Section; + if (lastTagData.EndTag != 0) + { + if (LookupTable[lastTagHash] != 0) + lastTagData.NextInBucket = LookupTable[lastTagHash]; + LookupTable[lastTagHash] = TagCount; + Tags.push_back(lastTagData); + } + + TagData const td(Stop - Section); + Tags.push_back(td); TrimRecord(false,End); return true; } @@ -331,8 +407,8 @@ void pkgTagSection::Trim() for (; Stop > Section + 2 && (Stop[-2] == '\n' || Stop[-2] == '\r'); Stop--); } /*}}}*/ -// TagSection::Exists - return True if a tag exists /*{{{*/ -bool pkgTagSection::Exists(const char* const Tag) +// TagSection::Exists - return True if a tag exists /*{{{*/ +bool pkgTagSection::Exists(const char* const Tag) const { unsigned int tmp; return Find(Tag, tmp); @@ -343,73 +419,43 @@ bool pkgTagSection::Exists(const char* const Tag) /* This searches the section for a tag that matches the given string. */ bool pkgTagSection::Find(const char *Tag,unsigned int &Pos) const { - unsigned int Length = strlen(Tag); - unsigned int I = AlphaIndexes[AlphaHash(Tag)]; - if (I == 0) + size_t const Length = strlen(Tag); + unsigned int Bucket = LookupTable[AlphaHash(Tag, Length)]; + if (Bucket == 0) return false; - I--; - - for (unsigned int Counter = 0; Counter != TagCount; Counter++, - I = (I+1)%TagCount) + + for (; Bucket != 0; Bucket = Tags[Bucket - 1].NextInBucket) { - const char *St; - St = Section + Indexes[I]; - if (strncasecmp(Tag,St,Length) != 0) + if ((Tags[Bucket - 1].EndTag - Tags[Bucket - 1].StartTag) != Length) continue; - // Make sure the colon is in the right place - const char *C = St + Length; - for (; isspace(*C) != 0; C++); - if (*C != ':') + char const * const St = Section + Tags[Bucket - 1].StartTag; + if (strncasecmp(Tag,St,Length) != 0) continue; - Pos = I; + + Pos = Bucket - 1; return true; } Pos = 0; return false; } - /*}}}*/ -// TagSection::Find - Locate a tag /*{{{*/ -// --------------------------------------------------------------------- -/* This searches the section for a tag that matches the given string. */ bool pkgTagSection::Find(const char *Tag,const char *&Start, const char *&End) const { - unsigned int Length = strlen(Tag); - unsigned int I = AlphaIndexes[AlphaHash(Tag)]; - if (I == 0) + unsigned int Pos; + if (Find(Tag, Pos) == false) return false; - I--; - - for (unsigned int Counter = 0; Counter != TagCount; Counter++, - I = (I+1)%TagCount) - { - const char *St; - St = Section + Indexes[I]; - if (strncasecmp(Tag,St,Length) != 0) - continue; - - // Make sure the colon is in the right place - const char *C = St + Length; - for (; isspace(*C) != 0; C++); - if (*C != ':') - continue; - // Strip off the gunk from the start end - Start = C; - End = Section + Indexes[I+1]; - if (Start >= End) - return _error->Error("Internal parsing error"); - - for (; (isspace(*Start) != 0 || *Start == ':') && Start < End; Start++); - for (; isspace(End[-1]) != 0 && End > Start; End--); - - return true; - } - - Start = End = 0; - return false; + Start = Section + Tags[Pos].StartValue; + // Strip off the gunk from the end + End = Section + Tags[Pos + 1].StartTag; + if (unlikely(Start > End)) + return _error->Error("Internal parsing error"); + + for (; isspace(End[-1]) != 0 && End > Start; --End); + + return true; } /*}}}*/ // TagSection::FindS - Find a string /*{{{*/ @@ -504,6 +550,13 @@ bool pkgTagSection::FindFlag(unsigned long &Flags, unsigned long Flag, return true; } /*}}}*/ +APT_PURE unsigned int pkgTagSection::Count() const { /*{{{*/ + if (Tags.empty() == true) + return 0; + // the last element is just marking the end and isn't a real one + return Tags.size() - 1; +} + /*}}}*/ // TFRewrite - Rewrite a control record /*{{{*/ // --------------------------------------------------------------------- /* This writes the control record to stdout rewriting it as necessary. The diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index d1a24ba45..b0cfab759 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -25,6 +25,8 @@ #include #include +#include +#include #ifndef APT_8_CLEANER_HEADERS #include @@ -35,23 +37,20 @@ class FileFd; class pkgTagSection { const char *Section; - // We have a limit of 256 tags per section. - unsigned int Indexes[256]; - unsigned int AlphaIndexes[0x100]; - unsigned int TagCount; + struct TagData { + unsigned int StartTag; + unsigned int EndTag; + unsigned int StartValue; + unsigned int NextInBucket; + + TagData(unsigned int const StartTag) : StartTag(StartTag), NextInBucket(0) {} + }; + std::vector Tags; + unsigned int LookupTable[0x100]; + // dpointer placeholder (for later in case we need it) void *d; - /* This very simple hash function for the last 8 letters gives - very good performance on the debian package files */ - inline static unsigned long AlphaHash(const char *Text, const char *End = 0) - { - unsigned long Res = 0; - for (; Text != End && *Text != ':' && *Text != 0; Text++) - Res = ((unsigned long)(*Text) & 0xDF) ^ (Res << 1); - return Res & 0xFF; - } - protected: const char *Stop; @@ -69,17 +68,39 @@ class pkgTagSection unsigned long Flag) const; bool static FindFlag(unsigned long &Flags, unsigned long Flag, const char* Start, const char* Stop); - bool Scan(const char *Start,unsigned long MaxLength); + + /** \brief searches the boundaries of the current section + * + * While parameter Start marks the beginning of the section, this method + * will search for the first double newline in the data stream which marks + * the end of the section. It also does a first pass over the content of + * the section parsing it as encountered for processing later on by Find + * + * @param Start is the beginning of the section + * @param MaxLength is the size of valid data in the stream pointed to by Start + * @param Restart if enabled internal state will be cleared, otherwise it is + * assumed that now more data is available in the stream and the parsing will + * start were it encountered insufficent data the last time. + * + * @return \b true if section end was found, \b false otherwise. + * Beware that internal state will be inconsistent if \b false is returned! + */ + APT_MUSTCHECK bool Scan(const char *Start, unsigned long MaxLength, bool const Restart = true); inline unsigned long size() const {return Stop - Section;}; void Trim(); virtual void TrimRecord(bool BeforeRecord, const char* &End); - - inline unsigned int Count() const {return TagCount;}; - bool Exists(const char* const Tag); - + + /** \brief amount of Tags in the current section + * + * Note: if a Tag is mentioned repeatly it will be counted multiple + * times, but only the last occurance is available via Find methods. + */ + unsigned int Count() const; + bool Exists(const char* const Tag) const; + inline void Get(const char *&Start,const char *&Stop,unsigned int I) const - {Start = Section + Indexes[I]; Stop = Section + Indexes[I+1];} - + {Start = Section + Tags[I].StartTag; Stop = Section + Tags[I+1].StartTag;} + inline void GetSection(const char *&Start,const char *&Stop) const { Start = Section; diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc index e4428e051..7be59b9f8 100644 --- a/cmdline/apt-extracttemplates.cc +++ b/cmdline/apt-extracttemplates.cc @@ -103,10 +103,12 @@ bool DebFile::DoItem(Item &I, int &Fd) if (strcmp(I.Name, "control") == 0) { delete [] Control; - Control = new char[I.Size+1]; - Control[I.Size] = 0; + Control = new char[I.Size+3]; + Control[I.Size] = '\n'; + Control[I.Size + 1] = '\n'; + Control[I.Size + 2] = '\0'; Which = IsControl; - ControlLen = I.Size; + ControlLen = I.Size + 3; // make it call the Process method below. this is so evil Fd = -2; } @@ -162,9 +164,10 @@ bool DebFile::Process(Item &/*I*/, const unsigned char *data, bool DebFile::ParseInfo() { if (Control == NULL) return false; - + pkgTagSection Section; - Section.Scan(Control, ControlLen); + if (Section.Scan(Control, ControlLen) == false) + return false; Package = Section.FindS("Package"); Version = GetInstalledVer(Package); diff --git a/test/integration/test-apt-ftparchive-src-cachedb b/test/integration/test-apt-ftparchive-src-cachedb index 1af193632..c850c739f 100755 --- a/test/integration/test-apt-ftparchive-src-cachedb +++ b/test/integration/test-apt-ftparchive-src-cachedb @@ -177,6 +177,11 @@ assert_correct_sources_file mkdir aptarchive/pool/invalid printf "meep" > aptarchive/pool/invalid/invalid_1.0.dsc testequal " +E: Could not find a record in the DSC 'aptarchive/pool/invalid/invalid_1.0.dsc'" aptftparchive sources aptarchive/pool/invalid +rm -f aptarchive/pool/invalid/invalid_1.0.dsc + +printf "meep: yes" > aptarchive/pool/invalid/invalid_1.0.dsc +testequal " E: Could not find a Source entry in the DSC 'aptarchive/pool/invalid/invalid_1.0.dsc'" aptftparchive sources aptarchive/pool/invalid rm -f aptarchive/pool/invalid/invalid_1.0.dsc diff --git a/test/libapt/tagfile_test.cc b/test/libapt/tagfile_test.cc index 1bac75b55..df618ea16 100644 --- a/test/libapt/tagfile_test.cc +++ b/test/libapt/tagfile_test.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -34,3 +35,181 @@ TEST(TagFileTest,SingleField) // There is only one section in this tag file EXPECT_FALSE(tfile.Step(section)); } + +TEST(TagFileTest,MultipleSections) +{ + FileFd fd; + createTemporaryFile("bigsection", fd, NULL, "Package: pkgA\n" + "Version: 1\n" + "Size: 100\n" + "Description: aaa\n" + " aaa\n" + "\n" + "Package: pkgB\n" + "Version: 1\n" + "Flag: no\n" + "Description: bbb\n" + "\n" + "Package: pkgC\n" + "Version: 2\n" + "Flag: yes\n" + "Description:\n" + " ccc\n" + ); + + pkgTagFile tfile(&fd); + pkgTagSection section; + EXPECT_FALSE(section.Exists("Version")); + + EXPECT_TRUE(tfile.Step(section)); + EXPECT_EQ(4, section.Count()); + EXPECT_TRUE(section.Exists("Version")); + EXPECT_TRUE(section.Exists("Package")); + EXPECT_TRUE(section.Exists("Size")); + EXPECT_FALSE(section.Exists("Flag")); + EXPECT_TRUE(section.Exists("Description")); + EXPECT_EQ("pkgA", section.FindS("Package")); + EXPECT_EQ("1", section.FindS("Version")); + EXPECT_EQ(1, section.FindULL("Version")); + EXPECT_EQ(100, section.FindULL("Size")); + unsigned long Flags = 1; + EXPECT_TRUE(section.FindFlag("Flag", Flags, 1)); + EXPECT_EQ(1, Flags); + Flags = 0; + EXPECT_TRUE(section.FindFlag("Flag", Flags, 1)); + EXPECT_EQ(0, Flags); + EXPECT_EQ("aaa\n aaa", section.FindS("Description")); + + + EXPECT_TRUE(tfile.Step(section)); + EXPECT_EQ(4, section.Count()); + EXPECT_TRUE(section.Exists("Version")); + EXPECT_TRUE(section.Exists("Package")); + EXPECT_FALSE(section.Exists("Size")); + EXPECT_TRUE(section.Exists("Flag")); + EXPECT_TRUE(section.Exists("Description")); + EXPECT_EQ("pkgB", section.FindS("Package")); + EXPECT_EQ("1", section.FindS("Version")); + EXPECT_EQ(1, section.FindULL("Version")); + EXPECT_EQ(0, section.FindULL("Size")); + Flags = 1; + EXPECT_TRUE(section.FindFlag("Flag", Flags, 1)); + EXPECT_EQ(0, Flags); + Flags = 0; + EXPECT_TRUE(section.FindFlag("Flag", Flags, 1)); + EXPECT_EQ(0, Flags); + EXPECT_EQ("bbb", section.FindS("Description")); + + EXPECT_TRUE(tfile.Step(section)); + EXPECT_EQ(4, section.Count()); + EXPECT_TRUE(section.Exists("Version")); + EXPECT_TRUE(section.Exists("Package")); + EXPECT_FALSE(section.Exists("Size")); + EXPECT_TRUE(section.Exists("Flag")); + EXPECT_TRUE(section.Exists("Description")); + EXPECT_EQ("pkgC", section.FindS("Package")); + EXPECT_EQ("2", section.FindS("Version")); + EXPECT_EQ(2, section.FindULL("Version")); + Flags = 0; + EXPECT_TRUE(section.FindFlag("Flag", Flags, 1)); + EXPECT_EQ(1, Flags); + Flags = 1; + EXPECT_TRUE(section.FindFlag("Flag", Flags, 1)); + EXPECT_EQ(1, Flags); + EXPECT_EQ("ccc", section.FindS("Description")); + + // There is no section left in this tag file + EXPECT_FALSE(tfile.Step(section)); +} + +TEST(TagFileTest,BigSection) +{ + size_t const count = 500; + std::stringstream content; + for (size_t i = 0; i < count; ++i) + content << "Field-" << i << ": " << (2000 + i) << std::endl; + + FileFd fd; + createTemporaryFile("bigsection", fd, NULL, content.str().c_str()); + + pkgTagFile tfile(&fd); + pkgTagSection section; + EXPECT_TRUE(tfile.Step(section)); + + EXPECT_EQ(count, section.Count()); + for (size_t i = 0; i < count; ++i) + { + std::stringstream name; + name << "Field-" << i; + EXPECT_TRUE(section.Exists(name.str().c_str())) << name.str() << " does not exist"; + EXPECT_EQ((2000 + i), section.FindULL(name.str().c_str())); + } + + // There is only one section in this tag file + EXPECT_FALSE(tfile.Step(section)); +} + +TEST(TagFileTest, PickedUpFromPreviousCall) +{ + size_t const count = 500; + std::stringstream contentstream; + for (size_t i = 0; i < count; ++i) + contentstream << "Field-" << i << ": " << (2000 + i) << std::endl; + contentstream << std::endl << std::endl; + std::string content = contentstream.str(); + + pkgTagSection section; + EXPECT_FALSE(section.Scan(content.c_str(), content.size()/2)); + EXPECT_NE(0, section.Count()); + EXPECT_NE(count, section.Count()); + EXPECT_TRUE(section.Scan(content.c_str(), content.size(), false)); + EXPECT_EQ(count, section.Count()); + + for (size_t i = 0; i < count; ++i) + { + std::stringstream name; + name << "Field-" << i; + EXPECT_TRUE(section.Exists(name.str().c_str())) << name.str() << " does not exist"; + EXPECT_EQ((2000 + i), section.FindULL(name.str().c_str())); + } +} + +TEST(TagFileTest, SpacesEverywhere) +{ + std::string content = + "Package: pkgA\n" + "Package: pkgB\n" + "NoSpaces:yes\n" + "TagSpaces\t :yes\n" + "ValueSpaces: \tyes\n" + "BothSpaces \t:\t yes\n" + "TrailingSpaces: yes\t \n" + "Naming Space: yes\n" + "Naming Spaces: yes\n" + "Package : pkgC \n" + "Multi-Colon::yes:\n" + "\n\n"; + + pkgTagSection section; + EXPECT_TRUE(section.Scan(content.c_str(), content.size())); + EXPECT_TRUE(section.Exists("Package")); + EXPECT_TRUE(section.Exists("NoSpaces")); + EXPECT_TRUE(section.Exists("TagSpaces")); + EXPECT_TRUE(section.Exists("ValueSpaces")); + EXPECT_TRUE(section.Exists("BothSpaces")); + EXPECT_TRUE(section.Exists("TrailingSpaces")); + EXPECT_TRUE(section.Exists("Naming Space")); + EXPECT_TRUE(section.Exists("Naming Spaces")); + EXPECT_TRUE(section.Exists("Multi-Colon")); + EXPECT_EQ("pkgC", section.FindS("Package")); + EXPECT_EQ("yes", section.FindS("NoSpaces")); + EXPECT_EQ("yes", section.FindS("TagSpaces")); + EXPECT_EQ("yes", section.FindS("ValueSpaces")); + EXPECT_EQ("yes", section.FindS("BothSpaces")); + EXPECT_EQ("yes", section.FindS("TrailingSpaces")); + EXPECT_EQ("yes", section.FindS("Naming Space")); + EXPECT_EQ("yes", section.FindS("Naming Spaces")); + EXPECT_EQ(":yes:", section.FindS("Multi-Colon")); + // overridden values are still present, but not really accessible + EXPECT_EQ(11, section.Count()); +} -- cgit v1.2.3-70-g09d2 From 8f418981337503ff7abedd872f788b51bcdbc886 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 22 Apr 2014 16:07:32 +0200 Subject: show upgradable packages after apt update Closes: 748389 --- apt-private/private-update.cc | 18 ++++++++++++++++++ cmdline/apt.cc | 1 + test/integration/test-apt-cli-update | 17 +++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100755 test/integration/test-apt-cli-update (limited to 'cmdline') diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc index da83d7741..fa827dea4 100644 --- a/apt-private/private-update.cc +++ b/apt-private/private-update.cc @@ -75,6 +75,24 @@ bool DoUpdate(CommandLine &CmdL) return false; } + // show basic stats (if the user whishes) + if (_config->FindB("APT::Cmd::Show-Update-Stats", false) == true) + { + int upgradable = 0; + Cache.Open(); + for (pkgCache::PkgIterator I = Cache->PkgBegin(); I.end() != true; ++I) + { + pkgDepCache::StateCache &state = Cache[I]; + if (I->CurrentVer != 0 && state.Upgradable()) + upgradable++; + } + const char *msg = ngettext( + "%i package can be upgraded. Run 'apt list --upgradable' to see it.\n", + "%i packages can be upgraded. Run 'apt list --upgradable' to see them.\n", + upgradable); + ioprintf(c1out, msg, upgradable); + } + return true; } /*}}}*/ diff --git a/cmdline/apt.cc b/cmdline/apt.cc index 4813861a4..8a6f96aea 100644 --- a/cmdline/apt.cc +++ b/cmdline/apt.cc @@ -112,6 +112,7 @@ int main(int argc, const char *argv[]) /*{{{*/ _config->CndSet("DPkg::Progress-Fancy", "1"); _config->CndSet("Apt::Color", "1"); _config->CndSet("APT::Get::Upgrade-Allow-New", true); + _config->CndSet("APT::Cmd::Show-Update-Stats", true); // Parse the command line and initialize the package library CommandLine CmdL(Args.data(), _config); diff --git a/test/integration/test-apt-cli-update b/test/integration/test-apt-cli-update new file mode 100755 index 000000000..8237bf03f --- /dev/null +++ b/test/integration/test-apt-cli-update @@ -0,0 +1,17 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +insertpackage 'unstable' 'foo' 'all' '2.0' +insertinstalledpackage 'foo' 'all' '1.0' + +setupaptarchive + +APTARCHIVE=$(readlink -f ./aptarchive) + +testequal "1 package can be upgraded. Run 'apt list --upgradable' to see it." apt update -q -- cgit v1.2.3-70-g09d2 From d287a035e07030be13f72265d4d0812086c323f5 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Sat, 17 May 2014 06:57:47 +0200 Subject: fix screen width detection for apt/apt-get lists 3163087b moved SigWinch(int) from apt-get.cc to private-output.cc without moving #include , making SigWinch a nop. Closes: 748430, 747942 --- apt-private/private-output.cc | 1 + cmdline/apt-get.cc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) (limited to 'cmdline') diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc index 7f3eef6c2..32f27a3dd 100644 --- a/apt-private/private-output.cc +++ b/apt-private/private-output.cc @@ -23,6 +23,7 @@ #include #include #include +#include #include /*}}}*/ diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 566103f8c..0f18b0e7c 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -76,7 +76,6 @@ #include #include #include -#include #include #include #include -- cgit v1.2.3-70-g09d2 From d01d91d397284fb2ab3402dddede1594d2685356 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 May 2014 13:49:52 +0200 Subject: cmdline/apt-extracttemplates.cc: remove unused private var Reported-By: clang -Wall --- cmdline/apt-extracttemplates.cc | 2 +- cmdline/apt-extracttemplates.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc index e4428e051..6dd53b78d 100644 --- a/cmdline/apt-extracttemplates.cc +++ b/cmdline/apt-extracttemplates.cc @@ -52,7 +52,7 @@ pkgCache *DebFile::Cache = 0; // --------------------------------------------------------------------- /* */ DebFile::DebFile(const char *debfile) - : File(debfile, FileFd::ReadOnly), Size(0), Control(NULL), ControlLen(0), + : File(debfile, FileFd::ReadOnly), Control(NULL), ControlLen(0), DepOp(0), PreDepOp(0), Config(0), Template(0), Which(None) { } diff --git a/cmdline/apt-extracttemplates.h b/cmdline/apt-extracttemplates.h index 9cc3f5f25..6d2870a02 100644 --- a/cmdline/apt-extracttemplates.h +++ b/cmdline/apt-extracttemplates.h @@ -20,7 +20,6 @@ class pkgCache; class DebFile : public pkgDirStream { FileFd File; - unsigned long Size; char *Control; unsigned long ControlLen; -- cgit v1.2.3-70-g09d2 From a93a32b5cda4f2a0572365a8b919329ea6af2c66 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 29 May 2014 10:22:09 +0200 Subject: add hashtable stats --- cmdline/apt-cache.cc | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 84b775390..a620740e1 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -373,7 +373,63 @@ static bool Stats(CommandLine &) Cache->Head().VerFileCount*Cache->Head().VerFileSz + Cache->Head().ProvidesCount*Cache->Head().ProvidesSz; cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; - + + // get the hash collisions average + long NumBuckets = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); + long UsedBuckets = 0; + long UnusedBuckets = 0; + long LongestBucket = 0; + long ShortestBucket = NumBuckets; + for (unsigned int i=0; i < NumBuckets; ++i) + { + pkgCache::Package *Pkg = Cache->PkgP + Cache->HeaderP->PkgHashTable[i]; + if(Pkg == 0 || Pkg == Cache->PkgP) + { + UnusedBuckets++; + continue; + } + long ThisBucketSize = 0; + for (; Pkg != Cache->PkgP; Pkg = Cache->PkgP + Pkg->NextPackage) + ThisBucketSize++; + LongestBucket = std::max(ThisBucketSize, LongestBucket); + ShortestBucket = std::min(ThisBucketSize, ShortestBucket); + UsedBuckets += ThisBucketSize; + } + cout << _("Pkg Hashtable stats:") << endl; + cout << _("Number of buckets: ") << SizeToStr(NumBuckets) << endl; + cout << _("Unused buckets: ") << SizeToStr(UnusedBuckets) << endl; + cout << _("Used buckets: ") << UsedBuckets << endl; + cout << _("Average num entries per bucket : ") << UsedBuckets/(double)NumBuckets << endl; + cout << _("Longest / Shortest bucket: ") << LongestBucket << " / " << ShortestBucket << endl; + + // get the hash collisions average + NumBuckets = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); + UsedBuckets = 0; + UnusedBuckets = 0; + LongestBucket = 0; + ShortestBucket = NumBuckets; + for (unsigned int i=0; i < NumBuckets; ++i) + { + pkgCache::Group *Grp = Cache->GrpP + Cache->HeaderP->GrpHashTable[i]; + if(Grp == 0 || Grp == Cache->GrpP) + { + UnusedBuckets++; + continue; + } + long ThisBucketSize = 0; + for (; Grp != Cache->GrpP; Grp = Cache->GrpP + Grp->Next) + ThisBucketSize++; + LongestBucket = std::max(ThisBucketSize, LongestBucket); + ShortestBucket = std::min(ThisBucketSize, ShortestBucket); + UsedBuckets += ThisBucketSize; + } + cout << _("Grp Hashtable stats:") << endl; + cout << _("Number of buckets: ") << SizeToStr(NumBuckets) << endl; + cout << _("Unused buckets: ") << SizeToStr(UnusedBuckets) << endl; + cout << _("Used buckets: ") << UsedBuckets << endl; + cout << _("Average num entries per bucket : ") << UsedBuckets/(double)NumBuckets << endl; + cout << _("Longest / Shortest bucket: ") << LongestBucket << " / " << ShortestBucket << endl; + return true; } /*}}}*/ -- cgit v1.2.3-70-g09d2 From 387fb4aea8415d09e177195841fbefb39bf5521b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 28 May 2014 13:49:52 +0200 Subject: cmdline/apt-extracttemplates.cc: remove unused private var Reported-By: clang -Wall --- cmdline/apt-extracttemplates.cc | 2 +- cmdline/apt-extracttemplates.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc index 7be59b9f8..0d568106f 100644 --- a/cmdline/apt-extracttemplates.cc +++ b/cmdline/apt-extracttemplates.cc @@ -52,7 +52,7 @@ pkgCache *DebFile::Cache = 0; // --------------------------------------------------------------------- /* */ DebFile::DebFile(const char *debfile) - : File(debfile, FileFd::ReadOnly), Size(0), Control(NULL), ControlLen(0), + : File(debfile, FileFd::ReadOnly), Control(NULL), ControlLen(0), DepOp(0), PreDepOp(0), Config(0), Template(0), Which(None) { } diff --git a/cmdline/apt-extracttemplates.h b/cmdline/apt-extracttemplates.h index 9cc3f5f25..6d2870a02 100644 --- a/cmdline/apt-extracttemplates.h +++ b/cmdline/apt-extracttemplates.h @@ -20,7 +20,6 @@ class pkgCache; class DebFile : public pkgDirStream { FileFd File; - unsigned long Size; char *Control; unsigned long ControlLen; -- cgit v1.2.3-70-g09d2 From 81b3540d26b4f47785e6678e22d1312b8cd6d704 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 30 May 2014 16:52:19 +0200 Subject: print error stack on failure of 'apt' solver Git-Dch: ignore --- cmdline/apt-internal-solver.cc | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index b85c07c33..e4cdf6381 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -31,6 +31,7 @@ #include #include #include +#include #include /*}}}*/ @@ -56,6 +57,12 @@ static bool ShowHelp(CommandLine &) { return true; } /*}}}*/ +APT_NORETURN static void DIE(std::string const &message) { /*{{{*/ + std::cerr << "ERROR: " << message << std::endl; + _error->DumpErrors(std::cerr); + exit(EXIT_FAILURE); +} + /*}}}*/ int main(int argc,const char *argv[]) /*{{{*/ { CommandLine::Args Args[] = { @@ -115,34 +122,29 @@ int main(int argc,const char *argv[]) /*{{{*/ EDSP::WriteProgress(0, "Start up solver…", output); - if (pkgInitSystem(*_config,_system) == false) { - std::cerr << "System could not be initialized!" << std::endl; - return 1; - } + if (pkgInitSystem(*_config,_system) == false) + DIE("System could not be initialized!"); EDSP::WriteProgress(1, "Read request…", output); if (WaitFd(input, false, 5) == false) - std::cerr << "WAIT timed out in the resolver" << std::endl; + DIE("WAIT timed out in the resolver"); std::list install, remove; bool upgrade, distUpgrade, autoRemove; - if (EDSP::ReadRequest(input, install, remove, upgrade, distUpgrade, autoRemove) == false) { - std::cerr << "Parsing the request failed!" << std::endl; - return 2; - } + if (EDSP::ReadRequest(input, install, remove, upgrade, distUpgrade, autoRemove) == false) + DIE("Parsing the request failed!"); EDSP::WriteProgress(5, "Read scenario…", output); pkgCacheFile CacheFile; - CacheFile.Open(NULL, false); + if (CacheFile.Open(NULL, false) == false) + DIE("Failed to open CacheFile!"); EDSP::WriteProgress(50, "Apply request on scenario…", output); - if (EDSP::ApplyRequest(install, remove, CacheFile) == false) { - std::cerr << "Failed to apply request to depcache!" << std::endl; - return 3; - } + if (EDSP::ApplyRequest(install, remove, CacheFile) == false) + DIE("Failed to apply request to depcache!"); pkgProblemResolver Fix(CacheFile); for (std::list::const_iterator i = remove.begin(); @@ -183,10 +185,8 @@ int main(int argc,const char *argv[]) /*{{{*/ EDSP::WriteProgress(95, "Write solution…", output); - if (EDSP::WriteSolution(CacheFile, output) == false) { - std::cerr << "Failed to output the solution!" << std::endl; - return 4; - } + if (EDSP::WriteSolution(CacheFile, output) == false) + DIE("Failed to output the solution!"); EDSP::WriteProgress(100, "Done", output); -- cgit v1.2.3-70-g09d2 From a3f1d60cb75ab83f63a52a43c056a4752c8fdeb2 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 30 May 2014 14:47:56 +0200 Subject: Show unauthenticated warning for source packages as well This will show the same unauthenticated warning for source packages as for binary packages and will not download a source package if it is unauthenticated. This can be overridden with --allow-unauthenticated Closes: #749795 --- apt-private/private-download.cc | 5 ++++ apt-private/private-download.h | 6 +++++ cmdline/apt-get.cc | 9 +++++++ test/integration/test-apt-get-source-authenticated | 31 ++++++++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100755 test/integration/test-apt-get-source-authenticated (limited to 'cmdline') diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc index a095f0c67..be7d23c31 100644 --- a/apt-private/private-download.cc +++ b/apt-private/private-download.cc @@ -28,6 +28,11 @@ bool CheckAuth(pkgAcquire& Fetcher, bool const PromptUser) if (UntrustedList == "") return true; + return AuthPrompt(UntrustedList, PromptUser); +} + +bool AuthPrompt(std::string UntrustedList, bool const PromptUser) +{ ShowList(c2out,_("WARNING: The following packages cannot be authenticated!"),UntrustedList,""); if (_config->FindB("APT::Get::AllowUnauthenticated",false) == true) diff --git a/apt-private/private-download.h b/apt-private/private-download.h index a108aa531..a90ac7eaa 100644 --- a/apt-private/private-download.h +++ b/apt-private/private-download.h @@ -5,7 +5,13 @@ class pkgAcquire; +// Check if all files in the fetcher are authenticated APT_PUBLIC bool CheckAuth(pkgAcquire& Fetcher, bool const PromptUser); + +// show a authentication warning prompt and return true if the system +// should continue +APT_PUBLIC bool AuthPrompt(std::string UntrustedList, bool const PromptUser); + APT_PUBLIC bool AcquireRun(pkgAcquire &Fetcher, int const PulseInterval, bool * const Failure, bool * const TransientNetworkFailure); #endif diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 0f18b0e7c..a58386eb0 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -76,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -755,6 +756,7 @@ static bool DoSource(CommandLine &CmdL) // Load the requestd sources into the fetcher unsigned J = 0; + std::string UntrustedList; for (const char **I = CmdL.FileList + 1; *I != 0; I++, J++) { string Src; @@ -763,6 +765,9 @@ static bool DoSource(CommandLine &CmdL) if (Last == 0) { return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); } + + if (Last->Index().IsTrusted() == false) + UntrustedList += Src + " "; string srec = Last->AsStr(); string::size_type pos = srec.find("\nVcs-"); @@ -846,6 +851,10 @@ static bool DoSource(CommandLine &CmdL) Last->Index().SourceInfo(*Last,*I),Src); } } + + // check authentication status of the source as well + if (UntrustedList != "" && !AuthPrompt(UntrustedList, false)) + return false; // Display statistics unsigned long long FetchBytes = Fetcher.FetchNeeded(); diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated new file mode 100755 index 000000000..2cee13923 --- /dev/null +++ b/test/integration/test-apt-get-source-authenticated @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Regression test for debian bug #749795. Ensure that we fail with +# a error if apt-get source foo will download a source that comes +# from a unauthenticated repository +# +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +# a "normal" package with source and binary +buildsimplenativepackage 'foo' 'all' '2.0' + +setupaptarchive --no-update + +APTARCHIVE=$(readlink -f ./aptarchive) +rm -f $APTARCHIVE/dists/unstable/*Release* + +# update without authenticated InRelease file +testsuccess aptget update + +# this all should fail +testfailure aptget install -y foo +testfailure aptget source foo + +# allow overriding the warning +testsuccess aptget source --allow-unauthenticated foo -- cgit v1.2.3-70-g09d2 From 0dd2cbc08aefb73421d5aa31bc62184181a28c32 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 6 Jun 2014 11:24:44 +0200 Subject: cmdline/apt-helper.cc: use less generic description/short-description --- cmdline/apt-helper.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index 2c1107d90..b0edafcbd 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -43,7 +43,8 @@ static bool DoDownloadFile(CommandLine &CmdL) std::string hash; if (CmdL.FileSize() > 3) hash = CmdL.FileList[3]; - new pkgAcqFile(&Fetcher, download_uri, hash, 0, "desc", "short-desc", + // we use download_uri as descr and targetfile as short-descr + new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, "dest-dir-ignored", targetfile); Fetcher.Run(); bool Failed = false; -- cgit v1.2.3-70-g09d2 From 4dde2b4285fc6288e44e915a0d7bc0faac114a2e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Jun 2014 14:12:12 +0200 Subject: support Acquire::GzipIndexes in dumpavail Closes: 742835 --- cmdline/apt-cache.cc | 2 +- test/integration/test-compressed-indexes | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 84b775390..1414617eb 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -507,7 +507,7 @@ static bool DumpAvail(CommandLine &) break; } - FileFd PkgF(File.FileName(),FileFd::ReadOnly); + FileFd PkgF(File.FileName(),FileFd::ReadOnly, FileFd::Extension); if (_error->PendingError() == true) break; diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes index 67ca0ba27..6671dd75a 100755 --- a/test/integration/test-compressed-indexes +++ b/test/integration/test-compressed-indexes @@ -67,6 +67,8 @@ testrun() { msgtest "\tdsc file is present"; testsuccess --nomsg test -f testpkg_1.0.dsc msgtest "\tdirectory is present"; testsuccess --nomsg test -d testpkg-1.0 rm -rf testpkg-1.0 + testequal "$(aptcache show testpkg -o Acquire::Languages=none) +" aptcache dumpavail } echo 'Acquire::GzipIndexes "false";' > rootdir/etc/apt/apt.conf.d/02compressindex -- cgit v1.2.3-70-g09d2 From fc1a78d8e9b958f3d65fe1c03494d785314f9816 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 11 Jun 2014 15:31:38 +0200 Subject: DropPrivs in the solvers (just to be on the safe side) --- apt-pkg/contrib/fileutl.cc | 17 +++++++++++++++++ apt-pkg/contrib/fileutl.h | 3 +++ cmdline/apt-dump-solver.cc | 2 ++ cmdline/apt-internal-solver.cc | 3 +++ 4 files changed, 25 insertions(+) (limited to 'cmdline') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 1ba4674e5..da81edbcc 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -2050,3 +2051,19 @@ bool Rename(std::string From, std::string To) } return true; } + +bool DropPrivs() +{ + if (getuid() != 0) + return true; + + const std::string nobody = _config->Find("APT::User::Nobody", "nobody"); + struct passwd *pw = getpwnam(nobody.c_str()); + if (pw == NULL) + return _error->Warning("No user %s, can not drop rights", nobody.c_str()); + if (setgid(pw->pw_gid) != 0) + return _error->Errno("setgid", "Failed to setgid"); + if (setuid(pw->pw_uid) != 0) + return _error->Errno("setuid", "Failed to setuid"); + return true; +} diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index cc1a98eae..683c04157 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -191,6 +191,9 @@ pid_t ExecFork(std::set keep_fds); void MergeKeepFdsFromConfiguration(std::set &keep_fds); bool ExecWait(pid_t Pid,const char *Name,bool Reap = false); +// process releated +bool DropPrivs(); + // File string manipulators std::string flNotDir(std::string File); std::string flNotFile(std::string File); diff --git a/cmdline/apt-dump-solver.cc b/cmdline/apt-dump-solver.cc index 04e13bde9..424764b3c 100644 --- a/cmdline/apt-dump-solver.cc +++ b/cmdline/apt-dump-solver.cc @@ -40,6 +40,8 @@ int main(int argc,const char *argv[]) /*{{{*/ ShowHelp(); return 0; } + // we really don't need anything + DropPrivs(); FILE* input = fdopen(STDIN_FILENO, "r"); FILE* output = fopen("/tmp/dump.edsp", "w"); diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index e4cdf6381..0cac12da2 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -74,6 +74,9 @@ int main(int argc,const char *argv[]) /*{{{*/ {'o',"option",0,CommandLine::ArbItem}, {0,0,0,0}}; + // we really don't need anything + DropPrivs(); + CommandLine CmdL(Args,_config); if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false) { -- cgit v1.2.3-70-g09d2 From eceb418be84f6271c10e374281c1a8379647679b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 18 Jun 2014 08:51:59 +0200 Subject: improve formating of the hash stats --- cmdline/apt-cache.cc | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index a620740e1..072cf4ef7 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -374,7 +374,7 @@ static bool Stats(CommandLine &) Cache->Head().ProvidesCount*Cache->Head().ProvidesSz; cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; - // get the hash collisions average + // hashtable stats for the PkgHashTable long NumBuckets = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); long UsedBuckets = 0; long UnusedBuckets = 0; @@ -395,14 +395,14 @@ static bool Stats(CommandLine &) ShortestBucket = std::min(ThisBucketSize, ShortestBucket); UsedBuckets += ThisBucketSize; } - cout << _("Pkg Hashtable stats:") << endl; - cout << _("Number of buckets: ") << SizeToStr(NumBuckets) << endl; - cout << _("Unused buckets: ") << SizeToStr(UnusedBuckets) << endl; - cout << _("Used buckets: ") << UsedBuckets << endl; - cout << _("Average num entries per bucket : ") << UsedBuckets/(double)NumBuckets << endl; - cout << _("Longest / Shortest bucket: ") << LongestBucket << " / " << ShortestBucket << endl; - - // get the hash collisions average + cout << "Total PkgHashTable buckets: " << SizeToStr(NumBuckets) << std::endl; + cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; + cout << " Used: " << UsedBuckets << std::endl; + cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; + cout << " Longest: " << LongestBucket << std::endl; + cout << " Shortest: " << ShortestBucket << std::endl; + + // hashtable stats for the GrpHashTable NumBuckets = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); UsedBuckets = 0; UnusedBuckets = 0; @@ -423,12 +423,12 @@ static bool Stats(CommandLine &) ShortestBucket = std::min(ThisBucketSize, ShortestBucket); UsedBuckets += ThisBucketSize; } - cout << _("Grp Hashtable stats:") << endl; - cout << _("Number of buckets: ") << SizeToStr(NumBuckets) << endl; - cout << _("Unused buckets: ") << SizeToStr(UnusedBuckets) << endl; - cout << _("Used buckets: ") << UsedBuckets << endl; - cout << _("Average num entries per bucket : ") << UsedBuckets/(double)NumBuckets << endl; - cout << _("Longest / Shortest bucket: ") << LongestBucket << " / " << ShortestBucket << endl; + cout << "Total GrpHashTable buckets: " << SizeToStr(NumBuckets) << std::endl; + cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; + cout << " Used: " << UsedBuckets << std::endl; + cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; + cout << " Longest: " << LongestBucket << std::endl; + cout << " Shortest: " << ShortestBucket << std::endl; return true; } -- cgit v1.2.3-70-g09d2 From 637c3b232223c17827a8842cb1c24655469329ba Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 18 Jun 2014 09:36:48 +0200 Subject: Provide ShowHashTableStats function --- cmdline/apt-cache.cc | 98 +++++++++++++++++++++++----------------------------- 1 file changed, 43 insertions(+), 55 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 072cf4ef7..35e9cc3a8 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -264,6 +264,44 @@ static bool DumpPackage(CommandLine &CmdL) return true; } /*}}}*/ +// ShowHashTableStats - Show stats about a hashtable /*{{{*/ +// --------------------------------------------------------------------- +/* */ +template +static void ShowHashTableStats(std::string Type, + T *StartP, + map_ptrloc *Hashtable, + unsigned long Size) +{ + // hashtable stats for the HashTable + long NumBuckets = Size; + long UsedBuckets = 0; + long UnusedBuckets = 0; + long LongestBucket = 0; + long ShortestBucket = NumBuckets; + for (unsigned int i=0; i < NumBuckets; ++i) + { + T *P = StartP + Hashtable[i]; + if(P == 0 || P == StartP) + { + UnusedBuckets++; + continue; + } + long ThisBucketSize = 0; + for (; P != StartP; P = StartP + P->Next) + ThisBucketSize++; + LongestBucket = std::max(ThisBucketSize, LongestBucket); + ShortestBucket = std::min(ThisBucketSize, ShortestBucket); + UsedBuckets += ThisBucketSize; + } + cout << "Total buckets " << Type << ": " << SizeToStr(NumBuckets) << std::endl; + cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; + cout << " Used: " << UsedBuckets << std::endl; + cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; + cout << " Longest: " << LongestBucket << std::endl; + cout << " Shortest: " << ShortestBucket << std::endl; +} + /*}}}*/ // Stats - Dump some nice statistics /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -374,61 +412,11 @@ static bool Stats(CommandLine &) Cache->Head().ProvidesCount*Cache->Head().ProvidesSz; cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; - // hashtable stats for the PkgHashTable - long NumBuckets = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); - long UsedBuckets = 0; - long UnusedBuckets = 0; - long LongestBucket = 0; - long ShortestBucket = NumBuckets; - for (unsigned int i=0; i < NumBuckets; ++i) - { - pkgCache::Package *Pkg = Cache->PkgP + Cache->HeaderP->PkgHashTable[i]; - if(Pkg == 0 || Pkg == Cache->PkgP) - { - UnusedBuckets++; - continue; - } - long ThisBucketSize = 0; - for (; Pkg != Cache->PkgP; Pkg = Cache->PkgP + Pkg->NextPackage) - ThisBucketSize++; - LongestBucket = std::max(ThisBucketSize, LongestBucket); - ShortestBucket = std::min(ThisBucketSize, ShortestBucket); - UsedBuckets += ThisBucketSize; - } - cout << "Total PkgHashTable buckets: " << SizeToStr(NumBuckets) << std::endl; - cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; - cout << " Used: " << UsedBuckets << std::endl; - cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; - cout << " Longest: " << LongestBucket << std::endl; - cout << " Shortest: " << ShortestBucket << std::endl; - - // hashtable stats for the GrpHashTable - NumBuckets = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); - UsedBuckets = 0; - UnusedBuckets = 0; - LongestBucket = 0; - ShortestBucket = NumBuckets; - for (unsigned int i=0; i < NumBuckets; ++i) - { - pkgCache::Group *Grp = Cache->GrpP + Cache->HeaderP->GrpHashTable[i]; - if(Grp == 0 || Grp == Cache->GrpP) - { - UnusedBuckets++; - continue; - } - long ThisBucketSize = 0; - for (; Grp != Cache->GrpP; Grp = Cache->GrpP + Grp->Next) - ThisBucketSize++; - LongestBucket = std::max(ThisBucketSize, LongestBucket); - ShortestBucket = std::min(ThisBucketSize, ShortestBucket); - UsedBuckets += ThisBucketSize; - } - cout << "Total GrpHashTable buckets: " << SizeToStr(NumBuckets) << std::endl; - cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; - cout << " Used: " << UsedBuckets << std::endl; - cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; - cout << " Longest: " << LongestBucket << std::endl; - cout << " Shortest: " << ShortestBucket << std::endl; + // hashtable stats + int HashTableSize = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); + ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->HeaderP->PkgHashTable, HashTableSize); + HashTableSize = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); + ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->HeaderP->GrpHashTable, HashTableSize); return true; } -- cgit v1.2.3-70-g09d2 From d39d7f885d61bfe296c131c83bdc042a2ab6b0d7 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 17 Jun 2014 17:45:33 +0200 Subject: show our broken packages message in 'apt' solver --- apt-private/private-output.cc | 204 ++++++++++++++++++++++------------------- apt-private/private-output.h | 3 +- cmdline/apt-internal-solver.cc | 24 +++-- cmdline/makefile | 20 ++-- 4 files changed, 134 insertions(+), 117 deletions(-) (limited to 'cmdline') diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc index 8f190a551..158bd5c71 100644 --- a/apt-private/private-output.cc +++ b/apt-private/private-output.cc @@ -344,129 +344,141 @@ bool ShowList(ostream &out,string Title,string List,string VersionsList) Depends: libldap2 (>= 2.0.2-2) but it is not going to be installed Depends: libsasl7 but it is not going to be installed */ -void ShowBroken(ostream &out,CacheFile &Cache,bool Now) +static void ShowBrokenPackage(ostream &out, pkgCacheFile * const Cache, pkgCache::PkgIterator const &Pkg, bool const Now) { - if (Cache->BrokenCount() == 0) + if (Now == true) + { + if ((*Cache)[Pkg].NowBroken() == false) + return; + } + else + { + if ((*Cache)[Pkg].InstBroken() == false) + return; + } + + // Print out each package and the failed dependencies + out << " " << Pkg.FullName(true) << " :"; + unsigned const Indent = Pkg.FullName(true).size() + 3; + bool First = true; + pkgCache::VerIterator Ver; + + if (Now == true) + Ver = Pkg.CurrentVer(); + else + Ver = (*Cache)[Pkg].InstVerIter(*Cache); + + if (Ver.end() == true) + { + out << endl; return; + } - out << _("The following packages have unmet dependencies:") << endl; - for (unsigned J = 0; J < Cache->Head().PackageCount; J++) + for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false;) { - pkgCache::PkgIterator I(Cache,Cache.List[J]); - + // Compute a single dependency element (glob or) + pkgCache::DepIterator Start; + pkgCache::DepIterator End; + D.GlobOr(Start,End); // advances D + + if ((*Cache)->IsImportantDep(End) == false) + continue; + if (Now == true) { - if (Cache[I].NowBroken() == false) + if (((*Cache)[End] & pkgDepCache::DepGNow) == pkgDepCache::DepGNow) continue; } else { - if (Cache[I].InstBroken() == false) + if (((*Cache)[End] & pkgDepCache::DepGInstall) == pkgDepCache::DepGInstall) continue; } - - // Print out each package and the failed dependencies - out << " " << I.FullName(true) << " :"; - unsigned const Indent = I.FullName(true).size() + 3; - bool First = true; - pkgCache::VerIterator Ver; - - if (Now == true) - Ver = I.CurrentVer(); - else - Ver = Cache[I].InstVerIter(Cache); - - if (Ver.end() == true) - { - out << endl; - continue; - } - - for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false;) + + bool FirstOr = true; + while (1) { - // Compute a single dependency element (glob or) - pkgCache::DepIterator Start; - pkgCache::DepIterator End; - D.GlobOr(Start,End); // advances D + if (First == false) + for (unsigned J = 0; J != Indent; J++) + out << ' '; + First = false; - if (Cache->IsImportantDep(End) == false) - continue; - - if (Now == true) + if (FirstOr == false) { - if ((Cache[End] & pkgDepCache::DepGNow) == pkgDepCache::DepGNow) - continue; + for (unsigned J = 0; J != strlen(End.DepType()) + 3; J++) + out << ' '; } else + out << ' ' << End.DepType() << ": "; + FirstOr = false; + + out << Start.TargetPkg().FullName(true); + + // Show a quick summary of the version requirements + if (Start.TargetVer() != 0) + out << " (" << Start.CompType() << " " << Start.TargetVer() << ")"; + + /* Show a summary of the target package if possible. In the case + of virtual packages we show nothing */ + pkgCache::PkgIterator Targ = Start.TargetPkg(); + if (Targ->ProvidesList == 0) { - if ((Cache[End] & pkgDepCache::DepGInstall) == pkgDepCache::DepGInstall) - continue; - } - - bool FirstOr = true; - while (1) - { - if (First == false) - for (unsigned J = 0; J != Indent; J++) - out << ' '; - First = false; + out << ' '; + pkgCache::VerIterator Ver = (*Cache)[Targ].InstVerIter(*Cache); + if (Now == true) + Ver = Targ.CurrentVer(); - if (FirstOr == false) + if (Ver.end() == false) { - for (unsigned J = 0; J != strlen(End.DepType()) + 3; J++) - out << ' '; + if (Now == true) + ioprintf(out,_("but %s is installed"),Ver.VerStr()); + else + ioprintf(out,_("but %s is to be installed"),Ver.VerStr()); } else - out << ' ' << End.DepType() << ": "; - FirstOr = false; - - out << Start.TargetPkg().FullName(true); - - // Show a quick summary of the version requirements - if (Start.TargetVer() != 0) - out << " (" << Start.CompType() << " " << Start.TargetVer() << ")"; - - /* Show a summary of the target package if possible. In the case - of virtual packages we show nothing */ - pkgCache::PkgIterator Targ = Start.TargetPkg(); - if (Targ->ProvidesList == 0) { - out << ' '; - pkgCache::VerIterator Ver = Cache[Targ].InstVerIter(Cache); - if (Now == true) - Ver = Targ.CurrentVer(); - - if (Ver.end() == false) + if ((*Cache)[Targ].CandidateVerIter(*Cache).end() == true) { - if (Now == true) - ioprintf(out,_("but %s is installed"),Ver.VerStr()); + if (Targ->ProvidesList == 0) + out << _("but it is not installable"); else - ioprintf(out,_("but %s is to be installed"),Ver.VerStr()); - } + out << _("but it is a virtual package"); + } else - { - if (Cache[Targ].CandidateVerIter(Cache).end() == true) - { - if (Targ->ProvidesList == 0) - out << _("but it is not installable"); - else - out << _("but it is a virtual package"); - } - else - out << (Now?_("but it is not installed"):_("but it is not going to be installed")); - } + out << (Now?_("but it is not installed"):_("but it is not going to be installed")); } - - if (Start != End) - out << _(" or"); - out << endl; - - if (Start == End) - break; - ++Start; - } - } - } + } + + if (Start != End) + out << _(" or"); + out << endl; + + if (Start == End) + break; + ++Start; + } + } +} +void ShowBroken(ostream &out, CacheFile &Cache, bool const Now) +{ + if (Cache->BrokenCount() == 0) + return; + + out << _("The following packages have unmet dependencies:") << endl; + for (unsigned J = 0; J < Cache->Head().PackageCount; J++) + { + pkgCache::PkgIterator const I(Cache,Cache.List[J]); + ShowBrokenPackage(out, &Cache, I, Now); + } +} +void ShowBroken(ostream &out, pkgCacheFile &Cache, bool const Now) +{ + if (Cache->BrokenCount() == 0) + return; + + out << _("The following packages have unmet dependencies:") << endl; + for (pkgCache::PkgIterator Pkg = Cache->PkgBegin(); Pkg.end() == false; ++Pkg) + ShowBrokenPackage(out, &Cache, Pkg, Now); } /*}}}*/ // ShowNew - Show packages to newly install /*{{{*/ diff --git a/apt-private/private-output.h b/apt-private/private-output.h index 9633d0c37..6f3a964d7 100644 --- a/apt-private/private-output.h +++ b/apt-private/private-output.h @@ -28,7 +28,8 @@ void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records, // helper to describe global state -APT_PUBLIC void ShowBroken(std::ostream &out,CacheFile &Cache,bool Now); +APT_PUBLIC void ShowBroken(std::ostream &out, CacheFile &Cache, bool const Now); +APT_PUBLIC void ShowBroken(std::ostream &out, pkgCacheFile &Cache, bool const Now); APT_PUBLIC bool ShowList(std::ostream &out, std::string Title, std::string List, std::string VersionsList); diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index e4cdf6381..5fda7b6a0 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -24,9 +24,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -168,18 +170,20 @@ int main(int argc,const char *argv[]) /*{{{*/ EDSP::WriteProgress(60, "Call problemresolver on current scenario…", output); + std::string failure; if (upgrade == true) { - if (pkgAllUpgrade(CacheFile) == false) { - EDSP::WriteError("ERR_UNSOLVABLE_UPGRADE", "An upgrade error occurred", output); - return 0; - } + if (pkgAllUpgrade(CacheFile) == false) + failure = "ERR_UNSOLVABLE_UPGRADE"; } else if (distUpgrade == true) { - if (pkgDistUpgrade(CacheFile) == false) { - EDSP::WriteError("ERR_UNSOLVABLE_DIST_UPGRADE", "An dist-upgrade error occurred", output); - return 0; - } - } else if (Fix.Resolve() == false) { - EDSP::WriteError("ERR_UNSOLVABLE", "An error occurred", output); + if (pkgDistUpgrade(CacheFile) == false) + failure = "ERR_UNSOLVABLE_DIST_UPGRADE"; + } else if (Fix.Resolve() == false) + failure = "ERR_UNSOLVABLE"; + + if (failure.empty() == false) { + std::ostringstream broken; + ShowBroken(broken, CacheFile, false); + EDSP::WriteError(failure.c_str(), broken.str(), output); return 0; } diff --git a/cmdline/makefile b/cmdline/makefile index c4a249cd6..b7c35ddd1 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -8,49 +8,49 @@ include ../buildlib/defaults.mak # The apt program PROGRAM=apt SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt.cc include $(PROGRAM_H) # The apt-cache program PROGRAM=apt-cache SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-cache.cc include $(PROGRAM_H) # The apt-get program PROGRAM=apt-get SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-get.cc include $(PROGRAM_H) # The apt-config program PROGRAM=apt-config SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-config.cc include $(PROGRAM_H) # The apt-cdrom program PROGRAM=apt-cdrom SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-cdrom.cc include $(PROGRAM_H) # The apt-mark program PROGRAM=apt-mark SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-mark.cc include $(PROGRAM_H) # The apt-helper PROGRAM=apt-helper SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-helper.cc include $(PROGRAM_H) @@ -75,14 +75,14 @@ include $(PROGRAM_H) # The apt-extracttemplates program PROGRAM=apt-extracttemplates SLIBS = -lapt-pkg -lapt-inst $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +LIB_MAKES = apt-pkg/makefile apt-inst/makefile SOURCE = apt-extracttemplates.cc include $(PROGRAM_H) # The internal solver acting as an external PROGRAM=apt-internal-solver -SLIBS = -lapt-pkg $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-internal-solver.cc include $(PROGRAM_H) -- cgit v1.2.3-70-g09d2 From 4ad8619bb1f0bf777d17c568bb7a6cf7f30aac34 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 12 Jun 2014 12:22:45 +0200 Subject: cleanup datatypes mix used in binary cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We had a wild mixture of (unsigned) int, long and long long here without much sense, so this commit adds a few typedefs to get some sense in the typesystem and ensures that a ID isn't sometimes computed as int, stored as long and compared with a long long… as this could potentially bite us later on as the size of the archive only increases over time. --- apt-pkg/deb/debindexfile.cc | 2 +- apt-pkg/deb/deblistparser.cc | 6 +- apt-pkg/deb/deblistparser.h | 8 +- apt-pkg/edsp/edspindexfile.cc | 2 +- apt-pkg/pkgcache.cc | 4 +- apt-pkg/pkgcache.h | 190 ++++++++++++++++++++++-------------------- apt-pkg/pkgcachegen.cc | 164 ++++++++++++++++++------------------ apt-pkg/pkgcachegen.h | 44 +++++----- cmdline/apt-cache.cc | 2 +- 9 files changed, 217 insertions(+), 205 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 3bdc551b4..c1c2b726a 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -619,7 +619,7 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); CFile->Size = Pkg.FileSize(); CFile->mtime = Pkg.ModificationTime(); - map_ptrloc const storage = Gen.WriteUniqString("now"); + map_stringitem_t const storage = Gen.WriteUniqString("now"); CFile->Archive = storage; if (Gen.MergeList(Parser) == false) diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 40d332196..4447b54dd 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -61,7 +61,7 @@ debListParser::debListParser(FileFd *File, string const &Arch) : Tags(File), // ListParser::UniqFindTagWrite - Find the tag and write a unq string /*{{{*/ // --------------------------------------------------------------------- /* */ -unsigned long debListParser::UniqFindTagWrite(const char *Tag) +map_stringitem_t debListParser::UniqFindTagWrite(const char *Tag) { const char *Start; const char *Stop; @@ -893,7 +893,7 @@ bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, { // apt-secure does no longer download individual (per-section) Release // file. to provide Component pinning we use the section name now - map_ptrloc const storage = WriteUniqString(component); + map_stringitem_t const storage = WriteUniqString(component); FileI->Component = storage; pkgTagFile TagFile(&File, File.Size()); @@ -906,7 +906,7 @@ bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, data = Section.FindS(TAG); \ if (data.empty() == false) \ { \ - map_ptrloc const storage = WriteUniqString(data); \ + map_stringitem_t const storage = WriteUniqString(data); \ STORE = storage; \ } APT_INRELEASE("Suite", FileI->Archive) diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h index 56a83b36e..f5ac47e1e 100644 --- a/apt-pkg/deb/deblistparser.h +++ b/apt-pkg/deb/deblistparser.h @@ -44,12 +44,12 @@ class debListParser : public pkgCacheGenerator::ListParser protected: pkgTagFile Tags; pkgTagSection Section; - unsigned long iOffset; + map_filesize_t iOffset; std::string Arch; std::vector Architectures; bool MultiArchEnabled; - unsigned long UniqFindTagWrite(const char *Tag); + map_stringitem_t UniqFindTagWrite(const char *Tag); virtual bool ParseStatus(pkgCache::PkgIterator &Pkg,pkgCache::VerIterator &Ver); bool ParseDepends(pkgCache::VerIterator &Ver,const char *Tag, unsigned int Type); @@ -77,8 +77,8 @@ class debListParser : public pkgCacheGenerator::ListParser #endif virtual bool UsePackage(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver); - virtual unsigned long Offset() {return iOffset;}; - virtual unsigned long Size() {return Section.size();}; + virtual map_filesize_t Offset() {return iOffset;}; + virtual map_filesize_t Size() {return Section.size();}; virtual bool Step(); diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc index 10313fd61..e013dd44c 100644 --- a/apt-pkg/edsp/edspindexfile.cc +++ b/apt-pkg/edsp/edspindexfile.cc @@ -56,7 +56,7 @@ bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); CFile->Size = Pkg.FileSize(); CFile->mtime = Pkg.ModificationTime(); - map_ptrloc const storage = Gen.WriteUniqString("edsp::scenario"); + map_stringitem_t const storage = Gen.WriteUniqString("edsp::scenario"); CFile->Archive = storage; if (Gen.MergeList(Parser) == false) diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 7b092f07e..8326741ed 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -188,7 +188,7 @@ bool pkgCache::ReMap(bool const &Errorchecks) /* This is used to generate the hash entries for the HashTable. With my package list from bo this function gets 94% table usage on a 512 item table (480 used items) */ -unsigned long pkgCache::sHash(const string &Str) const +map_id_t pkgCache::sHash(const string &Str) const { unsigned long Hash = 0; for (string::const_iterator I = Str.begin(); I != Str.end(); ++I) @@ -196,7 +196,7 @@ unsigned long pkgCache::sHash(const string &Str) const return Hash % HeaderP->HashTableSize; } -unsigned long pkgCache::sHash(const char *Str) const +map_id_t pkgCache::sHash(const char *Str) const { unsigned long Hash = tolower_ascii(*Str); for (const char *I = Str + 1; *I != 0; ++I) diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index b6b2894cf..f57c31b98 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -79,11 +79,23 @@ #include #include +#include #ifndef APT_8_CLEANER_HEADERS using std::string; #endif +// storing file sizes of indexes, which are way below 4 GB for now +typedef uint32_t map_filesize_t; +// each package/group/dependency gets an id +typedef uint32_t map_id_t; +// some files get an id, too, but in far less absolute numbers +typedef uint16_t map_fileid_t; +// relative pointer from cache start +typedef uint32_t map_pointer_t; +// same as the previous, but documented to be to a string item +typedef map_pointer_t map_stringitem_t; + class pkgVersioningSystem; class pkgCache /*{{{*/ { @@ -158,8 +170,8 @@ class pkgCache /*{{{*/ std::string CacheFile; MMap ⤅ - unsigned long sHash(const std::string &S) const APT_PURE; - unsigned long sHash(const char *S) const APT_PURE; + map_id_t sHash(const std::string &S) const APT_PURE; + map_id_t sHash(const char *S) const APT_PURE; public: @@ -183,8 +195,8 @@ class pkgCache /*{{{*/ inline void *DataEnd() {return ((unsigned char *)Map.Data()) + Map.Size();} // String hashing function (512 range) - inline unsigned long Hash(const std::string &S) const {return sHash(S);} - inline unsigned long Hash(const char *S) const {return sHash(S);} + inline map_id_t Hash(const std::string &S) const {return sHash(S);} + inline map_id_t Hash(const char *S) const {return sHash(S);} // Useful transformation things const char *Priority(unsigned char Priority); @@ -263,37 +275,37 @@ struct pkgCache::Header These indicate the number of each structure contained in the cache. PackageCount is especially useful for generating user state structures. See Package::Id for more info. */ - unsigned long GroupCount; - unsigned long PackageCount; - unsigned long VersionCount; - unsigned long DescriptionCount; - unsigned long DependsCount; - unsigned long PackageFileCount; - unsigned long VerFileCount; - unsigned long DescFileCount; - unsigned long ProvidesCount; + map_id_t GroupCount; + map_id_t PackageCount; + map_id_t VersionCount; + map_id_t DescriptionCount; + map_id_t DependsCount; + map_fileid_t PackageFileCount; + map_fileid_t VerFileCount; + map_fileid_t DescFileCount; + map_id_t ProvidesCount; /** \brief index of the first PackageFile structure The PackageFile structures are singly linked lists that represent all package files that have been merged into the cache. */ - map_ptrloc FileList; + map_pointer_t FileList; /** \brief index of the first StringItem structure The cache contains a list of all the unique strings (StringItems). The parser reads this list into memory so it can match strings against it.*/ - map_ptrloc StringList; + map_pointer_t StringList; /** \brief String representing the version system used */ - map_ptrloc VerSysName; + map_pointer_t VerSysName; /** \brief native architecture the cache was built against */ - map_ptrloc Architecture; + map_pointer_t Architecture; /** \brief all architectures the cache was built against */ - map_ptrloc Architectures; + map_pointer_t Architectures; /** \brief The maximum size of a raw entry from the original Package file */ - unsigned long MaxVerFileSize; + map_filesize_t MaxVerFileSize; /** \brief The maximum size of a raw entry from the original Translation file */ - unsigned long MaxDescFileSize; + map_filesize_t MaxDescFileSize; /** \brief The Pool structures manage the allocation pools that the generator uses @@ -316,11 +328,11 @@ struct pkgCache::Header these packages are stored as a sequence in the list. The size of both tables is the same. */ unsigned int HashTableSize; - map_ptrloc * PkgHashTable() const { return (map_ptrloc*) (this + 1); } - map_ptrloc * GrpHashTable() const { return PkgHashTable() + HashTableSize; } + map_pointer_t * PkgHashTable() const { return (map_pointer_t*) (this + 1); } + map_pointer_t * GrpHashTable() const { return PkgHashTable() + HashTableSize; } /** \brief Size of the complete cache file */ - unsigned long CacheFileSize; + unsigned long long CacheFileSize; bool CheckSizes(Header &Against) const APT_PURE; Header(); @@ -336,17 +348,17 @@ struct pkgCache::Header struct pkgCache::Group { /** \brief Name of the group */ - map_ptrloc Name; // StringItem + map_stringitem_t Name; // Linked List /** \brief Link to the first package which belongs to the group */ - map_ptrloc FirstPackage; // Package + map_pointer_t FirstPackage; // Package /** \brief Link to the last package which belongs to the group */ - map_ptrloc LastPackage; // Package + map_pointer_t LastPackage; // Package /** \brief Link to the next Group */ - map_ptrloc Next; // Group + map_pointer_t Next; // Group /** \brief unique sequel ID */ - unsigned int ID; + map_id_t ID; }; /*}}}*/ @@ -365,9 +377,9 @@ struct pkgCache::Group struct pkgCache::Package { /** \brief Name of the package */ - map_ptrloc Name; // StringItem + map_stringitem_t Name; /** \brief Architecture of the package */ - map_ptrloc Arch; // StringItem + map_stringitem_t Arch; /** \brief Base of a singly linked list of versions Each structure represents a unique version of the package. @@ -377,24 +389,24 @@ struct pkgCache::Package versions of a package can be cleanly handled by the system. Furthermore, this linked list is guaranteed to be sorted from Highest version to lowest version with no duplicate entries. */ - map_ptrloc VersionList; // Version + map_pointer_t VersionList; // Version /** \brief index to the installed version */ - map_ptrloc CurrentVer; // Version + map_pointer_t CurrentVer; // Version /** \brief indicates the deduced section Should be the index to the string "Unknown" or to the section of the last parsed item. */ - map_ptrloc Section; // StringItem + map_stringitem_t Section; /** \brief index of the group this package belongs to */ - map_ptrloc Group; // Group the Package belongs to + map_pointer_t Group; // Group the Package belongs to // Linked list /** \brief Link to the next package in the same bucket */ - map_ptrloc Next; // Package + map_pointer_t Next; // Package /** \brief List of all dependencies on this package */ - map_ptrloc RevDepends; // Dependency + map_pointer_t RevDepends; // Dependency /** \brief List of all "packages" this package provide */ - map_ptrloc ProvidesList; // Provides + map_pointer_t ProvidesList; // Provides // Install/Remove/Purge etc /** \brief state that the user wishes the package to be in */ @@ -414,7 +426,7 @@ struct pkgCache::Package This allows clients to create an array of size PackageCount and use it to store state information for the package map. For instance the status file emitter uses this to track which packages have been emitted already. */ - unsigned int ID; + map_id_t ID; /** \brief some useful indicators of the package's state */ unsigned long Flags; }; @@ -428,30 +440,30 @@ struct pkgCache::Package struct pkgCache::PackageFile { /** \brief physical disk file that this PackageFile represents */ - map_ptrloc FileName; // StringItem + map_pointer_t FileName; // StringItem /** \brief the release information Please see the files document for a description of what the release information means. */ - map_ptrloc Archive; // StringItem - map_ptrloc Codename; // StringItem - map_ptrloc Component; // StringItem - map_ptrloc Version; // StringItem - map_ptrloc Origin; // StringItem - map_ptrloc Label; // StringItem - map_ptrloc Architecture; // StringItem + map_stringitem_t Archive; + map_stringitem_t Codename; + map_stringitem_t Component; + map_stringitem_t Version; + map_stringitem_t Origin; + map_stringitem_t Label; + map_stringitem_t Architecture; /** \brief The site the index file was fetched from */ - map_ptrloc Site; // StringItem + map_stringitem_t Site; /** \brief indicates what sort of index file this is @TODO enumerate at least the possible indexes */ - map_ptrloc IndexType; // StringItem + map_stringitem_t IndexType; /** \brief Size of the file Used together with the modification time as a simple check to ensure that the Packages file has not been altered since Cache generation. */ - unsigned long Size; + map_filesize_t Size; /** \brief Modification time for the file */ time_t mtime; @@ -460,9 +472,9 @@ struct pkgCache::PackageFile // Linked list /** \brief Link to the next PackageFile in the Cache */ - map_ptrloc NextFile; // PackageFile + map_pointer_t NextFile; // PackageFile /** \brief unique sequel ID */ - unsigned int ID; + map_fileid_t ID; }; /*}}}*/ // VerFile structure /*{{{*/ @@ -473,13 +485,13 @@ struct pkgCache::PackageFile struct pkgCache::VerFile { /** \brief index of the package file that this version was found in */ - map_ptrloc File; // PackageFile + map_pointer_t File; // PackageFile /** \brief next step in the linked list */ - map_ptrloc NextFile; // PkgVerFile + map_pointer_t NextFile; // PkgVerFile /** \brief position in the package file */ - map_ptrloc Offset; // File offset + map_filesize_t Offset; // File offset /** @TODO document pkgCache::VerFile::Size */ - unsigned long Size; + map_filesize_t Size; }; /*}}}*/ // DescFile structure /*{{{*/ @@ -487,13 +499,13 @@ struct pkgCache::VerFile struct pkgCache::DescFile { /** \brief index of the file that this description was found in */ - map_ptrloc File; // PackageFile + map_pointer_t File; // PackageFile /** \brief next step in the linked list */ - map_ptrloc NextFile; // PkgVerFile + map_pointer_t NextFile; // PkgVerFile /** \brief position in the file */ - map_ptrloc Offset; // File offset + map_filesize_t Offset; // File offset /** @TODO document pkgCache::DescFile::Size */ - unsigned long Size; + map_filesize_t Size; }; /*}}}*/ // Version structure /*{{{*/ @@ -505,9 +517,9 @@ struct pkgCache::DescFile struct pkgCache::Version { /** \brief complete version string */ - map_ptrloc VerStr; // StringItem + map_stringitem_t VerStr; /** \brief section this version is filled in */ - map_ptrloc Section; // StringItem + map_stringitem_t Section; /** \brief Multi-Arch capabilities of a package version */ enum VerMultiArch { None = 0, /*!< is the default and doesn't trigger special behaviour */ @@ -529,33 +541,33 @@ struct pkgCache::Version applies to. If FileList is 0 then this is a blank version. The structure should also have a 0 in all other fields excluding pkgCache::Version::VerStr and Possibly pkgCache::Version::NextVer. */ - map_ptrloc FileList; // VerFile + map_pointer_t FileList; // VerFile /** \brief next (lower or equal) version in the linked list */ - map_ptrloc NextVer; // Version + map_pointer_t NextVer; // Version /** \brief next description in the linked list */ - map_ptrloc DescriptionList; // Description + map_pointer_t DescriptionList; // Description /** \brief base of the dependency list */ - map_ptrloc DependsList; // Dependency + map_pointer_t DependsList; // Dependency /** \brief links to the owning package This allows reverse dependencies to determine the package */ - map_ptrloc ParentPkg; // Package + map_pointer_t ParentPkg; // Package /** \brief list of pkgCache::Provides */ - map_ptrloc ProvidesList; // Provides + map_pointer_t ProvidesList; // Provides /** \brief archive size for this version For Debian this is the size of the .deb file. */ - unsigned long long Size; // These are the .deb size + uint64_t Size; // These are the .deb size /** \brief uncompressed size for this version */ - unsigned long long InstalledSize; + uint64_t InstalledSize; /** \brief characteristic value representing this version No two packages in existence should have the same VerStr and Hash with different contents. */ unsigned short Hash; /** \brief unique sequel ID */ - unsigned int ID; + map_id_t ID; /** \brief parsed priority value */ unsigned char Priority; }; @@ -568,22 +580,22 @@ struct pkgCache::Description If the value has a 0 length then this is read using the Package file else the Translation-CODE file is used. */ - map_ptrloc language_code; // StringItem + map_stringitem_t language_code; /** \brief MD5sum of the original description Used to map Translations of a description to a version and to check that the Translation is up-to-date. */ - map_ptrloc md5sum; // StringItem + map_stringitem_t md5sum; /** @TODO document pkgCache::Description::FileList */ - map_ptrloc FileList; // DescFile + map_pointer_t FileList; // DescFile /** \brief next translation for this description */ - map_ptrloc NextDesc; // Description + map_pointer_t NextDesc; // Description /** \brief the text is a description of this package */ - map_ptrloc ParentPkg; // Package + map_pointer_t ParentPkg; // Package /** \brief unique sequel ID */ - unsigned int ID; + map_id_t ID; }; /*}}}*/ // Dependency structure /*{{{*/ @@ -596,21 +608,21 @@ struct pkgCache::Description struct pkgCache::Dependency { /** \brief string of the version the dependency is applied against */ - map_ptrloc Version; // StringItem + map_stringitem_t Version; // StringItem /** \brief index of the package this depends applies to The generator will - if the package does not already exist - create a blank (no version records) package. */ - map_ptrloc Package; // Package + map_pointer_t Package; // Package /** \brief next dependency of this version */ - map_ptrloc NextDepends; // Dependency + map_pointer_t NextDepends; // Dependency /** \brief next reverse dependency of this package */ - map_ptrloc NextRevDepends; // Dependency + map_pointer_t NextRevDepends; // Dependency /** \brief version of the package which has the reverse depends */ - map_ptrloc ParentVer; // Version + map_pointer_t ParentVer; // Version /** \brief unique sequel ID */ - map_ptrloc ID; + map_id_t ID; /** \brief Dependency type - Depends, Recommends, Conflicts, etc */ unsigned char Type; /** \brief comparison operator specified on the depends line @@ -631,19 +643,19 @@ struct pkgCache::Dependency struct pkgCache::Provides { /** \brief index of the package providing this */ - map_ptrloc ParentPkg; // Package + map_pointer_t ParentPkg; // Package /** \brief index of the version this provide line applies to */ - map_ptrloc Version; // Version + map_pointer_t Version; // Version /** \brief version in the provides line (if any) This version allows dependencies to depend on specific versions of a Provides, as well as allowing Provides to override existing packages. This is experimental. Note that Debian doesn't allow versioned provides */ - map_ptrloc ProvideVersion; // StringItem + map_stringitem_t ProvideVersion; /** \brief next provides (based of package) */ - map_ptrloc NextProvides; // Provides + map_pointer_t NextProvides; // Provides /** \brief next provides (based of version) */ - map_ptrloc NextPkgProv; // Provides + map_pointer_t NextPkgProv; // Provides }; /*}}}*/ // StringItem structure /*{{{*/ @@ -658,9 +670,9 @@ struct pkgCache::Provides struct pkgCache::StringItem { /** \brief string this refers to */ - map_ptrloc String; // StringItem + map_stringitem_t String; /** \brief Next link in the chain */ - map_ptrloc NextItem; // StringItem + map_stringitem_t NextItem; }; /*}}}*/ diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 360f19736..6a3cb2637 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -75,16 +75,16 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : *Cache.HeaderP = pkgCache::Header(); // make room for the hashtables for packages and groups - if (Map.RawAllocate(2 * (Cache.HeaderP->HashTableSize * sizeof(map_ptrloc))) == 0) + if (Map.RawAllocate(2 * (Cache.HeaderP->HashTableSize * sizeof(map_pointer_t))) == 0) return; - map_ptrloc const idxVerSysName = WriteStringInMap(_system->VS->Label); + map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label); if (unlikely(idxVerSysName == 0)) return; Cache.HeaderP->VerSysName = idxVerSysName; // this pointer is set in ReMap, but we need it now for WriteUniqString Cache.StringItemP = (pkgCache::StringItem *)Map.Data(); - map_ptrloc const idxArchitecture = WriteUniqString(_config->Find("APT::Architecture")); + map_stringitem_t const idxArchitecture = WriteUniqString(_config->Find("APT::Architecture")); if (unlikely(idxArchitecture == 0)) return; Cache.HeaderP->Architecture = idxArchitecture; @@ -96,7 +96,7 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : std::string list = *a; for (++a; a != archs.end(); ++a) list.append(",").append(*a); - map_ptrloc const idxArchitectures = WriteStringInMap(list); + map_stringitem_t const idxArchitectures = WriteStringInMap(list); if (unlikely(idxArchitectures == 0)) return; Cache.HeaderP->Architectures = idxArchitectures; @@ -176,27 +176,27 @@ void pkgCacheGenerator::ReMap(void const * const oldMap, void const * const newM (*i)->ReMap(oldMap, newMap); } /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ -map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String, +map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String, const unsigned long &Len) { void const * const oldMap = Map.Data(); - map_ptrloc const index = Map.WriteString(String, Len); + map_stringitem_t const index = Map.WriteString(String, Len); if (index != 0) ReMap(oldMap, Map.Data()); return index; } /*}}}*/ // CacheGenerator::WriteStringInMap /*{{{*/ -map_ptrloc pkgCacheGenerator::WriteStringInMap(const char *String) { +map_stringitem_t pkgCacheGenerator::WriteStringInMap(const char *String) { void const * const oldMap = Map.Data(); - map_ptrloc const index = Map.WriteString(String); + map_stringitem_t const index = Map.WriteString(String); if (index != 0) ReMap(oldMap, Map.Data()); return index; } /*}}}*/ -map_ptrloc pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ +map_pointer_t pkgCacheGenerator::AllocateInMap(const unsigned long &size) {/*{{{*/ void const * const oldMap = Map.Data(); - map_ptrloc const index = Map.Allocate(size); + map_pointer_t const index = Map.Allocate(size); if (index != 0) ReMap(oldMap, Map.Data()); return index; @@ -276,16 +276,16 @@ bool pkgCacheGenerator::MergeList(ListParser &List, } } - if (Cache.HeaderP->PackageCount >= (1ULL<ID)*8)-1) + if (Cache.HeaderP->PackageCount >= std::numeric_limits::max()) return _error->Error(_("Wow, you exceeded the number of package " "names this APT is capable of.")); - if (Cache.HeaderP->VersionCount >= (1ULL<<(sizeof(Cache.VerP->ID)*8))-1) + if (Cache.HeaderP->VersionCount >= std::numeric_limits::max()) return _error->Error(_("Wow, you exceeded the number of versions " "this APT is capable of.")); - if (Cache.HeaderP->DescriptionCount >= (1ULL<<(sizeof(Cache.DescP->ID)*8))-1) + if (Cache.HeaderP->DescriptionCount >= std::numeric_limits::max()) return _error->Error(_("Wow, you exceeded the number of descriptions " "this APT is capable of.")); - if (Cache.HeaderP->DependsCount >= (1ULL<<(sizeof(Cache.DepP->ID)*8))-1ULL) + if (Cache.HeaderP->DependsCount >= std::numeric_limits::max()) return _error->Error(_("Wow, you exceeded the number of dependencies " "this APT is capable of.")); @@ -336,7 +336,7 @@ bool pkgCacheGenerator::MergeListPackage(ListParser &List, pkgCache::PkgIterator if (VerDesc.end() == true || MD5SumValue(VerDesc.md5()) != CurMd5) continue; - map_ptrloc md5idx = VerDesc->md5sum; + map_stringitem_t md5idx = VerDesc->md5sum; for (std::vector::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang) { // don't add a new description if we have one for the given @@ -360,7 +360,7 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator { pkgCache::VerIterator Ver = Pkg.VersionList(); Dynamic DynVer(Ver); - map_ptrloc *LastVer = &Pkg->VersionList; + map_pointer_t *LastVer = &Pkg->VersionList; void const * oldMap = Map.Data(); unsigned short const Hash = List.VersionHash(); @@ -405,13 +405,13 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator } // Add a new version - map_ptrloc const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer); + map_pointer_t const verindex = NewVersion(Ver, Version, Pkg.Index(), Hash, *LastVer); if (verindex == 0 && _error->PendingError()) return _error->Error(_("Error occurred while processing %s (%s%d)"), Pkg.Name(), "NewVersion", 1); if (oldMap != Map.Data()) - LastVer += (map_ptrloc const * const) Map.Data() - (map_ptrloc const * const) oldMap; + LastVer += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap; *LastVer = verindex; if (unlikely(List.NewVersion(Ver) == false)) @@ -472,7 +472,7 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator D.ParentPkg().Group() == Grp) continue; - map_ptrloc *OldDepLast = NULL; + map_pointer_t *OldDepLast = NULL; pkgCache::VerIterator ConVersion = D.ParentVer(); Dynamic DynV(ConVersion); // duplicate the Conflicts/Breaks/Replaces for :none arch @@ -514,7 +514,7 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator } // We haven't found reusable descriptions, so add the first description(s) - map_ptrloc md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum; + map_stringitem_t md5idx = Ver->DescriptionList == 0 ? 0 : Ver.DescriptionList()->md5sum; std::vector availDesc = List.AvailableDescriptionLanguages(); for (std::vector::const_iterator CurLang = availDesc.begin(); CurLang != availDesc.end(); ++CurLang) if (AddNewDescription(List, Ver, *CurLang, CurMd5, md5idx) == false) @@ -522,12 +522,12 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator return true; } /*}}}*/ -bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_ptrloc &md5idx) /*{{{*/ +bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx) /*{{{*/ { pkgCache::DescIterator Desc; Dynamic DynDesc(Desc); - map_ptrloc const descindex = NewDescription(Desc, lang, CurMd5, md5idx); + map_pointer_t const descindex = NewDescription(Desc, lang, CurMd5, md5idx); if (unlikely(descindex == 0 && _error->PendingError())) return _error->Error(_("Error occurred while processing %s (%s%d)"), Ver.ParentPkg().Name(), "NewDescription", 1); @@ -539,7 +539,7 @@ bool pkgCacheGenerator::AddNewDescription(ListParser &List, pkgCache::VerIterato // that to be able to efficiently share these lists pkgCache::DescIterator VerDesc = Ver.DescriptionList(); // old value might be invalid after ReMap for (;VerDesc.end() == false && VerDesc->NextDesc != 0; ++VerDesc); - map_ptrloc * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc; + map_pointer_t * const LastNextDesc = (VerDesc.end() == true) ? &Ver->DescriptionList : &VerDesc->NextDesc; *LastNextDesc = descindex; if (NewFileDesc(Desc,List) == false) @@ -611,19 +611,19 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) return true; // Get a structure - map_ptrloc const Group = AllocateInMap(sizeof(pkgCache::Group)); + map_pointer_t const Group = AllocateInMap(sizeof(pkgCache::Group)); if (unlikely(Group == 0)) return false; Grp = pkgCache::GrpIterator(Cache, Cache.GrpP + Group); - map_ptrloc const idxName = WriteStringInMap(Name); + map_pointer_t const idxName = WriteStringInMap(Name); if (unlikely(idxName == 0)) return false; Grp->Name = idxName; // Insert it into the hash table unsigned long const Hash = Cache.Hash(Name); - map_ptrloc *insertAt = &Cache.HeaderP->GrpHashTable()[Hash]; + map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTable()[Hash]; while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0) insertAt = &(Cache.GrpP + *insertAt)->Next; Grp->Next = *insertAt; @@ -648,7 +648,7 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name return true; // Get a structure - map_ptrloc const Package = AllocateInMap(sizeof(pkgCache::Package)); + map_pointer_t const Package = AllocateInMap(sizeof(pkgCache::Package)); if (unlikely(Package == 0)) return false; Pkg = pkgCache::PkgIterator(Cache,Cache.PkgP + Package); @@ -658,8 +658,8 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name { Grp->FirstPackage = Package; // Insert it into the hash table - unsigned long const Hash = Cache.Hash(Name); - map_ptrloc *insertAt = &Cache.HeaderP->PkgHashTable()[Hash]; + map_id_t const Hash = Cache.Hash(Name); + map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTable()[Hash]; while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.PkgP + *insertAt)->Name) > 0) insertAt = &(Cache.PkgP + *insertAt)->Next; Pkg->Next = *insertAt; @@ -678,7 +678,7 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name Pkg->Name = Grp->Name; Pkg->Group = Grp.Index(); // all is mapped to the native architecture - map_ptrloc const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str()); + map_stringitem_t const idxArch = (Arch == "all") ? Cache.HeaderP->Architecture : WriteUniqString(Arch.c_str()); if (unlikely(idxArch == 0)) return false; Pkg->Arch = idxArch; @@ -695,14 +695,14 @@ bool pkgCacheGenerator::AddImplicitDepends(pkgCache::GrpIterator &G, // copy P.Arch() into a string here as a cache remap // in NewDepends() later may alter the pointer location string Arch = P.Arch() == NULL ? "" : P.Arch(); - map_ptrloc *OldDepLast = NULL; + map_pointer_t *OldDepLast = NULL; /* MultiArch handling introduces a lot of implicit Dependencies: - MultiArch: same → Co-Installable if they have the same version - All others conflict with all other group members */ bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same); pkgCache::PkgIterator D = G.PackageList(); Dynamic DynD(D); - map_ptrloc const VerStrIdx = V->VerStr; + map_stringitem_t const VerStrIdx = V->VerStr; for (; D.end() != true; D = G.NextPkg(D)) { if (Arch == D.Arch() || D->VersionList == 0) @@ -735,11 +735,11 @@ bool pkgCacheGenerator::AddImplicitDepends(pkgCache::VerIterator &V, /* MultiArch handling introduces a lot of implicit Dependencies: - MultiArch: same → Co-Installable if they have the same version - All others conflict with all other group members */ - map_ptrloc *OldDepLast = NULL; + map_pointer_t *OldDepLast = NULL; bool const coInstall = ((V->MultiArch & pkgCache::Version::Same) == pkgCache::Version::Same); if (coInstall == true) { - map_ptrloc const VerStrIdx = V->VerStr; + map_stringitem_t const VerStrIdx = V->VerStr; // Replaces: ${self}:other ( << ${binary:Version}) NewDepends(D, V, VerStrIdx, pkgCache::Dep::Less, pkgCache::Dep::Replaces, @@ -768,7 +768,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, return true; // Get a structure - map_ptrloc const VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); + map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); if (VerFile == 0) return 0; @@ -776,7 +776,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, VF->File = CurrentFile - Cache.PkgFileP; // Link it to the end of the list - map_ptrloc *Last = &Ver->FileList; + map_pointer_t *Last = &Ver->FileList; for (pkgCache::VerFileIterator V = Ver.FileList(); V.end() == false; ++V) Last = &V->NextFile; VF->NextFile = *Last; @@ -794,14 +794,14 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, // CacheGenerator::NewVersion - Create a new Version /*{{{*/ // --------------------------------------------------------------------- /* This puts a version structure in the linked list */ -unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, +map_pointer_t pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, const string &VerStr, - map_ptrloc const ParentPkg, - unsigned long const Hash, - unsigned long Next) + map_pointer_t const ParentPkg, + unsigned short const Hash, + map_pointer_t const Next) { // Get a structure - map_ptrloc const Version = AllocateInMap(sizeof(pkgCache::Version)); + map_pointer_t const Version = AllocateInMap(sizeof(pkgCache::Version)); if (Version == 0) return 0; @@ -836,7 +836,7 @@ unsigned long pkgCacheGenerator::NewVersion(pkgCache::VerIterator &Ver, } } // haven't found the version string, so create - map_ptrloc const idxVerStr = WriteStringInMap(VerStr); + map_stringitem_t const idxVerStr = WriteStringInMap(VerStr); if (unlikely(idxVerStr == 0)) return 0; Ver->VerStr = idxVerStr; @@ -853,7 +853,7 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, return true; // Get a structure - map_ptrloc const DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); + map_pointer_t const DescFile = AllocateInMap(sizeof(pkgCache::DescFile)); if (DescFile == 0) return false; @@ -861,7 +861,7 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, DF->File = CurrentFile - Cache.PkgFileP; // Link it to the end of the list - map_ptrloc *Last = &Desc->FileList; + map_pointer_t *Last = &Desc->FileList; for (pkgCache::DescFileIterator D = Desc.FileList(); D.end() == false; ++D) Last = &D->NextFile; @@ -880,20 +880,20 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, // CacheGenerator::NewDescription - Create a new Description /*{{{*/ // --------------------------------------------------------------------- /* This puts a description structure in the linked list */ -map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, +map_pointer_t pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, const string &Lang, const MD5SumValue &md5sum, - map_ptrloc idxmd5str) + map_stringitem_t const idxmd5str) { // Get a structure - map_ptrloc const Description = AllocateInMap(sizeof(pkgCache::Description)); + map_pointer_t const Description = AllocateInMap(sizeof(pkgCache::Description)); if (Description == 0) return 0; // Fill it in Desc = pkgCache::DescIterator(Cache,Cache.DescP + Description); Desc->ID = Cache.HeaderP->DescriptionCount++; - map_ptrloc const idxlanguage_code = WriteUniqString(Lang); + map_stringitem_t const idxlanguage_code = WriteUniqString(Lang); if (unlikely(idxlanguage_code == 0)) return 0; Desc->language_code = idxlanguage_code; @@ -902,7 +902,7 @@ map_ptrloc pkgCacheGenerator::NewDescription(pkgCache::DescIterator &Desc, Desc->md5sum = idxmd5str; else { - map_ptrloc const idxmd5sum = WriteStringInMap(md5sum.Value()); + map_stringitem_t const idxmd5sum = WriteStringInMap(md5sum.Value()); if (unlikely(idxmd5sum == 0)) return 0; Desc->md5sum = idxmd5sum; @@ -920,9 +920,9 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, string const &Version, unsigned int const &Op, unsigned int const &Type, - map_ptrloc* &OldDepLast) + map_stringitem_t* &OldDepLast) { - map_ptrloc index = 0; + map_stringitem_t index = 0; if (Version.empty() == false) { int const CmpOp = Op & 0x0F; @@ -937,21 +937,21 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, if (unlikely(index == 0)) return false; if (OldDepLast != 0 && oldMap != Map.Data()) - OldDepLast += (map_ptrloc const * const) Map.Data() - (map_ptrloc const * const) oldMap; + OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap; } } return NewDepends(Pkg, Ver, index, Op, Type, OldDepLast); } bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver, - map_ptrloc const Version, + map_pointer_t const Version, unsigned int const &Op, unsigned int const &Type, - map_ptrloc* &OldDepLast) + map_pointer_t* &OldDepLast) { void const * const oldMap = Map.Data(); // Get a structure - map_ptrloc const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); + map_pointer_t const Dependency = AllocateInMap(sizeof(pkgCache::Dependency)); if (unlikely(Dependency == 0)) return false; @@ -976,7 +976,7 @@ bool pkgCacheGenerator::NewDepends(pkgCache::PkgIterator &Pkg, for (pkgCache::DepIterator D = Ver.DependsList(); D.end() == false; ++D) OldDepLast = &D->NextDepends; } else if (oldMap != Map.Data()) - OldDepLast += (map_ptrloc const * const) Map.Data() - (map_ptrloc const * const) oldMap; + OldDepLast += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap; Dep->NextDepends = *OldDepLast; *OldDepLast = Dep.Index(); @@ -1041,7 +1041,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, return true; // Get a structure - map_ptrloc const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); + map_pointer_t const Provides = Owner->AllocateInMap(sizeof(pkgCache::Provides)); if (unlikely(Provides == 0)) return false; Cache.HeaderP->ProvidesCount++; @@ -1053,7 +1053,7 @@ bool pkgCacheGenerator::ListParser::NewProvides(pkgCache::VerIterator &Ver, Prv->NextPkgProv = Ver->ProvidesList; Ver->ProvidesList = Prv.Index(); if (Version.empty() == false) { - map_ptrloc const idxProvideVersion = WriteString(Version); + map_stringitem_t const idxProvideVersion = WriteString(Version); Prv->ProvideVersion = idxProvideVersion; if (unlikely(idxProvideVersion == 0)) return false; @@ -1088,14 +1088,14 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, unsigned long Flags) { // Get some space for the structure - map_ptrloc const idxFile = AllocateInMap(sizeof(*CurrentFile)); + map_pointer_t const idxFile = AllocateInMap(sizeof(*CurrentFile)); if (unlikely(idxFile == 0)) return false; CurrentFile = Cache.PkgFileP + idxFile; // Fill it in - map_ptrloc const idxFileName = WriteStringInMap(File); - map_ptrloc const idxSite = WriteUniqString(Site); + map_stringitem_t const idxFileName = WriteStringInMap(File); + map_stringitem_t const idxSite = WriteUniqString(Site); if (unlikely(idxFileName == 0 || idxSite == 0)) return false; CurrentFile->FileName = idxFileName; @@ -1103,7 +1103,7 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, CurrentFile->NextFile = Cache.HeaderP->FileList; CurrentFile->Flags = Flags; CurrentFile->ID = Cache.HeaderP->PackageFileCount; - map_ptrloc const idxIndexType = WriteUniqString(Index.GetType()->Label); + map_stringitem_t const idxIndexType = WriteUniqString(Index.GetType()->Label); if (unlikely(idxIndexType == 0)) return false; CurrentFile->IndexType = idxIndexType; @@ -1120,7 +1120,7 @@ bool pkgCacheGenerator::SelectFile(const string &File,const string &Site, // --------------------------------------------------------------------- /* This is used to create handles to strings. Given the same text it always returns the same number */ -unsigned long pkgCacheGenerator::WriteUniqString(const char *S, +map_stringitem_t pkgCacheGenerator::WriteUniqString(const char *S, unsigned int Size) { /* We use a very small transient hash table here, this speeds up generation @@ -1133,7 +1133,7 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, // Search for an insertion point pkgCache::StringItem *I = Cache.StringItemP + Cache.HeaderP->StringList; int Res = 1; - map_ptrloc *Last = &Cache.HeaderP->StringList; + map_stringitem_t *Last = &Cache.HeaderP->StringList; for (; I != Cache.StringItemP; Last = &I->NextItem, I = Cache.StringItemP + I->NextItem) { @@ -1151,15 +1151,15 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S, // Get a structure void const * const oldMap = Map.Data(); - map_ptrloc const Item = AllocateInMap(sizeof(pkgCache::StringItem)); + map_pointer_t const Item = AllocateInMap(sizeof(pkgCache::StringItem)); if (Item == 0) return 0; - map_ptrloc const idxString = WriteStringInMap(S,Size); + map_stringitem_t const idxString = WriteStringInMap(S,Size); if (unlikely(idxString == 0)) return 0; if (oldMap != Map.Data()) { - Last += (map_ptrloc const * const) Map.Data() - (map_ptrloc const * const) oldMap; + Last += (map_pointer_t const * const) Map.Data() - (map_pointer_t const * const) oldMap; I += (pkgCache::StringItem const * const) Map.Data() - (pkgCache::StringItem const * const) oldMap; } *Last = Item; @@ -1280,9 +1280,9 @@ static bool CheckValidity(const string &CacheFile, // --------------------------------------------------------------------- /* Size is kind of an abstract notion that is only used for the progress meter */ -static unsigned long ComputeSize(FileIterator Start,FileIterator End) +static map_filesize_t ComputeSize(FileIterator Start,FileIterator End) { - unsigned long TotalSize = 0; + map_filesize_t TotalSize = 0; for (; Start < End; ++Start) { if ((*Start)->HasPackages() == false) @@ -1297,7 +1297,7 @@ static unsigned long ComputeSize(FileIterator Start,FileIterator End) /* */ static bool BuildCache(pkgCacheGenerator &Gen, OpProgress *Progress, - unsigned long &CurrentSize,unsigned long TotalSize, + map_filesize_t &CurrentSize,map_filesize_t TotalSize, FileIterator Start, FileIterator End) { FileIterator I; @@ -1316,7 +1316,7 @@ static bool BuildCache(pkgCacheGenerator &Gen, continue; } - unsigned long Size = (*I)->Size(); + map_filesize_t Size = (*I)->Size(); if (Progress != NULL) Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Reading package lists")); CurrentSize += Size; @@ -1333,7 +1333,7 @@ static bool BuildCache(pkgCacheGenerator &Gen, CurrentSize = 0; for (I = Start; I != End; ++I) { - unsigned long Size = (*I)->Size(); + map_filesize_t Size = (*I)->Size(); if (Progress != NULL) Progress->OverallProgress(CurrentSize,TotalSize,Size,_("Collecting File Provides")); CurrentSize += Size; @@ -1347,9 +1347,9 @@ static bool BuildCache(pkgCacheGenerator &Gen, /*}}}*/ // CacheGenerator::CreateDynamicMMap - load an mmap with configuration options /*{{{*/ DynamicMMap* pkgCacheGenerator::CreateDynamicMMap(FileFd *CacheF, unsigned long Flags) { - unsigned long const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024); - unsigned long const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024); - unsigned long const MapLimit = _config->FindI("APT::Cache-Limit", 0); + map_filesize_t const MapStart = _config->FindI("APT::Cache-Start", 24*1024*1024); + map_filesize_t const MapGrow = _config->FindI("APT::Cache-Grow", 1*1024*1024); + map_filesize_t const MapLimit = _config->FindI("APT::Cache-Limit", 0); Flags |= MMap::Moveable; if (_config->FindB("APT::Cache-Fallback", false) == true) Flags |= MMap::Fallback; @@ -1387,7 +1387,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress Files.push_back (*j); } - unsigned long const EndOfSource = Files.size(); + map_filesize_t const EndOfSource = Files.size(); if (_system->AddStatusFiles(Files) == false) return false; @@ -1477,8 +1477,8 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress } // Lets try the source cache. - unsigned long CurrentSize = 0; - unsigned long TotalSize = 0; + map_filesize_t CurrentSize = 0; + map_filesize_t TotalSize = 0; if (CheckValidity(SrcCacheFile, List, Files.begin(), Files.begin()+EndOfSource) == true) { @@ -1486,7 +1486,7 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl; // Preload the map with the source cache FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly); - unsigned long const alloc = Map->RawAllocate(SCacheF.Size()); + map_pointer_t const alloc = Map->RawAllocate(SCacheF.Size()); if ((alloc == 0 && _error->PendingError()) || SCacheF.Read((unsigned char *)Map->Data() + alloc, SCacheF.Size()) == false) @@ -1573,13 +1573,13 @@ APT_DEPRECATED bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **Ou bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap) { std::vector Files; - unsigned long EndOfSource = Files.size(); + map_filesize_t EndOfSource = Files.size(); if (_system->AddStatusFiles(Files) == false) return false; SPtr Map = CreateDynamicMMap(NULL); - unsigned long CurrentSize = 0; - unsigned long TotalSize = 0; + map_filesize_t CurrentSize = 0; + map_filesize_t TotalSize = 0; TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end()); diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index d275c1e42..42da7b12d 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -38,10 +38,10 @@ class pkgCacheGenerator /*{{{*/ private: pkgCache::StringItem *UniqHash[26]; - APT_HIDDEN map_ptrloc WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; - APT_HIDDEN map_ptrloc WriteStringInMap(const char *String); - APT_HIDDEN map_ptrloc WriteStringInMap(const char *String, const unsigned long &Len); - APT_HIDDEN map_ptrloc AllocateInMap(const unsigned long &size); + APT_HIDDEN map_stringitem_t WriteStringInMap(std::string const &String) { return WriteStringInMap(String.c_str()); }; + APT_HIDDEN map_stringitem_t WriteStringInMap(const char *String); + APT_HIDDEN map_stringitem_t WriteStringInMap(const char *String, const unsigned long &Len); + APT_HIDDEN map_pointer_t AllocateInMap(const unsigned long &size); public: @@ -78,21 +78,21 @@ class pkgCacheGenerator /*{{{*/ bool NewFileDesc(pkgCache::DescIterator &Desc,ListParser &List); bool NewDepends(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver, std::string const &Version, unsigned int const &Op, - unsigned int const &Type, map_ptrloc* &OldDepLast); + unsigned int const &Type, map_pointer_t* &OldDepLast); bool NewDepends(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver, - map_ptrloc const Version, unsigned int const &Op, - unsigned int const &Type, map_ptrloc* &OldDepLast); - unsigned long NewVersion(pkgCache::VerIterator &Ver,const std::string &VerStr,unsigned long Next) APT_DEPRECATED + map_pointer_t const Version, unsigned int const &Op, + unsigned int const &Type, map_pointer_t* &OldDepLast); + map_pointer_t NewVersion(pkgCache::VerIterator &Ver,const std::string &VerStr,map_pointer_t const Next) APT_DEPRECATED { return NewVersion(Ver, VerStr, 0, 0, Next); } - unsigned long NewVersion(pkgCache::VerIterator &Ver,const std::string &VerStr, - map_ptrloc const ParentPkg, unsigned long const Hash, - unsigned long Next); - map_ptrloc NewDescription(pkgCache::DescIterator &Desc,const std::string &Lang,const MD5SumValue &md5sum,map_ptrloc Next); + map_pointer_t NewVersion(pkgCache::VerIterator &Ver,const std::string &VerStr, + map_pointer_t const ParentPkg, unsigned short const Hash, + map_pointer_t const Next); + map_pointer_t NewDescription(pkgCache::DescIterator &Desc,const std::string &Lang,const MD5SumValue &md5sum,map_stringitem_t const idxmd5str); public: - unsigned long WriteUniqString(const char *S,unsigned int Size); - inline unsigned long WriteUniqString(const std::string &S) {return WriteUniqString(S.c_str(),S.length());}; + map_stringitem_t WriteUniqString(const char *S,unsigned int const Size); + inline map_stringitem_t WriteUniqString(const std::string &S) {return WriteUniqString(S.c_str(),S.length());}; void DropProgress() {Progress = 0;}; bool SelectFile(const std::string &File,const std::string &Site,pkgIndexFile const &Index, @@ -127,7 +127,7 @@ class pkgCacheGenerator /*{{{*/ APT_HIDDEN bool AddImplicitDepends(pkgCache::VerIterator &V, pkgCache::PkgIterator &D); APT_HIDDEN bool AddNewDescription(ListParser &List, pkgCache::VerIterator &Ver, - std::string const &lang, MD5SumValue const &CurMd5, map_ptrloc &md5idx); + std::string const &lang, MD5SumValue const &CurMd5, map_stringitem_t &md5idx); }; /*}}}*/ // This is the abstract package list parser class. /*{{{*/ @@ -138,17 +138,17 @@ class pkgCacheGenerator::ListParser // Some cache items pkgCache::VerIterator OldDepVer; - map_ptrloc *OldDepLast; + map_pointer_t *OldDepLast; // Flag file dependencies bool FoundFileDeps; protected: - inline unsigned long WriteUniqString(std::string S) {return Owner->WriteUniqString(S);}; - inline unsigned long WriteUniqString(const char *S,unsigned int Size) {return Owner->WriteUniqString(S,Size);}; - inline unsigned long WriteString(const std::string &S) {return Owner->WriteStringInMap(S);}; - inline unsigned long WriteString(const char *S,unsigned int Size) {return Owner->WriteStringInMap(S,Size);}; + inline map_stringitem_t WriteUniqString(std::string S) {return Owner->WriteUniqString(S);}; + inline map_stringitem_t WriteUniqString(const char *S,unsigned int Size) {return Owner->WriteUniqString(S,Size);}; + inline map_stringitem_t WriteString(const std::string &S) {return Owner->WriteStringInMap(S);}; + inline map_stringitem_t WriteString(const char *S,unsigned int Size) {return Owner->WriteStringInMap(S,Size);}; bool NewDepends(pkgCache::VerIterator &Ver,const std::string &Package, const std::string &Arch, const std::string &Version,unsigned int Op, unsigned int Type); @@ -178,8 +178,8 @@ class pkgCacheGenerator::ListParser APT_PURE bool SameVersion(unsigned short const Hash, pkgCache::VerIterator const &Ver); virtual bool UsePackage(pkgCache::PkgIterator &Pkg, pkgCache::VerIterator &Ver) = 0; - virtual unsigned long Offset() = 0; - virtual unsigned long Size() = 0; + virtual map_filesize_t Offset() = 0; + virtual map_filesize_t Size() = 0; virtual bool Step() = 0; diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 2ed1bf5d4..620cb6c01 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1275,7 +1275,7 @@ static bool DisplayRecord(pkgCacheFile &CacheFile, pkgCache::VerIterator V) struct ExDescFile { pkgCache::DescFile *Df; - map_ptrloc ID; + map_id_t ID; }; // Search - Perform a search /*{{{*/ -- cgit v1.2.3-70-g09d2 From ac2c559b33db7a2dbec3a95760ee2497baf7017a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 13 Jun 2014 09:10:11 +0200 Subject: correct 'apt-cache stats' to include more It still doesn't reflect the size the cache has on the disk compared to what is given as total size (90 vs 103 MB), but by counting all structs in we are at least a bit closer to the reality. Git-Dch: ignore --- apt-pkg/pkgcachegen.cc | 2 +- cmdline/apt-cache.cc | 56 ++++++++++++++++++++++++++++---------------------- 2 files changed, 33 insertions(+), 25 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 6a3cb2637..a4d5d8783 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -770,7 +770,7 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, // Get a structure map_pointer_t const VerFile = AllocateInMap(sizeof(pkgCache::VerFile)); if (VerFile == 0) - return 0; + return false; pkgCache::VerFileIterator VF(Cache,Cache.VerFileP + VerFile); VF->File = CurrentFile - Cache.PkgFileP; diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 620cb6c01..88a323280 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -270,34 +270,36 @@ static bool DumpPackage(CommandLine &CmdL) template static void ShowHashTableStats(std::string Type, T *StartP, - map_ptrloc *Hashtable, + map_pointer_t *Hashtable, unsigned long Size) { // hashtable stats for the HashTable - long NumBuckets = Size; - long UsedBuckets = 0; - long UnusedBuckets = 0; - long LongestBucket = 0; - long ShortestBucket = NumBuckets; + unsigned long NumBuckets = Size; + unsigned long UsedBuckets = 0; + unsigned long UnusedBuckets = 0; + unsigned long LongestBucket = 0; + unsigned long ShortestBucket = NumBuckets; + unsigned long Entries = 0; for (unsigned int i=0; i < NumBuckets; ++i) { T *P = StartP + Hashtable[i]; if(P == 0 || P == StartP) { - UnusedBuckets++; + ++UnusedBuckets; continue; } - long ThisBucketSize = 0; + ++UsedBuckets; + unsigned long ThisBucketSize = 0; for (; P != StartP; P = StartP + P->Next) - ThisBucketSize++; + ++ThisBucketSize; + Entries += ThisBucketSize; LongestBucket = std::max(ThisBucketSize, LongestBucket); ShortestBucket = std::min(ThisBucketSize, ShortestBucket); - UsedBuckets += ThisBucketSize; } - cout << "Total buckets " << Type << ": " << SizeToStr(NumBuckets) << std::endl; - cout << " Unused: " << SizeToStr(UnusedBuckets) << std::endl; + cout << "Total buckets in " << Type << ": " << NumBuckets << std::endl; + cout << " Unused: " << UnusedBuckets << std::endl; cout << " Used: " << UsedBuckets << std::endl; - cout << " Average entries: " << UsedBuckets/(double)NumBuckets << std::endl; + cout << " Average entries: " << Entries/(double)NumBuckets << std::endl; cout << " Longest: " << LongestBucket << std::endl; cout << " Shortest: " << ShortestBucket << std::endl; } @@ -398,25 +400,31 @@ static bool Stats(CommandLine &) } } cout << _("Total dependency version space: ") << SizeToStr(DepVerSize) << endl; - + unsigned long Slack = 0; for (int I = 0; I != 7; I++) Slack += Cache->Head().Pools[I].ItemSize*Cache->Head().Pools[I].Count; cout << _("Total slack space: ") << SizeToStr(Slack) << endl; - + unsigned long Total = 0; - Total = Slack + Size + Cache->Head().DependsCount*Cache->Head().DependencySz + - Cache->Head().VersionCount*Cache->Head().VersionSz + - Cache->Head().PackageCount*Cache->Head().PackageSz + - Cache->Head().VerFileCount*Cache->Head().VerFileSz + - Cache->Head().ProvidesCount*Cache->Head().ProvidesSz; +#define APT_CACHESIZE(X,Y) (Cache->Head().X * Cache->Head().Y) + Total = Slack + Size + + APT_CACHESIZE(GroupCount, GroupSz) + + APT_CACHESIZE(PackageCount, PackageSz) + + APT_CACHESIZE(VersionCount, VersionSz) + + APT_CACHESIZE(DescriptionCount, DescriptionSz) + + APT_CACHESIZE(DependsCount, DependencySz) + + APT_CACHESIZE(PackageFileCount, PackageFileSz) + + APT_CACHESIZE(VerFileCount, VerFileSz) + + APT_CACHESIZE(DescFileCount, DescFileSz) + + APT_CACHESIZE(ProvidesCount, ProvidesSz) + + (2 * Cache->Head().HashTableSize * sizeof(map_id_t)); cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; +#undef APT_CACHESIZE // hashtable stats - int HashTableSize = sizeof(Cache->HeaderP->PkgHashTable)/sizeof(map_ptrloc); - ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->HeaderP->PkgHashTable, HashTableSize); - HashTableSize = sizeof(Cache->HeaderP->GrpHashTable)/sizeof(map_ptrloc); - ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->HeaderP->GrpHashTable, HashTableSize); + ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->Head().PkgHashTable(), Cache->Head().HashTableSize); + ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->Head().GrpHashTable(), Cache->Head().HashTableSize); return true; } -- cgit v1.2.3-70-g09d2 From 3621b1c70f532c61ccfee6a65c413ee573ed4900 Mon Sep 17 00:00:00 2001 From: Guillem Jover Date: Wed, 2 Jul 2014 03:10:21 +0200 Subject: Fix ar and tar code to be LFS-safe This is an ABI break. Closes: #742882 --- apt-inst/contrib/arfile.h | 2 +- apt-inst/contrib/extracttar.cc | 13 ++++++------- apt-inst/contrib/extracttar.h | 4 ++-- apt-inst/deb/debfile.cc | 4 ++-- apt-inst/deb/debfile.h | 4 ++-- apt-inst/dirstream.h | 4 ++-- cmdline/apt-extracttemplates.cc | 2 +- cmdline/apt-extracttemplates.h | 2 +- debian/libapt-inst1.5.symbols | 8 ++++---- 9 files changed, 21 insertions(+), 22 deletions(-) (limited to 'cmdline') diff --git a/apt-inst/contrib/arfile.h b/apt-inst/contrib/arfile.h index 0f62a34a0..5aa38aedc 100644 --- a/apt-inst/contrib/arfile.h +++ b/apt-inst/contrib/arfile.h @@ -61,7 +61,7 @@ struct ARArchive::Member unsigned long long Size; // Location of the data. - unsigned long Start; + unsigned long long Start; Member *Next; Member() : Start(0), Next(0) {}; diff --git a/apt-inst/contrib/extracttar.cc b/apt-inst/contrib/extracttar.cc index 0ba3f0521..2c86d0d01 100644 --- a/apt-inst/contrib/extracttar.cc +++ b/apt-inst/contrib/extracttar.cc @@ -60,9 +60,8 @@ struct ExtractTar::TarHeader // ExtractTar::ExtractTar - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -ExtractTar::ExtractTar(FileFd &Fd,unsigned long Max,string DecompressionProgram) : File(Fd), - MaxInSize(Max), DecompressProg(DecompressionProgram) - +ExtractTar::ExtractTar(FileFd &Fd,unsigned long long Max,string DecompressionProgram) + : File(Fd), MaxInSize(Max), DecompressProg(DecompressionProgram) { GZPid = -1; Eof = false; @@ -267,7 +266,7 @@ bool ExtractTar::Go(pkgDirStream &Stream) case GNU_LongLink: { - unsigned long Length = Itm.Size; + unsigned long long Length = Itm.Size; unsigned char Block[512]; while (Length > 0) { @@ -286,7 +285,7 @@ bool ExtractTar::Go(pkgDirStream &Stream) case GNU_LongName: { - unsigned long Length = Itm.Size; + unsigned long long Length = Itm.Size; unsigned char Block[512]; while (Length > 0) { @@ -315,11 +314,11 @@ bool ExtractTar::Go(pkgDirStream &Stream) return false; // Copy the file over the FD - unsigned long Size = Itm.Size; + unsigned long long Size = Itm.Size; while (Size != 0) { unsigned char Junk[32*1024]; - unsigned long Read = min(Size,(unsigned long)sizeof(Junk)); + unsigned long Read = min(Size, (unsigned long long)sizeof(Junk)); if (InFd.Read(Junk,((Read+511)/512)*512) == false) return false; diff --git a/apt-inst/contrib/extracttar.h b/apt-inst/contrib/extracttar.h index 4b29df314..472e018f4 100644 --- a/apt-inst/contrib/extracttar.h +++ b/apt-inst/contrib/extracttar.h @@ -39,7 +39,7 @@ class ExtractTar GNU_LongLink = 'K',GNU_LongName = 'L'}; FileFd &File; - unsigned long MaxInSize; + unsigned long long MaxInSize; int GZPid; FileFd InFd; bool Eof; @@ -53,7 +53,7 @@ class ExtractTar bool Go(pkgDirStream &Stream); - ExtractTar(FileFd &Fd,unsigned long Max,std::string DecompressionProgram); + ExtractTar(FileFd &Fd,unsigned long long Max,std::string DecompressionProgram); virtual ~ExtractTar(); }; diff --git a/apt-inst/deb/debfile.cc b/apt-inst/deb/debfile.cc index a63cb6716..4853a13c7 100644 --- a/apt-inst/deb/debfile.cc +++ b/apt-inst/deb/debfile.cc @@ -203,7 +203,7 @@ bool debDebFile::MemControlExtract::DoItem(Item &Itm,int &Fd) /* Just memcopy the block from the tar extractor and put it in the right place in the pre-allocated memory block. */ bool debDebFile::MemControlExtract::Process(Item &/*Itm*/,const unsigned char *Data, - unsigned long Size,unsigned long Pos) + unsigned long long Size,unsigned long long Pos) { memcpy(Control + Pos, Data,Size); return true; @@ -232,7 +232,7 @@ bool debDebFile::MemControlExtract::Read(debDebFile &Deb) // --------------------------------------------------------------------- /* The given memory block is loaded into the parser and parsed as a control record. */ -bool debDebFile::MemControlExtract::TakeControl(const void *Data,unsigned long Size) +bool debDebFile::MemControlExtract::TakeControl(const void *Data,unsigned long long Size) { delete [] Control; Control = new char[Size+2]; diff --git a/apt-inst/deb/debfile.h b/apt-inst/deb/debfile.h index 880bcf6c5..b068efcec 100644 --- a/apt-inst/deb/debfile.h +++ b/apt-inst/deb/debfile.h @@ -81,12 +81,12 @@ class debDebFile::MemControlExtract : public pkgDirStream // Members from DirStream virtual bool DoItem(Item &Itm,int &Fd); virtual bool Process(Item &Itm,const unsigned char *Data, - unsigned long Size,unsigned long Pos); + unsigned long long Size,unsigned long long Pos); // Helpers bool Read(debDebFile &Deb); - bool TakeControl(const void *Data,unsigned long Size); + bool TakeControl(const void *Data,unsigned long long Size); MemControlExtract() : IsControl(false), Control(0), Length(0), Member("control") {}; MemControlExtract(std::string Member) : IsControl(false), Control(0), Length(0), Member(Member) {}; diff --git a/apt-inst/dirstream.h b/apt-inst/dirstream.h index 1be2688a1..571fe86a5 100644 --- a/apt-inst/dirstream.h +++ b/apt-inst/dirstream.h @@ -37,10 +37,10 @@ class pkgDirStream Directory, FIFO} Type; char *Name; char *LinkTarget; + unsigned long long Size; unsigned long Mode; unsigned long UID; unsigned long GID; - unsigned long Size; unsigned long MTime; unsigned long Major; unsigned long Minor; @@ -50,7 +50,7 @@ class pkgDirStream virtual bool Fail(Item &Itm,int Fd); virtual bool FinishedFile(Item &Itm,int Fd); virtual bool Process(Item &/*Itm*/,const unsigned char * /*Data*/, - unsigned long /*Size*/,unsigned long /*Pos*/) {return true;}; + unsigned long long /*Size*/,unsigned long long /*Pos*/) {return true;}; virtual ~pkgDirStream() {}; }; diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc index 0d568106f..f95b9e5ba 100644 --- a/cmdline/apt-extracttemplates.cc +++ b/cmdline/apt-extracttemplates.cc @@ -140,7 +140,7 @@ bool DebFile::DoItem(Item &I, int &Fd) // --------------------------------------------------------------------- /* */ bool DebFile::Process(Item &/*I*/, const unsigned char *data, - unsigned long size, unsigned long pos) + unsigned long long size, unsigned long long pos) { switch (Which) { diff --git a/cmdline/apt-extracttemplates.h b/cmdline/apt-extracttemplates.h index 6d2870a02..829cdae75 100644 --- a/cmdline/apt-extracttemplates.h +++ b/cmdline/apt-extracttemplates.h @@ -28,7 +28,7 @@ public: ~DebFile(); bool DoItem(Item &I, int &fd); bool Process(pkgDirStream::Item &I, const unsigned char *data, - unsigned long size, unsigned long pos); + unsigned long long size, unsigned long long pos); bool Go(); bool ParseInfo(); diff --git a/debian/libapt-inst1.5.symbols b/debian/libapt-inst1.5.symbols index 8ce707287..c5ab7f4ed 100644 --- a/debian/libapt-inst1.5.symbols +++ b/debian/libapt-inst1.5.symbols @@ -3,7 +3,7 @@ libapt-inst.so.1.5 libapt-inst1.5 #MINVER# (c++)"ExtractTar::Done(bool)@Base" 0.8.0 (c++)"ExtractTar::Go(pkgDirStream&)@Base" 0.8.0 (c++)"ExtractTar::StartGzip()@Base" 0.8.0 - (c++)"ExtractTar::ExtractTar(FileFd&, unsigned long, std::basic_string, std::allocator >)@Base" 0.8.0 + (c++)"ExtractTar::ExtractTar(FileFd&, unsigned long long, std::basic_string, std::allocator >)@Base" 1.0.5 (c++)"ExtractTar::~ExtractTar()@Base" 0.8.0 (c++)"debDebFile::GotoMember(char const*)@Base" 0.8.0 (c++)"debDebFile::CheckMember(char const*)@Base" 0.8.0 @@ -11,10 +11,10 @@ libapt-inst.so.1.5 libapt-inst1.5 #MINVER# (c++)"debDebFile::ControlExtract::~ControlExtract()@Base" 0.8.0 (c++)"debDebFile::ExtractTarMember(pkgDirStream&, char const*)@Base" 0.9.15.4 (c++)"debDebFile::ExtractArchive(pkgDirStream&)@Base" 0.8.0 - (c++)"debDebFile::MemControlExtract::TakeControl(void const*, unsigned long)@Base" 0.8.0 + (c++)"debDebFile::MemControlExtract::TakeControl(void const*, unsigned long long)@Base" 1.0.5 (c++)"debDebFile::MemControlExtract::Read(debDebFile&)@Base" 0.8.0 (c++)"debDebFile::MemControlExtract::DoItem(pkgDirStream::Item&, int&)@Base" 0.8.0 - (c++)"debDebFile::MemControlExtract::Process(pkgDirStream::Item&, unsigned char const*, unsigned long, unsigned long)@Base" 0.8.0 + (c++)"debDebFile::MemControlExtract::Process(pkgDirStream::Item&, unsigned char const*, unsigned long long, unsigned long long)@Base" 1.0.5 (c++)"debDebFile::MemControlExtract::~MemControlExtract()@Base" 0.8.0 (c++)"debDebFile::debDebFile(FileFd&)@Base" 0.8.0 (c++)"pkgExtract::FinishedFile(pkgDirStream::Item&, int)@Base" 0.8.0 @@ -41,7 +41,7 @@ libapt-inst.so.1.5 libapt-inst1.5 #MINVER# (c++)"pkgDirStream::FinishedFile(pkgDirStream::Item&, int)@Base" 0.8.0 (c++)"pkgDirStream::Fail(pkgDirStream::Item&, int)@Base" 0.8.0 (c++)"pkgDirStream::DoItem(pkgDirStream::Item&, int&)@Base" 0.8.0 - (c++)"pkgDirStream::Process(pkgDirStream::Item&, unsigned char const*, unsigned long, unsigned long)@Base" 0.8.0 + (c++)"pkgDirStream::Process(pkgDirStream::Item&, unsigned char const*, unsigned long long, unsigned long long)@Base" 1.0.5 (c++)"pkgDirStream::~pkgDirStream()@Base" 0.8.0 (c++|optional)"pkgCache::DepIterator::operator++(int)@Base" 0.8.0 (c++|optional)"pkgCache::DepIterator::operator++()@Base" 0.8.0 -- cgit v1.2.3-70-g09d2 From a5bb5e1e747ceb7b5a9defb6b1a8d9282a6e0957 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 8 Jul 2014 15:11:14 +0200 Subject: Only allow "apt-get build-dep path" when path starts with ./ or / This avoid the subtle problem that someone might have a directory with the same package name as the build-depends he/she is trying to fetch. Also print a note that the specific file/dir is used. --- apt-pkg/contrib/strutl.cc | 7 +++++++ apt-pkg/contrib/strutl.h | 1 + cmdline/apt-get.cc | 9 +++++++-- test/integration/test-apt-get-build-dep | 3 +++ test/libapt/strutil_test.cc | 9 +++++++++ 5 files changed, 27 insertions(+), 2 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc index 0f48860b1..7948673dc 100644 --- a/apt-pkg/contrib/strutl.cc +++ b/apt-pkg/contrib/strutl.cc @@ -62,6 +62,13 @@ bool Endswith(const std::string &s, const std::string &end) return (s.substr(s.size() - end.size(), s.size()) == end); } +bool Startswith(const std::string &s, const std::string &start) +{ + if (start.size() > s.size()) + return false; + return (s.substr(0, start.size()) == start); +} + } } /*}}}*/ diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h index 5733fd6e2..da8bebdb5 100644 --- a/apt-pkg/contrib/strutl.h +++ b/apt-pkg/contrib/strutl.h @@ -40,6 +40,7 @@ namespace APT { namespace String { std::string Strip(const std::string &s); bool Endswith(const std::string &s, const std::string &ending); + bool Startswith(const std::string &s, const std::string &starting); } } diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index bd866bc8c..cdbfa4708 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1065,9 +1065,12 @@ static bool DoBuildDep(CommandLine &CmdL) string Src; pkgSrcRecords::Parser *Last = 0; - // a unpacked debian source tree - if (DirectoryExists(*I)) + // an unpacked debian source tree + using APT::String::Startswith; + if ((Startswith(*I, "./") || Startswith(*I, "/")) && + DirectoryExists(*I)) { + ioprintf(c1out, _("Note, using directory '%s' to get the build dependencies\n"), *I); // FIXME: how can we make this more elegant? std::string TypeName = "debian/control File Source Index"; pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); @@ -1077,6 +1080,8 @@ static bool DoBuildDep(CommandLine &CmdL) // if its a local file (e.g. .dsc) use this else if (FileExists(*I)) { + ioprintf(c1out, _("Note, using file '%s' to get the build dependencies\n"), *I); + // see if we can get a parser for this pkgIndexFile type string TypeName = flExtension(*I) + " File Source Index"; pkgIndexFile::Type *Type = pkgIndexFile::Type::GetType(TypeName.c_str()); diff --git a/test/integration/test-apt-get-build-dep b/test/integration/test-apt-get-build-dep index f71beae9c..87ec6e54d 100755 --- a/test/integration/test-apt-get-build-dep +++ b/test/integration/test-apt-get-build-dep @@ -34,6 +34,7 @@ EOF testequal "Reading package lists... Building dependency tree... +Note, using file '2vcard_0.5-3.dsc' to get the build dependencies The following NEW packages will be installed: build-essential debhelper 0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded. @@ -76,6 +77,7 @@ EOF testequal "Reading package lists... Building dependency tree... +Note, using file '2vcard_0.5-3.dsc' to get the build dependencies The following NEW packages will be installed: build-essential debhelper 0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded. @@ -117,6 +119,7 @@ EOF testequal "Reading package lists... Building dependency tree... +Note, using directory './foo-1.0' to get the build dependencies The following NEW packages will be installed: build-essential debhelper 0 upgraded, 2 newly installed, 0 to remove and 0 not upgraded. diff --git a/test/libapt/strutil_test.cc b/test/libapt/strutil_test.cc index e9b778c6b..1c2f0abac 100644 --- a/test/libapt/strutil_test.cc +++ b/test/libapt/strutil_test.cc @@ -70,6 +70,15 @@ TEST(StrUtilTest,EndsWith) EXPECT_FALSE(Endswith("abcd", "x")); EXPECT_FALSE(Endswith("abcd", "abcndefg")); } +TEST(StrUtilTest,StartWith) +{ + using APT::String::Startswith; + EXPECT_TRUE(Startswith("abcd", "a")); + EXPECT_TRUE(Startswith("abcd", "ab")); + EXPECT_TRUE(Startswith("abcd", "abcd")); + EXPECT_FALSE(Startswith("abcd", "x")); + EXPECT_FALSE(Startswith("abcd", "abcndefg")); +} TEST(StrUtilTest,SubstVar) { EXPECT_EQ("", SubstVar("", "fails", "passes")); -- cgit v1.2.3-70-g09d2 From c53cf70da31a3c59675af8c5425b54433793dc8d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 16 Aug 2014 13:29:10 +0200 Subject: add dpkg::source-options for dpkg-source invocation dpkg-source can be told to enforce signature checks with --require-valid-signature, but while this isn't feasible as default for Debian itself at the moment, a local admin should be able to use it. This commit also fixes the size limit on the construction of the command being called for dpkg-source and dpkg-buildpackage. Closes: 757534 --- cmdline/apt-get.cc | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index a58386eb0..845d67d2b 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -953,18 +953,19 @@ static bool DoSource(CommandLine &CmdL) else { // Call dpkg-source - char S[500]; - snprintf(S,sizeof(S),"%s -x %s", + std::string const sourceopts = _config->Find("DPkg::Source-Options", "-x"); + std::string S; + strprintf(S, "%s %s %s", _config->Find("Dir::Bin::dpkg-source","dpkg-source").c_str(), - Dsc[I].Dsc.c_str()); - if (system(S) != 0) + sourceopts.c_str(), Dsc[I].Dsc.c_str()); + if (system(S.c_str()) != 0) { - fprintf(stderr,_("Unpack command '%s' failed.\n"),S); - fprintf(stderr,_("Check if the 'dpkg-dev' package is installed.\n")); + fprintf(stderr, _("Unpack command '%s' failed.\n"), S.c_str()); + fprintf(stderr, _("Check if the 'dpkg-dev' package is installed.\n")); _exit(1); - } + } } - + // Try to compile it with dpkg-buildpackage if (_config->FindB("APT::Get::Compile",false) == true) { @@ -980,20 +981,20 @@ static bool DoSource(CommandLine &CmdL) buildopts.append(_config->Find("DPkg::Build-Options","-b -uc")); // Call dpkg-buildpackage - char S[500]; - snprintf(S,sizeof(S),"cd %s && %s %s", + std::string S; + strprintf(S, "cd %s && %s %s", Dir.c_str(), _config->Find("Dir::Bin::dpkg-buildpackage","dpkg-buildpackage").c_str(), buildopts.c_str()); - - if (system(S) != 0) + + if (system(S.c_str()) != 0) { - fprintf(stderr,_("Build command '%s' failed.\n"),S); + fprintf(stderr, _("Build command '%s' failed.\n"), S.c_str()); _exit(1); - } - } + } + } } - + _exit(0); } -- cgit v1.2.3-70-g09d2 From c6ee61eab54edf6cc3fbe118d304d72a860e1451 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 2 Sep 2014 15:50:19 +0200 Subject: Make Proxy-Auto-Detect check for each host When doing Acquire::http{,s}::Proxy-Auto-Detect, run the auto-detect command for each host instead of only once. This should make using "proxy" from libproxy-tools feasible which can then be used for PAC style or other proxy configurations. Closes: #759264 --- apt-pkg/contrib/proxy.cc | 82 ++++++++++++++++++++++++++++++++++++++++ apt-pkg/contrib/proxy.h | 16 ++++++++ cmdline/apt-helper.cc | 16 ++++++++ methods/http.cc | 62 +----------------------------- methods/http.h | 3 -- methods/https.cc | 4 ++ test/integration/test-apt-helper | 80 ++++++++++++++++++++++++++++----------- 7 files changed, 178 insertions(+), 85 deletions(-) create mode 100644 apt-pkg/contrib/proxy.cc create mode 100644 apt-pkg/contrib/proxy.h (limited to 'cmdline') diff --git a/apt-pkg/contrib/proxy.cc b/apt-pkg/contrib/proxy.cc new file mode 100644 index 000000000..b58db8478 --- /dev/null +++ b/apt-pkg/contrib/proxy.cc @@ -0,0 +1,82 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Proxy - Proxy releated functions + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include + +#include +#include + +#include "proxy.h" + + +// AutoDetectProxy - auto detect proxy /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool AutoDetectProxy(URI &URL) +{ + // we support both http/https debug options + bool Debug = _config->FindB("Debug::Acquire::"+URL.Access,false); + + // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old + // name without the dash ("-") + std::string AutoDetectProxyCmd = _config->Find("Acquire::"+URL.Access+"::Proxy-Auto-Detect", + _config->Find("Acquire::"+URL.Access+"::ProxyAutoDetect")); + + if (AutoDetectProxyCmd.empty()) + return true; + + if (Debug) + std::clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << std::endl; + + int Pipes[2] = {-1,-1}; + if (pipe(Pipes) != 0) + return _error->Errno("pipe", "Failed to create Pipe"); + + pid_t Process = ExecFork(); + if (Process == 0) + { + close(Pipes[0]); + dup2(Pipes[1],STDOUT_FILENO); + SetCloseExec(STDOUT_FILENO,false); + + std::string foo = URL; + const char *Args[4]; + Args[0] = AutoDetectProxyCmd.c_str(); + Args[1] = foo.c_str(); + Args[2] = 0; + execv(Args[0],(char **)Args); + std::cerr << "Failed to exec method " << Args[0] << std::endl; + _exit(100); + } + char buf[512]; + int InFd = Pipes[0]; + close(Pipes[1]); + int res = read(InFd, buf, sizeof(buf)-1); + ExecWait(Process, "ProxyAutoDetect", true); + + if (res < 0) + return _error->Errno("read", "Failed to read"); + if (res == 0) + return _error->Warning("ProxyAutoDetect returned no data"); + + // add trailing \0 + buf[res] = 0; + + if (Debug) + std::clog << "auto detect command returned: '" << buf << "'" << std::endl; + + if (strstr(buf, URL.Access.c_str()) == buf) + _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, _strstrip(buf)); + + return true; +} + /*}}}*/ diff --git a/apt-pkg/contrib/proxy.h b/apt-pkg/contrib/proxy.h new file mode 100644 index 000000000..2cbcd07b4 --- /dev/null +++ b/apt-pkg/contrib/proxy.h @@ -0,0 +1,16 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Proxy - Proxy operations + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_PROXY_H +#define PKGLIB_PROXY_H + +class URI; +bool AutoDetectProxy(URI &URL); + + +#endif diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index b0edafcbd..dd43ea1bc 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,19 @@ #include /*}}}*/ +static bool DoAutoDetectProxy(CommandLine &CmdL) +{ + if (CmdL.FileSize() != 2) + return _error->Error(_("Need one URL as argument")); + URI ServerURL(CmdL.FileList[1]); + AutoDetectProxy(ServerURL); + std::string SpecificProxy = _config->Find("Acquire::"+ServerURL.Access+"::Proxy::" + ServerURL.Host); + ioprintf(std::cout, "Using proxy '%s' for URL '%s'\n", + SpecificProxy.c_str(), std::string(ServerURL).c_str()); + + return true; +} + static bool DoDownloadFile(CommandLine &CmdL) { if (CmdL.FileSize() <= 2) @@ -70,6 +84,7 @@ static bool ShowHelp(CommandLine &) "\n" "Commands:\n" " download-file - download the given uri to the target-path\n" + " auto-detect-proxy - detect proxy using apt.conf\n" "\n" " This APT helper has Super Meep Powers.\n"); return true; @@ -80,6 +95,7 @@ int main(int argc,const char *argv[]) /*{{{*/ { CommandLine::Dispatch Cmds[] = {{"help",&ShowHelp}, {"download-file", &DoDownloadFile}, + {"auto-detect-proxy", &DoAutoDetectProxy}, {0,0}}; std::vector Args = getCommandArgs( diff --git a/methods/http.cc b/methods/http.cc index 7c7949eac..f2a4a4db6 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -304,6 +305,7 @@ bool HttpServerState::Open() Persistent = true; // Determine the proxy setting + AutoDetectProxy(ServerName); string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host); if (!SpecificProxy.empty()) { @@ -762,66 +764,6 @@ bool HttpMethod::Configuration(string Message) PipelineDepth); Debug = _config->FindB("Debug::Acquire::http",false); - // Get the proxy to use - AutoDetectProxy(); - - return true; -} - /*}}}*/ -// HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool HttpMethod::AutoDetectProxy() -{ - // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old - // name without the dash ("-") - AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect", - _config->Find("Acquire::http::ProxyAutoDetect")); - - if (AutoDetectProxyCmd.empty()) - return true; - - if (Debug) - clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl; - - int Pipes[2] = {-1,-1}; - if (pipe(Pipes) != 0) - return _error->Errno("pipe", "Failed to create Pipe"); - - pid_t Process = ExecFork(); - if (Process == 0) - { - close(Pipes[0]); - dup2(Pipes[1],STDOUT_FILENO); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[2]; - Args[0] = AutoDetectProxyCmd.c_str(); - Args[1] = 0; - execv(Args[0],(char **)Args); - cerr << "Failed to exec method " << Args[0] << endl; - _exit(100); - } - char buf[512]; - int InFd = Pipes[0]; - close(Pipes[1]); - int res = read(InFd, buf, sizeof(buf)-1); - ExecWait(Process, "ProxyAutoDetect", true); - - if (res < 0) - return _error->Errno("read", "Failed to read"); - if (res == 0) - return _error->Warning("ProxyAutoDetect returned no data"); - - // add trailing \0 - buf[res] = 0; - - if (Debug) - clog << "auto detect command returned: '" << buf << "'" << endl; - - if (strstr(buf, "http://") == buf) - _config->Set("Acquire::http::proxy", _strstrip(buf)); - return true; } /*}}}*/ diff --git a/methods/http.h b/methods/http.h index 5406ce4a7..1df9fa07d 100644 --- a/methods/http.h +++ b/methods/http.h @@ -124,9 +124,6 @@ class HttpMethod : public ServerMethod public: virtual void SendReq(FetchItem *Itm); - /** \brief Try to AutoDetect the proxy */ - bool AutoDetectProxy(); - virtual bool Configuration(std::string Message); virtual ServerState * CreateServerState(URI uri); diff --git a/methods/https.cc b/methods/https.cc index e0348ab58..0499af0c5 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -107,6 +108,9 @@ void HttpsMethod::SetupProxy() /*{{{*/ { URI ServerName = Queue->Uri; + // Determine the proxy setting + AutoDetectProxy(ServerName); + // Curl should never read proxy settings from the environment, as // we determine which proxy to use. Do this for consistency among // methods and prevent an environment variable overriding a diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index 6505b5956..c749224ca 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -9,31 +9,67 @@ configarchitecture "i386" changetohttpswebserver -echo "foo" > aptarchive/foo +test_apt_helper_download() { + echo "foo" > aptarchive/foo -msgtest 'apt-file download-file md5sum' -apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail -testfileequal foo2 'foo' + msgtest 'apt-file download-file md5sum' + apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail + testfileequal foo2 'foo' -msgtest 'apt-file download-file sha1' -apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail -testfileequal foo1 'foo' + msgtest 'apt-file download-file sha1' + apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail + testfileequal foo1 'foo' -msgtest 'apt-file download-file sha256' -apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail -testfileequal foo3 'foo' + msgtest 'apt-file download-file sha256' + apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail + testfileequal foo3 'foo' -msgtest 'apt-file download-file no-hash' -apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail -testfileequal foo4 'foo' - -msgtest 'apt-file download-file wrong hash' -if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then - msgpass -else - msgfail -fi -testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo Hash Sum mismatch + msgtest 'apt-file download-file no-hash' + apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail + testfileequal foo4 'foo' + + msgtest 'apt-file download-file wrong hash' + if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then + msgpass + else + msgfail + fi + testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo Hash Sum mismatch E: Download Failed' -testfileequal foo5.FAILED 'foo' + testfileequal foo5.FAILED 'foo' +} + +test_apt_helper_detect_proxy() { + # no proxy + testequal "Using proxy '' for URL 'http://example.com/'" apthelper auto-detect-proxy http://example.com/ + + + # http auto detect proxy script + cat > apt-proxy-detect <<'EOF' +#!/bin/sh -e +echo "http://some-proxy" +EOF + chmod 755 apt-proxy-detect + echo "Acquire::http::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect + + testequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com + + + # https auto detect proxy script + cat > apt-proxy-detect <<'EOF' +#!/bin/sh -e +echo "https://https-proxy" +EOF + chmod 755 apt-proxy-detect + echo "Acquire::https::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect + + testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com + + + +} + +test_apt_helper_download +test_apt_helper_detect_proxy + -- cgit v1.2.3-70-g09d2 From 097248092ea4ff4543dfb17deb4e0d31dd697c71 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 2 Sep 2014 17:06:52 +0200 Subject: Use heap to allocate PatternMatch to avoid potential stack overflow When apt-cache search with many args (> 130) is given the allocation of PatternMatch on the stack may fail resulting in a segmentation fault. By using the heap the max size is much bigger and we also get a bad_alloc expection instead of a segfault (which we can catch *if* this ever becomes a pratical problem). No test for the crash as its not reproducable with the MALLOC_ settings in framework. Closes: 759612 --- cmdline/apt-cache.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 1414617eb..ac0d48a36 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1278,8 +1278,8 @@ static bool Search(CommandLine &CmdL) ExDescFile *DFList = new ExDescFile[descCount]; memset(DFList,0,sizeof(*DFList) * descCount); - bool PatternMatch[descCount * NumPatterns]; - memset(PatternMatch,false,sizeof(PatternMatch)); + bool *PatternMatch = new bool[descCount * NumPatterns]; + memset(PatternMatch,false,sizeof(*PatternMatch) * descCount * NumPatterns); // Map versions that we want to write out onto the VerList array. for (pkgCache::GrpIterator G = Cache->GrpBegin(); G.end() == false; ++G) @@ -1389,6 +1389,7 @@ static bool Search(CommandLine &CmdL) } delete [] DFList; + delete [] PatternMatch; for (unsigned I = 0; I != NumPatterns; I++) regfree(&Patterns[I]); if (ferror(stdout)) -- cgit v1.2.3-70-g09d2 From b0f4b486e6850c5f98520ccf19da71d0ed748ae4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 21 Sep 2014 10:18:03 +0200 Subject: generalize Acquire::GzipIndex --- apt-pkg/acquire-item.cc | 44 ++++++++++------ apt-pkg/deb/debindexfile.cc | 60 ++++++++++++++-------- apt-private/private-update.cc | 9 +++- cmdline/apt-get.cc | 2 +- methods/copy.cc | 11 ++-- .../test-bug-595691-empty-and-broken-archive-files | 2 +- test/integration/test-compressed-indexes | 20 ++++---- 7 files changed, 90 insertions(+), 58 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 5df43726b..da57f8d3b 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -969,8 +969,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); if (comprExt == "uncompressed") Desc.URI = URI; - else + else { Desc.URI = URI + '.' + comprExt; + DestFile = DestFile + '.' + comprExt; + } Desc.Description = URIDesc; Desc.Owner = this; @@ -984,10 +986,11 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S /* The only header we use is the last-modified header. */ string pkgAcqIndex::Custom600Headers() { + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + Final += compExt; string msg = "\nIndex-File: true"; // FIXME: this really should use "IndexTarget::IsOptional()" but that @@ -1027,8 +1030,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, { std::string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(URI); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - FinalFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false) == true) + FinalFile += '.' + compExt; return FinalFile; } /*}}}*/ @@ -1036,8 +1039,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) { std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - DestFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false) == true) + DestFile += compExt; string FinalFile = GetFinalFilename(RealURI, compExt); Rename(FinalFile, FileName); @@ -1080,7 +1083,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, /* Always verify the index file for correctness (all indexes must * have a Package field) (LP: #346386) (Closes: #627642) */ - FileFd fd(DestFile, FileFd::ReadOnlyGzip); + FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); // Only test for correctness if the file is not empty (empty is ok) if (fd.Size() > 0) { @@ -1104,8 +1107,8 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, will work OK */ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(RealURI); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - DestFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false)) + DestFile += '.' + compExt; // Remove the compressed version. if (Erase == true) @@ -1146,16 +1149,23 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, // matching the Release file if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { + // set destfile to the final destfile + if(_config->FindB("Acquire::GzipIndexes",false) == false) + { + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + } + ReverifyAfterIMS(FileName); return; } string decompProg; // If we enable compressed indexes, queue for hash verification - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) + if (_config->FindB("Acquire::GzipIndexes",false)) { DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI) + ".gz"; + DestFile += URItoFileName(RealURI) + '.' + compExt; Decompression = true; Desc.URI = "copy:" + FileName; @@ -1201,11 +1211,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, // --------------------------------------------------------------------- string pkgAcqIndexTrans::Custom600Headers() { + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); - if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + Final += compExt; struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) @@ -1552,8 +1562,12 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ URItoFileName((*Target)->URI); unlink(index.c_str()); // and also old gzipindexes - index += ".gz"; - unlink(index.c_str()); + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + index += '.' + (*t); + unlink(index.c_str()); + } } } diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index a0dd15cd8..5b4289e92 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -80,14 +80,18 @@ pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const { string SourcesURI = _config->FindDir("Dir::State::lists") + URItoFileName(IndexURI("Sources")); - string SourcesURIgzip = SourcesURI + ".gz"; - if (!FileExists(SourcesURI) && !FileExists(SourcesURIgzip)) - return NULL; - else if (!FileExists(SourcesURI) && FileExists(SourcesURIgzip)) - SourcesURI = SourcesURIgzip; - - return new debSrcRecordParser(SourcesURI,this); + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p; + p = SourcesURI + '.' + *t; + if (FileExists(p)) + return new debSrcRecordParser(p, this); + } + if (FileExists(SourcesURI)) + return new debSrcRecordParser(SourcesURI, this); + return NULL; } /*}}}*/ // SourcesIndex::Describe - Give a descriptive path to the index /*{{{*/ @@ -129,11 +133,15 @@ string debSourcesIndex::Info(const char *Type) const inline string debSourcesIndex::IndexFile(const char *Type) const { string s = URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debSourcesIndex::IndexURI(const char *Type) const @@ -259,11 +267,15 @@ string debPackagesIndex::Info(const char *Type) const inline string debPackagesIndex::IndexFile(const char *Type) const { string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debPackagesIndex::IndexURI(const char *Type) const { @@ -411,11 +423,15 @@ debTranslationsIndex::debTranslationsIndex(string URI,string Dist,string Section inline string debTranslationsIndex::IndexFile(const char *Type) const { string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debTranslationsIndex::IndexURI(const char *Type) const { diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc index 860d84b86..1cf3012ed 100644 --- a/apt-private/private-update.cc +++ b/apt-private/private-update.cc @@ -56,10 +56,17 @@ bool DoUpdate(CommandLine &CmdL) if (List->GetIndexes(&Fetcher,true) == false) return false; + std::string compExt = APT::Configuration::getCompressionTypes()[0]; pkgAcquire::UriIterator I = Fetcher.UriBegin(); for (; I != Fetcher.UriEnd(); ++I) - c1out << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' << + { + std::string FileName = flNotDir(I->Owner->DestFile); + if(compExt.empty() == false && + APT::String::Endswith(FileName, compExt)) + FileName = FileName.substr(0, FileName.size() - compExt.size() - 1); + c1out << '\'' << I->URI << "' " << FileName << ' ' << I->Owner->FileSize << ' ' << I->Owner->HashSum() << std::endl; + } return true; } diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 845d67d2b..2e283da5a 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -665,7 +665,7 @@ static bool DoDownload(CommandLine &CmdL) { pkgAcquire::UriIterator I = Fetcher.UriBegin(); for (; I != Fetcher.UriEnd(); ++I) - cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' << + cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' << I->Owner->FileSize << ' ' << I->Owner->HashSum() << endl; return true; } diff --git a/methods/copy.cc b/methods/copy.cc index b78053d36..40f8f85ec 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -37,15 +37,12 @@ class CopyMethod : public pkgAcqMethod void CopyMethod::CalculateHashes(FetchResult &Res) { - // For gzip indexes we need to look inside the gzip for the hash - // We can not use the extension here as its not used in partial - // on a IMS hit - FileFd::OpenMode OpenMode = FileFd::ReadOnly; + Hashes Hash; + FileFd::CompressMode CompressMode = FileFd::None; if (_config->FindB("Acquire::GzipIndexes", false) == true) - OpenMode = FileFd::ReadOnlyGzip; + CompressMode = FileFd::Extension; - Hashes Hash; - FileFd Fd(Res.Filename, OpenMode); + FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode); Hash.AddFD(Fd); Res.TakeHashes(Hash); } diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index a05ed5fa6..23a638801 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -121,7 +121,7 @@ Reading package lists..." "empty archive Packages.$COMPRESS over http" testaptgetupdate "Get: http://localhost:8080 Packages Err http://localhost:8080 Packages Empty files can't be valid archives -W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives +W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages.${COMPRESS}) Empty files can't be valid archives E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http" } diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes index 6671dd75a..819cbd35e 100755 --- a/test/integration/test-compressed-indexes +++ b/test/integration/test-compressed-indexes @@ -5,7 +5,7 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configcompression '.' 'gz' # only gz is supported for this, so ensure it is used +configcompression '.' 'xz' 'gz' configarchitecture "i386" buildsimplenativepackage "testpkg" "i386" "1.0" @@ -32,13 +32,13 @@ testrun() { if [ "$1" = "compressed" ]; then ! test -e rootdir/var/lib/apt/lists/*_Packages || F=1 ! test -e rootdir/var/lib/apt/lists/*_Sources || F=1 - test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1 - test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1 + test -e rootdir/var/lib/apt/lists/*_Packages.xz || F=1 + test -e rootdir/var/lib/apt/lists/*_Sources.xz || F=1 else test -e rootdir/var/lib/apt/lists/*_Packages || F=1 test -e rootdir/var/lib/apt/lists/*_Sources || F=1 - ! test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1 - ! test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1 + ! test -e rootdir/var/lib/apt/lists/*_Packages.xz || F=1 + ! test -e rootdir/var/lib/apt/lists/*_Sources.xz || F=1 fi if [ -n "$F" ]; then ls -laR rootdir/var/lib/apt/lists/ @@ -84,10 +84,9 @@ msgmsg "File: Test with uncompressed indexes (update unchanged without pdiffs)" testrun rm -rf rootdir/var/lib/apt/lists -echo 'Acquire::CompressionTypes::Order:: "gz"; -Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex +echo 'Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex -testsuccess aptget update +testsuccess aptget update -o Debug::pkgAcquire::worker=1 msgmsg "File: Test with compressed indexes" testrun "compressed" @@ -118,14 +117,13 @@ msgmsg "HTTP: Test with uncompressed indexes (update unchanged without pdiffs)" testrun rm -rf rootdir/var/lib/apt/lists -echo 'Acquire::CompressionTypes::Order:: "gz"; -Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex +echo 'Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex testsuccess aptget update msgmsg "HTTP: Test with compressed indexes" testrun "compressed" -testsuccess aptget update -o Acquire::Pdiffs=1 +testsuccess aptget update -o Acquire::Pdiffs=1 -o debug::pkgAcquire::Worker=1 -o debug::pkgAcquire::Auth=1 msgmsg "HTTP: Test with compressed indexes (update unchanged with pdiffs)" testrun "compressed" -- cgit v1.2.3-70-g09d2 From a3a91fd7bab34983a008854728baf26034d4033e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 20 Jun 2014 20:28:54 +0200 Subject: count strings more accurately for stats So far, only the few strings stored in stringitems were counted, but many more strings are directly inserted into the cache. We account for this now by identifying all these different strings and measure their length. We are still not at the correct size of the cache in 'stats' this way, but we are now again a bit closer. Git-Dch: Ignore --- cmdline/apt-cache.cc | 53 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 14 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 1447feb81..4030ce4e6 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -375,31 +375,56 @@ static bool Stats(CommandLine &) SizeToStr(Cache->Head().DescFileCount*Cache->Head().DescFileSz) << ')' << endl; cout << _("Total Provides mappings: ") << Cache->Head().ProvidesCount << " (" << SizeToStr(Cache->Head().ProvidesCount*Cache->Head().ProvidesSz) << ')' << endl; - - // String list stats - unsigned long Size = 0; - unsigned long Count = 0; - for (pkgCache::StringItem *I = Cache->StringItemP + Cache->Head().StringList; - I!= Cache->StringItemP; I = Cache->StringItemP + I->NextItem) - { - Count++; - Size += strlen(Cache->StrP + I->String) + 1; - } - cout << _("Total globbed strings: ") << Count << " (" << SizeToStr(Size) << ')' << endl; - unsigned long DepVerSize = 0; + // String list stats + std::set stritems; + for (pkgCache::GrpIterator G = Cache->GrpBegin(); G.end() == false; ++G) + stritems.insert(G->Name); for (pkgCache::PkgIterator P = Cache->PkgBegin(); P.end() == false; ++P) { + stritems.insert(P->Arch); for (pkgCache::VerIterator V = P.VersionList(); V.end() == false; ++V) { + if (V->VerStr != 0) + stritems.insert(V->VerStr); + if (V->Section != 0) + stritems.insert(V->Section); for (pkgCache::DepIterator D = V.DependsList(); D.end() == false; ++D) { if (D->Version != 0) - DepVerSize += strlen(D.TargetVer()) + 1; + stritems.insert(D->Version); + } + for (pkgCache::DescIterator D = V.DescriptionList(); D.end() == false; ++D) + { + stritems.insert(D->md5sum); + stritems.insert(D->language_code); } } + for (pkgCache::PrvIterator Prv = P.ProvidesList(); Prv.end() == false; ++Prv) + { + if (Prv->ProvideVersion != 0) + stritems.insert(Prv->ProvideVersion); + } } - cout << _("Total dependency version space: ") << SizeToStr(DepVerSize) << endl; + for (pkgCache::PkgFileIterator F = Cache->FileBegin(); F != Cache->FileEnd(); ++F) + { + stritems.insert(F->FileName); + stritems.insert(F->Archive); + stritems.insert(F->Codename); + stritems.insert(F->Component); + stritems.insert(F->Version); + stritems.insert(F->Origin); + stritems.insert(F->Label); + stritems.insert(F->Architecture); + stritems.insert(F->Site); + stritems.insert(F->IndexType); + } + unsigned long Size = 0; + for (std::set::const_iterator i = stritems.begin(); i != stritems.end(); ++i) + Size += strlen(Cache->StrP + *i) + 1; + + cout << _("Total globbed strings: ") << stritems.size() << " (" << SizeToStr(Size) << ')' << endl; + stritems.clear(); unsigned long Slack = 0; for (int I = 0; I != 7; I++) -- cgit v1.2.3-70-g09d2 From 67caa2e6538f13ff7a0d77e98ad6c58af998376d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 17 Aug 2014 13:27:59 +0200 Subject: mark pkg(All|Dist)Upgrade as deprecated The comment above their definition marks them already as such, so this is only a formalisation of the deprecation and fixes the occurances we have in our own code together with removing a magic number. Git-Dch: Ignore --- apt-pkg/upgrade.cc | 11 +++++++++-- apt-pkg/upgrade.h | 6 ++++-- apt-private/private-install.cc | 12 ++++-------- apt-private/private-upgrade.cc | 2 +- cmdline/apt-get.cc | 2 +- cmdline/apt-internal-solver.cc | 4 ++-- 6 files changed, 21 insertions(+), 16 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/upgrade.cc b/apt-pkg/upgrade.cc index 20a38ecee..ca670bdf5 100644 --- a/apt-pkg/upgrade.cc +++ b/apt-pkg/upgrade.cc @@ -282,7 +282,12 @@ bool pkgMinimizeUpgrade(pkgDepCache &Cache) // APT::Upgrade::Upgrade - Upgrade using a specific strategy /*{{{*/ bool APT::Upgrade::Upgrade(pkgDepCache &Cache, int mode, OpProgress * const Progress) { - if (mode == 0) +#if __GNUC__ >= 4 + // the deprecated methods will become our privates, so that is fine + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + if (mode == ALLOW_EVERYTHING) return pkgDistUpgrade(Cache, Progress); else if ((mode & ~FORBID_REMOVE_PACKAGES) == 0) return pkgAllUpgradeWithNewPackages(Cache, Progress); @@ -290,7 +295,9 @@ bool APT::Upgrade::Upgrade(pkgDepCache &Cache, int mode, OpProgress * const Prog return pkgAllUpgradeNoNewPackages(Cache, Progress); else _error->Error("pkgAllUpgrade called with unsupported mode %i", mode); - +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif return false; } /*}}}*/ diff --git a/apt-pkg/upgrade.h b/apt-pkg/upgrade.h index 894f0625e..a3f693d86 100644 --- a/apt-pkg/upgrade.h +++ b/apt-pkg/upgrade.h @@ -11,6 +11,7 @@ #define PKGLIB_UPGRADE_H #include +#include class pkgDepCache; class OpProgress; @@ -19,6 +20,7 @@ namespace APT { namespace Upgrade { // FIXME: make this "enum class UpgradeMode {" once we enable c++11 enum UpgradeMode { + ALLOW_EVERYTHING = 0, FORBID_REMOVE_PACKAGES = 1, FORBID_INSTALL_NEW_PACKAGES = 2 }; @@ -27,8 +29,8 @@ namespace APT { } // please use APT::Upgrade::Upgrade() instead -bool pkgDistUpgrade(pkgDepCache &Cache, OpProgress * const Progress = NULL); -bool pkgAllUpgrade(pkgDepCache &Cache, OpProgress * const Progress = NULL); +APT_DEPRECATED bool pkgDistUpgrade(pkgDepCache &Cache, OpProgress * const Progress = NULL); +APT_DEPRECATED bool pkgAllUpgrade(pkgDepCache &Cache, OpProgress * const Progress = NULL); bool pkgMinimizeUpgrade(pkgDepCache &Cache); #endif diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index 656b97233..c172f5ee7 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -619,18 +619,14 @@ bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, CacheFile &Cache, if (Fix != NULL) { // Call the scored problem resolver - bool resolver_fail = false; OpTextProgress Progress(*_config); bool const distUpgradeMode = strcmp(CmdL.FileList[0], "dist-upgrade") == 0 || strcmp(CmdL.FileList[0], "full-upgrade") == 0; - if (UpgradeMode == 0) - { - if (distUpgradeMode == true) - resolver_fail = APT::Upgrade::Upgrade(Cache, 0, &Progress); - else - resolver_fail = Fix->Resolve(true, &Progress); - } else + bool resolver_fail = false; + if (distUpgradeMode == true || UpgradeMode != APT::Upgrade::ALLOW_EVERYTHING) resolver_fail = APT::Upgrade::Upgrade(Cache, UpgradeMode, &Progress); + else + resolver_fail = Fix->Resolve(true, &Progress); if (resolver_fail == false && Cache->BrokenCount() == 0) return false; diff --git a/apt-private/private-upgrade.cc b/apt-private/private-upgrade.cc index 2a6dcc153..d13a6af49 100644 --- a/apt-private/private-upgrade.cc +++ b/apt-private/private-upgrade.cc @@ -33,7 +33,7 @@ static bool UpgradeHelper(CommandLine &CmdL, int UpgradeFlags) /* Intelligent upgrader that will install and remove packages at will */ bool DoDistUpgrade(CommandLine &CmdL) { - return UpgradeHelper(CmdL, 0); + return UpgradeHelper(CmdL, APT::Upgrade::ALLOW_EVERYTHING); } /*}}}*/ bool DoUpgrade(CommandLine &CmdL) /*{{{*/ diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index d6cd94f72..aed1beb4d 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -540,7 +540,7 @@ static bool DoDSelectUpgrade(CommandLine &) } // Now upgrade everything - if (pkgAllUpgrade(Cache) == false) + if (APT::Upgrade::Upgrade(Cache, APT::Upgrade::FORBID_REMOVE_PACKAGES | APT::Upgrade::FORBID_INSTALL_NEW_PACKAGES) == false) { ShowBroken(c1out,Cache,false); return _error->Error(_("Internal error, problem resolver broke stuff")); diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index c24a96cdf..0f2ec6283 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -175,10 +175,10 @@ int main(int argc,const char *argv[]) /*{{{*/ std::string failure; if (upgrade == true) { - if (pkgAllUpgrade(CacheFile) == false) + if (APT::Upgrade::Upgrade(CacheFile, APT::Upgrade::FORBID_REMOVE_PACKAGES | APT::Upgrade::FORBID_INSTALL_NEW_PACKAGES) == false) failure = "ERR_UNSOLVABLE_UPGRADE"; } else if (distUpgrade == true) { - if (pkgDistUpgrade(CacheFile) == false) + if (APT::Upgrade::Upgrade(CacheFile, APT::Upgrade::ALLOW_EVERYTHING) == false) failure = "ERR_UNSOLVABLE_DIST_UPGRADE"; } else if (Fix.Resolve() == false) failure = "ERR_UNSOLVABLE"; -- cgit v1.2.3-70-g09d2 From fdba4d53d6b9b594531c34792798f4044a25157e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 3 Sep 2014 18:16:16 +0200 Subject: rework cachesets API to allow future extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The introduction of Fnmatch showed that each new selector would require multiple new virtual methods in the CacheSetHelper to work correctly, which isn't that great. We now flip to a single virtual method which handles all cases separated by an enum – as new enum values can be added without an ABI break. Great care was taken to make old code work with the new way of organisation, which means in return that you might be bombarded with deprecation warnings now if you don't adapt, but code should still compile and work as before as can be seen in apt itself with this commit. Git-Dch: Ignore --- apt-pkg/cacheset.cc | 264 +++++++++++++++++++++-------- apt-pkg/cacheset.h | 364 +++++++++++++++++++++++++++++++--------- apt-private/private-cacheset.cc | 8 +- cmdline/apt-cache.cc | 12 +- 4 files changed, 494 insertions(+), 154 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/cacheset.cc b/apt-pkg/cacheset.cc index d95a32aff..3c38895e9 100644 --- a/apt-pkg/cacheset.cc +++ b/apt-pkg/cacheset.cc @@ -55,7 +55,7 @@ bool PackageContainerInterface::FromTask(PackageContainerInterface * const pci, bool const wasEmpty = pci->empty(); if (wasEmpty == true) - pci->setConstructor(TASK); + pci->setConstructor(CacheSetHelper::TASK); // get the records pkgRecords Recs(Cache); @@ -91,19 +91,19 @@ bool PackageContainerInterface::FromTask(PackageContainerInterface * const pci, continue; pci->insert(Pkg); - helper.showTaskSelection(Pkg, pattern); + helper.showPackageSelection(Pkg, CacheSetHelper::TASK, pattern); found = true; } regfree(&Pattern); if (found == false) { - helper.canNotFindTask(pci, Cache, pattern); - pci->setConstructor(UNKNOWN); + helper.canNotFindPackage(CacheSetHelper::TASK, pci, Cache, pattern); + pci->setConstructor(CacheSetHelper::UNKNOWN); return false; } - if (wasEmpty == false && pci->getConstructor() != UNKNOWN) - pci->setConstructor(UNKNOWN); + if (wasEmpty == false && pci->getConstructor() != CacheSetHelper::UNKNOWN) + pci->setConstructor(CacheSetHelper::UNKNOWN); return true; } @@ -116,7 +116,7 @@ bool PackageContainerInterface::FromRegEx(PackageContainerInterface * const pci, bool const wasEmpty = pci->empty(); if (wasEmpty == true) - pci->setConstructor(REGEX); + pci->setConstructor(CacheSetHelper::REGEX); size_t archfound = pattern.find_last_of(':'); std::string arch = "native"; @@ -150,18 +150,18 @@ bool PackageContainerInterface::FromRegEx(PackageContainerInterface * const pci, } pci->insert(Pkg); - helper.showRegExSelection(Pkg, pattern); + helper.showPackageSelection(Pkg, CacheSetHelper::REGEX, pattern); found = true; } if (found == false) { - helper.canNotFindRegEx(pci, Cache, pattern); - pci->setConstructor(UNKNOWN); + helper.canNotFindPackage(CacheSetHelper::REGEX, pci, Cache, pattern); + pci->setConstructor(CacheSetHelper::UNKNOWN); return false; } - if (wasEmpty == false && pci->getConstructor() != UNKNOWN) - pci->setConstructor(UNKNOWN); + if (wasEmpty == false && pci->getConstructor() != CacheSetHelper::UNKNOWN) + pci->setConstructor(CacheSetHelper::UNKNOWN); return true; } @@ -179,7 +179,7 @@ PackageContainerInterface::FromFnmatch(PackageContainerInterface * const pci, bool const wasEmpty = pci->empty(); if (wasEmpty == true) - pci->setConstructor(FNMATCH); + pci->setConstructor(CacheSetHelper::FNMATCH); size_t archfound = pattern.find_last_of(':'); std::string arch = "native"; @@ -213,26 +213,18 @@ PackageContainerInterface::FromFnmatch(PackageContainerInterface * const pci, } pci->insert(Pkg); -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) - helper.showFnmatchSelection(Pkg, pattern); -#else - helper.showRegExSelection(Pkg, pattern); -#endif + helper.showPackageSelection(Pkg, CacheSetHelper::FNMATCH, pattern); found = true; } if (found == false) { -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) - helper.canNotFindFnmatch(pci, Cache, pattern); -#else - helper.canNotFindRegEx(pci, Cache, pattern); -#endif - pci->setConstructor(UNKNOWN); + helper.canNotFindPackage(CacheSetHelper::FNMATCH, pci, Cache, pattern); + pci->setConstructor(CacheSetHelper::UNKNOWN); return false; } - if (wasEmpty == false && pci->getConstructor() != UNKNOWN) - pci->setConstructor(UNKNOWN); + if (wasEmpty == false && pci->getConstructor() != CacheSetHelper::UNKNOWN) + pci->setConstructor(CacheSetHelper::UNKNOWN); return true; } @@ -321,13 +313,11 @@ bool PackageContainerInterface::FromString(PackageContainerInterface * const pci if (FromGroup(pci, Cache, str, helper) == false && FromTask(pci, Cache, str, helper) == false && -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) - // FIXME: hm, hm, regexp/fnmatch incompatible? + // FIXME: hm, hm, regexp/fnmatch incompatible? FromFnmatch(pci, Cache, str, helper) == false && -#endif FromRegEx(pci, Cache, str, helper) == false) { - helper.canNotFindPackage(pci, Cache, str); + helper.canNotFindPackage(CacheSetHelper::PACKAGENAME, pci, Cache, str); found = false; } @@ -391,7 +381,7 @@ bool VersionContainerInterface::FromModifierCommandLine(unsigned short &modID, pkgCacheFile &Cache, const char * cmdline, std::list const &mods, CacheSetHelper &helper) { - Version select = NEWEST; + CacheSetHelper::VerSelector select = CacheSetHelper::NEWEST; std::string str = cmdline; if (unlikely(str.empty() == true)) return false; @@ -434,7 +424,8 @@ bool VersionContainerInterface::FromModifierCommandLine(unsigned short &modID, // FromCommandLine - Return all versions specified on commandline /*{{{*/ bool VersionContainerInterface::FromCommandLine(VersionContainerInterface * const vci, pkgCacheFile &Cache, const char **cmdline, - Version const &fallback, CacheSetHelper &helper) { + CacheSetHelper::VerSelector const fallback, + CacheSetHelper &helper) { bool found = false; for (const char **I = cmdline; *I != 0; ++I) found |= VersionContainerInterface::FromString(vci, Cache, *I, fallback, helper); @@ -444,7 +435,8 @@ bool VersionContainerInterface::FromCommandLine(VersionContainerInterface * cons // FromString - Returns all versions spedcified by a string /*{{{*/ bool VersionContainerInterface::FromString(VersionContainerInterface * const vci, pkgCacheFile &Cache, std::string pkg, - Version const &fallback, CacheSetHelper &helper, + CacheSetHelper::VerSelector const fallback, + CacheSetHelper &helper, bool const onlyFromName) { PackageSet pkgset; if(FileExists(pkg)) @@ -470,7 +462,7 @@ bool VersionContainerInterface::FromString(VersionContainerInterface * const vci } bool errors = true; - if (pkgset.getConstructor() != PackageSet::UNKNOWN) + if (pkgset.getConstructor() != CacheSetHelper::UNKNOWN) errors = helper.showErrors(false); bool found = false; @@ -489,7 +481,7 @@ bool VersionContainerInterface::FromString(VersionContainerInterface * const vci if (P->VersionList != 0) V = P.VersionList(); else - V = helper.canNotFindNewestVer(Cache, P); + V = helper.canNotGetVersion(CacheSetHelper::NEWEST, Cache, P); } else { pkgVersionMatch Match(ver, (verIsRel == true ? pkgVersionMatch::Release : pkgVersionMatch::Version)); @@ -506,11 +498,14 @@ bool VersionContainerInterface::FromString(VersionContainerInterface * const vci } if (V.end() == true) continue; - helper.showSelectedVersion(P, V, ver, verIsRel); + if (verIsRel == true) + helper.showVersionSelection(P, V, CacheSetHelper::RELEASE, ver); + else + helper.showVersionSelection(P, V, CacheSetHelper::VERSIONNUMBER, ver); vci->insert(V); found = true; } - if (pkgset.getConstructor() != PackageSet::UNKNOWN) + if (pkgset.getConstructor() != CacheSetHelper::UNKNOWN) helper.showErrors(errors); return found; } @@ -519,30 +514,30 @@ bool VersionContainerInterface::FromString(VersionContainerInterface * const vci bool VersionContainerInterface::FromPackage(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &P, - Version const &fallback, + CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper) { pkgCache::VerIterator V; bool showErrors; bool found = false; switch(fallback) { - case ALL: + case CacheSetHelper::ALL: if (P->VersionList != 0) for (V = P.VersionList(); V.end() != true; ++V) found |= vci->insert(V); else - helper.canNotFindAllVer(vci, Cache, P); + helper.canNotFindVersion(CacheSetHelper::ALL, vci, Cache, P); break; - case CANDANDINST: + case CacheSetHelper::CANDANDINST: found |= vci->insert(getInstalledVer(Cache, P, helper)); found |= vci->insert(getCandidateVer(Cache, P, helper)); break; - case CANDIDATE: + case CacheSetHelper::CANDIDATE: found |= vci->insert(getCandidateVer(Cache, P, helper)); break; - case INSTALLED: + case CacheSetHelper::INSTALLED: found |= vci->insert(getInstalledVer(Cache, P, helper)); break; - case CANDINST: + case CacheSetHelper::CANDINST: showErrors = helper.showErrors(false); V = getCandidateVer(Cache, P, helper); if (V.end() == true) @@ -551,9 +546,9 @@ bool VersionContainerInterface::FromPackage(VersionContainerInterface * const vc if (V.end() == false) found |= vci->insert(V); else - helper.canNotFindInstCandVer(vci, Cache, P); + helper.canNotFindVersion(CacheSetHelper::CANDINST, vci, Cache, P); break; - case INSTCAND: + case CacheSetHelper::INSTCAND: showErrors = helper.showErrors(false); V = getInstalledVer(Cache, P, helper); if (V.end() == true) @@ -562,14 +557,18 @@ bool VersionContainerInterface::FromPackage(VersionContainerInterface * const vc if (V.end() == false) found |= vci->insert(V); else - helper.canNotFindInstCandVer(vci, Cache, P); + helper.canNotFindVersion(CacheSetHelper::INSTCAND, vci, Cache, P); break; - case NEWEST: + case CacheSetHelper::NEWEST: if (P->VersionList != 0) found |= vci->insert(P.VersionList()); else - helper.canNotFindNewestVer(Cache, P); + helper.canNotFindVersion(CacheSetHelper::NEWEST, vci, Cache, P); break; + case CacheSetHelper::RELEASE: + case CacheSetHelper::VERSIONNUMBER: + // both make no sense here, so always false + return false; } return found; } @@ -586,7 +585,7 @@ pkgCache::VerIterator VersionContainerInterface::getCandidateVer(pkgCacheFile &C Cand = Cache[Pkg].CandidateVerIter(Cache); } if (Cand.end() == true) - return helper.canNotFindCandidateVer(Cache, Pkg); + return helper.canNotGetVersion(CacheSetHelper::CANDIDATE, Cache, Pkg); return Cand; } /*}}}*/ @@ -594,19 +593,30 @@ pkgCache::VerIterator VersionContainerInterface::getCandidateVer(pkgCacheFile &C pkgCache::VerIterator VersionContainerInterface::getInstalledVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg, CacheSetHelper &helper) { if (Pkg->CurrentVer == 0) - return helper.canNotFindInstalledVer(Cache, Pkg); + return helper.canNotGetVersion(CacheSetHelper::INSTALLED, Cache, Pkg); return Pkg.CurrentVer(); } /*}}}*/ -// canNotFindPkgName - handle the case no package has this name /*{{{*/ -pkgCache::PkgIterator CacheSetHelper::canNotFindPkgName(pkgCacheFile &Cache, - std::string const &str) { - if (ShowError == true) - _error->Insert(ErrorType, _("Unable to locate package %s"), str.c_str()); - return pkgCache::PkgIterator(Cache, 0); +// canNotFindPackage - with the given selector and pattern /*{{{*/ +void CacheSetHelper::canNotFindPackage(enum PkgSelector const select, + PackageContainerInterface * const pci, pkgCacheFile &Cache, + std::string const &pattern) { + switch (select) { +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + case REGEX: canNotFindRegEx(pci, Cache, pattern); break; + case TASK: canNotFindTask(pci, Cache, pattern); break; + case FNMATCH: canNotFindFnmatch(pci, Cache, pattern); break; + case PACKAGENAME: canNotFindPackage(pci, Cache, pattern); break; + case UNKNOWN: break; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + } } - /*}}}*/ // canNotFindTask - handle the case no package is found for a task /*{{{*/ void CacheSetHelper::canNotFindTask(PackageContainerInterface * const /*pci*/, pkgCacheFile &/*Cache*/, std::string pattern) { if (ShowError == true) @@ -618,17 +628,50 @@ void CacheSetHelper::canNotFindRegEx(PackageContainerInterface * const /*pci*/, if (ShowError == true) _error->Insert(ErrorType, _("Couldn't find any package by regex '%s'"), pattern.c_str()); } -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) + /*}}}*/ // canNotFindFnmatch - handle the case no package is found by a fnmatch /*{{{*/ void CacheSetHelper::canNotFindFnmatch(PackageContainerInterface * const /*pci*/, pkgCacheFile &/*Cache*/, std::string pattern) { if (ShowError == true) _error->Insert(ErrorType, _("Couldn't find any package by glob '%s'"), pattern.c_str()); } -#endif /*}}}*/ + /*}}}*/ // canNotFindPackage - handle the case no package is found from a string/*{{{*/ APT_CONST void CacheSetHelper::canNotFindPackage(PackageContainerInterface * const /*pci*/, pkgCacheFile &/*Cache*/, std::string const &/*str*/) { } /*}}}*/ + /*}}}*/ +// canNotFindPkgName - handle the case no package has this name /*{{{*/ +pkgCache::PkgIterator CacheSetHelper::canNotFindPkgName(pkgCacheFile &Cache, + std::string const &str) { + if (ShowError == true) + _error->Insert(ErrorType, _("Unable to locate package %s"), str.c_str()); + return pkgCache::PkgIterator(Cache, 0); +} + /*}}}*/ +// canNotFindVersion - for package by selector /*{{{*/ +void CacheSetHelper::canNotFindVersion(enum VerSelector const select, VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) +{ + switch (select) { +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + case ALL: canNotFindAllVer(vci, Cache, Pkg); break; + case INSTCAND: canNotFindInstCandVer(vci, Cache, Pkg); break; + case CANDINST: canNotFindCandInstVer(vci, Cache, Pkg); break; + case NEWEST: canNotFindNewestVer(Cache, Pkg); break; + case CANDIDATE: canNotFindCandidateVer(Cache, Pkg); break; + case INSTALLED: canNotFindInstalledVer(Cache, Pkg); break; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + case CANDANDINST: canNotGetCandInstVer(Cache, Pkg); break; + case RELEASE: + case VERSIONNUMBER: + // invalid in this branch + break; + } +} // canNotFindAllVer /*{{{*/ void CacheSetHelper::canNotFindAllVer(VersionContainerInterface * const /*vci*/, pkgCacheFile &/*Cache*/, pkgCache::PkgIterator const &Pkg) { @@ -637,19 +680,42 @@ void CacheSetHelper::canNotFindAllVer(VersionContainerInterface * const /*vci*/, } /*}}}*/ // canNotFindInstCandVer /*{{{*/ -void CacheSetHelper::canNotFindInstCandVer(VersionContainerInterface * const /*vci*/, pkgCacheFile &/*Cache*/, +void CacheSetHelper::canNotFindInstCandVer(VersionContainerInterface * const /*vci*/, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { - if (ShowError == true) - _error->Insert(ErrorType, _("Can't select installed nor candidate version from package '%s' as it has neither of them"), Pkg.FullName(true).c_str()); + canNotGetInstCandVer(Cache, Pkg); } /*}}}*/ // canNotFindInstCandVer /*{{{*/ -void CacheSetHelper::canNotFindCandInstVer(VersionContainerInterface * const /*vci*/, pkgCacheFile &/*Cache*/, +void CacheSetHelper::canNotFindCandInstVer(VersionContainerInterface * const /*vci*/, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { - if (ShowError == true) - _error->Insert(ErrorType, _("Can't select installed nor candidate version from package '%s' as it has neither of them"), Pkg.FullName(true).c_str()); + canNotGetCandInstVer(Cache, Pkg); } /*}}}*/ + /*}}}*/ +// canNotGetVersion - for package by selector /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotGetVersion(enum VerSelector const select, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { + switch (select) { +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + case NEWEST: return canNotFindNewestVer(Cache, Pkg); + case CANDIDATE: return canNotFindCandidateVer(Cache, Pkg); + case INSTALLED: return canNotFindInstalledVer(Cache, Pkg); +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + case CANDINST: return canNotGetCandInstVer(Cache, Pkg); + case INSTCAND: return canNotGetInstCandVer(Cache, Pkg); + case ALL: + case CANDANDINST: + case RELEASE: + case VERSIONNUMBER: + // invalid in this branch + return pkgCache::VerIterator(Cache, 0); + } + return pkgCache::VerIterator(Cache, 0); +} // canNotFindNewestVer /*{{{*/ pkgCache::VerIterator CacheSetHelper::canNotFindNewestVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { @@ -674,6 +740,41 @@ pkgCache::VerIterator CacheSetHelper::canNotFindInstalledVer(pkgCacheFile &Cache return pkgCache::VerIterator(Cache, 0); } /*}}}*/ +// canNotFindInstCandVer /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotGetInstCandVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + if (ShowError == true) + _error->Insert(ErrorType, _("Can't select installed nor candidate version from package '%s' as it has neither of them"), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} + /*}}}*/ +// canNotFindInstCandVer /*{{{*/ +pkgCache::VerIterator CacheSetHelper::canNotGetCandInstVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg) { + if (ShowError == true) + _error->Insert(ErrorType, _("Can't select installed nor candidate version from package '%s' as it has neither of them"), Pkg.FullName(true).c_str()); + return pkgCache::VerIterator(Cache, 0); +} + /*}}}*/ + /*}}}*/ +// showPackageSelection - by selector and given pattern /*{{{*/ +APT_CONST void CacheSetHelper::showPackageSelection(pkgCache::PkgIterator const &pkg, enum PkgSelector const select, + std::string const &pattern) { + switch (select) { +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + case REGEX: showRegExSelection(pkg, pattern); break; + case TASK: showTaskSelection(pkg, pattern); break; + case FNMATCH: showFnmatchSelection(pkg, pattern); break; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + case PACKAGENAME: /* no suprises here */ break; + case UNKNOWN: break; + } +} // showTaskSelection /*{{{*/ APT_CONST void CacheSetHelper::showTaskSelection(pkgCache::PkgIterator const &/*pkg*/, std::string const &/*pattern*/) { @@ -684,14 +785,41 @@ APT_CONST void CacheSetHelper::showRegExSelection(pkgCache::PkgIterator const &/ std::string const &/*pattern*/) { } /*}}}*/ -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) // showFnmatchSelection /*{{{*/ APT_CONST void CacheSetHelper::showFnmatchSelection(pkgCache::PkgIterator const &/*pkg*/, std::string const &/*pattern*/) { } /*}}}*/ + /*}}}*/ +// showVersionSelection /*{{{*/ +APT_CONST void CacheSetHelper::showVersionSelection(pkgCache::PkgIterator const &Pkg, + pkgCache::VerIterator const &Ver, enum VerSelector const select, std::string const &pattern) { + switch (select) { +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + case RELEASE: + showSelectedVersion(Pkg, Ver, pattern, true); + break; + case VERSIONNUMBER: + showSelectedVersion(Pkg, Ver, pattern, false); + break; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif -// showSelectedVersion /*{{{*/ + case NEWEST: + case CANDIDATE: + case INSTALLED: + case CANDINST: + case INSTCAND: + case ALL: + case CANDANDINST: + // not really suprises, but in fact: just not implemented + break; + } +} APT_CONST void CacheSetHelper::showSelectedVersion(pkgCache::PkgIterator const &/*Pkg*/, pkgCache::VerIterator const /*Ver*/, std::string const &/*ver*/, diff --git a/apt-pkg/cacheset.h b/apt-pkg/cacheset.h index 2656727d4..57a956808 100644 --- a/apt-pkg/cacheset.h +++ b/apt-pkg/cacheset.h @@ -53,36 +53,109 @@ public: /*{{{*/ ShowError(ShowError), ErrorType(ErrorType) {} virtual ~CacheSetHelper() {} - virtual void showTaskSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); - virtual void showRegExSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) - virtual void showFnmatchSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); -#endif - virtual void showSelectedVersion(pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const Ver, - std::string const &ver, bool const verIsRel); + enum PkgSelector { UNKNOWN, PACKAGENAME, REGEX, TASK, FNMATCH }; + + /** \brief be notified about the package being selected via pattern + * + * Main use is probably to show a message to the user what happened + * + * \param pkg is the package which was selected + * \param select is the selection method which choose the package + * \param pattern is the string used by the selection method to pick the package + */ + virtual void showPackageSelection(pkgCache::PkgIterator const &pkg, PkgSelector const select, std::string const &pattern); + // use the method above instead, react only on the type you need and let the base handle the rest if need be + // this allows use to add new selection methods without breaking the ABI constantly with new virtual methods + APT_DEPRECATED virtual void showTaskSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); + APT_DEPRECATED virtual void showRegExSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); + APT_DEPRECATED virtual void showFnmatchSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern); + + /** \brief be notified if a package can't be found via pattern + * + * Can be used to show a message as well as to try something else to make it match + * + * \param select is the method tried for selection + * \param pci is the container the package should be inserted in + * \param Cache is the package universe available + * \param pattern is the string not matching anything + */ + virtual void canNotFindPackage(enum PkgSelector const select, PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern); + // same as above for showPackageSelection + APT_DEPRECATED virtual void canNotFindTask(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); + APT_DEPRECATED virtual void canNotFindRegEx(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); + APT_DEPRECATED virtual void canNotFindFnmatch(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); + APT_DEPRECATED virtual void canNotFindPackage(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &str); + + /** \brief specifies which version(s) we want to refer to */ + enum VerSelector { + /** by release string */ + RELEASE, + /** by version number string */ + VERSIONNUMBER, + /** All versions */ + ALL, + /** Candidate and installed version */ + CANDANDINST, + /** Candidate version */ + CANDIDATE, + /** Installed version */ + INSTALLED, + /** Candidate or if non installed version */ + CANDINST, + /** Installed or if non candidate version */ + INSTCAND, + /** Newest version */ + NEWEST + }; - virtual void canNotFindTask(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); - virtual void canNotFindRegEx(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) - virtual void canNotFindFnmatch(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern); -#endif - virtual void canNotFindPackage(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &str); + /** \brief be notified about the version being selected via pattern + * + * Main use is probably to show a message to the user what happened + * Note that at the moment this method is only called for RELEASE + * and VERSION selections, not for the others. + * + * \param Pkg is the package which was selected for + * \param Ver is the version selected + * \param select is the selection method which choose the version + * \param pattern is the string used by the selection method to pick the version + */ + virtual void showVersionSelection(pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const &Ver, + enum VerSelector const select, std::string const &pattern); + // renamed to have a similar interface to showPackageSelection + APT_DEPRECATED virtual void showSelectedVersion(pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const Ver, + std::string const &ver, bool const verIsRel); - virtual void canNotFindAllVer(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); - virtual void canNotFindInstCandVer(VersionContainerInterface * const vci, pkgCacheFile &Cache, + /** \brief be notified if a version can't be found for a package + * + * Main use is probably to show a message to the user what happened + * + * \param select is the method tried for selection + * \param vci is the container the version should be inserted in + * \param Cache is the package universe available + * \param Pkg is the package we wanted a version from + */ + virtual void canNotFindVersion(enum VerSelector const select, VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + // same as above for showPackageSelection + APT_DEPRECATED virtual void canNotFindAllVer(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + APT_DEPRECATED virtual void canNotFindInstCandVer(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); - virtual void canNotFindCandInstVer(VersionContainerInterface * const vci, + APT_DEPRECATED virtual void canNotFindCandInstVer(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); - virtual pkgCache::PkgIterator canNotFindPkgName(pkgCacheFile &Cache, std::string const &str); - virtual pkgCache::VerIterator canNotFindNewestVer(pkgCacheFile &Cache, + // the difference between canNotFind and canNotGet is that the later is more low-level + // and called from other places: In this case looking into the code is the only real answer… + virtual pkgCache::VerIterator canNotGetVersion(enum VerSelector const select, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + // same as above for showPackageSelection + APT_DEPRECATED virtual pkgCache::VerIterator canNotFindNewestVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); - virtual pkgCache::VerIterator canNotFindCandidateVer(pkgCacheFile &Cache, + APT_DEPRECATED virtual pkgCache::VerIterator canNotFindCandidateVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); - virtual pkgCache::VerIterator canNotFindInstalledVer(pkgCacheFile &Cache, + APT_DEPRECATED virtual pkgCache::VerIterator canNotFindInstalledVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg); + virtual pkgCache::PkgIterator canNotFindPkgName(pkgCacheFile &Cache, std::string const &str); + bool showErrors() const { return ShowError; } bool showErrors(bool const newValue) { if (ShowError == newValue) return ShowError; else return ((ShowError = newValue) == false); } GlobalError::MsgType errorType() const { return ErrorType; } @@ -100,6 +173,11 @@ public: /*{{{*/ protected: bool ShowError; GlobalError::MsgType ErrorType; + + pkgCache::VerIterator canNotGetInstCandVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg); + pkgCache::VerIterator canNotGetCandInstVer(pkgCacheFile &Cache, + pkgCache::PkgIterator const &Pkg); }; /*}}}*/ class PackageContainerInterface { /*{{{*/ @@ -154,9 +232,24 @@ public: virtual bool empty() const = 0; virtual void clear() = 0; - enum Constructor { UNKNOWN, REGEX, TASK, FNMATCH }; - virtual void setConstructor(Constructor const &con) = 0; - virtual Constructor getConstructor() const = 0; + // FIXME: This is a bloody hack removed soon. Use CacheSetHelper::PkgSelector ! + enum APT_DEPRECATED Constructor { UNKNOWN = CacheSetHelper::UNKNOWN, + REGEX = CacheSetHelper::REGEX, + TASK = CacheSetHelper::TASK, + FNMATCH = CacheSetHelper::FNMATCH }; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + void setConstructor(Constructor const by) { ConstructedBy = (CacheSetHelper::PkgSelector)by; } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + + void setConstructor(CacheSetHelper::PkgSelector const by) { ConstructedBy = by; } + CacheSetHelper::PkgSelector getConstructor() const { return ConstructedBy; } + PackageContainerInterface() : ConstructedBy(CacheSetHelper::UNKNOWN) {} + PackageContainerInterface(CacheSetHelper::PkgSelector const by) : ConstructedBy(by) {} static bool FromTask(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern, CacheSetHelper &helper); static bool FromRegEx(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern, CacheSetHelper &helper); @@ -177,6 +270,9 @@ public: static bool FromModifierCommandLine(unsigned short &modID, PackageContainerInterface * const pci, pkgCacheFile &Cache, const char * cmdline, std::list const &mods, CacheSetHelper &helper); + +private: + CacheSetHelper::PkgSelector ConstructedBy; }; /*}}}*/ template class PackageContainer : public PackageContainerInterface {/*{{{*/ @@ -240,11 +336,16 @@ public: /*{{{*/ iterator end() { return iterator(_cont.end()); } const_iterator find(pkgCache::PkgIterator const &P) const { return const_iterator(_cont.find(P)); } - void setConstructor(Constructor const &by) { ConstructedBy = by; } - Constructor getConstructor() const { return ConstructedBy; } - - PackageContainer() : ConstructedBy(UNKNOWN) {} - PackageContainer(Constructor const &by) : ConstructedBy(by) {} + PackageContainer() : PackageContainerInterface() {} + PackageContainer(CacheSetHelper::PkgSelector const &by) : PackageContainerInterface(by) {} +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + APT_DEPRECATED PackageContainer(Constructor const &by) : PackageContainerInterface((CacheSetHelper::PkgSelector)by) {} +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif /** \brief sort all included versions with given comparer @@ -267,7 +368,7 @@ public: /*{{{*/ \param pattern name of the task \param helper responsible for error and message handling */ static PackageContainer FromTask(pkgCacheFile &Cache, std::string const &pattern, CacheSetHelper &helper) { - PackageContainer cont(TASK); + PackageContainer cont(CacheSetHelper::TASK); PackageContainerInterface::FromTask(&cont, Cache, pattern, helper); return cont; } @@ -284,8 +385,8 @@ public: /*{{{*/ \param Cache the packages are in \param pattern regular expression for package names \param helper responsible for error and message handling */ - static PackageContainer FromRegEx(pkgCacheFile &Cache, std::string pattern, CacheSetHelper &helper) { - PackageContainer cont(REGEX); + static PackageContainer FromRegEx(pkgCacheFile &Cache, std::string const &pattern, CacheSetHelper &helper) { + PackageContainer cont(CacheSetHelper::REGEX); PackageContainerInterface::FromRegEx(&cont, Cache, pattern, helper); return cont; } @@ -295,8 +396,8 @@ public: /*{{{*/ return FromRegEx(Cache, pattern, helper); } - static PackageContainer FromFnmatch(pkgCacheFile &Cache, std::string pattern, CacheSetHelper &helper) { - PackageContainer cont(FNMATCH); + static PackageContainer FromFnmatch(pkgCacheFile &Cache, std::string const &pattern, CacheSetHelper &helper) { + PackageContainer cont(CacheSetHelper::FNMATCH); PackageContainerInterface::FromFnmatch(&cont, Cache, pattern, helper); return cont; } @@ -386,9 +487,6 @@ public: /*{{{*/ mods, fallback, helper); } /*}}}*/ -private: /*{{{*/ - Constructor ConstructedBy; - /*}}}*/ }; /*}}}*/ // specialisations for push_back containers: std::list & std::vector /*{{{*/ template<> template void PackageContainer >::insert(PackageContainer const &pkgcont) { @@ -461,9 +559,6 @@ private: iterator& erase(iterator &iter) { return iter; } size_t erase(const pkgCache::PkgIterator) { return 0; } void erase(iterator, iterator) { } - - void setConstructor(Constructor const &) { } - Constructor getConstructor() const { return UNKNOWN; } }; /*}}}*/ typedef PackageContainer > PackageSet; @@ -510,45 +605,83 @@ public: virtual void clear() = 0; /** \brief specifies which version(s) will be returned if non is given */ - enum Version { - /** All versions */ - ALL, - /** Candidate and installed version */ - CANDANDINST, - /** Candidate version */ - CANDIDATE, - /** Installed version */ - INSTALLED, - /** Candidate or if non installed version */ - CANDINST, - /** Installed or if non candidate version */ - INSTCAND, - /** Newest version */ - NEWEST + enum APT_DEPRECATED Version { + ALL = CacheSetHelper::ALL, + CANDANDINST = CacheSetHelper::CANDANDINST, + CANDIDATE = CacheSetHelper::CANDIDATE, + INSTALLED = CacheSetHelper::INSTALLED, + CANDINST = CacheSetHelper::CANDINST, + INSTCAND = CacheSetHelper::INSTCAND, + NEWEST = CacheSetHelper::NEWEST }; struct Modifier { - enum Position { NONE, PREFIX, POSTFIX }; - unsigned short ID; + unsigned short const ID; const char * const Alias; - Position Pos; - Version SelectVersion; + enum Position { NONE, PREFIX, POSTFIX } const Pos; + enum CacheSetHelper::VerSelector const SelectVersion; Modifier (unsigned short const &id, const char * const alias, Position const &pos, - Version const &select) : ID(id), Alias(alias), Pos(pos), + enum CacheSetHelper::VerSelector const select) : ID(id), Alias(alias), Pos(pos), SelectVersion(select) {} +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + APT_DEPRECATED Modifier(unsigned short const &id, const char * const alias, Position const &pos, + Version const &select) : ID(id), Alias(alias), Pos(pos), + SelectVersion((CacheSetHelper::VerSelector)select) {} +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif }; static bool FromCommandLine(VersionContainerInterface * const vci, pkgCacheFile &Cache, - const char **cmdline, Version const &fallback, + const char **cmdline, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper); +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + APT_DEPRECATED static bool FromCommandLine(VersionContainerInterface * const vci, pkgCacheFile &Cache, + const char **cmdline, Version const &fallback, + CacheSetHelper &helper) { + return FromCommandLine(vci, Cache, cmdline, (CacheSetHelper::VerSelector)fallback, helper); + } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif static bool FromString(VersionContainerInterface * const vci, pkgCacheFile &Cache, - std::string pkg, Version const &fallback, CacheSetHelper &helper, + std::string pkg, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper, bool const onlyFromName = false); +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + APT_DEPRECATED static bool FromString(VersionContainerInterface * const vci, pkgCacheFile &Cache, + std::string pkg, Version const &fallback, CacheSetHelper &helper, + bool const onlyFromName = false) { + return FromString(vci, Cache, pkg, (CacheSetHelper::VerSelector)fallback, helper, onlyFromName); + } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif static bool FromPackage(VersionContainerInterface * const vci, pkgCacheFile &Cache, - pkgCache::PkgIterator const &P, Version const &fallback, + pkgCache::PkgIterator const &P, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper); +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + APT_DEPRECATED static bool FromPackage(VersionContainerInterface * const vci, pkgCacheFile &Cache, + pkgCache::PkgIterator const &P, Version const &fallback, + CacheSetHelper &helper) { + return FromPackage(vci, Cache, P, (CacheSetHelper::VerSelector)fallback, helper); + } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif static bool FromModifierCommandLine(unsigned short &modID, VersionContainerInterface * const vci, @@ -560,8 +693,22 @@ public: static bool FromDependency(VersionContainerInterface * const vci, pkgCacheFile &Cache, pkgCache::DepIterator const &D, - Version const &selector, + CacheSetHelper::VerSelector const selector, CacheSetHelper &helper); +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + APT_DEPRECATED static bool FromDependency(VersionContainerInterface * const vci, + pkgCacheFile &Cache, + pkgCache::DepIterator const &D, + Version const &selector, + CacheSetHelper &helper) { + return FromDependency(vci, Cache, D, (CacheSetHelper::VerSelector)selector, helper); + } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif protected: /*{{{*/ @@ -664,35 +811,64 @@ public: /*{{{*/ \param fallback version specification \param helper responsible for error and message handling */ static VersionContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline, - Version const &fallback, CacheSetHelper &helper) { + CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper) { VersionContainer vercon; VersionContainerInterface::FromCommandLine(&vercon, Cache, cmdline, fallback, helper); return vercon; } static VersionContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline, - Version const &fallback) { + CacheSetHelper::VerSelector const fallback) { CacheSetHelper helper; return FromCommandLine(Cache, cmdline, fallback, helper); } static VersionContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline) { - return FromCommandLine(Cache, cmdline, CANDINST); + return FromCommandLine(Cache, cmdline, CacheSetHelper::CANDINST); } - static VersionContainer FromString(pkgCacheFile &Cache, std::string const &pkg, - Version const &fallback, CacheSetHelper &helper, + CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper, bool const /*onlyFromName = false*/) { VersionContainer vercon; VersionContainerInterface::FromString(&vercon, Cache, pkg, fallback, helper); return vercon; } static VersionContainer FromString(pkgCacheFile &Cache, std::string pkg, - Version const &fallback) { + CacheSetHelper::VerSelector const fallback) { CacheSetHelper helper; return FromString(Cache, pkg, fallback, helper); } static VersionContainer FromString(pkgCacheFile &Cache, std::string pkg) { - return FromString(Cache, pkg, CANDINST); + return FromString(Cache, pkg, CacheSetHelper::CANDINST); + } +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + static VersionContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline, + Version const &fallback, CacheSetHelper &helper) { + VersionContainer vercon; + VersionContainerInterface::FromCommandLine(&vercon, Cache, cmdline, (CacheSetHelper::VerSelector)fallback, helper); + return vercon; + } + static VersionContainer FromCommandLine(pkgCacheFile &Cache, const char **cmdline, + Version const &fallback) { + CacheSetHelper helper; + return FromCommandLine(Cache, cmdline, (CacheSetHelper::VerSelector)fallback, helper); } + static VersionContainer FromString(pkgCacheFile &Cache, std::string const &pkg, + Version const &fallback, CacheSetHelper &helper, + bool const /*onlyFromName = false*/) { + VersionContainer vercon; + VersionContainerInterface::FromString(&vercon, Cache, pkg, (CacheSetHelper::VerSelector)fallback, helper); + return vercon; + } + static VersionContainer FromString(pkgCacheFile &Cache, std::string pkg, + Version const &fallback) { + CacheSetHelper helper; + return FromString(Cache, pkg, (CacheSetHelper::VerSelector)fallback, helper); + } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif /** \brief returns all versions specified for the package @@ -701,18 +877,36 @@ public: /*{{{*/ \param fallback the version(s) you want to get \param helper the helper used for display and error handling */ static VersionContainer FromPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &P, - Version const &fallback, CacheSetHelper &helper) { + CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper) { VersionContainer vercon; VersionContainerInterface::FromPackage(&vercon, Cache, P, fallback, helper); return vercon; } static VersionContainer FromPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &P, - Version const &fallback) { + CacheSetHelper::VerSelector const fallback) { CacheSetHelper helper; return FromPackage(Cache, P, fallback, helper); } +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + static VersionContainer FromPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &P, + Version const &fallback, CacheSetHelper &helper) { + VersionContainer vercon; + VersionContainerInterface::FromPackage(&vercon, Cache, P, (CacheSetHelper::VerSelector)fallback, helper); + return vercon; + } + static VersionContainer FromPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &P, + Version const &fallback) { + CacheSetHelper helper; + return FromPackage(Cache, P, (CacheSetHelper::VerSelector)fallback, helper); + } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif static VersionContainer FromPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &P) { - return FromPackage(Cache, P, CANDIDATE); + return FromPackage(Cache, P, CacheSetHelper::CANDIDATE); } static std::map GroupedFromCommandLine( @@ -741,18 +935,36 @@ public: /*{{{*/ } static VersionContainer FromDependency(pkgCacheFile &Cache, pkgCache::DepIterator const &D, - Version const &selector, CacheSetHelper &helper) { + CacheSetHelper::VerSelector const selector, CacheSetHelper &helper) { VersionContainer vercon; VersionContainerInterface::FromDependency(&vercon, Cache, D, selector, helper); return vercon; } static VersionContainer FromDependency(pkgCacheFile &Cache, pkgCache::DepIterator const &D, - Version const &selector) { + CacheSetHelper::VerSelector const selector) { CacheSetHelper helper; return FromPackage(Cache, D, selector, helper); } +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + static VersionContainer FromDependency(pkgCacheFile &Cache, pkgCache::DepIterator const &D, + Version const &selector, CacheSetHelper &helper) { + VersionContainer vercon; + VersionContainerInterface::FromDependency(&vercon, Cache, D, (CacheSetHelper::VerSelector)selector, helper); + return vercon; + } + static VersionContainer FromDependency(pkgCacheFile &Cache, pkgCache::DepIterator const &D, + Version const &selector) { + CacheSetHelper helper; + return FromPackage(Cache, D, (CacheSetHelper::VerSelector)selector, helper); + } +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif static VersionContainer FromDependency(pkgCacheFile &Cache, pkgCache::DepIterator const &D) { - return FromPackage(Cache, D, CANDIDATE); + return FromPackage(Cache, D, CacheSetHelper::CANDIDATE); } /*}}}*/ }; /*}}}*/ diff --git a/apt-private/private-cacheset.cc b/apt-private/private-cacheset.cc index eb77be274..cb68024db 100644 --- a/apt-private/private-cacheset.cc +++ b/apt-private/private-cacheset.cc @@ -60,22 +60,22 @@ bool GetLocalitySortedVersionSet(pkgCacheFile &CacheFile, if (insertCurrentVer == true) { if (P->CurrentVer != 0) - vci->FromPackage(vci, CacheFile, P, APT::VersionContainerInterface::INSTALLED, helper); + vci->FromPackage(vci, CacheFile, P, APT::CacheSetHelper::INSTALLED, helper); } else if (insertUpgradable == true) { if(P.CurrentVer() && state.Upgradable()) - vci->FromPackage(vci, CacheFile, P, APT::VersionContainerInterface::CANDIDATE, helper); + vci->FromPackage(vci, CacheFile, P, APT::CacheSetHelper::CANDIDATE, helper); } else if (insertManualInstalled == true) { if (P.CurrentVer() && ((*DepCache)[P].Flags & pkgCache::Flag::Auto) == false) - vci->FromPackage(vci, CacheFile, P, APT::VersionContainerInterface::CANDIDATE, helper); + vci->FromPackage(vci, CacheFile, P, APT::CacheSetHelper::CANDIDATE, helper); } else { - if (vci->FromPackage(vci, CacheFile, P, APT::VersionContainerInterface::CANDIDATE, helper) == false) + if (vci->FromPackage(vci, CacheFile, P, APT::CacheSetHelper::CANDIDATE, helper) == false) { // no candidate, this may happen for packages in // dpkg "deinstall ok config-file" state - we pick the first ver diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 4030ce4e6..bd10a41ef 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -838,9 +838,9 @@ static bool XVcg(CommandLine &CmdL) // Load the list of packages from the command line into the show list APT::CacheSetHelper helper(true, GlobalError::NOTICE); - std::list mods; - mods.push_back(APT::PackageSet::Modifier(0, ",", APT::PackageSet::Modifier::POSTFIX)); - mods.push_back(APT::PackageSet::Modifier(1, "^", APT::PackageSet::Modifier::POSTFIX)); + std::list mods; + mods.push_back(APT::CacheSetHelper::PkgModifier(0, ",", APT::PackageSet::Modifier::POSTFIX)); + mods.push_back(APT::CacheSetHelper::PkgModifier(1, "^", APT::PackageSet::Modifier::POSTFIX)); std::map pkgsets = APT::PackageSet::GroupedFromCommandLine(CacheFile, CmdL.FileList + 1, mods, 0, helper); @@ -1050,9 +1050,9 @@ static bool Dotty(CommandLine &CmdL) // Load the list of packages from the command line into the show list APT::CacheSetHelper helper(true, GlobalError::NOTICE); - std::list mods; - mods.push_back(APT::PackageSet::Modifier(0, ",", APT::PackageSet::Modifier::POSTFIX)); - mods.push_back(APT::PackageSet::Modifier(1, "^", APT::PackageSet::Modifier::POSTFIX)); + std::list mods; + mods.push_back(APT::CacheSetHelper::PkgModifier(0, ",", APT::PackageSet::Modifier::POSTFIX)); + mods.push_back(APT::CacheSetHelper::PkgModifier(1, "^", APT::PackageSet::Modifier::POSTFIX)); std::map pkgsets = APT::PackageSet::GroupedFromCommandLine(CacheFile, CmdL.FileList + 1, mods, 0, helper); -- cgit v1.2.3-70-g09d2 From e6f0c9bca4b052d20a2e48ce9715b89e187b671a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 3 Sep 2014 19:02:05 +0200 Subject: adapt to the new CacheSetHelper API Git-Dch: Ignore --- apt-private/private-cacheset.h | 25 +++++++++++-------------- apt-private/private-install.cc | 4 ++-- apt-private/private-show.cc | 8 ++++---- cmdline/apt-cache.cc | 12 ++++++------ 4 files changed, 23 insertions(+), 26 deletions(-) (limited to 'cmdline') diff --git a/apt-private/private-cacheset.h b/apt-private/private-cacheset.h index ca8f4be5d..059c7637e 100644 --- a/apt-private/private-cacheset.h +++ b/apt-private/private-cacheset.h @@ -76,19 +76,16 @@ class CacheSetHelperVirtuals: public APT::CacheSetHelper { public: APT::PackageSet virtualPkgs; - virtual pkgCache::VerIterator canNotFindCandidateVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { - virtualPkgs.insert(Pkg); - return CacheSetHelper::canNotFindCandidateVer(Cache, Pkg); + virtual pkgCache::VerIterator canNotGetVersion(enum CacheSetHelper::VerSelector const select, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { + if (select == NEWEST || select == CANDIDATE || select == ALL) + virtualPkgs.insert(Pkg); + return CacheSetHelper::canNotGetVersion(select, Cache, Pkg); } - virtual pkgCache::VerIterator canNotFindNewestVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { - virtualPkgs.insert(Pkg); - return CacheSetHelper::canNotFindNewestVer(Cache, Pkg); - } - - virtual void canNotFindAllVer(APT::VersionContainerInterface * vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { - virtualPkgs.insert(Pkg); - CacheSetHelper::canNotFindAllVer(vci, Cache, Pkg); + virtual void canNotFindVersion(enum CacheSetHelper::VerSelector const select, APT::VersionContainerInterface * vci, pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { + if (select == NEWEST || select == CANDIDATE || select == ALL) + virtualPkgs.insert(Pkg); + return CacheSetHelper::canNotFindVersion(select, vci, Cache, Pkg); } CacheSetHelperVirtuals(bool const ShowErrors = true, GlobalError::MsgType const &ErrorType = GlobalError::NOTICE) : CacheSetHelper(ShowErrors, ErrorType) {} @@ -190,7 +187,7 @@ public: } virtual pkgCache::VerIterator canNotFindCandidateVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { - APT::VersionSet const verset = tryVirtualPackage(Cache, Pkg, APT::VersionSet::CANDIDATE); + APT::VersionSet const verset = tryVirtualPackage(Cache, Pkg, CacheSetHelper::CANDIDATE); if (verset.empty() == false) return *(verset.begin()); else if (ShowError == true) { @@ -203,7 +200,7 @@ public: virtual pkgCache::VerIterator canNotFindNewestVer(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg) { if (Pkg->ProvidesList != 0) { - APT::VersionSet const verset = tryVirtualPackage(Cache, Pkg, APT::VersionSet::NEWEST); + APT::VersionSet const verset = tryVirtualPackage(Cache, Pkg, CacheSetHelper::NEWEST); if (verset.empty() == false) return *(verset.begin()); if (ShowError == true) @@ -231,7 +228,7 @@ public: } APT::VersionSet tryVirtualPackage(pkgCacheFile &Cache, pkgCache::PkgIterator const &Pkg, - APT::VersionSet::Version const &select) { + CacheSetHelper::VerSelector const select) { /* This is a pure virtual package and there is a single available candidate providing it. */ if (unlikely(Cache[Pkg].CandidateVer != 0) || Pkg->ProvidesList == 0) diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index c172f5ee7..86ba52857 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -560,9 +560,9 @@ bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, CacheFile &Cache, std::list mods; mods.push_back(APT::VersionSet::Modifier(MOD_INSTALL, "+", - APT::VersionSet::Modifier::POSTFIX, APT::VersionSet::CANDIDATE)); + APT::VersionSet::Modifier::POSTFIX, APT::CacheSetHelper::CANDIDATE)); mods.push_back(APT::VersionSet::Modifier(MOD_REMOVE, "-", - APT::VersionSet::Modifier::POSTFIX, APT::VersionSet::NEWEST)); + APT::VersionSet::Modifier::POSTFIX, APT::CacheSetHelper::NEWEST)); CacheSetHelperAPTGet helper(c0out); verset = APT::VersionSet::GroupedFromCommandLine(Cache, CmdL.FileList + 1, mods, fallback, helper); diff --git a/apt-private/private-show.cc b/apt-private/private-show.cc index 8ae6a6dac..289f035a6 100644 --- a/apt-private/private-show.cc +++ b/apt-private/private-show.cc @@ -141,16 +141,16 @@ bool ShowPackage(CommandLine &CmdL) /*{{{*/ { pkgCacheFile CacheFile; CacheSetHelperVirtuals helper(true, GlobalError::NOTICE); - APT::VersionList::Version const select = _config->FindB("APT::Cache::AllVersions", false) ? - APT::VersionList::ALL : APT::VersionList::CANDIDATE; + APT::CacheSetHelper::VerSelector const select = _config->FindB("APT::Cache::AllVersions", false) ? + APT::CacheSetHelper::ALL : APT::CacheSetHelper::CANDIDATE; APT::VersionList const verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, select, helper); for (APT::VersionList::const_iterator Ver = verset.begin(); Ver != verset.end(); ++Ver) if (DisplayRecord(CacheFile, Ver, c1out) == false) return false; - if (select == APT::VersionList::CANDIDATE) + if (select == APT::CacheSetHelper::CANDIDATE) { - APT::VersionList const verset_all = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, APT::VersionList::ALL, helper); + APT::VersionList const verset_all = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, APT::CacheSetHelper::ALL, helper); int const records = verset_all.size() - verset.size(); if (records > 0) _error->Notice(P_("There is %i additional record. Please use the '-a' switch to see it", "There are %i additional records. Please use the '-a' switch to see them.", records), records); diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index bd10a41ef..5a5dde088 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -191,7 +191,7 @@ static bool UnMet(CommandLine &CmdL) { CacheSetHelperVirtuals helper(true, GlobalError::NOTICE); APT::VersionList verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, - APT::VersionList::CANDIDATE, helper); + APT::CacheSetHelper::CANDIDATE, helper); for (APT::VersionList::iterator V = verset.begin(); V != verset.end(); ++V) if (ShowUnMet(V, Important) == false) return false; @@ -656,7 +656,7 @@ static bool ShowDepends(CommandLine &CmdL, bool const RevDepends) return false; CacheSetHelperVirtuals helper(false); - APT::VersionList verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, APT::VersionList::CANDIDATE, helper); + APT::VersionList verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper); if (verset.empty() == true && helper.virtualPkgs.empty() == true) return _error->Error(_("No packages found")); std::vector Shown(Cache->Head().PackageCount); @@ -724,7 +724,7 @@ static bool ShowDepends(CommandLine &CmdL, bool const RevDepends) if (Recurse == true && Shown[Trg->ID] == false) { Shown[Trg->ID] = true; - verset.insert(APT::VersionSet::FromPackage(CacheFile, Trg, APT::VersionSet::CANDIDATE, helper)); + verset.insert(APT::VersionSet::FromPackage(CacheFile, Trg, APT::CacheSetHelper::CANDIDATE, helper)); } } @@ -743,7 +743,7 @@ static bool ShowDepends(CommandLine &CmdL, bool const RevDepends) if (Recurse == true && Shown[V.ParentPkg()->ID] == false) { Shown[V.ParentPkg()->ID] = true; - verset.insert(APT::VersionSet::FromPackage(CacheFile, V.ParentPkg(), APT::VersionSet::CANDIDATE, helper)); + verset.insert(APT::VersionSet::FromPackage(CacheFile, V.ParentPkg(), APT::CacheSetHelper::CANDIDATE, helper)); } } @@ -1506,8 +1506,8 @@ static bool ShowPackage(CommandLine &CmdL) { pkgCacheFile CacheFile; CacheSetHelperVirtuals helper(true, GlobalError::NOTICE); - APT::VersionList::Version const select = _config->FindB("APT::Cache::AllVersions", true) ? - APT::VersionList::ALL : APT::VersionList::CANDIDATE; + APT::CacheSetHelper::VerSelector const select = _config->FindB("APT::Cache::AllVersions", true) ? + APT::CacheSetHelper::ALL : APT::CacheSetHelper::CANDIDATE; APT::VersionList const verset = APT::VersionList::FromCommandLine(CacheFile, CmdL.FileList + 1, select, helper); for (APT::VersionList::const_iterator Ver = verset.begin(); Ver != verset.end(); ++Ver) if (DisplayRecord(CacheFile, Ver) == false) -- cgit v1.2.3-70-g09d2 From 3e032cda2d3558c076a559697e664546da56c243 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 24 Jan 2014 15:02:39 +0100 Subject: remove leftover debug output from multikey softlink Git-Dch: Ignore --- cmdline/apt-key.in | 1 - 1 file changed, 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 0774cf4b7..59b69b89c 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -198,7 +198,6 @@ remove_key_from_keyring() { REALTARGET="$(readlink -f "$1")" mv -f "$1" "${1}.dpkg-tmp" cp -a "$REALTARGET" "$1" - ls "$(dirname $1)" fi # delete the key from the keyring $GPG --batch --delete-key --yes "$2" -- cgit v1.2.3-70-g09d2 From 3d0def059874a6e63566ce13579de296ccd26932 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 24 Jan 2014 16:43:19 +0100 Subject: add a (hidden) --quiet option for apt-key --- cmdline/apt-key.in | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 59b69b89c..d50ad40ac 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -39,6 +39,8 @@ ARCHIVE_KEYRING_URI='&keyring-uri;' eval $(apt-config shell ARCHIVE_KEYRING_URI APT::Key::ArchiveKeyringURI) TMP_KEYRING=${APT_DIR}/var/lib/apt/keyrings/maybe-import-keyring.gpg +aptkey_echo() { echo "$@"; } + requires_root() { if [ "$(id -u)" -ne 0 ]; then echo >&1 "ERROR: This command can only be used by root." @@ -142,7 +144,7 @@ net_update() { fi new_mtime=$(stat -c %Y $keyring) if [ $new_mtime -ne $old_mtime ]; then - echo "Checking for new archive signing keys now" + aptkey_echo "Checking for new archive signing keys now" add_keys_with_verify_against_master_keyring $keyring $MASTER_KEYRING fi } @@ -227,7 +229,7 @@ remove_key() { done fi fi - echo "OK" + aptkey_echo "OK" } @@ -267,6 +269,10 @@ while [ -n "$1" ]; do requires_root() { true; } shift ;; + --quiet) + aptkey_echo() { true; } + shift + ;; --*) echo >&2 "Unknown option: $1" usage @@ -316,7 +322,7 @@ case "$command" in requires_root init_keyring "$TRUSTEDFILE" $GPG --quiet --batch --import "$1" - echo "OK" + aptkey_echo "OK" ;; del|rm|remove) init_keyring "$TRUSTEDFILE" @@ -348,7 +354,7 @@ case "$command" in ;; adv*) init_keyring "$TRUSTEDFILE" - echo "Executing: $GPG $*" + aptkey_echo "Executing: $GPG $*" $GPG $* ;; help) -- cgit v1.2.3-70-g09d2 From 84b286f65c55e21b4734a17474f1cba464cbbd9c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 24 Jan 2014 16:44:49 +0100 Subject: all errors should be printed to stderr Git-Dch: Ignore --- cmdline/apt-key.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index d50ad40ac..e592925d6 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -43,7 +43,7 @@ aptkey_echo() { echo "$@"; } requires_root() { if [ "$(id -u)" -ne 0 ]; then - echo >&1 "ERROR: This command can only be used by root." + echo >&2 "ERROR: This command can only be used by root." exit 1 fi } @@ -63,11 +63,11 @@ add_keys_with_verify_against_master_keyring() { MASTER=$2 if [ ! -f "$ADD_KEYRING" ]; then - echo "ERROR: '$ADD_KEYRING' not found" + echo >&2 "ERROR: '$ADD_KEYRING' not found" return - fi + fi if [ ! -f "$MASTER" ]; then - echo "ERROR: '$MASTER' not found" + echo >&2 "ERROR: '$MASTER' not found" return fi @@ -175,7 +175,7 @@ update() { fi done else - echo "Warning: removed keys keyring $REMOVED_KEYS missing or not readable" >&2 + echo >&2 "Warning: removed keys keyring $REMOVED_KEYS missing or not readable" fi } -- cgit v1.2.3-70-g09d2 From 9fda3be1ae000e81e15dc8840271946e140765c9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 24 Jan 2014 17:14:52 +0100 Subject: only create new trusted.gpg if directory is writeable --- cmdline/apt-key.in | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index e592925d6..66e26ef0c 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -48,16 +48,6 @@ requires_root() { fi } -# gpg defaults to mode 0600 for new keyrings. Create one with 0644 instead. -init_keyring() { - for path; do - if ! [ -e "$path" ]; then - touch -- "$path" - chmod 0644 -- "$path" - fi - done -} - add_keys_with_verify_against_master_keyring() { ADD_KEYRING=$1 MASTER=$2 @@ -311,49 +301,50 @@ if [ -z "$command" ]; then fi shift -if [ "$command" != "help" ] && ! which gpg >/dev/null 2>&1; then - echo >&2 "Warning: gnupg does not seem to be installed." - echo >&2 "Warning: apt-key requires gnupg for most operations." - echo >&2 +if [ "$command" != "help" ]; then + if ! which gpg >/dev/null 2>&1; then + echo >&2 "Warning: gnupg does not seem to be installed." + echo >&2 "Warning: apt-key requires gnupg for most operations." + echo >&2 + fi + + # gpg defaults to mode 0600 for new keyrings. Create one with 0644 instead. + if ! [ -e "$TRUSTEDFILE" ]; then + if [ -w "$(dirname "$TRUSTEDFILE")" ]; then + touch -- "$TRUSTEDFILE" + chmod 0644 -- "$TRUSTEDFILE" + fi + fi fi case "$command" in add) requires_root - init_keyring "$TRUSTEDFILE" $GPG --quiet --batch --import "$1" aptkey_echo "OK" ;; del|rm|remove) - init_keyring "$TRUSTEDFILE" remove_key "$1" ;; update) - init_keyring "$TRUSTEDFILE" update ;; net-update) - init_keyring "$TRUSTEDFILE" net_update ;; list) - init_keyring "$TRUSTEDFILE" $GPG --batch --list-keys ;; finger*) - init_keyring "$TRUSTEDFILE" $GPG --batch --fingerprint ;; export) - init_keyring "$TRUSTEDFILE" $GPG --armor --export "$1" ;; exportall) - init_keyring "$TRUSTEDFILE" $GPG --armor --export ;; adv*) - init_keyring "$TRUSTEDFILE" aptkey_echo "Executing: $GPG $*" $GPG $* ;; -- cgit v1.2.3-70-g09d2 From 4f51a496d56807c73586220dd736173d7783c8b3 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 24 Jan 2014 17:24:56 +0100 Subject: support (multiple) arguments properly in apt-key --- cmdline/apt-key.in | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 66e26ef0c..ab62d4c8f 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -202,24 +202,27 @@ remove_key_from_keyring() { remove_key() { requires_root - # if a --keyring was given, just remove from there - if [ -n "$FORCED_KEYRING" ]; then - remove_key_from_keyring "$FORCED_KEYRING" "$1" - else - # otherwise all known keyrings are up for inspection - local TRUSTEDFILE="/etc/apt/trusted.gpg" - eval $(apt-config shell TRUSTEDFILE Apt::GPGV::TrustedKeyring) - eval $(apt-config shell TRUSTEDFILE Dir::Etc::Trusted/f) - remove_key_from_keyring "$TRUSTEDFILE" "$1" - TRUSTEDPARTS="/etc/apt/trusted.gpg.d" - eval $(apt-config shell TRUSTEDPARTS Dir::Etc::TrustedParts/d) - if [ -d "$TRUSTEDPARTS" ]; then - for trusted in $(run-parts --list "$TRUSTEDPARTS" --regex '^.*\.gpg$'); do - remove_key_from_keyring "$trusted" "$1" - done + while [ -n "$1" ]; do + # if a --keyring was given, just remove from there + if [ -n "$FORCED_KEYRING" ]; then + remove_key_from_keyring "$FORCED_KEYRING" "$1" + else + # otherwise all known keyrings are up for inspection + local TRUSTEDFILE="/etc/apt/trusted.gpg" + eval $(apt-config shell TRUSTEDFILE Apt::GPGV::TrustedKeyring) + eval $(apt-config shell TRUSTEDFILE Dir::Etc::Trusted/f) + remove_key_from_keyring "$TRUSTEDFILE" "$1" + TRUSTEDPARTS="/etc/apt/trusted.gpg.d" + eval $(apt-config shell TRUSTEDPARTS Dir::Etc::TrustedParts/d) + if [ -d "$TRUSTEDPARTS" ]; then + for trusted in $(run-parts --list "$TRUSTEDPARTS" --regex '^.*\.gpg$'); do + remove_key_from_keyring "$trusted" "$1" + done + fi fi - fi - aptkey_echo "OK" + aptkey_echo "OK" + shift + done } @@ -320,11 +323,11 @@ fi case "$command" in add) requires_root - $GPG --quiet --batch --import "$1" + $GPG --quiet --batch --import "$@" aptkey_echo "OK" ;; del|rm|remove) - remove_key "$1" + remove_key "$@" ;; update) update @@ -333,20 +336,17 @@ case "$command" in net_update ;; list) - $GPG --batch --list-keys + $GPG --batch --list-keys "$@" ;; finger*) - $GPG --batch --fingerprint - ;; - export) - $GPG --armor --export "$1" + $GPG --batch --fingerprint "$@" ;; - exportall) - $GPG --armor --export + export|exportall) + $GPG --armor --export "$@" ;; adv*) aptkey_echo "Executing: $GPG $*" - $GPG $* + $GPG "$@" ;; help) usage -- cgit v1.2.3-70-g09d2 From 08cfcd73f517e909823c100143c4938b9997343e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 24 Jan 2014 17:41:35 +0100 Subject: set a primary-keyring only if we have access to it --- cmdline/apt-key.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index ab62d4c8f..9da5fceb8 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -281,8 +281,8 @@ if [ -z "$TRUSTEDFILE" ]; then eval $(apt-config shell TRUSTEDFILE Dir::Etc::Trusted/f) if [ -r "$TRUSTEDFILE" ]; then GPG="$GPG --keyring $TRUSTEDFILE" + GPG="$GPG --primary-keyring $TRUSTEDFILE" fi - GPG="$GPG --primary-keyring $TRUSTEDFILE" TRUSTEDPARTS="/etc/apt/trusted.gpg.d" eval $(apt-config shell TRUSTEDPARTS Dir::Etc::TrustedParts/d) if [ -d "$TRUSTEDPARTS" ]; then @@ -316,6 +316,8 @@ if [ "$command" != "help" ]; then if [ -w "$(dirname "$TRUSTEDFILE")" ]; then touch -- "$TRUSTEDFILE" chmod 0644 -- "$TRUSTEDFILE" + GPG="$GPG --keyring $TRUSTEDFILE" + GPG="$GPG --primary-keyring $TRUSTEDFILE" fi fi fi -- cgit v1.2.3-70-g09d2 From 4b30c1dc053278a6b9bdb50f0d91b3f934e8613d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 24 Jan 2014 18:24:43 +0100 Subject: refactor key removal code to reuse it in next step Git-Dch: Ignore --- cmdline/apt-key.in | 103 ++++++++++++++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 48 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 9da5fceb8..9adbd6443 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -170,62 +170,69 @@ update() { } remove_key_from_keyring() { - local GPG="$GPG_CMD --keyring $1" - # check if the key is in this keyring: the key id is in the 5 column at the end - if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+$2:"; then - return - fi - if [ ! -w "$1" ]; then - echo >&2 "Key ${2} is in keyring ${1}, but can't be removed as it is read only." - return - fi - # check if it is the only key in the keyring and if so remove the keyring altogether - if [ '1' = "$($GPG --with-colons --list-keys | grep "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+:" | wc -l)" ]; then - mv -f "$1" "${1}~" # behave like gpg - return - fi - # we can't just modify pointed to files as these might be in /usr or something - local REALTARGET - if [ -L "$1" ]; then - REALTARGET="$(readlink -f "$1")" - mv -f "$1" "${1}.dpkg-tmp" - cp -a "$REALTARGET" "$1" - fi - # delete the key from the keyring - $GPG --batch --delete-key --yes "$2" - if [ -n "$REALTARGET" ]; then - # the real backup is the old link, not the copy we made - mv -f "${1}.dpkg-tmp" "${1}~" - fi + local KEYRINGFILE="$1" + shift + local GPG="$GPG_CMD --keyring $KEYRINGFILE" + while [ -n "$1" ]; do + local KEY="$1" + shift + # check if the key is in this keyring: the key id is in the 5 column at the end + if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+${KEY}:"; then + continue + fi + if [ ! -w "$KEYRINGFILE" ]; then + echo >&2 "Key ${KEY} is in keyring ${KEYRINGFILE}, but can't be removed as it is read only." + continue + fi + # check if it is the only key in the keyring and if so remove the keyring altogether + if [ '1' = "$($GPG --with-colons --list-keys | grep "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+:" | wc -l)" ]; then + mv -f "$KEYRINGFILE" "${KEYRINGFILE}~" # behave like gpg + return + fi + # we can't just modify pointed to files as these might be in /usr or something + local REALTARGET + if [ -L "$KEYRINGFILE" ]; then + REALTARGET="$(readlink -f "$KEYRINGFILE")" + mv -f "$KEYRINGFILE" "${KEYRINGFILE}.dpkg-tmp" + cp -a "$REALTARGET" "$KEYRINGFILE" + fi + # delete the key from the keyring + $GPG --batch --delete-key --yes "$KEY" + if [ -n "$REALTARGET" ]; then + # the real backup is the old link, not the copy we made + mv -f "${KEYRINGFILE}.dpkg-tmp" "${KEYRINGFILE}~" + fi + done } remove_key() { requires_root + foreach_keyring_do 'remove_key_from_keyring' "$@" + aptkey_echo "OK" + } - while [ -n "$1" ]; do - # if a --keyring was given, just remove from there - if [ -n "$FORCED_KEYRING" ]; then - remove_key_from_keyring "$FORCED_KEYRING" "$1" - else - # otherwise all known keyrings are up for inspection - local TRUSTEDFILE="/etc/apt/trusted.gpg" - eval $(apt-config shell TRUSTEDFILE Apt::GPGV::TrustedKeyring) - eval $(apt-config shell TRUSTEDFILE Dir::Etc::Trusted/f) - remove_key_from_keyring "$TRUSTEDFILE" "$1" - TRUSTEDPARTS="/etc/apt/trusted.gpg.d" - eval $(apt-config shell TRUSTEDPARTS Dir::Etc::TrustedParts/d) - if [ -d "$TRUSTEDPARTS" ]; then - for trusted in $(run-parts --list "$TRUSTEDPARTS" --regex '^.*\.gpg$'); do - remove_key_from_keyring "$trusted" "$1" - done - fi +foreach_keyring_do() { + local ACTION="$1" + shift + # if a --keyring was given, just remove from there + if [ -n "$FORCED_KEYRING" ]; then + $ACTION "$FORCED_KEYRING" "$@" + else + # otherwise all known keyrings are up for inspection + local TRUSTEDFILE="/etc/apt/trusted.gpg" + eval $(apt-config shell TRUSTEDFILE Apt::GPGV::TrustedKeyring) + eval $(apt-config shell TRUSTEDFILE Dir::Etc::Trusted/f) + $ACTION "$TRUSTEDFILE" "$@" + local TRUSTEDPARTS="/etc/apt/trusted.gpg.d" + eval $(apt-config shell TRUSTEDPARTS Dir::Etc::TrustedParts/d) + if [ -d "$TRUSTEDPARTS" ]; then + for trusted in $(run-parts --list "$TRUSTEDPARTS" --regex '^.*\.gpg$'); do + $ACTION "$trusted" "$@" + done fi - aptkey_echo "OK" - shift - done + fi } - usage() { echo "Usage: apt-key [--keyring file] [command] [arguments]" echo -- cgit v1.2.3-70-g09d2 From 5beb682d2de2003e1c022cb298d6c2ec0cf91c0d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 24 Jan 2014 22:40:52 +0100 Subject: merge fragment keyrings in apt-key to avoid hitting gpg limits gnupg has a hardlimit of 40 (at the moment) keyrings per invocation, which can be exceeded with (many) repositories. That is rather misfortune as the longrun goal was to drop gnupg dependency at some point in the future, but this can now be considered missed and dropped. It also means that 'apt-key adv' commands might not have the behaviour one would expect it to have as it mainly operates on a big temporary keyring, so commands modifying keys will break. Doing this was never a good idea anyway through, so lets just hope nothing break too badly. Closes: 733028 --- cmdline/apt-key.in | 134 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 77 insertions(+), 57 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 9adbd6443..9d8e60ec0 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -23,7 +23,6 @@ GPG_CMD="$GPG_CMD --homedir $GPGHOMEDIR" $GPG_CMD --quiet --check-trustdb --keyring $SECRETKEYRING >/dev/null 2>&1 # tell gpg that it shouldn't try to maintain a trustdb file GPG_CMD="$GPG_CMD --no-auto-check-trustdb --trust-model always" - GPG="$GPG_CMD" APT_DIR="/" @@ -113,7 +112,6 @@ net_update() { echo >&2 "ERROR: Your distribution is not supported in net-update as no uri for the archive-keyring is set" exit 1 fi - requires_root # in theory we would need to depend on wget for this, but this feature # isn't useable in debian anyway as we have no keyring uri nor a master key if ! which wget >/dev/null 2>&1; then @@ -145,7 +143,6 @@ update() { echo >&2 "Is the &keyring-package; package installed?" exit 1 fi - requires_root # add new keys from the package; @@ -158,11 +155,8 @@ update() { if [ -r "$REMOVED_KEYS" ]; then # remove no-longer supported/used keys - keys=`$GPG_CMD --keyring $REMOVED_KEYS --with-colons --list-keys | grep ^pub | cut -d: -f5` - for key in $keys; do - if $GPG --list-keys --with-colons | grep ^pub | cut -d: -f5 | grep -q $key; then - $GPG --quiet --batch --delete-key --yes ${key} - fi + $GPG_CMD --keyring $REMOVED_KEYS --with-colons --list-keys | grep ^pub | cut -d: -f5 | while read key; do + foreach_keyring_do 'remove_key_from_keyring' "$key" done else echo >&2 "Warning: removed keys keyring $REMOVED_KEYS missing or not readable" @@ -172,12 +166,17 @@ update() { remove_key_from_keyring() { local KEYRINGFILE="$1" shift + # non-existent keyrings have by definition no keys + if [ ! -e "$KEYRINGFILE" ]; then + return + fi + local GPG="$GPG_CMD --keyring $KEYRINGFILE" while [ -n "$1" ]; do local KEY="$1" shift # check if the key is in this keyring: the key id is in the 5 column at the end - if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+${KEY}:"; then + if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*${KEY}:"; then continue fi if [ ! -w "$KEYRINGFILE" ]; then @@ -205,12 +204,6 @@ remove_key_from_keyring() { done } -remove_key() { - requires_root - foreach_keyring_do 'remove_key_from_keyring' "$@" - aptkey_echo "OK" - } - foreach_keyring_do() { local ACTION="$1" shift @@ -219,20 +212,62 @@ foreach_keyring_do() { $ACTION "$FORCED_KEYRING" "$@" else # otherwise all known keyrings are up for inspection - local TRUSTEDFILE="/etc/apt/trusted.gpg" - eval $(apt-config shell TRUSTEDFILE Apt::GPGV::TrustedKeyring) - eval $(apt-config shell TRUSTEDFILE Dir::Etc::Trusted/f) - $ACTION "$TRUSTEDFILE" "$@" + if [ -s "$TRUSTEDFILE" ]; then + $ACTION "$TRUSTEDFILE" "$@" + fi local TRUSTEDPARTS="/etc/apt/trusted.gpg.d" eval $(apt-config shell TRUSTEDPARTS Dir::Etc::TrustedParts/d) if [ -d "$TRUSTEDPARTS" ]; then + # strip / suffix as gpg will double-slash in that case (#665411) + local STRIPPED_TRUSTEDPARTS="${TRUSTEDPARTS%/}" + if [ "${STRIPPED_TRUSTEDPARTS}/" = "$TRUSTEDPARTS" ]; then + TRUSTEDPARTS="$STRIPPED_TRUSTEDPARTS" + fi for trusted in $(run-parts --list "$TRUSTEDPARTS" --regex '^.*\.gpg$'); do - $ACTION "$trusted" "$@" + if [ -s "$trusted" ]; then + $ACTION "$trusted" "$@" + fi done fi fi } +list_keys_from_keyring() { + local KEYRINGFILE="$1" + shift + # don't show the error message if this keyring doesn't include the key + $GPG_CMD --keyring "$KEYRINGFILE" --batch --list-keys "$@" 2>/dev/null || true +} + +fingerprint_keys_from_keyring() { + local KEYRINGFILE="$1" + shift + # don't show the error message if this keyring doesn't include the fingerprint + $GPG_CMD --keyring "$KEYRINGFILE" --batch --fingerprint "$@" 2>/dev/null || true +} + +import_keys_from_keyring() { + local IMPORT="$1" + local KEYRINGFILE="$2" + $GPG_CMD --keyring "$KEYRINGFILE" --batch --import "$IMPORT" >/dev/null 2>&1 +} + +setup_merged_keyring() { + local TRUSTEDFILE_BAK="$TRUSTEDFILE" + TRUSTEDFILE='/dev/null' + foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/trusted.gpg" + TRUSTEDFILE="$TRUSTEDFILE_BAK" + # mark it as non-writeable so users get errors if gnupg tries to modify it + if [ -s "${GPGHOMEDIR}/trusted.gpg" ]; then + chmod -w "${GPGHOMEDIR}/trusted.gpg" + GPG="$GPG --keyring ${GPGHOMEDIR}/trusted.gpg" + fi + if [ -r "$TRUSTEDFILE" ]; then + GPG="$GPG --keyring $TRUSTEDFILE --primary-keyring $TRUSTEDFILE" + fi +} + + usage() { echo "Usage: apt-key [--keyring file] [command] [arguments]" echo @@ -257,12 +292,6 @@ while [ -n "$1" ]; do shift TRUSTEDFILE="$1" FORCED_KEYRING="$1" - if [ -r "$TRUSTEDFILE" ] || [ "$2" = 'add' ] || [ "$2" = 'adv' ]; then - GPG="$GPG --keyring $TRUSTEDFILE --primary-keyring $TRUSTEDFILE" - else - echo >&2 "Error: The specified keyring »$TRUSTEDFILE« is missing or not readable" - exit 1 - fi shift ;; --fakeroot) @@ -286,22 +315,6 @@ if [ -z "$TRUSTEDFILE" ]; then TRUSTEDFILE="/etc/apt/trusted.gpg" eval $(apt-config shell TRUSTEDFILE Apt::GPGV::TrustedKeyring) eval $(apt-config shell TRUSTEDFILE Dir::Etc::Trusted/f) - if [ -r "$TRUSTEDFILE" ]; then - GPG="$GPG --keyring $TRUSTEDFILE" - GPG="$GPG --primary-keyring $TRUSTEDFILE" - fi - TRUSTEDPARTS="/etc/apt/trusted.gpg.d" - eval $(apt-config shell TRUSTEDPARTS Dir::Etc::TrustedParts/d) - if [ -d "$TRUSTEDPARTS" ]; then - # strip / suffix as gpg will double-slash in that case (#665411) - STRIPPED_TRUSTEDPARTS="${TRUSTEDPARTS%/}" - if [ "${STRIPPED_TRUSTEDPARTS}/" = "$TRUSTEDPARTS" ]; then - TRUSTEDPARTS="$STRIPPED_TRUSTEDPARTS" - fi - for trusted in $(run-parts --list "$TRUSTEDPARTS" --regex '^.*\.gpg$'); do - GPG="$GPG --keyring $trusted" - done - fi fi command="$1" @@ -323,40 +336,47 @@ if [ "$command" != "help" ]; then if [ -w "$(dirname "$TRUSTEDFILE")" ]; then touch -- "$TRUSTEDFILE" chmod 0644 -- "$TRUSTEDFILE" - GPG="$GPG --keyring $TRUSTEDFILE" - GPG="$GPG --primary-keyring $TRUSTEDFILE" fi fi fi case "$command" in add) - requires_root - $GPG --quiet --batch --import "$@" - aptkey_echo "OK" + requires_root + setup_merged_keyring + $GPG --quiet --batch --import "$@" + aptkey_echo "OK" ;; del|rm|remove) - remove_key "$@" + requires_root + foreach_keyring_do 'remove_key_from_keyring' "$@" + aptkey_echo "OK" ;; update) + requires_root + setup_merged_keyring update ;; net-update) + requires_root + setup_merged_keyring net_update ;; list) - $GPG --batch --list-keys "$@" - ;; + foreach_keyring_do 'list_keys_from_keyring' "$@" + ;; finger*) - $GPG --batch --fingerprint "$@" - ;; + foreach_keyring_do 'fingerprint_keys_from_keyring' "$@" + ;; export|exportall) - $GPG --armor --export "$@" - ;; + foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/trusted.gpg" + $GPG_CMD --keyring "${GPGHOMEDIR}/trusted.gpg" --armor --export "$@" + ;; adv*) - aptkey_echo "Executing: $GPG $*" - $GPG "$@" - ;; + setup_merged_keyring + aptkey_echo "Executing: $GPG $*" + $GPG "$@" + ;; help) usage ;; -- cgit v1.2.3-70-g09d2 From 059911800900a42c4246f1e209b51656055215b2 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 26 Jan 2014 17:37:00 +0100 Subject: delay gnupg setup in apt-key until it is needed 'apt-key help' and incorrect usage do not need a functioning gnupg setup, as well as we shouldn't try to setup gnupg before we actually test if it is available (and print a message if it is not). --- cmdline/apt-key.in | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 9d8e60ec0..b8fdfe121 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -3,28 +3,6 @@ set -e unset GREP_OPTIONS -GPG_CMD="gpg --ignore-time-conflict --no-options --no-default-keyring" - -# gpg needs (in different versions more or less) files to function correctly, -# so we give it its own homedir and generate some valid content for it -GPGHOMEDIR="$(mktemp -d)" -CURRENTTRAP="${CURRENTTRAP} rm -rf '${GPGHOMEDIR}';" -trap "${CURRENTTRAP}" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM -chmod 700 "$GPGHOMEDIR" -# We don't use a secret keyring, of course, but gpg panics and -# implodes if there isn't one available - and writeable for imports -SECRETKEYRING="${GPGHOMEDIR}/secring.gpg" -touch $SECRETKEYRING -GPG_CMD="$GPG_CMD --homedir $GPGHOMEDIR" -# create the trustdb with an (empty) dummy keyring -# older gpgs required it, newer gpgs even warn that it isn't needed, -# but require it nonetheless for some commands, so we just play safe -# here for the foreseeable future and create a dummy one -$GPG_CMD --quiet --check-trustdb --keyring $SECRETKEYRING >/dev/null 2>&1 -# tell gpg that it shouldn't try to maintain a trustdb file -GPG_CMD="$GPG_CMD --no-auto-check-trustdb --trust-model always" -GPG="$GPG_CMD" - APT_DIR="/" eval $(apt-config shell APT_DIR Dir) @@ -331,6 +309,28 @@ if [ "$command" != "help" ]; then echo >&2 fi + GPG_CMD="gpg --ignore-time-conflict --no-options --no-default-keyring" + + # gpg needs (in different versions more or less) files to function correctly, + # so we give it its own homedir and generate some valid content for it + GPGHOMEDIR="$(mktemp -d)" + CURRENTTRAP="${CURRENTTRAP} rm -rf '${GPGHOMEDIR}';" + trap "${CURRENTTRAP}" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM + chmod 700 "$GPGHOMEDIR" + # We don't use a secret keyring, of course, but gpg panics and + # implodes if there isn't one available - and writeable for imports + SECRETKEYRING="${GPGHOMEDIR}/secring.gpg" + touch $SECRETKEYRING + GPG_CMD="$GPG_CMD --homedir $GPGHOMEDIR" + # create the trustdb with an (empty) dummy keyring + # older gpgs required it, newer gpgs even warn that it isn't needed, + # but require it nonetheless for some commands, so we just play safe + # here for the foreseeable future and create a dummy one + $GPG_CMD --quiet --check-trustdb --keyring $SECRETKEYRING >/dev/null 2>&1 + # tell gpg that it shouldn't try to maintain a trustdb file + GPG_CMD="$GPG_CMD --no-auto-check-trustdb --trust-model always" + GPG="$GPG_CMD" + # gpg defaults to mode 0600 for new keyrings. Create one with 0644 instead. if ! [ -e "$TRUSTEDFILE" ]; then if [ -w "$(dirname "$TRUSTEDFILE")" ]; then -- cgit v1.2.3-70-g09d2 From 93d0d08cdd6854f9bfb779c13b5b78cd6ed263aa Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 26 Jan 2014 18:28:50 +0100 Subject: support gnupg2 as drop-in replacement for gnupg If both are available APT will still prefer gpg over gpg2 as it is a bit more lightweight, but it shouldn't be a problem to use one or the other (at least at the moment, who knows what will happen in the future). --- cmdline/apt-key.in | 17 +++- debian/control | 2 +- test/integration/test-apt-key | 191 ++++++++++++++++++++++++------------------ 3 files changed, 122 insertions(+), 88 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index b8fdfe121..12aee9750 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -303,13 +303,22 @@ fi shift if [ "$command" != "help" ]; then - if ! which gpg >/dev/null 2>&1; then - echo >&2 "Warning: gnupg does not seem to be installed." - echo >&2 "Warning: apt-key requires gnupg for most operations." + eval $(apt-config shell GPG_EXE Apt::Key::gpgcommand) + + if [ -n "$GPG_EXE" ] && which "$GPG_EXE" >/dev/null 2>&1; then + true + elif which gpg >/dev/null 2>&1; then + GPG_EXE="gpg" + elif which gpg2 >/dev/null 2>&1; then + GPG_EXE="gpg2" + else + echo >&2 "Error: gnupg or gnupg2 do not seem to be installed," + echo >&2 "Error: but apt-key requires gnupg or gnupg2 for operation." echo >&2 + exit 255 fi - GPG_CMD="gpg --ignore-time-conflict --no-options --no-default-keyring" + GPG_CMD="$GPG_EXE --ignore-time-conflict --no-options --no-default-keyring" # gpg needs (in different versions more or less) files to function correctly, # so we give it its own homedir and generate some valid content for it diff --git a/debian/control b/debian/control index 51bb85d30..b86e828c5 100644 --- a/debian/control +++ b/debian/control @@ -18,7 +18,7 @@ XS-Testsuite: autopkgtest Package: apt Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends}, ${apt:keyring}, gnupg +Depends: ${shlibs:Depends}, ${misc:Depends}, ${apt:keyring}, gnupg | gnupg2 Replaces: manpages-pl (<< 20060617-3~), manpages-it (<< 2.80-4~), sun-java6-jdk (>> 0), sun-java5-jdk (>> 0), openjdk-6-jdk (<< 6b24-1.11-0ubuntu1~) Breaks: manpages-pl (<< 20060617-3~), manpages-it (<< 2.80-4~), sun-java6-jdk (>> 0), sun-java5-jdk (>> 0), openjdk-6-jdk (<< 6b24-1.11-0ubuntu1~) Conflicts: python-apt (<< 0.7.93.2~) diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index 68b3f9710..99ce855d4 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -7,101 +7,126 @@ TESTDIR=$(readlink -f $(dirname $0)) setupenvironment configarchitecture 'amd64' -msgtest 'Check that paths in list output are not' 'double-slashed' -aptkey list 2>&1 | grep -q '//' && msgfail || msgpass - -msgtest 'Check that paths in finger output are not' 'double-slashed' -aptkey finger 2>&1 | grep -q '//' && msgfail || msgpass +# start from a clean plate again +cleanplate() { + rm -rf rootdir/etc/apt/trusted.gpg.d/ rootdir/etc/apt/trusted.gpg + mkdir rootdir/etc/apt/trusted.gpg.d/ +} echo 'APT::Key::ArchiveKeyring "./keys/joesixpack.pub"; APT::Key::RemovedKeys "./keys/rexexpired.pub";' > rootdir/etc/apt/apt.conf.d/aptkey.conf -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' +testrun() { + cleanplate + ln -sf ${TMPWORKINGDIRECTORY}/keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg -testequal 'gpg: key DBAC8DAE: "Joe Sixpack (APT Testcases Dummy) " not changed -gpg: Total number processed: 1 -gpg: unchanged: 1' aptkey --fakeroot update + msgtest 'Check that paths in list output are not' 'double-slashed' + aptkey list 2>&1 | grep -q '//' && msgfail || msgpass -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + msgtest 'Check that paths in finger output are not' 'double-slashed' + aptkey finger 2>&1 | grep -q '//' && msgfail || msgpass -testsuccess aptkey --fakeroot add ./keys/rexexpired.pub - -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/27CE74F9 2013-07-12 [expired: 2013-07-13] -pub 2048R/DBAC8DAE 2010-08-18' + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' -msgtest 'Execute update again to trigger removal of' 'Rex Expired key' -testsuccess --nomsg aptkey --fakeroot update + testequal 'gpg: key DBAC8DAE: "Joe Sixpack (APT Testcases Dummy) " not changed +gpg: Total number processed: 1 +gpg: unchanged: 1' aptkey --fakeroot update -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' -msgtest "Try to remove a key which exists, but isn't in the" 'forced keyring' -testsuccess --nomsg aptkey --fakeroot --keyring rootdir/etc/apt/trusted.gpg del DBAC8DAE + testsuccess aptkey --fakeroot add ./keys/rexexpired.pub -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/27CE74F9 2013-07-12 [expired: 2013-07-13] +pub 2048R/DBAC8DAE 2010-08-18' -testsuccess aptkey --fakeroot del DBAC8DAE -testempty aptkey list + msgtest 'Execute update again to trigger removal of' 'Rex Expired key' + testsuccess --nomsg aptkey --fakeroot update + + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + + msgtest "Try to remove a key which exists, but isn't in the" 'forced keyring' + testsuccess --nomsg aptkey --fakeroot --keyring rootdir/etc/apt/trusted.gpg del DBAC8DAE + + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + + testsuccess aptkey --fakeroot del DBAC8DAE + testempty aptkey list + + msgtest 'Test key removal with' 'single key in real file' + cleanplate + cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + testsuccess --nomsg aptkey --fakeroot del DBAC8DAE + testempty aptkey list + testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ + + msgtest 'Test key removal with' 'single key in softlink' + cleanplate + ln -s $(readlink -f ./keys/joesixpack.pub) rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + testsuccess --nomsg aptkey --fakeroot del DBAC8DAE + testempty aptkey list + testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + testsuccess test -L rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ + + cleanplate + testsuccess aptkey --fakeroot add ./keys/joesixpack.pub + testsuccess aptkey --fakeroot add ./keys/marvinparanoid.pub + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18 +pub 2048R/528144E2 2011-01-16' + cp -a rootdir/etc/apt/trusted.gpg keys/testcase-multikey.pub # store for reuse + + msgtest 'Test key removal with' 'multi key in real file' + cleanplate + cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg + testsuccess --nomsg aptkey --fakeroot del DBAC8DAE + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' + testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ + + msgtest 'Test key removal with' 'multi key in softlink' + cleanplate + ln -s $(readlink -f ./keys/testcase-multikey.pub) rootdir/etc/apt/trusted.gpg.d/multikey.gpg + testsuccess --nomsg aptkey --fakeroot del DBAC8DAE + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' + testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ + testsuccess test ! -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg + testsuccess test -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ + + msgtest 'Test key removal with' 'multiple files including key' + cleanplate + cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg + testsuccess --nomsg aptkey --fakeroot del DBAC8DAE + aptkey list | grep '^pub' > aptkey.list + testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' + testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ + testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ +} -# start from a clean plate again -cleanplate() { - rm -rf rootdir/etc/apt/trusted.gpg.d/ rootdir/etc/apt/trusted.gpg - mkdir rootdir/etc/apt/trusted.gpg.d/ +setupgpgcommand() { + echo "APT::Key::GPGCommand \"$1\";" > rootdir/etc/apt/apt.conf.d/00gpgcmd + msgtest 'Test that apt-key uses for the following tests command' "$1" + aptkey adv --version >aptkey.version 2>&1 + if grep -q "^Executing: $1 --" aptkey.version; then + msgpass + else + cat aptkey.version + msgfail + fi } -msgtest 'Test key removal with' 'single key in real file' -cleanplate -cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg -testsuccess --nomsg aptkey --fakeroot del DBAC8DAE -testempty aptkey list -testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg -testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ - -msgtest 'Test key removal with' 'single key in softlink' -cleanplate -ln -s $(readlink -f ./keys/joesixpack.pub) rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg -testsuccess --nomsg aptkey --fakeroot del DBAC8DAE -testempty aptkey list -testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg -testsuccess test -L rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ - -cleanplate -testsuccess aptkey --fakeroot add ./keys/joesixpack.pub -testsuccess aptkey --fakeroot add ./keys/marvinparanoid.pub -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18 -pub 2048R/528144E2 2011-01-16' -cp -a rootdir/etc/apt/trusted.gpg keys/testcase-multikey.pub # store for reuse - -msgtest 'Test key removal with' 'multi key in real file' -cleanplate -cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg -testsuccess --nomsg aptkey --fakeroot del DBAC8DAE -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' -testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ - -msgtest 'Test key removal with' 'multi key in softlink' -cleanplate -ln -s $(readlink -f ./keys/testcase-multikey.pub) rootdir/etc/apt/trusted.gpg.d/multikey.gpg -testsuccess --nomsg aptkey --fakeroot del DBAC8DAE -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' -testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ -testsuccess test ! -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg -testsuccess test -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ - -msgtest 'Test key removal with' 'multiple files including key' -cleanplate -cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg -cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg -testsuccess --nomsg aptkey --fakeroot del DBAC8DAE -aptkey list | grep '^pub' > aptkey.list -testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' -testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg -testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ -testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ +# run with default (whatever this is) +testrun +# run with … +setupgpgcommand 'gpg' +testrun +setupgpgcommand 'gpg2' +testrun -- cgit v1.2.3-70-g09d2 From 0740a31033739ba30e7cb6754111f968167cbbf5 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 26 Jan 2014 19:14:13 +0100 Subject: respect --keyring also in merged keyring commands Git-Dch: Ignore --- cmdline/apt-key.in | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 12aee9750..c54b608e1 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -231,14 +231,16 @@ import_keys_from_keyring() { } setup_merged_keyring() { - local TRUSTEDFILE_BAK="$TRUSTEDFILE" - TRUSTEDFILE='/dev/null' - foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/trusted.gpg" - TRUSTEDFILE="$TRUSTEDFILE_BAK" - # mark it as non-writeable so users get errors if gnupg tries to modify it - if [ -s "${GPGHOMEDIR}/trusted.gpg" ]; then - chmod -w "${GPGHOMEDIR}/trusted.gpg" - GPG="$GPG --keyring ${GPGHOMEDIR}/trusted.gpg" + if [ -z "$FORCED_KEYRING" ]; then + local TRUSTEDFILE_BAK="$TRUSTEDFILE" + TRUSTEDFILE='/dev/null' + foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/pubring.gpg" + TRUSTEDFILE="$TRUSTEDFILE_BAK" + # mark it as non-writeable so users get errors if gnupg tries to modify it + if [ -s "${GPGHOMEDIR}/pubring.gpg" ]; then + chmod -w "${GPGHOMEDIR}/pubring.gpg" + GPG="$GPG --keyring ${GPGHOMEDIR}/pubring.gpg" + fi fi if [ -r "$TRUSTEDFILE" ]; then GPG="$GPG --keyring $TRUSTEDFILE --primary-keyring $TRUSTEDFILE" -- cgit v1.2.3-70-g09d2 From 38005d8b24bb81f4862d2c2a228e4a49a2af4ccd Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 27 Jan 2014 16:59:46 +0100 Subject: add a test for apt-key export{,all} Git-Dch: Ignore --- cmdline/apt-key.in | 4 ++-- test/integration/test-apt-key | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index c54b608e1..a3f8dde3a 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -380,8 +380,8 @@ case "$command" in foreach_keyring_do 'fingerprint_keys_from_keyring' "$@" ;; export|exportall) - foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/trusted.gpg" - $GPG_CMD --keyring "${GPGHOMEDIR}/trusted.gpg" --armor --export "$@" + foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/pubring.gpg" + $GPG_CMD --keyring "${GPGHOMEDIR}/pubring.gpg" --armor --export "$@" ;; adv*) setup_merged_keyring diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index 99ce855d4..e863e54a4 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -42,6 +42,13 @@ gpg: unchanged: 1' aptkey --fakeroot update testfileequal ./aptkey.list 'pub 2048R/27CE74F9 2013-07-12 [expired: 2013-07-13] pub 2048R/DBAC8DAE 2010-08-18' + msgtest 'Check that Sixpack key can be' 'exported' + aptkey export 'Sixpack' > aptkey.export + aptkey --keyring rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg exportall > aptkey.exportall + testsuccess --nomsg cmp aptkey.export aptkey.exportall + testsuccess test -s aptkey.export + testsuccess test -s aptkey.exportall + msgtest 'Execute update again to trigger removal of' 'Rex Expired key' testsuccess --nomsg aptkey --fakeroot update -- cgit v1.2.3-70-g09d2 From ba72845c07b2682f251dc7661869d20095260f8f Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 27 Jan 2014 17:04:53 +0100 Subject: allow to specify fingerprints in 'apt-key del' --- cmdline/apt-key.in | 21 +++++++++++++++++---- test/integration/test-apt-key | 8 ++++++++ test/integration/test-apt-key-net-update | 2 +- 3 files changed, 26 insertions(+), 5 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index a3f8dde3a..74ca4d135 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -25,6 +25,19 @@ requires_root() { fi } +get_fingerprints_of_keyring() { + $GPG_CMD --keyring "$1" --with-colons --fingerprint | while read publine; do + # search for a public key + if [ "${publine%%:*}" != 'pub' ]; then continue; fi + # search for the associated fingerprint (should be the very next line) + while read fprline; do + if [ "${fprline%%:*}" = 'sub' ]; then break; # should never happen + elif [ "${fprline%%:*}" != 'fpr' ]; then continue; fi + echo "$fprline" | cut -d':' -f 10 + done + done +} + add_keys_with_verify_against_master_keyring() { ADD_KEYRING=$1 MASTER=$2 @@ -42,7 +55,7 @@ add_keys_with_verify_against_master_keyring() { # is honored. so: # all keys that are exported must have a valid signature # from a key in the $distro-master-keyring - add_keys=`$GPG_CMD --keyring $ADD_KEYRING --with-colons --list-keys | grep ^pub | cut -d: -f5` + add_keys="$(get_fingerprints_of_keyring "$ADD_KEYRING")" all_add_keys=`$GPG_CMD --keyring $ADD_KEYRING --with-colons --list-keys | grep ^[ps]ub | cut -d: -f5` master_keys=`$GPG_CMD --keyring $MASTER --with-colons --list-keys | grep ^pub | cut -d: -f5` @@ -133,7 +146,7 @@ update() { if [ -r "$REMOVED_KEYS" ]; then # remove no-longer supported/used keys - $GPG_CMD --keyring $REMOVED_KEYS --with-colons --list-keys | grep ^pub | cut -d: -f5 | while read key; do + get_fingerprints_of_keyring "$REMOVED_KEYS" | while read key; do foreach_keyring_do 'remove_key_from_keyring' "$key" done else @@ -154,7 +167,7 @@ remove_key_from_keyring() { local KEY="$1" shift # check if the key is in this keyring: the key id is in the 5 column at the end - if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*${KEY}:"; then + if ! get_fingerprints_of_keyring "$KEYRINGFILE" | grep -q "^[0-9A-F]*${KEY}$"; then continue fi if [ ! -w "$KEYRINGFILE" ]; then @@ -162,7 +175,7 @@ remove_key_from_keyring() { continue fi # check if it is the only key in the keyring and if so remove the keyring altogether - if [ '1' = "$($GPG --with-colons --list-keys | grep "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+:" | wc -l)" ]; then + if [ '1' = "$(get_fingerprints_of_keyring "$KEYRINGFILE" | wc -l)" ]; then mv -f "$KEYRINGFILE" "${KEYRINGFILE}~" # behave like gpg return fi diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index e863e54a4..6bece40d7 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -72,6 +72,14 @@ pub 2048R/DBAC8DAE 2010-08-18' testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ + msgtest 'Test key removal with' 'fingerprint' + cleanplate + cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + testsuccess --nomsg aptkey --fakeroot del 34A8E9D18DB320F367E8EAA05A90D141DBAC8DAE + testempty aptkey list + testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ + msgtest 'Test key removal with' 'single key in softlink' cleanplate ln -s $(readlink -f ./keys/joesixpack.pub) rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg diff --git a/test/integration/test-apt-key-net-update b/test/integration/test-apt-key-net-update index b5fb796d0..b3c118555 100755 --- a/test/integration/test-apt-key-net-update +++ b/test/integration/test-apt-key-net-update @@ -41,7 +41,7 @@ echo 'APT::Key::Net-Update-Enabled "1";' >> ./aptconfig.conf # test against the "real" webserver testequal "Checking for new archive signing keys now -Key 'E8525D47528144E2' not added. It is not signed with a master key" aptkey --fakeroot net-update +Key 'DE66AECA9151AFA1877EC31DE8525D47528144E2' not added. It is not signed with a master key" aptkey --fakeroot net-update aptkey list | grep '^pub' > aptkey.list testfileequal ./aptkey.list 'pub 1024R/F68C85A3 2013-12-19 -- cgit v1.2.3-70-g09d2 From bd7fb5aa31f58917e8630f2981e78d190d465198 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 27 Jan 2014 18:26:44 +0100 Subject: add --secret-keyring option for apt-key For some advanced usecases it might be handy to specify the secret keyring to be used (e.g. as it is used in the testcases), but specifying it via a normal option for gnupg might not be available forever: http://lists.gnupg.org/pipermail/gnupg-users/2013-August/047180.html Git-Dch: Ignore --- cmdline/apt-key.in | 11 +++++++++++ test/integration/framework | 6 +++--- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 74ca4d135..36824b6ec 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -287,6 +287,11 @@ while [ -n "$1" ]; do FORCED_KEYRING="$1" shift ;; + --secret-keyring) + shift + FORCED_SECRET_KEYRING="$1" + shift + ;; --fakeroot) requires_root() { true; } shift @@ -355,6 +360,12 @@ if [ "$command" != "help" ]; then GPG_CMD="$GPG_CMD --no-auto-check-trustdb --trust-model always" GPG="$GPG_CMD" + # for advanced operations, we might really need a secret keyring after all + if [ -n "$FORCED_SECRET_KEYRING" ] && [ -r "$FORCED_SECRET_KEYRING" ]; then + rm -f "$SECRETKEYRING" + cp -a "$FORCED_SECRET_KEYRING" "$SECRETKEYRING" + fi + # gpg defaults to mode 0600 for new keyrings. Create one with 0644 instead. if ! [ -e "$TRUSTEDFILE" ]; then if [ -w "$(dirname "$TRUSTEDFILE")" ]; then diff --git a/test/integration/framework b/test/integration/framework index 1ab01b20a..50f16fb46 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -429,8 +429,8 @@ Package: $NAME" >> ${BUILDDIR}/debian/control | while read SRC; do echo "pool/${SRC}" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.srclist # if expr match "${SRC}" '.*\.dsc' >/dev/null 2>&1; then -# aptkey --keyring ./keys/joesixpack.pub --quiet adv --yes \ -# --secret-keyring ./keys/joesixpack.sec --default-key 'Joe Sixpack' \ +# aptkey --keyring ./keys/joesixpack.pub --secret-keyring ./keys/joesixpack.sec --quiet \ +# adv --yes --default-key 'Joe Sixpack' \ # --clearsign -o "${BUILDDIR}/../${SRC}.sign" "${BUILDDIR}/../$SRC" # mv "${BUILDDIR}/../${SRC}.sign" "${BUILDDIR}/../$SRC" # fi @@ -816,7 +816,7 @@ setupaptarchive() { signreleasefiles() { local SIGNER="${1:-Joe Sixpack}" local KEY="keys/$(echo "$SIGNER" | tr 'A-Z' 'a-z' | sed 's# ##g')" - local GPG="aptkey --quiet --keyring ${KEY}.pub adv --batch --yes --secret-keyring ${KEY}.sec" + local GPG="aptkey --quiet --keyring ${KEY}.pub --secret-keyring ${KEY}.sec adv --batch --yes" msgninfo "\tSign archive with $SIGNER key $KEY… " local REXKEY='keys/rexexpired' local SECEXPIREBAK="${REXKEY}.sec.bak" -- cgit v1.2.3-70-g09d2 From 0dae96a2b5e8ecd80a1b6e44961f1692ad4aec15 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 27 Jan 2014 22:07:16 +0100 Subject: use only one --keyring in gpg interactions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We were down to at most two keyrings before, but gnupg upstream plans dropping support for multiple keyrings in the longrun, so with a single keyring we hope to be future proof – and 'apt-key adv' isn't a problem anymore as every change to the keys is merged back, so we have now the same behavior as before, but support an unlimited amount of trusted.gpg.d keyrings. --- cmdline/apt-key.in | 105 +++++++++++++++++++++++++++++++----------- test/integration/test-apt-key | 57 +++++++++++++++-------- 2 files changed, 116 insertions(+), 46 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 36824b6ec..9259fac0d 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -14,7 +14,6 @@ REMOVED_KEYS='&keyring-removed-filename;' eval $(apt-config shell REMOVED_KEYS APT::Key::RemovedKeys) ARCHIVE_KEYRING_URI='&keyring-uri;' eval $(apt-config shell ARCHIVE_KEYRING_URI APT::Key::ArchiveKeyringURI) -TMP_KEYRING=${APT_DIR}/var/lib/apt/keyrings/maybe-import-keyring.gpg aptkey_echo() { echo "$@"; } @@ -68,24 +67,28 @@ add_keys_with_verify_against_master_keyring() { fi done done - + for add_key in $add_keys; do # export the add keyring one-by-one - rm -f $TMP_KEYRING - $GPG_CMD --keyring $ADD_KEYRING --output $TMP_KEYRING --export $add_key - # check if signed with the master key and only add in this case - ADDED=0 + local TMP_KEYRING="${GPGHOMEDIR}/tmp-keyring.gpg" + $GPG_CMD --batch --yes --keyring "$ADD_KEYRING" --output "$TMP_KEYRING" --export "$add_key" + if ! $GPG_CMD --batch --yes --keyring "$TMP_KEYRING" --import "$MASTER" > "${GPGHOMEDIR}/gpgoutput.log" 2>&1; then + cat "${GPGHOMEDIR}/gpgoutput.log" + false + fi + # check if signed with the master key and only add in this case + ADDED=0 for master_key in $master_keys; do - if $GPG_CMD --keyring $MASTER --keyring $TMP_KEYRING --check-sigs --with-colons $add_key | grep '^sig:!:' | cut -d: -f5 | grep -q $master_key; then - $GPG --import $TMP_KEYRING + if $GPG_CMD --keyring $TMP_KEYRING --check-sigs --with-colons $add_key | grep '^sig:!:' | cut -d: -f5 | grep -q $master_key; then + $GPG_CMD --batch --yes --keyring "$ADD_KEYRING" --export "$add_key" | $GPG --batch --yes --import ADDED=1 fi done if [ $ADDED = 0 ]; then echo >&2 "Key '$add_key' not added. It is not signed with a master key" fi + rm -f "${TMP_KEYRING}" done - rm -f $TMP_KEYRING } # update the current archive signing keyring from a network URI @@ -240,26 +243,75 @@ fingerprint_keys_from_keyring() { import_keys_from_keyring() { local IMPORT="$1" local KEYRINGFILE="$2" - $GPG_CMD --keyring "$KEYRINGFILE" --batch --import "$IMPORT" >/dev/null 2>&1 + if ! $GPG_CMD --keyring "$KEYRINGFILE" --batch --import "$IMPORT" > "${GPGHOMEDIR}/gpgoutput.log" 2>&1; then + cat "${GPGHOMEDIR}/gpgoutput.log" + false + fi +} + +merge_keys_into_keyrings() { + local KEYRINGFILE="$1" + local IMPORT="$2" + if ! $GPG_CMD --keyring "$KEYRINGFILE" --batch --import --import-options 'merge-only' "$IMPORT" > "${GPGHOMEDIR}/gpgoutput.log" 2>&1; then + cat "${GPGHOMEDIR}/gpgoutput.log" + false + fi +} + +merge_back_changes() { + if [ -n "$FORCED_KEYRING" ]; then + # if the keyring was forced merge is already done + return + fi + if [ -s "${GPGHOMEDIR}/pubring.gpg" ]; then + # merge all updated keys + foreach_keyring_do 'merge_keys_into_keyrings' "${GPGHOMEDIR}/pubring.gpg" + fi + # no look for keys which were added or removed + get_fingerprints_of_keyring "${GPGHOMEDIR}/pubring.orig.gpg" > "${GPGHOMEDIR}/pubring.orig.keylst" + get_fingerprints_of_keyring "${GPGHOMEDIR}/pubring.gpg" > "${GPGHOMEDIR}/pubring.keylst" + #echo >&2 "MERGE BACK" + sort "${GPGHOMEDIR}/pubring.keylst" "${GPGHOMEDIR}/pubring.orig.keylst" | uniq --unique | while read key; do + if grep -q "^${key}$" "${GPGHOMEDIR}/pubring.orig.keylst"; then + # key isn't part of new keyring, so remove + foreach_keyring_do 'remove_key_from_keyring' "$key" + elif grep -q "^${key}$" "${GPGHOMEDIR}/pubring.keylst"; then + # key is part of new keyring, so we need to import it + create_new_keyring "$TRUSTEDFILE" + if ! $GPG --batch --yes --export "$key" | $GPG_CMD --keyring "$TRUSTEDFILE" --batch --yes --import > "${GPGHOMEDIR}/gpgoutput.log" 2>&1; then + cat "${GPGHOMEDIR}/gpgoutput.log" + false + fi + else + echo >&2 "Errror: Key ${key} (dis)appeared out of nowhere" + fi + done } setup_merged_keyring() { if [ -z "$FORCED_KEYRING" ]; then - local TRUSTEDFILE_BAK="$TRUSTEDFILE" - TRUSTEDFILE='/dev/null' foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/pubring.gpg" - TRUSTEDFILE="$TRUSTEDFILE_BAK" - # mark it as non-writeable so users get errors if gnupg tries to modify it - if [ -s "${GPGHOMEDIR}/pubring.gpg" ]; then - chmod -w "${GPGHOMEDIR}/pubring.gpg" - GPG="$GPG --keyring ${GPGHOMEDIR}/pubring.gpg" + if [ -r "${GPGHOMEDIR}/pubring.gpg" ]; then + cp -a "${GPGHOMEDIR}/pubring.gpg" "${GPGHOMEDIR}/pubring.orig.gpg" + else + touch "${GPGHOMEDIR}/pubring.gpg" "${GPGHOMEDIR}/pubring.orig.gpg" fi - fi - if [ -r "$TRUSTEDFILE" ]; then - GPG="$GPG --keyring $TRUSTEDFILE --primary-keyring $TRUSTEDFILE" + GPG="$GPG --keyring ${GPGHOMEDIR}/pubring.gpg" + else + GPG="$GPG --keyring $TRUSTEDFILE" + create_new_keyring "$TRUSTEDFILE" fi } +create_new_keyring() { + # gpg defaults to mode 0600 for new keyrings. Create one with 0644 instead. + if ! [ -e "$TRUSTEDFILE" ]; then + if [ -w "$(dirname "$TRUSTEDFILE")" ]; then + touch -- "$TRUSTEDFILE" + chmod 0644 -- "$TRUSTEDFILE" + fi + fi +} usage() { echo "Usage: apt-key [--keyring file] [command] [arguments]" @@ -365,14 +417,6 @@ if [ "$command" != "help" ]; then rm -f "$SECRETKEYRING" cp -a "$FORCED_SECRET_KEYRING" "$SECRETKEYRING" fi - - # gpg defaults to mode 0600 for new keyrings. Create one with 0644 instead. - if ! [ -e "$TRUSTEDFILE" ]; then - if [ -w "$(dirname "$TRUSTEDFILE")" ]; then - touch -- "$TRUSTEDFILE" - chmod 0644 -- "$TRUSTEDFILE" - fi - fi fi case "$command" in @@ -380,22 +424,26 @@ case "$command" in requires_root setup_merged_keyring $GPG --quiet --batch --import "$@" + merge_back_changes aptkey_echo "OK" ;; del|rm|remove) requires_root foreach_keyring_do 'remove_key_from_keyring' "$@" + merge_back_changes aptkey_echo "OK" ;; update) requires_root setup_merged_keyring update + merge_back_changes ;; net-update) requires_root setup_merged_keyring net_update + merge_back_changes ;; list) foreach_keyring_do 'list_keys_from_keyring' "$@" @@ -411,6 +459,7 @@ case "$command" in setup_merged_keyring aptkey_echo "Executing: $GPG $*" $GPG "$@" + merge_back_changes ;; help) usage diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index 6bece40d7..337b16a59 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -13,6 +13,13 @@ cleanplate() { mkdir rootdir/etc/apt/trusted.gpg.d/ } +testaptkeys() { + if ! aptkey list | grep '^pub' > aptkey.list; then + echo -n > aptkey.list + fi + testequal "$1" cat ./aptkey.list +} + echo 'APT::Key::ArchiveKeyring "./keys/joesixpack.pub"; APT::Key::RemovedKeys "./keys/rexexpired.pub";' > rootdir/etc/apt/apt.conf.d/aptkey.conf @@ -26,20 +33,17 @@ testrun() { msgtest 'Check that paths in finger output are not' 'double-slashed' aptkey finger 2>&1 | grep -q '//' && msgfail || msgpass - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + testaptkeys 'pub 2048R/DBAC8DAE 2010-08-18' testequal 'gpg: key DBAC8DAE: "Joe Sixpack (APT Testcases Dummy) " not changed gpg: Total number processed: 1 gpg: unchanged: 1' aptkey --fakeroot update - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + testaptkeys 'pub 2048R/DBAC8DAE 2010-08-18' testsuccess aptkey --fakeroot add ./keys/rexexpired.pub - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/27CE74F9 2013-07-12 [expired: 2013-07-13] + testaptkeys 'pub 2048R/27CE74F9 2013-07-12 [expired: 2013-07-13] pub 2048R/DBAC8DAE 2010-08-18' msgtest 'Check that Sixpack key can be' 'exported' @@ -52,14 +56,12 @@ pub 2048R/DBAC8DAE 2010-08-18' msgtest 'Execute update again to trigger removal of' 'Rex Expired key' testsuccess --nomsg aptkey --fakeroot update - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + testaptkeys 'pub 2048R/DBAC8DAE 2010-08-18' msgtest "Try to remove a key which exists, but isn't in the" 'forced keyring' testsuccess --nomsg aptkey --fakeroot --keyring rootdir/etc/apt/trusted.gpg del DBAC8DAE - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18' + testaptkeys 'pub 2048R/DBAC8DAE 2010-08-18' testsuccess aptkey --fakeroot del DBAC8DAE testempty aptkey list @@ -91,8 +93,7 @@ pub 2048R/DBAC8DAE 2010-08-18' cleanplate testsuccess aptkey --fakeroot add ./keys/joesixpack.pub testsuccess aptkey --fakeroot add ./keys/marvinparanoid.pub - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/DBAC8DAE 2010-08-18 + testaptkeys 'pub 2048R/DBAC8DAE 2010-08-18 pub 2048R/528144E2 2011-01-16' cp -a rootdir/etc/apt/trusted.gpg keys/testcase-multikey.pub # store for reuse @@ -100,16 +101,14 @@ pub 2048R/528144E2 2011-01-16' cleanplate cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg testsuccess --nomsg aptkey --fakeroot del DBAC8DAE - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' + testaptkeys 'pub 2048R/528144E2 2011-01-16' testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ msgtest 'Test key removal with' 'multi key in softlink' cleanplate ln -s $(readlink -f ./keys/testcase-multikey.pub) rootdir/etc/apt/trusted.gpg.d/multikey.gpg testsuccess --nomsg aptkey --fakeroot del DBAC8DAE - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' + testaptkeys 'pub 2048R/528144E2 2011-01-16' testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ testsuccess test ! -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg testsuccess test -L rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ @@ -119,11 +118,33 @@ pub 2048R/528144E2 2011-01-16' cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg testsuccess --nomsg aptkey --fakeroot del DBAC8DAE - aptkey list | grep '^pub' > aptkey.list - testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' + testaptkeys 'pub 2048R/528144E2 2011-01-16' testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ + + cleanplate + cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg + cp -a keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg + testaptkeys 'pub 2048R/DBAC8DAE 2010-08-18 +pub 2048R/DBAC8DAE 2010-08-18 +pub 2048R/528144E2 2011-01-16' + msgtest 'Test merge-back of' 'added keys' + testsuccess --nomsg aptkey adv --batch --yes --import keys/rexexpired.pub + testaptkeys 'pub 2048R/27CE74F9 2013-07-12 [expired: 2013-07-13] +pub 2048R/DBAC8DAE 2010-08-18 +pub 2048R/DBAC8DAE 2010-08-18 +pub 2048R/528144E2 2011-01-16' + + msgtest 'Test merge-back of' 'removed keys' + testsuccess --nomsg aptkey adv --batch --yes --delete-keys 27CE74F9 + testaptkeys 'pub 2048R/DBAC8DAE 2010-08-18 +pub 2048R/DBAC8DAE 2010-08-18 +pub 2048R/528144E2 2011-01-16' + + msgtest 'Test merge-back of' 'removed duplicate keys' + testsuccess --nomsg aptkey adv --batch --yes --delete-keys DBAC8DAE + testaptkeys 'pub 2048R/528144E2 2011-01-16' } setupgpgcommand() { -- cgit v1.2.3-70-g09d2 From 33a2267214eed2a11281c9f93b8cf10b4c436d94 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 6 Feb 2014 17:56:28 +0100 Subject: add --readonly option for apt-key adv Some advanced commands can be executed without the keyring being modified like --verify, so this adds an option to disable the mergeback and uses it for our gpg calling code. Git-Dch: Ignore --- apt-pkg/contrib/gpgv.cc | 1 + cmdline/apt-key.in | 9 ++++----- test/integration/framework | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc index 56e26ab99..cd17cd536 100644 --- a/apt-pkg/contrib/gpgv.cc +++ b/apt-pkg/contrib/gpgv.cc @@ -54,6 +54,7 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, Args.push_back(aptkey.c_str()); Args.push_back("--quiet"); + Args.push_back("--readonly"); Args.push_back("adv"); char statusfdstr[10]; diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 9259fac0d..21d692631 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -337,20 +337,19 @@ while [ -n "$1" ]; do shift TRUSTEDFILE="$1" FORCED_KEYRING="$1" - shift ;; --secret-keyring) shift FORCED_SECRET_KEYRING="$1" - shift + ;; + --readonly) + merge_back_changes() { true; } ;; --fakeroot) requires_root() { true; } - shift ;; --quiet) aptkey_echo() { true; } - shift ;; --*) echo >&2 "Unknown option: $1" @@ -359,6 +358,7 @@ while [ -n "$1" ]; do *) break;; esac + shift done if [ -z "$TRUSTEDFILE" ]; then @@ -430,7 +430,6 @@ case "$command" in del|rm|remove) requires_root foreach_keyring_do 'remove_key_from_keyring' "$@" - merge_back_changes aptkey_echo "OK" ;; update) diff --git a/test/integration/framework b/test/integration/framework index 50f16fb46..fcdca34ce 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -429,7 +429,7 @@ Package: $NAME" >> ${BUILDDIR}/debian/control | while read SRC; do echo "pool/${SRC}" >> ${BUILDDIR}/../${RELEASE}.${DISTSECTION}.srclist # if expr match "${SRC}" '.*\.dsc' >/dev/null 2>&1; then -# aptkey --keyring ./keys/joesixpack.pub --secret-keyring ./keys/joesixpack.sec --quiet \ +# aptkey --keyring ./keys/joesixpack.pub --secret-keyring ./keys/joesixpack.sec --quiet --readonly \ # adv --yes --default-key 'Joe Sixpack' \ # --clearsign -o "${BUILDDIR}/../${SRC}.sign" "${BUILDDIR}/../$SRC" # mv "${BUILDDIR}/../${SRC}.sign" "${BUILDDIR}/../$SRC" @@ -816,7 +816,7 @@ setupaptarchive() { signreleasefiles() { local SIGNER="${1:-Joe Sixpack}" local KEY="keys/$(echo "$SIGNER" | tr 'A-Z' 'a-z' | sed 's# ##g')" - local GPG="aptkey --quiet --keyring ${KEY}.pub --secret-keyring ${KEY}.sec adv --batch --yes" + local GPG="aptkey --quiet --keyring ${KEY}.pub --secret-keyring ${KEY}.sec --readonly adv --batch --yes" msgninfo "\tSign archive with $SIGNER key $KEY… " local REXKEY='keys/rexexpired' local SECEXPIREBAK="${REXKEY}.sec.bak" -- cgit v1.2.3-70-g09d2 From 0b94a7bc2ebb42453d9e22dc58dde76b9261de50 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 6 Feb 2014 18:43:55 +0100 Subject: miscellaneous small cleanups in apt-key Git-Dch: Ignore --- cmdline/apt-key.in | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 21d692631..a9a729cce 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -166,9 +166,7 @@ remove_key_from_keyring() { fi local GPG="$GPG_CMD --keyring $KEYRINGFILE" - while [ -n "$1" ]; do - local KEY="$1" - shift + for KEY in "$@"; do # check if the key is in this keyring: the key id is in the 5 column at the end if ! get_fingerprints_of_keyring "$KEYRINGFILE" | grep -q "^[0-9A-F]*${KEY}$"; then continue @@ -226,18 +224,11 @@ foreach_keyring_do() { fi } -list_keys_from_keyring() { +run_cmd_on_keyring() { local KEYRINGFILE="$1" shift - # don't show the error message if this keyring doesn't include the key - $GPG_CMD --keyring "$KEYRINGFILE" --batch --list-keys "$@" 2>/dev/null || true -} - -fingerprint_keys_from_keyring() { - local KEYRINGFILE="$1" - shift - # don't show the error message if this keyring doesn't include the fingerprint - $GPG_CMD --keyring "$KEYRINGFILE" --batch --fingerprint "$@" 2>/dev/null || true + # fingerprint and co will fail if key isn't in this keyring + $GPG_CMD --keyring "$KEYRINGFILE" --batch "$@" 2>/dev/null || true } import_keys_from_keyring() { @@ -267,10 +258,9 @@ merge_back_changes() { # merge all updated keys foreach_keyring_do 'merge_keys_into_keyrings' "${GPGHOMEDIR}/pubring.gpg" fi - # no look for keys which were added or removed + # look for keys which were added or removed get_fingerprints_of_keyring "${GPGHOMEDIR}/pubring.orig.gpg" > "${GPGHOMEDIR}/pubring.orig.keylst" get_fingerprints_of_keyring "${GPGHOMEDIR}/pubring.gpg" > "${GPGHOMEDIR}/pubring.keylst" - #echo >&2 "MERGE BACK" sort "${GPGHOMEDIR}/pubring.keylst" "${GPGHOMEDIR}/pubring.orig.keylst" | uniq --unique | while read key; do if grep -q "^${key}$" "${GPGHOMEDIR}/pubring.orig.keylst"; then # key isn't part of new keyring, so remove @@ -445,10 +435,10 @@ case "$command" in merge_back_changes ;; list) - foreach_keyring_do 'list_keys_from_keyring' "$@" + foreach_keyring_do 'run_cmd_on_keyring' --list-keys "$@" ;; finger*) - foreach_keyring_do 'fingerprint_keys_from_keyring' "$@" + foreach_keyring_do 'run_cmd_on_keyring' --fingerprint "$@" ;; export|exportall) foreach_keyring_do 'import_keys_from_keyring' "${GPGHOMEDIR}/pubring.gpg" -- cgit v1.2.3-70-g09d2 From c46a36adaf51fc28464ea1a0e826c754ee60672b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 14 Apr 2014 18:24:17 +0200 Subject: add and use 'apt-key verify' which prefers gpgv over gpg gnupg/gnupg2 can do verify just fine of course, so we don't need to use gpgv here, but it is what we always used in the past, so there might be scripts expecting a certain output and more importantly the output of apt-cdrom contains messages from gpg and even with all the settings we activate to prevent it, it still shows (in some versions) a quiet scary: "gpg: WARNING: Using untrusted key!" message. Keeping the use of gpgv is the simplest way to prevent it. We are increasing also the "Breaks: apt" version from libapt as it requires a newer apt-key than might be installed in partial upgrades. --- apt-pkg/contrib/gpgv.cc | 3 +-- cmdline/apt-key.in | 8 ++++++++ debian/control | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc index cd17cd536..9d798cca9 100644 --- a/apt-pkg/contrib/gpgv.cc +++ b/apt-pkg/contrib/gpgv.cc @@ -55,7 +55,7 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, Args.push_back(aptkey.c_str()); Args.push_back("--quiet"); Args.push_back("--readonly"); - Args.push_back("adv"); + Args.push_back("verify"); char statusfdstr[10]; if (statusfd != -1) @@ -77,7 +77,6 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG, Args.push_back(Opts->Value.c_str()); } } - Args.push_back("--verify"); enum { DETACHED, CLEARSIGNED } releaseSignature = (FileGPG != File) ? DETACHED : CLEARSIGNED; std::vector dataHeader; diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index a9a729cce..83a7a31b9 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -450,6 +450,14 @@ case "$command" in $GPG "$@" merge_back_changes ;; + verify) + setup_merged_keyring + if which gpgv >/dev/null 2>&1; then + gpgv --homedir "${GPGHOMEDIR}" --keyring "${GPGHOMEDIR}/pubring.gpg" --ignore-time-conflict "$@" + else + $GPG --verify "$@" + fi + ;; help) usage ;; diff --git a/debian/control b/debian/control index b86e828c5..6d29b0250 100644 --- a/debian/control +++ b/debian/control @@ -43,7 +43,7 @@ Architecture: any Multi-Arch: same Pre-Depends: ${misc:Pre-Depends} Depends: ${shlibs:Depends}, ${misc:Depends} -Breaks: apt (<< 0.9.4~), libapt-inst1.5 (<< 0.9.9~) +Breaks: apt (<< 1.0.2~), libapt-inst1.5 (<< 0.9.9~) Section: libs Description: package management runtime library This library provides the common functionality for searching and -- cgit v1.2.3-70-g09d2 From a221efc331693f8905da870141756c892911c433 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 20 Jun 2014 19:34:40 +0200 Subject: store source name and version in binary cache Accessing the package records to acquire this information is pretty costly, so that information wasn't used so far in many places. The most noticeable user by far is EDSP at the moment, but there are ideas to change that which this commit tries to enable. --- apt-pkg/cacheiterators.h | 6 +++++ apt-pkg/deb/deblistparser.cc | 51 +++++++++++++++++++++++++++++++++++ apt-pkg/deb/dpkgpm.cc | 7 +---- apt-pkg/edsp.cc | 6 +---- apt-pkg/pkgcache.h | 6 +++++ cmdline/apt-cache.cc | 2 ++ cmdline/apt-get.cc | 64 +++++++++++++++----------------------------- 7 files changed, 89 insertions(+), 53 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index 12a03eb07..b0c02d4a2 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -215,6 +215,12 @@ class pkgCache::VerIterator : public Iterator { // Accessors inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;} inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;} + /** \brief source package name this version comes from + Always contains the name, even if it is the same as the binary name */ + inline const char *SourcePkgName() const {return Owner->StrP + S->SourcePkgName;} + /** \brief source version this version comes from + Always contains the version string, even if it is the same as the binary version */ + inline const char *SourceVerStr() const {return Owner->StrP + S->SourceVerStr;} inline const char *Arch() const { if ((S->MultiArch & pkgCache::Version::All) == pkgCache::Version::All) return "all"; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 9919d2036..103b126de 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -141,6 +141,57 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) map_stringitem_t const idx = StoreString(pkgCacheGenerator::SECTION, Start, Stop - Start); Ver->Section = idx; } + // Parse the source package name + pkgCache::GrpIterator const G = Ver.ParentPkg().Group(); + Ver->SourcePkgName = G->Name; + Ver->SourceVerStr = Ver->VerStr; + if (Section.Find("Source",Start,Stop) == true) + { + const char * const Space = (const char * const) memchr(Start, ' ', Stop - Start); + pkgCache::VerIterator V; + + if (Space != NULL) + { + Stop = Space; + const char * const Open = (const char * const) memchr(Space, '(', Stop - Space); + if (likely(Open != NULL)) + { + const char * const Close = (const char * const) memchr(Open, ')', Stop - Open); + if (likely(Close != NULL)) + { + std::string const version(Open + 1, (Close - Open) - 1); + if (version != Ver.VerStr()) + { + map_stringitem_t const idx = StoreString(pkgCacheGenerator::VERSION, version); + Ver->SourceVerStr = idx; + } + } + } + } + + std::string const pkgname(Start, Stop - Start); + if (pkgname != G.Name()) + { + for (pkgCache::PkgIterator P = G.PackageList(); P.end() == false; P = G.NextPkg(P)) + { + for (V = P.VersionList(); V.end() == false; ++V) + { + if (pkgname == V.SourcePkgName()) + { + Ver->SourcePkgName = V->SourcePkgName; + break; + } + } + if (V.end() == false) + break; + } + if (V.end() == true) + { + map_stringitem_t const idx = StoreString(pkgCacheGenerator::PKGNAME, pkgname); + Ver->SourcePkgName = idx; + } + } + } Ver->MultiArch = ParseMultiArch(true); // Archive Size diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index a5c05d5ea..95fae9a28 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1754,11 +1754,6 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) if (Ver.end() == true) return; pkgver = Ver.VerStr() == NULL ? "unknown" : Ver.VerStr(); - pkgRecords Recs(Cache); - pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); - srcpkgname = Parse.SourcePkg(); - if(srcpkgname.empty()) - srcpkgname = pkgname; // if the file exists already, we check: // - if it was reported already (touched by apport). @@ -1809,7 +1804,7 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) time_t now = time(NULL); fprintf(report, "Date: %s" , ctime(&now)); fprintf(report, "Package: %s %s\n", pkgname.c_str(), pkgver.c_str()); - fprintf(report, "SourcePackage: %s\n", srcpkgname.c_str()); + fprintf(report, "SourcePackage: %s\n", Ver.SourcePkgName()); fprintf(report, "ErrorMessage:\n %s\n", errormsg); // ensure that the log is flushed diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc index 0d0418e06..2ba914b16 100644 --- a/apt-pkg/edsp.cc +++ b/apt-pkg/edsp.cc @@ -95,12 +95,8 @@ bool EDSP::WriteLimitedScenario(pkgDepCache &Cache, FILE* output, void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgIterator const &Pkg, pkgCache::VerIterator const &Ver) { - pkgRecords Recs(Cache); - pkgRecords::Parser &rec = Recs.Lookup(Ver.FileList()); - string srcpkg = rec.SourcePkg().empty() ? Pkg.Name() : rec.SourcePkg(); - fprintf(output, "Package: %s\n", Pkg.Name()); - fprintf(output, "Source: %s\n", srcpkg.c_str()); + fprintf(output, "Source: %s\n", Ver.SourcePkgName()); fprintf(output, "Architecture: %s\n", Ver.Arch()); fprintf(output, "Version: %s\n", Ver.VerStr()); if (Pkg.CurrentVer() == Ver) diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 4284a318c..6a89eabd7 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -510,6 +510,12 @@ struct pkgCache::Version map_stringitem_t VerStr; /** \brief section this version is filled in */ map_stringitem_t Section; + /** \brief source package name this version comes from + Always contains the name, even if it is the same as the binary name */ + map_stringitem_t SourcePkgName; + /** \brief source version this version comes from + Always contains the version string, even if it is the same as the binary version */ + map_stringitem_t SourceVerStr; /** \brief Multi-Arch capabilities of a package version */ enum VerMultiArch { None = 0, /*!< is the default and doesn't trigger special behaviour */ diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 5a5dde088..0f4f7e1ce 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -389,6 +389,8 @@ static bool Stats(CommandLine &) stritems.insert(V->VerStr); if (V->Section != 0) stritems.insert(V->Section); + stritems.insert(V->SourcePkgName); + stritems.insert(V->SourceVerStr); for (pkgCache::DepIterator D = V.DependsList(); D.end() == false; ++D) { if (D->Version != 0) diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index aed1beb4d..a5cafc39d 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -195,7 +195,7 @@ static std::string GetReleaseForSourceRecord(pkgSourceList *SrcList, // FindSrc - Find a source record /*{{{*/ // --------------------------------------------------------------------- /* */ -static pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, +static pkgSrcRecords::Parser *FindSrc(const char *Name, pkgSrcRecords &SrcRecs,string &Src, CacheFile &CacheFile) { @@ -303,16 +303,10 @@ static pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, (VF.File().Archive() != 0 && VF.File().Archive() == RelTag) || (VF.File().Codename() != 0 && VF.File().Codename() == RelTag)) { - pkgRecords::Parser &Parse = Recs.Lookup(VF); - Src = Parse.SourcePkg(); - // no SourcePkg name, so it is the "binary" name - if (Src.empty() == true) - Src = TmpSrc; + Src = Ver.SourcePkgName(); // the Version we have is possibly fuzzy or includes binUploads, - // so we use the Version of the SourcePkg (empty if same as package) - VerTag = Parse.SourceVer(); - if (VerTag.empty() == true) - VerTag = Ver.VerStr(); + // so we use the Version of the SourcePkg + VerTag = Ver.SourceVerStr(); break; } } @@ -343,10 +337,10 @@ static pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, pkgCache::VerIterator Ver = Cache->GetCandidateVer(Pkg); if (Ver.end() == false) { - pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); - Src = Parse.SourcePkg(); - if (VerTag.empty() == true) - VerTag = Parse.SourceVer(); + if (strcmp(Ver.SourcePkgName(),Ver.ParentPkg().Name()) != 0) + Src = Ver.SourcePkgName(); + if (VerTag.empty() == true && strcmp(Ver.SourceVerStr(),Ver.VerStr()) != 0) + VerTag = Ver.SourceVerStr(); } } } @@ -731,7 +725,6 @@ static bool DoSource(CommandLine &CmdL) pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers - pkgRecords Recs(Cache); pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@ -760,7 +753,7 @@ static bool DoSource(CommandLine &CmdL) for (const char **I = CmdL.FileList + 1; *I != 0; I++, J++) { string Src; - pkgSrcRecords::Parser *Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); + pkgSrcRecords::Parser *Last = FindSrc(*I,SrcRecs,Src,Cache); if (Last == 0) { return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); @@ -1037,7 +1030,6 @@ static bool DoBuildDep(CommandLine &CmdL) pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers - pkgRecords Recs(Cache); pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@ -1090,7 +1082,7 @@ static bool DoBuildDep(CommandLine &CmdL) Last = Type->CreateSrcPkgParser(*I); } else { // normal case, search the cache for the source file - Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); + Last = FindSrc(*I,SrcRecs,Src,Cache); } if (Last == 0) @@ -1441,21 +1433,15 @@ static bool DoBuildDep(CommandLine &CmdL) * pool/ next to the deb itself) * Example return: "pool/main/a/apt/apt_0.8.8ubuntu3" */ -static string GetChangelogPath(CacheFile &Cache, - pkgCache::PkgIterator Pkg, +static string GetChangelogPath(CacheFile &Cache, pkgCache::VerIterator Ver) { - string path; - pkgRecords Recs(Cache); pkgRecords::Parser &rec=Recs.Lookup(Ver.FileList()); - string srcpkg = rec.SourcePkg().empty() ? Pkg.Name() : rec.SourcePkg(); - string ver = Ver.VerStr(); - // if there is a source version it always wins - if (rec.SourceVer() != "") - ver = rec.SourceVer(); - path = flNotFile(rec.FileName()); - path += srcpkg + "_" + StripEpoch(ver); + string path = flNotFile(rec.FileName()); + path.append(Ver.SourcePkgName()); + path.append("_"); + path.append(StripEpoch(Ver.SourceVerStr())); return path; } /*}}}*/ @@ -1469,7 +1455,6 @@ static string GetChangelogPath(CacheFile &Cache, * http://packages.medibuntu.org/pool/non-free/m/mplayer/mplayer_1.0~rc4~try1.dsfg1-1ubuntu1+medibuntu1.changelog */ static bool GuessThirdPartyChangelogUri(CacheFile &Cache, - pkgCache::PkgIterator Pkg, pkgCache::VerIterator Ver, string &out_uri) { @@ -1484,7 +1469,7 @@ static bool GuessThirdPartyChangelogUri(CacheFile &Cache, return false; // get archive uri for the binary deb - string path_without_dot_changelog = GetChangelogPath(Cache, Pkg, Ver); + string path_without_dot_changelog = GetChangelogPath(Cache, Ver); out_uri = index->ArchiveURI(path_without_dot_changelog + ".changelog"); // now strip away the filename and add srcpkg_srcver.changelog @@ -1502,25 +1487,20 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, * GuessThirdPartyChangelogUri for details how) */ { - string path; - string descr; - string server; - string changelog_uri; - - // data structures we need - pkgCache::PkgIterator Pkg = Ver.ParentPkg(); - // make the server root configurable - server = _config->Find("Apt::Changelogs::Server", + string const server = _config->Find("Apt::Changelogs::Server", "http://packages.debian.org/changelogs"); - path = GetChangelogPath(CacheFile, Pkg, Ver); + string const path = GetChangelogPath(CacheFile, Ver); + string changelog_uri; strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str()); if (_config->FindB("APT::Get::Print-URIs", false) == true) { std::cout << '\'' << changelog_uri << '\'' << std::endl; return true; } + pkgCache::PkgIterator const Pkg = Ver.ParentPkg(); + string descr; strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), changelog_uri.c_str()); // queue it new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); @@ -1531,7 +1511,7 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, if (!FileExists(targetfile)) { string third_party_uri; - if (GuessThirdPartyChangelogUri(CacheFile, Pkg, Ver, third_party_uri)) + if (GuessThirdPartyChangelogUri(CacheFile, Ver, third_party_uri)) { strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), third_party_uri.c_str()); new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); -- cgit v1.2.3-70-g09d2 From 43acd01979039b248cb7f033b82e36d778d0ebec Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 27 Sep 2014 19:45:30 +0200 Subject: allow fetcher setup without directory creation apt-get download and changelog as well as apt-helper reuse the acquire system for their own proposes without requiring the directories the fetcher wants to create, which is a problem if you run them as non-root and the directories do not exist as it greets you with: E: Archives directory /var/cache/apt/archives/partial is missing. - Acquire (13: Permission denied) Closes: 762898 --- apt-pkg/acquire.cc | 30 +++++++++++++++++------------- apt-pkg/acquire.h | 5 ++++- cmdline/apt-get.cc | 9 +++++---- cmdline/apt-helper.cc | 3 ++- test/integration/test-apt-get-changelog | 6 ++++++ test/integration/test-apt-get-download | 27 +++++++++++++++++++++++++-- test/integration/test-apt-helper | 14 +++++++------- 7 files changed, 66 insertions(+), 28 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index f0a88a538..8119e4487 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -73,23 +73,27 @@ pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Wor // --------------------------------------------------------------------- /* Do everything needed to be a complete Acquire object and report the success (or failure) back so the user knows that something is wrong… */ -bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) +bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock, + bool const createDirectories) { Log = Progress; // check for existence and possibly create auxiliary directories - string const listDir = _config->FindDir("Dir::State::lists"); - string const partialListDir = listDir + "partial/"; - string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - string const partialArchivesDir = archivesDir + "partial/"; - - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && - CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false) - return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); - - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::Cache"), partialArchivesDir) == false && - CreateAPTDirectoryIfNeeded(archivesDir, partialArchivesDir) == false) - return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + if (createDirectories == true) + { + string const listDir = _config->FindDir("Dir::State::lists"); + string const partialListDir = listDir + "partial/"; + string const archivesDir = _config->FindDir("Dir::Cache::Archives"); + string const partialArchivesDir = archivesDir + "partial/"; + + if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && + CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + + if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::Cache"), partialArchivesDir) == false && + CreateAPTDirectoryIfNeeded(archivesDir, partialArchivesDir) == false) + return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + } if (Lock.empty() == true || _config->FindB("Debug::NoLocking", false) == true) return true; diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index a92ecc24c..7bceb4323 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -352,8 +352,11 @@ class pkgAcquire * \param Lock defines a lock file that should be acquired to ensure * only one Acquire class is in action at the time or an empty string * if no lock file should be used. + * \param createDirectories can be used to disable the creation of directories, + * e.g. if the fetcher is used with different directories later on */ - bool Setup(pkgAcquireStatus *Progress = NULL, std::string const &Lock = ""); + bool Setup(pkgAcquireStatus *Progress = NULL, std::string const &Lock = "", + bool const createDirectories = true); void SetLog(pkgAcquireStatus *Progress) { Log = Progress; } diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index a5cafc39d..6d03c7eec 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -626,14 +626,14 @@ static bool DoDownload(CommandLine &CmdL) APT::CacheSetHelper helper(c0out); APT::VersionSet verset = APT::VersionSet::FromCommandLine(Cache, - CmdL.FileList + 1, APT::VersionSet::CANDIDATE, helper); + CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper); if (verset.empty() == true) return false; AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet", 0)); pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat) == false) + if (Fetcher.Setup(&Stat, "", false) == false) return false; pkgRecords Recs(Cache); @@ -1536,7 +1536,7 @@ static bool DoChangelog(CommandLine &CmdL) APT::CacheSetHelper helper(c0out); APT::VersionList verset = APT::VersionList::FromCommandLine(Cache, - CmdL.FileList + 1, APT::VersionList::CANDIDATE, helper); + CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper); if (verset.empty() == true) return false; pkgAcquire Fetcher; @@ -1551,7 +1551,8 @@ static bool DoChangelog(CommandLine &CmdL) } AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); - Fetcher.Setup(&Stat); + if (Fetcher.Setup(&Stat, "",false) == false) + return false; bool const downOnly = _config->FindB("APT::Get::Download-Only", false); diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index dd43ea1bc..b89df61d6 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -51,7 +51,8 @@ static bool DoDownloadFile(CommandLine &CmdL) pkgAcquire Fetcher; AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); - Fetcher.Setup(&Stat); + if (Fetcher.Setup(&Stat, "", false) == false) + return false; std::string download_uri = CmdL.FileList[1]; std::string targetfile = CmdL.FileList[2]; std::string hash; diff --git a/test/integration/test-apt-get-changelog b/test/integration/test-apt-get-changelog index 4ee113482..02d6c39ab 100755 --- a/test/integration/test-apt-get-changelog +++ b/test/integration/test-apt-get-changelog @@ -13,6 +13,12 @@ setupaptarchive --no-update changetowebserver testsuccess aptget update +# simulate normal user with non-existent root-owned directories +rm -rf rootdir/var/cache/apt/archives/ +mkdir rootdir/var/cache/apt/archives/ +addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/cache/apt/archives || true;" +chmod -R -w rootdir/var/cache/apt/archives + echo 'Apt::Changelogs::Server "http://localhost:8080/";' > rootdir/etc/apt/apt.conf.d/changelog.conf testequal "'http://localhost:8080//pool/apt_1.0/changelog'" aptget changelog apt --print-uris diff --git a/test/integration/test-apt-get-download b/test/integration/test-apt-get-download index be3144e1f..58ed44f8f 100755 --- a/test/integration/test-apt-get-download +++ b/test/integration/test-apt-get-download @@ -20,10 +20,19 @@ testdownload() { fi msgtest "Test download of package file $1 with" "$APT" testsuccess --nomsg aptget download ${APT} - testsuccess test -f $1 - rm $1 + testsuccess test -f "$1" + rm -f "$1" } +# normal case as "root" +testdownload apt_2.0_all.deb apt + +# simulate normal user with non-existent root-owned directories +rm -rf rootdir/var/cache/apt/archives/ +mkdir rootdir/var/cache/apt/archives/ +addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/cache/apt/archives || true;" +chmod -R -w rootdir/var/cache/apt/archives + # normal case(es) testdownload apt_1.0_all.deb apt stable testdownload apt_2.0_all.deb apt @@ -45,3 +54,17 @@ rm -f apt_1.0_all.deb apt_2.0_all.deb testsuccess aptget download apt apt apt/unstable apt=2.0 testsuccess test -s apt_2.0_all.deb +# restore "root" rights +chmod -f -R +w $PWD/rootdir/var/cache/apt/archives +rm -rf rootdir/var/cache/apt/archives/ + +# file: debs aren't copied to archives, so change to http which obviously are +changetowebserver +testsuccess aptget update + +# test with already stored deb +testsuccess aptget install -d apt +testsuccess test -s rootdir/var/cache/apt/archives/apt_2.0_all.deb +mv aptarchive/pool/apt_2.0_all.deb aptarchive/pool/apt_2.0_all.deb.gone +testdownload apt_2.0_all.deb apt +mv aptarchive/pool/apt_2.0_all.deb.gone aptarchive/pool/apt_2.0_all.deb diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index c749224ca..42c40bb9e 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -5,30 +5,30 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configarchitecture "i386" +configarchitecture 'i386' changetohttpswebserver test_apt_helper_download() { - echo "foo" > aptarchive/foo + echo 'foo' > aptarchive/foo - msgtest 'apt-file download-file md5sum' + msgtest 'apt-file download-file' 'md5sum' apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail testfileequal foo2 'foo' - msgtest 'apt-file download-file sha1' + msgtest 'apt-file download-file' 'sha1' apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail testfileequal foo1 'foo' - msgtest 'apt-file download-file sha256' + msgtest 'apt-file download-file' 'sha256' apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail testfileequal foo3 'foo' - msgtest 'apt-file download-file no-hash' + msgtest 'apt-file download-file' 'no-hash' apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail testfileequal foo4 'foo' - msgtest 'apt-file download-file wrong hash' + msgtest 'apt-file download-file' 'wrong hash' if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then msgpass else -- cgit v1.2.3-70-g09d2 From 20e6965ad6388b964b59b89c31ff8b81cbcb8f8c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 27 Sep 2014 20:09:44 +0200 Subject: cleanup partial directory of lists in apt-get clean Not really the intended usecase for apt-get clean, but users expect it to help them in recovery and it can't really hurt as this directory should be empty if everything was fine and proper anyway. Closes: #762889 --- cmdline/apt-get.cc | 26 ++++++++++++++++++++------ test/integration/test-apt-get-clean | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) create mode 100755 test/integration/test-apt-get-clean (limited to 'cmdline') diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 6d03c7eec..c8c3ca56b 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -549,30 +549,44 @@ static bool DoDSelectUpgrade(CommandLine &) static bool DoClean(CommandLine &) { std::string const archivedir = _config->FindDir("Dir::Cache::archives"); - std::string const pkgcache = _config->FindFile("Dir::cache::pkgcache"); - std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache"); + std::string const listsdir = _config->FindDir("Dir::state::lists"); if (_config->FindB("APT::Get::Simulate") == true) { + std::string const pkgcache = _config->FindFile("Dir::cache::pkgcache"); + std::string const srcpkgcache = _config->FindFile("Dir::cache::srcpkgcache"); cout << "Del " << archivedir << "* " << archivedir << "partial/*"<< endl + << "Del " << listsdir << "partial/*" << endl << "Del " << pkgcache << " " << srcpkgcache << endl; return true; } - + + bool const NoLocking = _config->FindB("Debug::NoLocking",false); // Lock the archive directory FileFd Lock; - if (_config->FindB("Debug::NoLocking",false) == false) + if (NoLocking == false) { int lock_fd = GetLock(archivedir + "lock"); if (lock_fd < 0) - return _error->Error(_("Unable to lock the download directory")); + return _error->Error(_("Unable to lock directory %s"), archivedir.c_str()); Lock.Fd(lock_fd); } - + pkgAcquire Fetcher; Fetcher.Clean(archivedir); Fetcher.Clean(archivedir + "partial/"); + if (NoLocking == false) + { + Lock.Close(); + int lock_fd = GetLock(listsdir + "lock"); + if (lock_fd < 0) + return _error->Error(_("Unable to lock directory %s"), listsdir.c_str()); + Lock.Fd(lock_fd); + } + + Fetcher.Clean(listsdir + "partial/"); + pkgCacheFile::RemoveCaches(); return true; diff --git a/test/integration/test-apt-get-clean b/test/integration/test-apt-get-clean new file mode 100755 index 000000000..646ea31be --- /dev/null +++ b/test/integration/test-apt-get-clean @@ -0,0 +1,34 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' + +insertpackage 'testing' 'foo' 'all' '1' +insertpackage 'unstable' 'foo' 'all' '2' +insertinstalledpackage 'foo' 'all' '3' + +setupaptarchive + +# nothing to do always works +testsuccess aptget clean + +# generate some dirt and clean it up +touch rootdir/var/lib/apt/lists/partial/http.debian.net_debian_dists_sid_main_i18n_Translation-en +touch rootdir/var/cache/apt/archives/foo_1_all.deb +touch rootdir/var/cache/apt/archives/foo_2_all.deb +touch rootdir/var/cache/apt/archives/foo_3_all.deb +touch rootdir/var/cache/apt/archives/foo_4_all.deb + +testsuccess aptget clean + +testsuccess test ! -e rootdir/var/lib/apt/lists/partial/http.debian.net_debian_dists_sid_main_i18n_Translation-en +testsuccess test ! -e rootdir/var/cache/apt/archives/foo_1_all.deb +testsuccess test ! -e rootdir/var/cache/apt/archives/foo_2_all.deb +testsuccess test ! -e rootdir/var/cache/apt/archives/foo_3_all.deb +testsuccess test ! -e rootdir/var/cache/apt/archives/foo_4_all.deb + + -- cgit v1.2.3-70-g09d2 From 7ffac4c10144a8458fec6de03c58f3b49d081e5c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 29 Sep 2014 11:43:37 +0200 Subject: Test if TMPDIR is a directory in apt-key and if not unset it This prevents a failure in mktemp -d - it will blindly trust TMPDIR and not use something else if the dir is not there. --- cmdline/apt-key.in | 3 +++ test/integration/test-bug-728500-tempdir | 1 + 2 files changed, 4 insertions(+) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 83a7a31b9..7a3852ee8 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -384,6 +384,9 @@ if [ "$command" != "help" ]; then # gpg needs (in different versions more or less) files to function correctly, # so we give it its own homedir and generate some valid content for it + if [ ! -d "$TMPDIR" ]; then + unset TMPDIR + fi GPGHOMEDIR="$(mktemp -d)" CURRENTTRAP="${CURRENTTRAP} rm -rf '${GPGHOMEDIR}';" trap "${CURRENTTRAP}" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM diff --git a/test/integration/test-bug-728500-tempdir b/test/integration/test-bug-728500-tempdir index 0451fc1ed..bdc38c3ca 100755 --- a/test/integration/test-bug-728500-tempdir +++ b/test/integration/test-bug-728500-tempdir @@ -27,3 +27,4 @@ fi unset TMPDIR testequal 'coolstuff' aptcache pkgnames +testsuccess ls rootdir/var/lib/apt/lists/*InRelease -- cgit v1.2.3-70-g09d2 From 4c370aad7539b7e3bc0028aa538f34b95a526cff Mon Sep 17 00:00:00 2001 From: Guillem Jover Date: Thu, 2 Oct 2014 17:48:13 +0200 Subject: apt-get: Create the temporary downloaded changelog inside tmpdir The code is creating a secure temporary directory, but then creates the changelog alongside the tmpdir in the same base directory. This defeats the secure tmpdir creation, making the filename predictable. Inject a '/' between the tmpdir and the changelog filename. --- cmdline/apt-get.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 2e283da5a..cfa79339b 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1563,7 +1563,7 @@ static bool DoChangelog(CommandLine &CmdL) { string changelogfile; if (downOnly == false) - changelogfile.append(tmpname).append("changelog"); + changelogfile.append(tmpname).append("/changelog"); else changelogfile.append(Ver.ParentPkg().Name()).append(".changelog"); if (DownloadChangelog(Cache, Fetcher, Ver, changelogfile) && downOnly == false) -- cgit v1.2.3-70-g09d2 From 339bc464875616d214cc3b73f5d41ced10c820b9 Mon Sep 17 00:00:00 2001 From: Guillem Jover Date: Thu, 2 Oct 2014 17:48:13 +0200 Subject: apt-get: Create the temporary downloaded changelog inside tmpdir The code is creating a secure temporary directory, but then creates the changelog alongside the tmpdir in the same base directory. This defeats the secure tmpdir creation, making the filename predictable. Inject a '/' between the tmpdir and the changelog filename. --- cmdline/apt-get.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 0cea05cb3..13a85d34b 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1588,7 +1588,7 @@ static bool DoChangelog(CommandLine &CmdL) { string changelogfile; if (downOnly == false) - changelogfile.append(tmpname).append("changelog"); + changelogfile.append(tmpname).append("/changelog"); else changelogfile.append(Ver.ParentPkg().Name()).append(".changelog"); if (DownloadChangelog(Cache, Fetcher, Ver, changelogfile) && downOnly == false) -- cgit v1.2.3-70-g09d2 From 04a54261afd1c99686109f102afc83346c01c930 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 6 Oct 2014 11:15:03 +0200 Subject: ensure partial dirs are 0700 and owned by _apt:root Reworks the API involved in creating and setting up the fetcher to be a bit more pleasent to look at and work with as e.g. an empty string for no lock isn't very nice. With the lock we can also stop creating all our partial directories "just in case". This way we can also be a bit more aggressive with the partial directory itself as with a lock, we know we will gone need it. --- apt-pkg/acquire.cc | 78 ++++++++++++++++++++++++++----------- apt-pkg/acquire.h | 19 ++++++--- apt-pkg/update.cc | 4 +- apt-private/private-install.cc | 6 +-- apt-private/private-update.cc | 4 +- cmdline/apt-get.cc | 18 ++------- cmdline/apt-helper.cc | 6 +-- test/integration/test-apt-get-clean | 1 + 8 files changed, 81 insertions(+), 55 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index ec565fcfa..9dee1b3cf 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -27,17 +27,20 @@ #include #include #include +#include + #include #include #include #include -#include - +#include +#include #include #include #include #include #include +#include #include /*}}}*/ @@ -57,8 +60,8 @@ pkgAcquire::pkgAcquire() : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NU if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; } -pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), - Configs(0), Log(Progress), ToFetch(0), +pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), + Configs(0), Log(NULL), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) { @@ -67,40 +70,69 @@ pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Wor QueueMode = QueueHost; if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; - Setup(Progress, ""); + SetLog(Progress); } /*}}}*/ -// Acquire::Setup - Delayed Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* Do everything needed to be a complete Acquire object and report the - success (or failure) back so the user knows that something is wrong… */ -bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock, - bool const createDirectories) +// Acquire::GetLock - lock directory and prepare for action /*{{{*/ +static bool SetupAPTPartialDirectory(std::string const &grand, std::string const &parent) { - Log = Progress; + std::string const partial = parent + "partial"; + if (CreateAPTDirectoryIfNeeded(grand, partial) == false && + CreateAPTDirectoryIfNeeded(parent, partial) == false) + return false; - // check for existence and possibly create auxiliary directories - if (createDirectories == true) + if (getuid() == 0) // if we aren't root, we can't chown, so don't try it + { + struct passwd *pw = getpwnam("_apt"); + struct group *gr = getgrnam("root"); + if (pw != NULL && gr != NULL && chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to _apt:root of directory %s failed", partial.c_str()); + } + if (chmod(partial.c_str(), 0700) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str()); + + return true; +} +bool pkgAcquire::Setup(pkgAcquireStatus *Progress, string const &Lock) +{ + Log = Progress; + if (Lock.empty()) { string const listDir = _config->FindDir("Dir::State::lists"); - string const partialListDir = listDir + "partial/"; + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - string const partialArchivesDir = archivesDir + "partial/"; + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) + return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); + return true; + } + return GetLock(Lock); +} +bool pkgAcquire::GetLock(std::string const &Lock) +{ + if (Lock.empty() == true) + return false; - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::State"), partialListDir) == false && - CreateAPTDirectoryIfNeeded(listDir, partialListDir) == false) - return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + // check for existence and possibly create auxiliary directories + string const listDir = _config->FindDir("Dir::State::lists"); + string const archivesDir = _config->FindDir("Dir::Cache::Archives"); - if (CreateAPTDirectoryIfNeeded(_config->FindDir("Dir::Cache"), partialArchivesDir) == false && - CreateAPTDirectoryIfNeeded(archivesDir, partialArchivesDir) == false) + if (Lock == listDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::State"), listDir) == false) + return _error->Errno("Acquire", _("List directory %spartial is missing."), listDir.c_str()); + } + if (Lock == archivesDir) + { + if (SetupAPTPartialDirectory(_config->FindDir("Dir::Cache"), archivesDir) == false) return _error->Errno("Acquire", _("Archives directory %spartial is missing."), archivesDir.c_str()); } - if (Lock.empty() == true || _config->FindB("Debug::NoLocking", false) == true) + if (_config->FindB("Debug::NoLocking", false) == true) return true; // Lock the directory this acquire object will work in - LockFD = GetLock(flCombine(Lock, "lock")); + LockFD = ::GetLock(flCombine(Lock, "lock")); if (LockFD == -1) return _error->Error(_("Unable to lock directory %s"), Lock.c_str()); diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 7bceb4323..f9eeb1641 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -351,17 +351,24 @@ class pkgAcquire * long as the pkgAcquire object does. * \param Lock defines a lock file that should be acquired to ensure * only one Acquire class is in action at the time or an empty string - * if no lock file should be used. - * \param createDirectories can be used to disable the creation of directories, - * e.g. if the fetcher is used with different directories later on + * if no lock file should be used. If set also all needed directories + * will be created. */ - bool Setup(pkgAcquireStatus *Progress = NULL, std::string const &Lock = "", - bool const createDirectories = true); + APT_DEPRECATED bool Setup(pkgAcquireStatus *Progress = NULL, std::string const &Lock = ""); void SetLog(pkgAcquireStatus *Progress) { Log = Progress; } + /** \brief acquire lock and perform directory setup + * + * \param Lock defines a lock file that should be acquired to ensure + * only one Acquire class is in action at the time or an empty string + * if no lock file should be used. If set also all needed directories + * will be created and setup. + */ + bool GetLock(std::string const &Lock); + /** \brief Construct a new pkgAcquire. */ - pkgAcquire(pkgAcquireStatus *Log) APT_DEPRECATED; + pkgAcquire(pkgAcquireStatus *Log); pkgAcquire(); /** \brief Destroy this pkgAcquire object. diff --git a/apt-pkg/update.cc b/apt-pkg/update.cc index 5d5b19626..2908a4820 100644 --- a/apt-pkg/update.cc +++ b/apt-pkg/update.cc @@ -27,8 +27,8 @@ bool ListUpdate(pkgAcquireStatus &Stat, pkgSourceList &List, int PulseInterval) { - pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat, _config->FindDir("Dir::State::Lists")) == false) + pkgAcquire Fetcher(&Stat); + if (Fetcher.GetLock(_config->FindDir("Dir::State::Lists")) == false) return false; // Populate it with the source selection diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index 86ba52857..c06caeedd 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -119,14 +119,14 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety) return false; // Create the download object - pkgAcquire Fetcher; - AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); + AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); + pkgAcquire Fetcher(&Stat); if (_config->FindB("APT::Get::Print-URIs", false) == true) { // force a hashsum for compatibility reasons _config->CndSet("Acquire::ForceHash", "md5sum"); } - else if (Fetcher.Setup(&Stat, _config->FindDir("Dir::Cache::Archives")) == false) + else if (Fetcher.GetLock(_config->FindDir("Dir::Cache::Archives")) == false) return false; // Read the source list diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc index 1cf3012ed..df77ac33a 100644 --- a/apt-private/private-update.cc +++ b/apt-private/private-update.cc @@ -47,9 +47,7 @@ bool DoUpdate(CommandLine &CmdL) _config->CndSet("Acquire::ForceHash", "md5sum"); // get a fetcher - pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat) == false) - return false; + pkgAcquire Fetcher(&Stat); // Populate it with the source selection and get all Indexes // (GetAll=true) diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 0cea05cb3..15696e19f 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -646,9 +646,7 @@ static bool DoDownload(CommandLine &CmdL) return false; AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet", 0)); - pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat, "", false) == false) - return false; + pkgAcquire Fetcher(&Stat); pkgRecords Recs(Cache); pkgSourceList *SrcList = Cache.GetSourceList(); @@ -744,9 +742,8 @@ static bool DoSource(CommandLine &CmdL) return false; // Create the download object - AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); - pkgAcquire Fetcher; - Fetcher.SetLog(&Stat); + AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); + pkgAcquire Fetcher(&Stat); SPtrArray Dsc = new DscFile[CmdL.FileSize()]; @@ -1048,12 +1045,6 @@ static bool DoBuildDep(CommandLine &CmdL) if (_error->PendingError() == true) return false; - // Create the download object - AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0)); - pkgAcquire Fetcher; - if (Fetcher.Setup(&Stat) == false) - return false; - bool StripMultiArch; string hostArch = _config->Find("APT::Get::Host-Architecture"); if (hostArch.empty() == false) @@ -1565,8 +1556,7 @@ static bool DoChangelog(CommandLine &CmdL) } AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); - if (Fetcher.Setup(&Stat, "",false) == false) - return false; + Fetcher.SetLog(&Stat); bool const downOnly = _config->FindB("APT::Get::Download-Only", false); diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index b89df61d6..c240008aa 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -48,11 +48,9 @@ static bool DoDownloadFile(CommandLine &CmdL) if (CmdL.FileSize() <= 2) return _error->Error(_("Must specify at least one pair url/filename")); - - pkgAcquire Fetcher; AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); - if (Fetcher.Setup(&Stat, "", false) == false) - return false; + pkgAcquire Fetcher(&Stat); + std::string download_uri = CmdL.FileList[1]; std::string targetfile = CmdL.FileList[2]; std::string hash; diff --git a/test/integration/test-apt-get-clean b/test/integration/test-apt-get-clean index 646ea31be..98f7c84d0 100755 --- a/test/integration/test-apt-get-clean +++ b/test/integration/test-apt-get-clean @@ -18,6 +18,7 @@ testsuccess aptget clean # generate some dirt and clean it up touch rootdir/var/lib/apt/lists/partial/http.debian.net_debian_dists_sid_main_i18n_Translation-en +mkdir -p rootdir/var/cache/apt/archives touch rootdir/var/cache/apt/archives/foo_1_all.deb touch rootdir/var/cache/apt/archives/foo_2_all.deb touch rootdir/var/cache/apt/archives/foo_3_all.deb -- cgit v1.2.3-70-g09d2 From 373fa2b4b2caae977c41b2c10ea27e41308a05c3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 13:30:27 +0200 Subject: Rename DropPrivs() to DropPrivileges() Git-Dch: ignore --- apt-pkg/acquire-method.cc | 2 +- apt-pkg/contrib/fileutl.cc | 2 +- apt-pkg/contrib/fileutl.h | 2 +- cmdline/apt-dump-solver.cc | 2 +- cmdline/apt-internal-solver.cc | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 82f2fb3ce..d116994e6 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -124,7 +124,7 @@ void pkgAcqMethod::Fail(string Err,bool Transient) /* */ void pkgAcqMethod::DropPrivsOrDie() { - if (!DropPrivs()) { + if (!DropPrivileges()) { Fail(false); exit(112); /* call the european emergency number */ } diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index df409fa36..1e1fb5957 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -2171,7 +2171,7 @@ bool Popen(const char* Args[], FileFd &Fd, pid_t &Child, FileFd::OpenMode Mode) return true; } -bool DropPrivs() +bool DropPrivileges() { // uid will be 0 in the end, but gid might be different anyway uid_t old_uid = getuid(); diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 9dd29eb9e..a64d6cb98 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -210,7 +210,7 @@ bool StartsWithGPGClearTextSignature(std::string const &FileName); * * \return true on success, false on failure with _error set */ -bool DropPrivs(); +bool DropPrivileges(); // File string manipulators std::string flNotDir(std::string File); diff --git a/cmdline/apt-dump-solver.cc b/cmdline/apt-dump-solver.cc index 424764b3c..f765234c5 100644 --- a/cmdline/apt-dump-solver.cc +++ b/cmdline/apt-dump-solver.cc @@ -41,7 +41,7 @@ int main(int argc,const char *argv[]) /*{{{*/ return 0; } // we really don't need anything - DropPrivs(); + DropPrivileges(); FILE* input = fdopen(STDIN_FILENO, "r"); FILE* output = fopen("/tmp/dump.edsp", "w"); diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index 0f2ec6283..92a4429e5 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -77,7 +77,7 @@ int main(int argc,const char *argv[]) /*{{{*/ {0,0,0,0}}; // we really don't need anything - DropPrivs(); + DropPrivileges(); CommandLine CmdL(Args,_config); if (pkgInitConfig(*_config) == false || -- cgit v1.2.3-70-g09d2 From 460601d53039b1d1b5688a8cd58bae10fb746f57 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Oct 2014 02:43:44 +0200 Subject: don't drop privileges if _apt has not enough rights MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Privilege dropping breaks download/source/changelog commands as they require the _apt user to have write permissions in the current directory, which is e.g. the case in /tmp, but not in /root, so we disable the privilege dropping if we deal with such a directory based on idea and code by Michael Vogt. The alternative would be to download always to a temp directory and move it then done, but this breaks partial file support. To resolve this, we could move to one of our partial/ directories, but this would require a lock which would block root from using two of these commands in parallel. As both seems unacceptable we instead let the user choose what to do: Either a directory is setupped for _apt, downloading as root is accepted or – which is potentially even better – an unprivileged user is used for the commands. --- apt-private/private-download.cc | 50 ++++++++++++++++++++++++++++++++++++++++- apt-private/private-download.h | 6 ++++- cmdline/apt-get.cc | 19 +++++++++++----- cmdline/apt-helper.cc | 5 ++++- 4 files changed, 72 insertions(+), 8 deletions(-) (limited to 'cmdline') diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc index be7d23c31..3ded9e0b9 100644 --- a/apt-private/private-download.cc +++ b/apt-private/private-download.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -14,9 +15,56 @@ #include #include +#include +#include +#include +#include + #include /*}}}*/ +bool CheckDropPrivsMustBeDisabled(pkgAcquire &Fetcher) /*{{{*/ +{ + // no need/possibility to drop privs + if(getuid() != 0) + return true; + + // the user does not want to drop privs + std::string SandboxUser = _config->Find("APT::Sandbox::User"); + if (SandboxUser.empty()) + return true; + + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + if (pw == NULL) + return true; + + if (seteuid(pw->pw_uid) != 0) + return _error->Errno("seteuid", "seteuid %u failed", pw->pw_uid); + + bool res = true; + // check if we can write to destfile + for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin(); + I != Fetcher.ItemsEnd() && res == true; ++I) + { + int fd = open((*I)->DestFile.c_str(), O_CREAT | O_RDWR, 0600); + if (fd < 0) + { + res = false; + std::string msg; + strprintf(msg, _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."), + (*I)->DestFile.c_str(), SandboxUser.c_str()); + c0out << msg << std::endl; + _config->Set("APT::Sandbox::User", ""); + } + close(fd); + } + + if (seteuid(0) != 0) + return _error->Errno("seteuid", "seteuid %u failed", 0); + + return res; +} + /*}}}*/ // CheckAuth - check if each download comes form a trusted source /*{{{*/ bool CheckAuth(pkgAcquire& Fetcher, bool const PromptUser) { @@ -31,7 +79,7 @@ bool CheckAuth(pkgAcquire& Fetcher, bool const PromptUser) return AuthPrompt(UntrustedList, PromptUser); } -bool AuthPrompt(std::string UntrustedList, bool const PromptUser) +bool AuthPrompt(std::string const &UntrustedList, bool const PromptUser) { ShowList(c2out,_("WARNING: The following packages cannot be authenticated!"),UntrustedList,""); diff --git a/apt-private/private-download.h b/apt-private/private-download.h index a90ac7eaa..809650a97 100644 --- a/apt-private/private-download.h +++ b/apt-private/private-download.h @@ -3,14 +3,18 @@ #include +#include + class pkgAcquire; +APT_PUBLIC bool CheckDropPrivsMustBeDisabled(pkgAcquire &Fetcher); + // Check if all files in the fetcher are authenticated APT_PUBLIC bool CheckAuth(pkgAcquire& Fetcher, bool const PromptUser); // show a authentication warning prompt and return true if the system // should continue -APT_PUBLIC bool AuthPrompt(std::string UntrustedList, bool const PromptUser); +APT_PUBLIC bool AuthPrompt(std::string const &UntrustedList, bool const PromptUser); APT_PUBLIC bool AcquireRun(pkgAcquire &Fetcher, int const PulseInterval, bool * const Failure, bool * const TransientNetworkFailure); diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 8448638db..8c0c50f83 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -676,6 +676,9 @@ static bool DoDownload(CommandLine &CmdL) return true; } + // Disable drop-privs if "_apt" can not write to the target dir + CheckDropPrivsMustBeDisabled(Fetcher); + if (_error->PendingError() == true || CheckAuth(Fetcher, false) == false) return false; @@ -858,10 +861,6 @@ static bool DoSource(CommandLine &CmdL) } } - // check authentication status of the source as well - if (UntrustedList != "" && !AuthPrompt(UntrustedList, false)) - return false; - // Display statistics unsigned long long FetchBytes = Fetcher.FetchNeeded(); unsigned long long FetchPBytes = Fetcher.PartialPresent(); @@ -908,7 +907,7 @@ static bool DoSource(CommandLine &CmdL) ioprintf(cout,_("Fetch source %s\n"),Dsc[I].Package.c_str()); return true; } - + // Just print out the uris an exit if the --print-uris flag was used if (_config->FindB("APT::Get::Print-URIs") == true) { @@ -919,6 +918,13 @@ static bool DoSource(CommandLine &CmdL) return true; } + // Disable drop-privs if "_apt" can not write to the target dir + CheckDropPrivsMustBeDisabled(Fetcher); + + // check authentication status of the source as well + if (UntrustedList != "" && !AuthPrompt(UntrustedList, false)) + return false; + // Run it bool Failed = false; if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true) @@ -1510,6 +1516,9 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, // queue it new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); + // Disable drop-privs if "_apt" can not write to the target dir + CheckDropPrivsMustBeDisabled(Fetcher); + // try downloading it, if that fails, try third-party-changelogs location // FIXME: Fetcher.Run() is "Continue" even if I get a 404?!? Fetcher.Run(); diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index c240008aa..27abb2013 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -59,7 +59,10 @@ static bool DoDownloadFile(CommandLine &CmdL) // we use download_uri as descr and targetfile as short-descr new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, "dest-dir-ignored", targetfile); - Fetcher.Run(); + + // Disable drop-privs if "_apt" can not write to the target dir + CheckDropPrivsMustBeDisabled(Fetcher); + bool Failed = false; if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true || FileExists(targetfile) == false) -- cgit v1.2.3-70-g09d2 From 9c81f8de38df940559d13a3ea9591d63cbe970bb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 15 Oct 2014 04:18:07 +0200 Subject: check for available space, excluding root reserved blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are checking the space requirements for ages, but the check uses the free blocks count, which includes the blocks reserved for usage by root. Now that we use an unprivileged user it has no access to these blocks anymore – and more importantly these blocks are a reserve, they shouldn't be used by apt without special encouragement by the user as it would be bad to have dpkg run out of diskspace and maintainerscripts like man-db skip certain actions if not enough space is available freely. --- apt-private/private-download.cc | 39 +++++++++++++++++++++++++++++++++++++++ apt-private/private-download.h | 2 ++ apt-private/private-install.cc | 33 +++------------------------------ apt-private/private-install.h | 2 -- cmdline/apt-get.cc | 28 +++------------------------- 5 files changed, 47 insertions(+), 57 deletions(-) (limited to 'cmdline') diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc index 3ded9e0b9..8cabf14b5 100644 --- a/apt-private/private-download.cc +++ b/apt-private/private-download.cc @@ -19,6 +19,9 @@ #include #include #include +#include +#include +#include #include /*}}}*/ @@ -146,3 +149,39 @@ bool AcquireRun(pkgAcquire &Fetcher, int const PulseInterval, bool * const Failu return true; } /*}}}*/ +bool CheckFreeSpaceBeforeDownload(std::string const &Dir, unsigned long long FetchBytes)/*{{{*/ +{ + uint32_t const RAMFS_MAGIC = 0x858458f6; + /* Check for enough free space, but only if we are actually going to + download */ + if (_config->FindB("APT::Get::Print-URIs", false) == true || + _config->FindB("APT::Get::Download", true) == false) + return true; + + struct statvfs Buf; + if (statvfs(Dir.c_str(),&Buf) != 0) { + if (errno == EOVERFLOW) + return _error->WarningE("statvfs",_("Couldn't determine free space in %s"), + Dir.c_str()); + else + return _error->Errno("statvfs",_("Couldn't determine free space in %s"), + Dir.c_str()); + } + else + { + unsigned long long const FreeBlocks = _config->Find("APT::Sandbox::User").empty() ? Buf.f_bfree : Buf.f_bavail; + if (FreeBlocks < (FetchBytes / Buf.f_bsize)) + { + struct statfs Stat; + if (statfs(Dir.c_str(),&Stat) != 0 +#if HAVE_STRUCT_STATFS_F_TYPE + || Stat.f_type != RAMFS_MAGIC +#endif + ) + return _error->Error(_("You don't have enough free space in %s."), + Dir.c_str()); + } + } + return true; +} + /*}}}*/ diff --git a/apt-private/private-download.h b/apt-private/private-download.h index 809650a97..0a0ac6b95 100644 --- a/apt-private/private-download.h +++ b/apt-private/private-download.h @@ -18,4 +18,6 @@ APT_PUBLIC bool AuthPrompt(std::string const &UntrustedList, bool const PromptUs APT_PUBLIC bool AcquireRun(pkgAcquire &Fetcher, int const PulseInterval, bool * const Failure, bool * const TransientNetworkFailure); +APT_PUBLIC bool CheckFreeSpaceBeforeDownload(std::string const &Dir, unsigned long long FetchBytes); + #endif diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index c06caeedd..2a4c3eea5 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -22,11 +22,8 @@ #include #include -#include #include #include -#include -#include #include #include #include @@ -177,33 +174,9 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety) if (_error->PendingError() == true) return false; - /* Check for enough free space, but only if we are actually going to - download */ - if (_config->FindB("APT::Get::Print-URIs") == false && - _config->FindB("APT::Get::Download",true) == true) - { - struct statvfs Buf; - std::string OutputDir = _config->FindDir("Dir::Cache::Archives"); - if (statvfs(OutputDir.c_str(),&Buf) != 0) { - if (errno == EOVERFLOW) - return _error->WarningE("statvfs",_("Couldn't determine free space in %s"), - OutputDir.c_str()); - else - return _error->Errno("statvfs",_("Couldn't determine free space in %s"), - OutputDir.c_str()); - } else if (unsigned(Buf.f_bfree) < (FetchBytes - FetchPBytes)/Buf.f_bsize) - { - struct statfs Stat; - if (statfs(OutputDir.c_str(),&Stat) != 0 -#if HAVE_STRUCT_STATFS_F_TYPE - || unsigned(Stat.f_type) != RAMFS_MAGIC -#endif - ) - return _error->Error(_("You don't have enough free space in %s."), - OutputDir.c_str()); - } - } - + if (CheckFreeSpaceBeforeDownload(_config->FindDir("Dir::Cache::Archives"), (FetchBytes - FetchPBytes)) == false) + return false; + // Fail safe check if (_config->FindI("quiet",0) >= 2 || _config->FindB("APT::Get::Assume-Yes",false) == true) diff --git a/apt-private/private-install.h b/apt-private/private-install.h index 8daa4a776..62276fbff 100644 --- a/apt-private/private-install.h +++ b/apt-private/private-install.h @@ -16,8 +16,6 @@ class CacheFile; class CommandLine; class pkgProblemResolver; -#define RAMFS_MAGIC 0x858458f6 - APT_PUBLIC bool DoInstall(CommandLine &Cmd); bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, CacheFile &Cache, diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 8c0c50f83..e176a3350 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -78,8 +78,6 @@ #include #include #include -#include -#include #include #include #include @@ -866,29 +864,9 @@ static bool DoSource(CommandLine &CmdL) unsigned long long FetchPBytes = Fetcher.PartialPresent(); unsigned long long DebBytes = Fetcher.TotalNeeded(); - // Check for enough free space - struct statvfs Buf; - string OutputDir = "."; - if (statvfs(OutputDir.c_str(),&Buf) != 0) { - if (errno == EOVERFLOW) - return _error->WarningE("statvfs",_("Couldn't determine free space in %s"), - OutputDir.c_str()); - else - return _error->Errno("statvfs",_("Couldn't determine free space in %s"), - OutputDir.c_str()); - } else if (unsigned(Buf.f_bfree) < (FetchBytes - FetchPBytes)/Buf.f_bsize) - { - struct statfs Stat; - if (statfs(OutputDir.c_str(),&Stat) != 0 -#if HAVE_STRUCT_STATFS_F_TYPE - || unsigned(Stat.f_type) != RAMFS_MAGIC -#endif - ) { - return _error->Error(_("You don't have enough free space in %s"), - OutputDir.c_str()); - } - } - + if (CheckFreeSpaceBeforeDownload(".", (FetchBytes - FetchPBytes)) == false) + return false; + // Number of bytes if (DebBytes != FetchBytes) //TRANSLATOR: The required space between number and unit is already included -- cgit v1.2.3-70-g09d2 From 0d303f1764645284b33924c9be8bf29f0a32ca5c Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 20 Oct 2014 12:00:46 +0200 Subject: test if TMPDIR is accessible before using Private temporary directories as created by e.g. libpam-tmpdir are nice, but they are also very effective in preventing our priviledge dropping to work as TMPDIR will be set to a directory only root has access to, so working with it as _apt will fail. We circumvent this by extending our check for a usable TMPDIR setting by checking access rights. Closes: 765951 --- apt-pkg/contrib/fileutl.cc | 6 ++++-- cmdline/apt-key.in | 8 ++++++-- test/integration/framework | 4 ++++ test/integration/test-bug-604401-files-are-directories | 2 +- test/libapt/fileutl_test.cc | 10 +++++++++- 5 files changed, 24 insertions(+), 6 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index c51eee737..47033eadf 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -2066,9 +2066,11 @@ std::string GetTempDir() /*{{{*/ tmpdir = P_tmpdir; #endif - // check that tmpdir is set and exists struct stat st; - if (!tmpdir || strlen(tmpdir) == 0 || stat(tmpdir, &st) != 0) + if (!tmpdir || strlen(tmpdir) == 0 || // tmpdir is set + stat(tmpdir, &st) != 0 || (st.st_mode & S_IFDIR) == 0 || // exists and is directory + access(tmpdir, R_OK | W_OK | X_OK) != 0 // current user has rwx access to directory + ) tmpdir = "/tmp"; return string(tmpdir); diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 7a3852ee8..cf0b9a96f 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -384,8 +384,12 @@ if [ "$command" != "help" ]; then # gpg needs (in different versions more or less) files to function correctly, # so we give it its own homedir and generate some valid content for it - if [ ! -d "$TMPDIR" ]; then - unset TMPDIR + if [ -n "$TMPDIR" ]; then + # tmpdir is a directory and current user has rwx access to it + # same tests as in apt-pkg/contrib/fileutl.cc GetTempDir() + if [ ! -d "$TMPDIR" ] || [ ! -r "$TMPDIR" ] || [ ! -w "$TMPDIR" ] || [ ! -x "$TMPDIR" ]; then + unset TMPDIR + fi fi GPGHOMEDIR="$(mktemp -d)" CURRENTTRAP="${CURRENTTRAP} rm -rf '${GPGHOMEDIR}';" diff --git a/test/integration/framework b/test/integration/framework index 9ce300d55..d576712e5 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -176,6 +176,10 @@ addtrap() { } setupenvironment() { + # privilege dropping and testing doesn't work if /tmp isn't world-writeable (as e.g. with libpam-tmpdir) + if [ -n "$TMPDIR" ] && [ "$(id -u)" = '0' ] && [ "$(stat --format '%a' "$TMPDIR")" != '1777' ]; then + unset TMPDIR + fi TMPWORKINGDIRECTORY=$(mktemp -d) addtrap "cd /; rm -rf $TMPWORKINGDIRECTORY;" msgninfo "Preparing environment for ${CCMD}$(basename $0)${CINFO} in ${TMPWORKINGDIRECTORY}… " diff --git a/test/integration/test-bug-604401-files-are-directories b/test/integration/test-bug-604401-files-are-directories index e6913edcf..fe0ccc783 100755 --- a/test/integration/test-bug-604401-files-are-directories +++ b/test/integration/test-bug-604401-files-are-directories @@ -57,7 +57,7 @@ echo 'Package: apt Pin: release a=now Pin-Value: 1000' > rootdir/etc/apt/good-link.pref ln -s rootdir/etc/apt/good-link.pref rootdir/etc/apt/preferences -test -n "$(aptcache policy | grep 1000)" && msgfail || msgpass +test -n "$(aptcache policy | grep '1000 ')" && msgfail || msgpass rm rootdir/etc/apt/preferences msgtest "Broken link instead of a file as preferences ignored" diff --git a/test/libapt/fileutl_test.cc b/test/libapt/fileutl_test.cc index cdf7ea479..8d47c5098 100644 --- a/test/libapt/fileutl_test.cc +++ b/test/libapt/fileutl_test.cc @@ -217,8 +217,16 @@ TEST(FileUtlTest, GetTempDir) setenv("TMPDIR", "/not-there-no-really-not", 1); EXPECT_EQ("/tmp", GetTempDir()); + // here but not accessible for non-roots setenv("TMPDIR", "/usr", 1); - EXPECT_EQ("/usr", GetTempDir()); + EXPECT_EQ("/tmp", GetTempDir()); + + // files are no good for tmpdirs, too + setenv("TMPDIR", "/dev/null", 1); + EXPECT_EQ("/tmp", GetTempDir()); + + setenv("TMPDIR", "/var/tmp", 1); + EXPECT_EQ("/var/tmp", GetTempDir()); unsetenv("TMPDIR"); if (old_tmpdir.empty() == false) -- cgit v1.2.3-70-g09d2 From 03aa08472dcd689572a46ce6efdb1dccf6136334 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 23 Oct 2014 01:28:05 +0200 Subject: chown finished partial files earlier partial files are chowned by the Item baseclass to let the methods work with them. Now, this baseclass is also responsible for chowning the files back to root instead of having various deeper levels do this. The consequence is that all overloaded Failed() methods now call the Item::Failed base as their first step. The same is done for Done(). The effect is that even in partial files usually don't belong to _apt anymore, helping sneakernets and reducing possibilities of a bad method modifying files not belonging to them. The change is supported by the framework not only supporting being run as root, but with proper permission management, too, so that privilege dropping can be tested with them. --- apt-pkg/acquire-item.cc | 111 +++++++++++---------- apt-pkg/acquire-item.h | 1 + apt-pkg/acquire.cc | 40 +++++--- apt-pkg/acquire.h | 2 + apt-private/private-download.cc | 11 +- cmdline/apt-get.cc | 33 ++++-- test/integration/framework | 30 +++--- test/integration/test-apt-cdrom | 2 + test/integration/test-apt-get-changelog | 25 +++-- test/integration/test-apt-get-download | 12 ++- test/integration/test-apt-get-source-authenticated | 7 +- test/integration/test-apt-helper | 21 ++-- test/integration/test-apt-update-nofallback | 2 +- ...17690-allow-unauthenticated-makes-all-untrusted | 10 +- test/integration/test-bug-738785-switch-protocol | 12 +-- test/integration/test-compressed-indexes | 8 +- test/integration/test-partial-file-support | 31 +++--- test/integration/test-sourceslist-trusted-options | 10 +- .../test-ubuntu-bug-1098738-apt-get-source-md5sum | 2 + 19 files changed, 229 insertions(+), 141 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 704e285b5..11c522ae5 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -67,11 +67,11 @@ static void printHashSumComparision(std::string const &URI, HashStringList const /*}}}*/ static void ChangeOwnerAndPermissionOfFile(char const * const requester, char const * const file, char const * const user, char const * const group, mode_t const mode) /*{{{*/ { - // ensure the file is owned by root and has good permissions - struct passwd const * const pw = getpwnam(user); - struct group const * const gr = getgrnam(group); - if (getuid() == 0) // if we aren't root, we can't chown, so don't try it + if (getuid() == 0 && strlen(user) != 0 && strlen(group) != 0) // if we aren't root, we can't chown, so don't try it { + // ensure the file is owned by root and has good permissions + struct passwd const * const pw = getpwnam(user); + struct group const * const gr = getgrnam(group); if (pw != NULL && gr != NULL && chown(file, pw->pw_uid, gr->gr_gid) != 0) _error->WarningE(requester, "chown to %s:%s of file %s failed", user, group, file); } @@ -155,7 +155,10 @@ pkgAcquire::Item::~Item() fetch this object */ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - if(ErrorText == "") + if (RealFileExists(DestFile)) + ChangeOwnerAndPermissionOfFile("Item::Failed", DestFile.c_str(), "root", "root", 0644); + + if(ErrorText.empty()) ErrorText = LookupTag(Message,"Message"); UsedMirror = LookupTag(Message,"UsedMirror"); if (QueueCounter <= 1) @@ -179,9 +182,9 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Status = StatIdle; // check fail reason - string FailReason = LookupTag(Message, "FailReason"); + string const FailReason = LookupTag(Message, "FailReason"); if(FailReason == "MaximumSizeExceeded") - Rename(DestFile, DestFile+".FAILED"); + RenameOnError(MaximumSizeExceeded); // report mirror failure back to LP if we actually use a mirror if(FailReason.size() != 0) @@ -197,6 +200,7 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) void pkgAcquire::Item::Start(string /*Message*/,unsigned long long Size) { Status = StatFetching; + ErrorText.clear(); if (FileSize == 0 && Complete == false) FileSize = Size; } @@ -215,6 +219,8 @@ void pkgAcquire::Item::Done(string Message,unsigned long long Size,HashStringLis if (Owner->Log != 0) Owner->Log->Fetched(Size,atoi(LookupTag(Message,"Resume-Point","0").c_str())); } + if (RealFileExists(DestFile)) + ChangeOwnerAndPermissionOfFile("Item::Done", DestFile.c_str(), "root", "root", 0644); if (FileSize == 0) FileSize= Size; @@ -231,6 +237,7 @@ bool pkgAcquire::Item::Rename(string From,string To) { if (rename(From.c_str(),To.c_str()) == 0) return true; + ChangeOwnerAndPermissionOfFile("Item::Failed", To.c_str(), "root", "root", 0644); std::string S; strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno), @@ -258,7 +265,7 @@ void pkgAcquire::Item::Dequeue() /*{{{*/ /*}}}*/ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const error)/*{{{*/ { - if(FileExists(DestFile)) + if (RealFileExists(DestFile)) Rename(DestFile, DestFile + ".FAILED"); switch (error) @@ -283,7 +290,11 @@ bool pkgAcquire::Item::RenameOnError(pkgAcquire::Item::RenameOnErrorState const Status = StatError; break; case NotClearsigned: - ErrorText = _("Does not start with a cleartext signature"); + ErrorText = _("Does not start with a cleartext signature"); + Status = StatError; + break; + case MaximumSizeExceeded: + // the method is expected to report a good error for this Status = StatError; break; } @@ -702,14 +713,14 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ /*}}}*/ void pkgAcqDiffIndex::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/ { + Item::Failed(Message,Cnf); + Status = StatDone; + if(Debug) std::clog << "pkgAcqDiffIndex failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; new pkgAcqIndex(Owner, TransactionManager, Target, ExpectedHashes, MetaIndexParser); - - Item::Failed(Message,Cnf); - Status = StatDone; } /*}}}*/ void pkgAcqDiffIndex::Done(string Message,unsigned long long Size,HashStringList const &Hashes, /*{{{*/ @@ -792,8 +803,11 @@ pkgAcqIndexDiffs::pkgAcqIndexDiffs(pkgAcquire *Owner, } } /*}}}*/ -void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * /*Cnf*/)/*{{{*/ +void pkgAcqIndexDiffs::Failed(string Message,pkgAcquire::MethodConfig * Cnf)/*{{{*/ { + Item::Failed(Message,Cnf); + Status = StatDone; + if(Debug) std::clog << "pkgAcqIndexDiffs failed: " << Desc.URI << " with " << Message << std::endl << "Falling back to normal index file acquire" << std::endl; @@ -1281,11 +1295,14 @@ string pkgAcqIndex::Custom600Headers() const // pkgAcqIndex::Failed - getting the indexfile failed /*{{{*/ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { + Item::Failed(Message,Cnf); + size_t const nextExt = CompressionExtensions.find(' '); if (nextExt != std::string::npos) { CompressionExtensions = CompressionExtensions.substr(nextExt+1); Init(RealURI, Desc.Description, Desc.ShortDesc); + Status = StatIdle; return; } @@ -1295,8 +1312,6 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) unlink(EraseFileName.c_str()); } - Item::Failed(Message,Cnf); - /// cancel the entire transaction TransactionManager->AbortTransaction(); } @@ -1521,6 +1536,8 @@ string pkgAcqIndexTrans::Custom600Headers() const // AcqIndexTrans::Failed - Silence failure messages for missing files /*{{{*/ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { + Item::Failed(Message,Cnf); + size_t const nextExt = CompressionExtensions.find(' '); if (nextExt != std::string::npos) { @@ -1530,8 +1547,6 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) return; } - Item::Failed(Message,Cnf); - // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) @@ -1563,16 +1578,9 @@ void pkgAcqMetaBase::AbortTransaction() if ((*I)->Status == pkgAcquire::Item::StatIdle) (*I)->Status = pkgAcquire::Item::StatDone; - // kill failed files in partial - if ((*I)->Status == pkgAcquire::Item::StatError) - { - std::string const PartialFile = GetPartialFileName(flNotDir((*I)->DestFile)); - if(FileExists(PartialFile)) - Rename(PartialFile, PartialFile + ".FAILED"); - } - // fix permissions for existing files which were part of a reverify - // like InRelease files or files in partial we might work with next time - else if (FileExists((*I)->DestFile)) + // reverify might act on a file outside of partial + // (as it itself is good, but needed to verify others, like Release) + if ((*I)->DestFile == (*I)->PartialFile && RealFileExists((*I)->DestFile)) ChangeOwnerAndPermissionOfFile("AbortTransaction", (*I)->DestFile.c_str(), "root", "root", 0644); } Transaction.clear(); @@ -1608,8 +1616,6 @@ void pkgAcqMetaBase::CommitTransaction() << (*I)->DescURI() << std::endl; Rename((*I)->PartialFile, (*I)->DestFile); - ChangeOwnerAndPermissionOfFile("CommitTransaction", (*I)->DestFile.c_str(), "root", "root", 0644); - } else { if(_config->FindB("Debug::Acquire::Transaction", false) == true) std::clog << "rm " @@ -1758,16 +1764,17 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { - string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + Item::Failed(Message,Cnf); // check if we need to fail at this point if (AuthPass == true && CheckStopAuthentication(RealURI, Message)) return; // FIXME: meh, this is not really elegant - string InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12, + string const Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); + string const InReleaseURI = RealURI.replace(RealURI.rfind("Release.gpg"), 12, "InRelease"); - string FinalInRelease = _config->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI); + string const FinalInRelease = _config->FindDir("Dir::State::lists") + URItoFileName(InReleaseURI); if (RealFileExists(Final) || RealFileExists(FinalInRelease)) { @@ -1782,7 +1789,7 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ _error->Warning(_("This is normally not allowed, but the option " "Acquire::AllowDowngradeToInsecureRepositories was " "given to override it.")); - + Status = StatDone; } else { _error->Error("%s", downgrade_msg.c_str()); Rename(MetaIndexFile, MetaIndexFile+".FAILED"); @@ -1810,8 +1817,6 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ QueueIndexes(true); } - Item::Failed(Message,Cnf); - // FIXME: this is used often (e.g. in pkgAcqIndexTrans) so refactor if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) @@ -2226,10 +2231,12 @@ string pkgAcqMetaClearSig::Custom600Headers() const /*}}}*/ // pkgAcqMetaClearSig::Done - We got a file /*{{{*/ // --------------------------------------------------------------------- -void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long /*Size*/, - HashStringList const &/*Hashes*/, +void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size, + HashStringList const &Hashes, pkgAcquire::MethodConfig *Cnf) { + Item::Done(Message, Size, Hashes, Cnf); + // if we expect a ClearTextSignature (InRelase), ensure that // this is what we get and if not fail to queue a // Release/Release.gpg, see #346386 @@ -2567,7 +2574,6 @@ void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList string FinalFile = _config->FindDir("Dir::Cache::Archives"); FinalFile += flNotDir(StoreFilename); Rename(DestFile,FinalFile); - ChangeOwnerAndPermissionOfFile("pkgAcqArchive::Done", FinalFile.c_str(), "root", "root", 0644); StoreFilename = DestFile = FinalFile; Complete = true; } @@ -2577,8 +2583,8 @@ void pkgAcqArchive::Done(string Message,unsigned long long Size, HashStringList /* Here we try other sources */ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - ErrorText = LookupTag(Message,"Message"); - + Item::Failed(Message,Cnf); + /* We don't really want to retry on failed media swaps, this prevents that. An interesting observation is that permanent failures are not recorded. */ @@ -2588,10 +2594,10 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) // Vf = Version.FileList(); while (Vf.end() == false) ++Vf; StoreFilename = string(); - Item::Failed(Message,Cnf); return; } - + + Status = StatIdle; if (QueueNext() == false) { // This is the retry counter @@ -2604,9 +2610,9 @@ void pkgAcqArchive::Failed(string Message,pkgAcquire::MethodConfig *Cnf) if (QueueNext() == true) return; } - + StoreFilename = string(); - Item::Failed(Message,Cnf); + Status = StatError; } } /*}}}*/ @@ -2726,7 +2732,12 @@ void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList cons // Symlink the file if (symlink(FileName.c_str(),DestFile.c_str()) != 0) { - ErrorText = "Link to " + DestFile + " failure "; + _error->PushToStack(); + _error->Errno("pkgAcqFile::Done", "Symlinking file %s failed", DestFile.c_str()); + std::stringstream msg; + _error->DumpErrors(msg); + _error->RevertToStack(); + ErrorText = msg.str(); Status = StatError; Complete = false; } @@ -2738,19 +2749,19 @@ void pkgAcqFile::Done(string Message,unsigned long long Size,HashStringList cons /* Here we try other sources */ void pkgAcqFile::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { - ErrorText = LookupTag(Message,"Message"); - + Item::Failed(Message,Cnf); + // This is the retry counter if (Retries != 0 && Cnf->LocalOnly == false && StringToBool(LookupTag(Message,"Transient-Failure"),false) == true) { - Retries--; + --Retries; QueueURI(Desc); + Status = StatIdle; return; } - - Item::Failed(Message,Cnf); + } /*}}}*/ // AcqIndex::Custom600Headers - Insert custom request headers /*{{{*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 68d5a01ce..4b4ef5fca 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -323,6 +323,7 @@ class pkgAcquire::Item : public WeakPointable InvalidFormat, SignatureError, NotClearsigned, + MaximumSizeExceeded }; /** \brief Rename failed file and set error diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 1e20b74be..2c89c2dea 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -54,23 +54,38 @@ pkgAcquire::pkgAcquire() : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NU Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) { - string const Mode = _config->Find("Acquire::Queue-Mode","host"); - if (strcasecmp(Mode.c_str(),"host") == 0) - QueueMode = QueueHost; - if (strcasecmp(Mode.c_str(),"access") == 0) - QueueMode = QueueAccess; + Initialize(); } pkgAcquire::pkgAcquire(pkgAcquireStatus *Progress) : LockFD(-1), Queues(0), Workers(0), Configs(0), Log(NULL), ToFetch(0), Debug(_config->FindB("Debug::pkgAcquire",false)), Running(false) +{ + Initialize(); + SetLog(Progress); +} +void pkgAcquire::Initialize() { string const Mode = _config->Find("Acquire::Queue-Mode","host"); if (strcasecmp(Mode.c_str(),"host") == 0) QueueMode = QueueHost; if (strcasecmp(Mode.c_str(),"access") == 0) QueueMode = QueueAccess; - SetLog(Progress); + + // chown the auth.conf file as it will be accessed by our methods + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it + { + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + struct group const * const gr = getgrnam("root"); + if (pw != NULL && gr != NULL) + { + std::string const AuthConf = _config->FindFile("Dir::Etc::netrc"); + if(AuthConf.empty() == false && RealFileExists(AuthConf) && + chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str()); + } + } } /*}}}*/ // Acquire::GetLock - lock directory and prepare for action /*{{{*/ @@ -81,21 +96,16 @@ static bool SetupAPTPartialDirectory(std::string const &grand, std::string const CreateAPTDirectoryIfNeeded(parent, partial) == false) return false; - if (getuid() == 0) // if we aren't root, we can't chown, so don't try it + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it { - std::string SandboxUser = _config->Find("APT::Sandbox::User"); - struct passwd *pw = getpwnam(SandboxUser.c_str()); - struct group *gr = getgrnam("root"); + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + struct group const * const gr = getgrnam("root"); if (pw != NULL && gr != NULL) { // chown the partial dir if(chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str()); - // chown the auth.conf file - std::string const AuthConf = _config->FindFile("Dir::Etc::netrc"); - if(AuthConf.empty() == false && RealFileExists(AuthConf) && - chown(AuthConf.c_str(), pw->pw_uid, gr->gr_gid) != 0) - _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of file %s failed", SandboxUser.c_str(), AuthConf.c_str()); } } if (chmod(partial.c_str(), 0700) != 0) diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index a1a192d5f..f33362922 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -378,6 +378,8 @@ class pkgAcquire */ virtual ~pkgAcquire(); + private: + APT_HIDDEN void Initialize(); }; /** \brief Represents a single download source from which an item diff --git a/apt-private/private-download.cc b/apt-private/private-download.cc index 8cabf14b5..37fae18e9 100644 --- a/apt-private/private-download.cc +++ b/apt-private/private-download.cc @@ -49,16 +49,23 @@ bool CheckDropPrivsMustBeDisabled(pkgAcquire &Fetcher) /*{{{*/ for (pkgAcquire::ItemIterator I = Fetcher.ItemsBegin(); I != Fetcher.ItemsEnd() && res == true; ++I) { - int fd = open((*I)->DestFile.c_str(), O_CREAT | O_RDWR, 0600); + if ((*I)->DestFile.empty()) + continue; + // we assume that an existing (partial) file means that we have sufficient rights + if (RealFileExists((*I)->DestFile)) + continue; + int fd = open((*I)->DestFile.c_str(), O_CREAT | O_EXCL | O_RDWR, 0600); if (fd < 0) { res = false; std::string msg; strprintf(msg, _("Can't drop privileges for downloading as file '%s' couldn't be accessed by user '%s'."), (*I)->DestFile.c_str(), SandboxUser.c_str()); - c0out << msg << std::endl; + std::cerr << "W: " << msg << std::endl; _config->Set("APT::Sandbox::User", ""); + break; } + unlink((*I)->DestFile.c_str()); close(fd); } diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index e176a3350..b6786faf8 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -80,6 +80,9 @@ #include #include #include +#include +#include + #include #include #include @@ -659,6 +662,8 @@ static bool DoDownload(CommandLine &CmdL) Ver != verset.end(); ++Ver, ++i) { pkgAcquire::Item *I = new pkgAcqArchive(&Fetcher, SrcList, &Recs, *Ver, storefile[i]); + if (storefile[i].empty()) + continue; std::string const filename = cwd + flNotDir(storefile[i]); storefile[i].assign(filename); I->DestFile.assign(filename); @@ -1481,7 +1486,10 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, "http://packages.debian.org/changelogs"); string const path = GetChangelogPath(CacheFile, Ver); string changelog_uri; - strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str()); + if (APT::String::Endswith(server, "/") == true) + strprintf(changelog_uri, "%s%s/changelog", server.c_str(), path.c_str()); + else + strprintf(changelog_uri, "%s/%s/changelog", server.c_str(), path.c_str()); if (_config->FindB("APT::Get::Print-URIs", false) == true) { std::cout << '\'' << changelog_uri << '\'' << std::endl; @@ -1492,7 +1500,7 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, string descr; strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), changelog_uri.c_str()); // queue it - new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); + pkgAcquire::Item const * itm = new pkgAcqFile(&Fetcher, changelog_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); // Disable drop-privs if "_apt" can not write to the target dir CheckDropPrivsMustBeDisabled(Fetcher); @@ -1500,18 +1508,18 @@ static bool DownloadChangelog(CacheFile &CacheFile, pkgAcquire &Fetcher, // try downloading it, if that fails, try third-party-changelogs location // FIXME: Fetcher.Run() is "Continue" even if I get a 404?!? Fetcher.Run(); - if (!FileExists(targetfile)) + if (itm->Status != pkgAcquire::Item::StatDone) { string third_party_uri; if (GuessThirdPartyChangelogUri(CacheFile, Ver, third_party_uri)) { strprintf(descr, _("Changelog for %s (%s)"), Pkg.Name(), third_party_uri.c_str()); - new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); + itm = new pkgAcqFile(&Fetcher, third_party_uri, "", 0, descr, Pkg.Name(), "ignored", targetfile); Fetcher.Run(); } } - if (FileExists(targetfile)) + if (itm->Status == pkgAcquire::Item::StatDone) return true; // error @@ -1557,6 +1565,19 @@ static bool DoChangelog(CommandLine &CmdL) tmpdir = mkdtemp(tmpname); if (tmpdir == NULL) return _error->Errno("mkdtemp", "mkdtemp failed"); + + std::string const SandboxUser = _config->Find("APT::Sandbox::User"); + if (getuid() == 0 && SandboxUser.empty() == false) // if we aren't root, we can't chown, so don't try it + { + struct passwd const * const pw = getpwnam(SandboxUser.c_str()); + struct group const * const gr = getgrnam("root"); + if (pw != NULL && gr != NULL) + { + // chown the tmp dir directory we use to the sandbox user + if(chown(tmpdir, pw->pw_uid, gr->gr_gid) != 0) + _error->WarningE("DoChangelog", "chown to %s:%s of directory %s failed", SandboxUser.c_str(), "root", tmpdir); + } + } } for (APT::VersionList::const_iterator Ver = verset.begin(); @@ -1572,7 +1593,7 @@ static bool DoChangelog(CommandLine &CmdL) { DisplayFileInPager(changelogfile); // cleanup temp file - unlink(changelogfile.c_str()); + unlink(changelogfile.c_str()); } } // clenaup tmp dir diff --git a/test/integration/framework b/test/integration/framework index d576712e5..23ff0983b 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -32,7 +32,7 @@ msgprintf() { printf "$START " "$1" shift while [ -n "$1" ]; do - printf "$MIDDLE " "$(echo "$1" | sed -e 's#^apt\([cgfs]\)#apt-\1#')" + printf "$MIDDLE " "$(echo "$1" | sed -e 's#^apt\([cfghs]\)#apt-\1#')" shift done fi @@ -57,9 +57,9 @@ msgskip() { msgfail() { if [ $# -gt 0 ]; then printf "${CFAIL}FAIL: $*${CNORMAL}\n" >&2; else printf "${CFAIL}FAIL${CNORMAL}\n" >&2; fi - if [ -n "$APT_DEBUG_TESTS" ]; then - bash - fi + if [ -n "$APT_DEBUG_TESTS" ]; then + $SHELL + fi EXIT_CODE=$((EXIT_CODE+1)); } @@ -184,10 +184,12 @@ setupenvironment() { addtrap "cd /; rm -rf $TMPWORKINGDIRECTORY;" msgninfo "Preparing environment for ${CCMD}$(basename $0)${CINFO} in ${TMPWORKINGDIRECTORY}… " + mkdir -m 700 "${TMPWORKINGDIRECTORY}/downloaded" if [ "$(id -u)" = '0' ]; then # relax permissions so that running as root with user switching works umask 022 - chmod o+rx "$TMPWORKINGDIRECTORY" + chmod 711 "$TMPWORKINGDIRECTORY" + chown _apt:root "${TMPWORKINGDIRECTORY}/downloaded" fi TESTDIRECTORY=$(readlink -f $(dirname $0)) @@ -1061,7 +1063,7 @@ downloadfile() { } checkdiff() { - local DIFFTEXT="$(command diff -u "$@" | sed -e '/^---/ d' -e '/^+++/ d' -e '/^@@/ d')" + local DIFFTEXT="$(command diff -u "$@" 2>&1 | sed -e '/^---/ d' -e '/^+++/ d' -e '/^@@/ d')" if [ -n "$DIFFTEXT" ]; then echo >&2 echo >&2 "$DIFFTEXT" @@ -1307,7 +1309,7 @@ testaccessrights() { testwebserverlaststatuscode() { local DOWNLOG='rootdir/tmp/webserverstatus-testfile.log' - local STATUS='rootdir/tmp/webserverstatus-statusfile.log' + local STATUS='downloaded/webserverstatus-statusfile.log' rm -f "$DOWNLOG" "$STATUS" msgtest 'Test last status code from the webserver was' "$1" downloadfile "http://localhost:8080/_config/find/aptwebserver::last-status-code" "$STATUS" > "$DOWNLOG" @@ -1353,20 +1355,20 @@ aptautotest() { shift 3 # save and restore the *.output files from other tests # as we might otherwise override them in these automatic tests - rm -rf rootdir/tmp-before - mv rootdir/tmp rootdir/tmp-before - mkdir rootdir/tmp + rm -rf ${TMPWORKINGDIRECTORY}/rootdir/tmp-before + mv ${TMPWORKINGDIRECTORY}/rootdir/tmp ${TMPWORKINGDIRECTORY}/rootdir/tmp-before + mkdir ${TMPWORKINGDIRECTORY}/rootdir/tmp $AUTOTEST "$TESTCALL" "$@" - rm -rf rootdir/tmp-aptautotest - mv rootdir/tmp rootdir/tmp-aptautotest - mv rootdir/tmp-before rootdir/tmp + rm -rf ${TMPWORKINGDIRECTORY}/rootdir/tmp-aptautotest + mv ${TMPWORKINGDIRECTORY}/rootdir/tmp ${TMPWORKINGDIRECTORY}/rootdir/tmp-aptautotest + mv ${TMPWORKINGDIRECTORY}/rootdir/tmp-before ${TMPWORKINGDIRECTORY}/rootdir/tmp fi } aptautotest_aptget_update() { if ! test -d "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists"; then return; fi # all copied files are properly chmodded - find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f | while read file; do + for file in $(find "${TMPWORKINGDIRECTORY}/rootdir/var/lib/apt/lists" -maxdepth 1 -type f); do testfilestats "$file" '%U:%G:%a' '=' "${USER}:${USER}:644" done } diff --git a/test/integration/test-apt-cdrom b/test/integration/test-apt-cdrom index 2220a290c..4489143e4 100755 --- a/test/integration/test-apt-cdrom +++ b/test/integration/test-apt-cdrom @@ -99,6 +99,7 @@ Conf testing:i386 (0.8.15 stable [i386])' aptget install testing:i386 -s testsuccess aptget purge testing:i386 -y testdpkgnotinstalled testing:i386 + cd downloaded rm -f testing_0.8.15_amd64.deb testsuccess aptget download testing testsuccess test -s testing_0.8.15_amd64.deb @@ -108,6 +109,7 @@ Conf testing:i386 (0.8.15 stable [i386])' aptget install testing:i386 -s testsuccess aptget source testing --dsc-only -d testsuccess test -s testing_0.8.15.dsc rm -f testing_0.8.15.dsc + cd - >/dev/null } testcdromusage diff --git a/test/integration/test-apt-get-changelog b/test/integration/test-apt-get-changelog index 76a32a122..648dccf40 100755 --- a/test/integration/test-apt-get-changelog +++ b/test/integration/test-apt-get-changelog @@ -21,23 +21,28 @@ chmod -R -w rootdir/var/cache/apt/archives echo 'Apt::Changelogs::Server "http://localhost:8080/";' > rootdir/etc/apt/apt.conf.d/changelog.conf -testequal "'http://localhost:8080//pool/apt_1.0/changelog'" aptget changelog apt --print-uris +testequal "'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt --print-uris -testequal "'http://localhost:8080//pool/apt_1.0/changelog' -'http://localhost:8080//pool/apt_1.0/changelog'" aptget changelog apt apt --print-uris +testequal "'http://localhost:8080/pool/apt_1.0/changelog' +'http://localhost:8080/pool/apt_1.0/changelog'" aptget changelog apt apt --print-uris + +cd downloaded testsuccess aptget changelog apt -qq -testfileequal 'rootdir/tmp/testsuccess.output' "$(cat aptarchive/pool/apt_1.0/changelog)" +testfileequal '../rootdir/tmp/testsuccess.output' "$(cat ../aptarchive/pool/apt_1.0/changelog)" testsuccess aptget changelog apt -d -testfileequal 'apt.changelog' "$(cat aptarchive/pool/apt_1.0/changelog)" -rm -f apt.changelog aptarchive/pool/apt_1.0/changelog +testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0/changelog)" +testfilestats 'apt.changelog' '%U:%G:%a' '=' "${USER}:${USER}:644" +rm -f apt.changelog ../aptarchive/pool/apt_1.0/changelog -testequal "$(cat aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \ +testequal "$(cat ../aptarchive/pool/apt_1.0.changelog)" aptget changelog apt \ -qq -o APT::Changelogs::Server='http://not-on-the-main-server:8080/' testsuccess aptget changelog apt -d -testfileequal 'apt.changelog' "$(cat aptarchive/pool/apt_1.0.changelog)" -rm -f apt.changelog aptarchive/pool/apt_1.0.changelog +testfileequal 'apt.changelog' "$(cat ../aptarchive/pool/apt_1.0.changelog)" +testfilestats 'apt.changelog' '%U:%G:%a' '=' "${USER}:${USER}:644" +rm -f apt.changelog ../aptarchive/pool/apt_1.0.changelog -testequal 'E: changelog download failed' aptget changelog apt -qq -o APT::Changelogs::Server='http://not-on-the-main-server:8080/' +testequal 'E: changelog download failed' aptget changelog apt -qq -d -o APT::Changelogs::Server='http://not-on-the-main-server:8080/' +testfailure test -e apt.changelog diff --git a/test/integration/test-apt-get-download b/test/integration/test-apt-get-download index 48086d808..6503bbd1c 100755 --- a/test/integration/test-apt-get-download +++ b/test/integration/test-apt-get-download @@ -44,7 +44,10 @@ testdownload() { } # normal case as "root" +OLDPWD="$(pwd)" +cd downloaded testdownload apt_2.0_all.deb apt +cd "$OLDPWD" # simulate normal user with non-existent root-owned directories rm -rf rootdir/var/cache/apt/archives/ @@ -52,14 +55,18 @@ mkdir rootdir/var/cache/apt/archives/ addtrap 'prefix' "chmod -f -R +w $PWD/rootdir/var/cache/apt/archives || true;" chmod -R -w rootdir/var/cache/apt/archives +OLDPWD="$(pwd)" +cd downloaded + # normal case(es) testdownload apt_1.0_all.deb apt stable testdownload apt_2.0_all.deb apt -DEBFILE="$(readlink -f aptarchive)/pool/apt_2.0_all.deb" +DEBFILE="$(readlink -f ../aptarchive)/pool/apt_2.0_all.deb" testequal "'file://${DEBFILE}' apt_2.0_all.deb $(stat -c%s $DEBFILE) SHA512:$(sha512sum $DEBFILE | cut -d' ' -f 1)" aptget download apt --print-uris # deb:677887 +testequal "E: Can't find a source to download version '1.0' of 'vrms:i386'" aptget download vrms --print-uris testequal "E: Can't find a source to download version '1.0' of 'vrms:i386'" aptget download vrms # deb:736962 @@ -74,6 +81,7 @@ testsuccess aptget download apt apt apt/unstable apt=2.0 testsuccess test -s apt_2.0_all.deb # restore "root" rights +cd "$OLDPWD" chmod -f -R +w $PWD/rootdir/var/cache/apt/archives rm -rf rootdir/var/cache/apt/archives/ @@ -86,5 +94,7 @@ testsuccess aptget install -d apt testsuccess test -s rootdir/var/cache/apt/archives/apt_2.0_all.deb testaccessrights 'aptarchive/pool/apt_2.0_all.deb' '644' mv aptarchive/pool/apt_2.0_all.deb aptarchive/pool/apt_2.0_all.deb.gone +cd downloaded testdownload apt_2.0_all.deb apt +cd "$OLDPWD" mv aptarchive/pool/apt_2.0_all.deb.gone aptarchive/pool/apt_2.0_all.deb diff --git a/test/integration/test-apt-get-source-authenticated b/test/integration/test-apt-get-source-authenticated index fedfc8308..f68c32386 100755 --- a/test/integration/test-apt-get-source-authenticated +++ b/test/integration/test-apt-get-source-authenticated @@ -25,7 +25,10 @@ testwarning aptget update --allow-insecure-repositories # this all should fail testfailure aptget install -y foo +cd downloaded testfailure aptget source foo - +testfailure test -e foo_2.0.dsc # allow overriding the warning -testsuccess aptget source --allow-unauthenticated foo +testsuccess aptget source --allow-unauthenticated foo -o Debug::pkgAcquire::Worker=1 +testsuccess test -s foo_2.0.dsc -a -L foo_2.0.dsc +testaccessrights 'foo_2.0.dsc' '777' diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index 42c40bb9e..06e497ff7 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -13,23 +13,23 @@ test_apt_helper_download() { echo 'foo' > aptarchive/foo msgtest 'apt-file download-file' 'md5sum' - apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail - testfileequal foo2 'foo' + apthelper -qq download-file http://localhost:8080/foo downloaded/foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail + testfileequal downloaded/foo2 'foo' msgtest 'apt-file download-file' 'sha1' - apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail - testfileequal foo1 'foo' + apthelper -qq download-file http://localhost:8080/foo downloaded/foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail + testfileequal downloaded/foo1 'foo' msgtest 'apt-file download-file' 'sha256' - apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail - testfileequal foo3 'foo' + apthelper -qq download-file http://localhost:8080/foo downloaded/foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail + testfileequal downloaded/foo3 'foo' msgtest 'apt-file download-file' 'no-hash' - apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail - testfileequal foo4 'foo' + apthelper -qq download-file http://localhost:8080/foo downloaded/foo4 && msgpass || msgfail + testfileequal downloaded/foo4 'foo' msgtest 'apt-file download-file' 'wrong hash' - if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then + if ! apthelper -qq download-file http://localhost:8080/foo downloaded/foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then msgpass else msgfail @@ -37,7 +37,8 @@ test_apt_helper_download() { testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo Hash Sum mismatch E: Download Failed' - testfileequal foo5.FAILED 'foo' + testfileequal downloaded/foo5.FAILED 'foo' + testfailure test -e downloaded/foo5 } test_apt_helper_detect_proxy() { diff --git a/test/integration/test-apt-update-nofallback b/test/integration/test-apt-update-nofallback index 831fc67eb..e82a976a6 100755 --- a/test/integration/test-apt-update-nofallback +++ b/test/integration/test-apt-update-nofallback @@ -195,7 +195,7 @@ test_release_gpg_to_invalid_release_release_gpg() testequal "W: An error occurred during the signature verification. The repository is not updated and the previous index files will be used. GPG error: file: unstable Release.gpg: The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) -W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg +W: Failed to fetch file:${APTARCHIVE}/dists/unstable/Release.gpg The following signatures were invalid: BADSIG 5A90D141DBAC8DAE Joe Sixpack (APT Testcases Dummy) W: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq diff --git a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted index 164dca070..582e1bf5e 100755 --- a/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted +++ b/test/integration/test-bug-617690-allow-unauthenticated-makes-all-untrusted @@ -24,6 +24,7 @@ testfilemissing() { testrun() { rm -rf rootdir/var/lib/apt + cd downloaded if [ "$1" = 'trusted' ]; then testsuccess aptget update @@ -43,18 +44,19 @@ testrun() { testfileexists 'cool_1.0_i386.deb' fi - mv aptarchive/pool/cool_1.0_i386.deb aptarchive/pool/cool_1.0_i386.deb.bak - echo 'this is not a good package' > aptarchive/pool/cool_1.0_i386.deb + mv ../aptarchive/pool/cool_1.0_i386.deb ../aptarchive/pool/cool_1.0_i386.deb.bak + echo 'this is not a good package' > ../aptarchive/pool/cool_1.0_i386.deb testfailure aptget download cool testfilemissing cool_1.0_i386.deb testfailure aptget download cool --allow-unauthenticated # unauthenticated doesn't mean unchecked testfilemissing cool_1.0_i386.deb - rm -f aptarchive/pool/cool_1.0_i386.deb - mv aptarchive/pool/cool_1.0_i386.deb.bak aptarchive/pool/cool_1.0_i386.deb + rm -f ../aptarchive/pool/cool_1.0_i386.deb + mv ../aptarchive/pool/cool_1.0_i386.deb.bak ../aptarchive/pool/cool_1.0_i386.deb testsuccess aptget download cool --allow-unauthenticated testfileexists 'cool_1.0_i386.deb' + cd - >/dev/null } testrun 'trusted' diff --git a/test/integration/test-bug-738785-switch-protocol b/test/integration/test-bug-738785-switch-protocol index f450e5e5a..e50c84dbf 100755 --- a/test/integration/test-bug-738785-switch-protocol +++ b/test/integration/test-bug-738785-switch-protocol @@ -25,6 +25,7 @@ downloadfile 'http://localhost:8080/pool/apt_1.0/changelog' changelog >/dev/null echo 'Apt::Changelogs::Server "http://localhost:8080/redirectme";' > rootdir/etc/apt/apt.conf.d/changelog.conf testequal "'http://localhost:8080/redirectme/pool/apt_1.0/changelog'" aptget changelog apt --print-uris +cd downloaded testsuccess aptget changelog apt -d testsuccess test -s apt.changelog rm -f apt.changelog @@ -32,6 +33,7 @@ rm -f apt.changelog testsuccess aptget download apt testsuccess test -s apt_1.0_all.deb rm apt_1.0_all.deb +cd - >/dev/null testsuccess aptget install apt -y testdpkginstalled 'apt' @@ -49,15 +51,11 @@ rm https cd - >/dev/null echo "Dir::Bin::Methods \"${COPYMETHODS}\";" >> aptconfig.conf -if [ "$(id -u)" = '0' ]; then - testequal "Can't drop privileges for downloading as file '$(pwd)/apt_1.0_all.deb' couldn't be accessed by user '_apt'. -E: The method driver $(pwd)/rootdir/usr/lib/apt/methods/https could not be found. -N: Is the package apt-transport-https installed?" aptget download apt -q=0 -else - testequal "E: The method driver $(pwd)/rootdir/usr/lib/apt/methods/https could not be found. +cd downloaded +testequal "E: The method driver $(readlink -f './../')/rootdir/usr/lib/apt/methods/https could not be found. N: Is the package apt-transport-https installed?" aptget download apt -q=0 -fi testsuccess test ! -e apt_1.0_all.deb +cd - >/dev/null # revert to all methods rm -rf rootdir/$COPYMETHODS diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes index 72b262e84..92e7c0e84 100755 --- a/test/integration/test-compressed-indexes +++ b/test/integration/test-compressed-indexes @@ -58,9 +58,11 @@ testrun() { msgpass fi msgtest 'Check if package is downloadable' + cd downloaded testsuccess --nomsg aptget download testpkg msgtest '\tdeb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb rm testpkg_1.0_i386.deb + cd - >/dev/null testequal 'Reading package lists... Building dependency tree... The following NEW packages will be installed: @@ -79,10 +81,12 @@ Conf testpkg (1.0 unstable [i386])' aptget install testpkg -s testequal "$GOODSHOWSRC" aptcache showsrc testpkg aptget clean msgtest 'Check if the source is aptgetable' + cd downloaded testsuccess --nomsg aptget source testpkg - msgtest '\tdsc file is present'; testsuccess --nomsg test -f testpkg_1.0.dsc - msgtest '\tdirectory is present'; testsuccess --nomsg test -d testpkg-1.0 + testsuccess test -s testpkg_1.0.dsc + testsuccess test -d testpkg-1.0 rm -rf testpkg-1.0* + cd - >/dev/null testequal "$(aptcache show testpkg -o Acquire::Languages=none) " aptcache dumpavail } diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support index 5ab326def..98b2f242a 100755 --- a/test/integration/test-partial-file-support +++ b/test/integration/test-partial-file-support @@ -52,36 +52,37 @@ cp -a ${TESTDIR}/framework $TESTFILE testrun() { webserverconfig 'aptwebserver::support::range' 'true' + local DOWN='./downloaded/testfile' - copysource $TESTFILE 0 ./testfile - testdownloadfile 'no data' "${1}/testfile" './testfile' '=' + copysource $TESTFILE 0 $DOWN + testdownloadfile 'no data' "${1}/testfile" "$DOWN" '=' testwebserverlaststatuscode '200' "$DOWNLOADLOG" - copysource $TESTFILE 20 ./testfile - testdownloadfile 'valid partial data' "${1}/testfile" './testfile' '=' + copysource $TESTFILE 20 $DOWN + testdownloadfile 'valid partial data' "${1}/testfile" "$DOWN" '=' testwebserverlaststatuscode '206' "$DOWNLOADLOG" - copysource /dev/zero 20 ./testfile - testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!=' + copysource /dev/zero 20 $DOWN + testdownloadfile 'invalid partial data' "${1}/testfile" "$DOWN" '!=' testwebserverlaststatuscode '206' "$DOWNLOADLOG" - copysource $TESTFILE 1M ./testfile - testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '=' + copysource $TESTFILE 1M $DOWN + testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' testwebserverlaststatuscode '416' "$DOWNLOADLOG" - copysource /dev/zero 1M ./testfile - testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '=' + copysource /dev/zero 1M $DOWN + testdownloadfile 'too-big partial file' "${1}/testfile" "$DOWN" '=' testwebserverlaststatuscode '200' "$DOWNLOADLOG" - copysource /dev/zero 20 ./testfile - touch ./testfile - testdownloadfile 'old data' "${1}/testfile" './testfile' '=' + copysource /dev/zero 20 $DOWN + touch $DOWN + testdownloadfile 'old data' "${1}/testfile" "$DOWN" '=' testwebserverlaststatuscode '200' "$DOWNLOADLOG" webserverconfig 'aptwebserver::support::range' 'false' - copysource $TESTFILE 20 ./testfile - testdownloadfile 'no server support' "${1}/testfile" './testfile' '=' + copysource $TESTFILE 20 $DOWN + testdownloadfile 'no server support' "${1}/testfile" "$DOWN" '=' testwebserverlaststatuscode '200' "$DOWNLOADLOG" } diff --git a/test/integration/test-sourceslist-trusted-options b/test/integration/test-sourceslist-trusted-options index 28415dd62..55d4e0233 100755 --- a/test/integration/test-sourceslist-trusted-options +++ b/test/integration/test-sourceslist-trusted-options @@ -20,6 +20,7 @@ foo/testing 2 amd64 foo/stable 1 amd64 ' apt list foo -a + cd downloaded rm -f foo_1_amd64.deb foo_2_amd64.deb testsuccess aptget download foo "$@" testsuccess test -s foo_1_amd64.deb -o -s foo_2_amd64.deb @@ -27,6 +28,7 @@ foo/stable 1 amd64 rm -f foo_1.dsc foo_2.dsc testsuccess aptget source foo --dsc-only -d "$@" testsuccess test -s foo_1.dsc -o -s foo_2.dsc + cd - >/dev/null } everythingfails() { @@ -39,18 +41,22 @@ foo/stable 1 amd64 foo E: Some packages could not be authenticated' + cd downloaded rm -f foo_1_amd64.deb foo_2_amd64.deb testfailure aptget download foo "$@" - testequal "$WARNING" tail -n 3 rootdir/tmp/testfailure.output + testequal "$WARNING" tail -n 3 ../rootdir/tmp/testfailure.output testfailure test -s foo_1_amd64.deb -o -s foo_2_amd64.deb rm -f foo_1.dsc foo_2.dsc testfailure aptget source foo --dsc-only -d "$@" - testequal "$WARNING" tail -n 3 rootdir/tmp/testfailure.output + testequal "$WARNING" tail -n 3 ../rootdir/tmp/testfailure.output testfailure test -s foo_1.dsc -o -s foo_2.dsc + cd - >/dev/null } cp -a rootdir/etc/apt/sources.list.d/ rootdir/etc/apt/sources.list.d.bak/ +echo 'Debug::Acquire::Transaction "true"; +Debug::pkgAcquire::Worker "true";' > rootdir/etc/apt/apt.conf.d/00debugging aptgetupdate() { rm -rf rootdir/var/lib/apt/lists diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum index 8c9c9c767..574183b0a 100755 --- a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -141,6 +141,8 @@ setupaptarchive changetowebserver testsuccess aptget update +cd downloaded + testok() { rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz testequal "Reading package lists... -- cgit v1.2.3-70-g09d2 From f105aaba433f5a8b9c4326dd0d704501bf07d1e5 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 6 Nov 2014 12:53:59 +0100 Subject: better non-virtual metaIndex.LocalFileName() implementation We can't add a new virtual method without breaking the ABI, but we can freely add new methods, so for older ABIs we just implement this method with a dynamic_cast, so that clients can be more ignorant about the API here and especially don't need to pull a very dirty trick by assuming internal knowledge (like apt-get did here). --- apt-pkg/deb/debmetaindex.cc | 2 -- apt-pkg/deb/debmetaindex.h | 3 ++- apt-pkg/metaindex.cc | 40 ++++++++++++++++++++++++++++++++++++++++ apt-pkg/metaindex.h | 25 ++++++------------------- cmdline/apt-get.cc | 30 ++---------------------------- debian/libapt-pkg4.15.symbols | 1 + 6 files changed, 51 insertions(+), 50 deletions(-) create mode 100644 apt-pkg/metaindex.cc (limited to 'cmdline') diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index a6c88b393..d99fd8393 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -78,7 +78,6 @@ string debReleaseIndex::MetaIndexURI(const char *Type) const return Res; } -#if APT_PKG_ABI >= 0x0413 std::string debReleaseIndex::LocalFileName() const { // see if we have a InRelease file @@ -92,7 +91,6 @@ std::string debReleaseIndex::LocalFileName() const return ""; } -#endif string debReleaseIndex::IndexURISuffix(const char *Type, string const &Section, string const &Arch) const { diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 3d35401ec..4a8e454c7 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -54,8 +54,9 @@ class debReleaseIndex : public metaIndex { std::string MetaIndexURI(const char *Type) const; #if APT_PKG_ABI >= 413 - virtual std::string LocalFileName() const; + virtual #endif + std::string LocalFileName() const; std::string IndexURI(const char *Type, std::string const &Section, std::string const &Arch="native") const; std::string IndexURISuffix(const char *Type, std::string const &Section, std::string const &Arch="native") const; diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc new file mode 100644 index 000000000..31a8ec009 --- /dev/null +++ b/apt-pkg/metaindex.cc @@ -0,0 +1,40 @@ +// Include Files /*{{{*/ +#include +#include + +#include + +#include +#include + /*}}}*/ + +#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR >= 13) +std::string metaIndex::LocalFileName() const { return ""; } +#else +#include +std::string metaIndex::LocalFileName() const +{ + debReleaseIndex const * deb = dynamic_cast(this); + if (deb != NULL) + return deb->LocalFileName(); + + return ""; +} +#endif + +metaIndex::metaIndex(std::string const &URI, std::string const &Dist, + char const * const Type) +: Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(false) +{ + /* nothing */ +} + +metaIndex::~metaIndex() +{ + if (Indexes == 0) + return; + for (std::vector::iterator I = (*Indexes).begin(); + I != (*Indexes).end(); ++I) + delete *I; + delete Indexes; +} diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index ae73c27dd..6c3d2880b 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -41,8 +41,8 @@ class metaIndex // interface to to query it #if APT_PKG_ABI >= 413 - // returns the path of the local file (or "" if its not available) - virtual std::string LocalFileName() const {return "";}; + /** \return the path of the local file (or "" if its not available) */ + virtual std::string LocalFileName() const; #else std::string LocalFileName() const; #endif @@ -50,25 +50,12 @@ class metaIndex // Interface for acquire virtual std::string ArchiveURI(std::string const& File) const = 0; virtual bool GetIndexes(pkgAcquire *Owner, bool const &GetAll=false) const = 0; - virtual std::vector *GetIndexFiles() = 0; + virtual std::vector *GetIndexFiles() = 0; virtual bool IsTrusted() const = 0; - metaIndex(std::string const &URI, std::string const &Dist, - char const * const Type) - : Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(false) - { - /* nothing */ - } - - virtual ~metaIndex() - { - if (Indexes == 0) - return; - for (std::vector::iterator I = (*Indexes).begin(); - I != (*Indexes).end(); ++I) - delete *I; - delete Indexes; - } + metaIndex(std::string const &URI, std::string const &Dist, + char const * const Type); + virtual ~metaIndex(); }; #endif diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index b6786faf8..e9e38debc 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -136,28 +136,6 @@ static bool TryToInstallBuildDep(pkgCache::PkgIterator Pkg,pkgCacheFile &Cache, return true; } /*}}}*/ - - -// helper that can go wit hthe next ABI break -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR < 13) -static std::string MetaIndexFileNameOnDisk(metaIndex *metaindex) -{ - // FIXME: this cast is the horror, the horror - debReleaseIndex *r = (debReleaseIndex*)metaindex; - - // see if we have a InRelease file - std::string PathInRelease = r->MetaIndexFile("InRelease"); - if (FileExists(PathInRelease)) - return PathInRelease; - - // and if not return the normal one - if (FileExists(PathInRelease)) - return r->MetaIndexFile("Release"); - - return ""; -} -#endif - // GetReleaseForSourceRecord - Return Suite for the given srcrecord /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -176,12 +154,8 @@ static std::string GetReleaseForSourceRecord(pkgSourceList *SrcList, { if (&CurrentIndexFile == (*IF)) { -#if (APT_PKG_MAJOR >= 4 && APT_PKG_MINOR < 13) - std::string path = MetaIndexFileNameOnDisk(*S); -#else - std::string path = (*S)->LocalFileName(); -#endif - if (path != "") + std::string const path = (*S)->LocalFileName(); + if (path != "") { indexRecords records; records.Load(path); diff --git a/debian/libapt-pkg4.15.symbols b/debian/libapt-pkg4.15.symbols index 90f67b5cd..a6c5c21ea 100644 --- a/debian/libapt-pkg4.15.symbols +++ b/debian/libapt-pkg4.15.symbols @@ -1789,6 +1789,7 @@ libapt-pkg.so.4.15 libapt-pkg4.15 #MINVER# (c++)"indexRecords::IsAlwaysTrusted() const@Base" 1.1~exp5 (c++)"indexRecords::IsNeverTrusted() const@Base" 1.1~exp5 (c++)"indexRecords::SetTrusted(bool)@Base" 1.1~exp5 + (c++)"metaIndex::metaIndex(std::basic_string, std::allocator > const&, std::basic_string, std::allocator > const&, char const*)@Base" 1.1~exp9 ### demangle strangeness - buildd report it as MISSING and as new… (c++)"pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire*, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::vector > const*, indexRecords*)@Base" 0.8.0 (c++)"pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire*, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::vector > const*, indexRecords*)@Base" 0.8.0 -- cgit v1.2.3-70-g09d2 From 765190e493645e13b5651625d87fd9c8db910a85 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 7 Nov 2014 16:45:18 +0100 Subject: guard ABI changes for SourcePkg/Ver in pkgCache Git-Dch: Ignore --- apt-pkg/cacheiterators.h | 2 ++ apt-pkg/deb/deblistparser.cc | 2 ++ apt-pkg/deb/dpkgpm.cc | 11 +++++++++- apt-pkg/edsp.cc | 7 ++++++ apt-pkg/pkgcache.h | 2 ++ cmdline/apt-cache.cc | 2 ++ cmdline/apt-get.cc | 52 ++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 73 insertions(+), 5 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h index b0c02d4a2..f8f4c05ce 100644 --- a/apt-pkg/cacheiterators.h +++ b/apt-pkg/cacheiterators.h @@ -215,12 +215,14 @@ class pkgCache::VerIterator : public Iterator { // Accessors inline const char *VerStr() const {return S->VerStr == 0?0:Owner->StrP + S->VerStr;} inline const char *Section() const {return S->Section == 0?0:Owner->StrP + S->Section;} +#if APT_PKG_ABI >= 413 /** \brief source package name this version comes from Always contains the name, even if it is the same as the binary name */ inline const char *SourcePkgName() const {return Owner->StrP + S->SourcePkgName;} /** \brief source version this version comes from Always contains the version string, even if it is the same as the binary version */ inline const char *SourceVerStr() const {return Owner->StrP + S->SourceVerStr;} +#endif inline const char *Arch() const { if ((S->MultiArch & pkgCache::Version::All) == pkgCache::Version::All) return "all"; diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 42db341b3..462818a03 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -141,6 +141,7 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) map_stringitem_t const idx = StoreString(pkgCacheGenerator::SECTION, Start, Stop - Start); Ver->Section = idx; } +#if APT_PKG_ABI >= 413 // Parse the source package name pkgCache::GrpIterator const G = Ver.ParentPkg().Group(); Ver->SourcePkgName = G->Name; @@ -192,6 +193,7 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver) } } } +#endif Ver->MultiArch = ParseMultiArch(true); // Archive Size diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 0cded32e1..3f6039e0d 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1684,7 +1684,7 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) if (apportPkg.end() == true || apportPkg->CurrentVer == 0) return; - string pkgname, reportfile, srcpkgname, pkgver, arch; + string pkgname, reportfile, pkgver, arch; string::size_type pos; FILE *report; @@ -1823,7 +1823,16 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) time_t now = time(NULL); fprintf(report, "Date: %s" , ctime(&now)); fprintf(report, "Package: %s %s\n", pkgname.c_str(), pkgver.c_str()); +#if APT_PKG_ABI >= 413 fprintf(report, "SourcePackage: %s\n", Ver.SourcePkgName()); +#else + pkgRecords Recs(Cache); + pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); + std::string srcpkgname = Parse.SourcePkg(); + if(srcpkgname.empty()) + srcpkgname = pkgname; + fprintf(report, "SourcePackage: %s\n", srcpkgname.c_str()); +#endif fprintf(report, "ErrorMessage:\n %s\n", errormsg); // ensure that the log is flushed diff --git a/apt-pkg/edsp.cc b/apt-pkg/edsp.cc index 2ba914b16..3c6a7e30f 100644 --- a/apt-pkg/edsp.cc +++ b/apt-pkg/edsp.cc @@ -96,7 +96,14 @@ void EDSP::WriteScenarioVersion(pkgDepCache &Cache, FILE* output, pkgCache::PkgI pkgCache::VerIterator const &Ver) { fprintf(output, "Package: %s\n", Pkg.Name()); +#if APT_PKG_ABI >= 413 fprintf(output, "Source: %s\n", Ver.SourcePkgName()); +#else + pkgRecords Recs(Cache); + pkgRecords::Parser &rec = Recs.Lookup(Ver.FileList()); + string srcpkg = rec.SourcePkg().empty() ? Pkg.Name() : rec.SourcePkg(); + fprintf(output, "Source: %s\n", srcpkg.c_str()); +#endif fprintf(output, "Architecture: %s\n", Ver.Arch()); fprintf(output, "Version: %s\n", Ver.VerStr()); if (Pkg.CurrentVer() == Ver) diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 4f8568205..a7b7fa539 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -510,12 +510,14 @@ struct pkgCache::Version map_stringitem_t VerStr; /** \brief section this version is filled in */ map_stringitem_t Section; +#if APT_PKG_ABI >= 413 /** \brief source package name this version comes from Always contains the name, even if it is the same as the binary name */ map_stringitem_t SourcePkgName; /** \brief source version this version comes from Always contains the version string, even if it is the same as the binary version */ map_stringitem_t SourceVerStr; +#endif /** \brief Multi-Arch capabilities of a package version */ enum VerMultiArch { None = 0, /*!< is the default and doesn't trigger special behaviour */ diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 0f4f7e1ce..1bd75dfba 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -389,8 +389,10 @@ static bool Stats(CommandLine &) stritems.insert(V->VerStr); if (V->Section != 0) stritems.insert(V->Section); +#if APT_PKG_ABI >= 413 stritems.insert(V->SourcePkgName); stritems.insert(V->SourceVerStr); +#endif for (pkgCache::DepIterator D = V.DependsList(); D.end() == false; ++D) { if (D->Version != 0) diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index e9e38debc..eca4a723b 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -170,7 +170,11 @@ static std::string GetReleaseForSourceRecord(pkgSourceList *SrcList, // FindSrc - Find a source record /*{{{*/ // --------------------------------------------------------------------- /* */ +#if APT_PKG_ABI >= 413 static pkgSrcRecords::Parser *FindSrc(const char *Name, +#else +static pkgSrcRecords::Parser *FindSrc(const char *Name,pkgRecords &Recs, +#endif pkgSrcRecords &SrcRecs,string &Src, CacheFile &CacheFile) { @@ -278,10 +282,21 @@ static pkgSrcRecords::Parser *FindSrc(const char *Name, (VF.File().Archive() != 0 && VF.File().Archive() == RelTag) || (VF.File().Codename() != 0 && VF.File().Codename() == RelTag)) { - Src = Ver.SourcePkgName(); // the Version we have is possibly fuzzy or includes binUploads, - // so we use the Version of the SourcePkg + // so we use the Version of the SourcePkg (empty if same as package) +#if APT_PKG_ABI >= 413 + Src = Ver.SourcePkgName(); VerTag = Ver.SourceVerStr(); +#else + pkgRecords::Parser &Parse = Recs.Lookup(VF); + Src = Parse.SourcePkg(); + // no SourcePkg name, so it is the "binary" name + if (Src.empty() == true) + Src = TmpSrc; + VerTag = Parse.SourceVer(); + if (VerTag.empty() == true) + VerTag = Ver.VerStr(); +#endif break; } } @@ -312,10 +327,17 @@ static pkgSrcRecords::Parser *FindSrc(const char *Name, pkgCache::VerIterator Ver = Cache->GetCandidateVer(Pkg); if (Ver.end() == false) { +#if APT_PKG_ABI >= 413 if (strcmp(Ver.SourcePkgName(),Ver.ParentPkg().Name()) != 0) Src = Ver.SourcePkgName(); if (VerTag.empty() == true && strcmp(Ver.SourceVerStr(),Ver.VerStr()) != 0) VerTag = Ver.SourceVerStr(); +#else + pkgRecords::Parser &Parse = Recs.Lookup(Ver.FileList()); + Src = Parse.SourcePkg(); + if (VerTag.empty() == true) + VerTag = Parse.SourceVer(); +#endif } } } @@ -717,6 +739,9 @@ static bool DoSource(CommandLine &CmdL) pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers +#if APT_PKG_ABI < 413 + pkgRecords Recs(Cache); +#endif pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@ -744,8 +769,11 @@ static bool DoSource(CommandLine &CmdL) for (const char **I = CmdL.FileList + 1; *I != 0; I++, J++) { string Src; +#if APT_PKG_ABI >= 413 pkgSrcRecords::Parser *Last = FindSrc(*I,SrcRecs,Src,Cache); - +#else + pkgSrcRecords::Parser *Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); +#endif if (Last == 0) { return _error->Error(_("Unable to find a source package for %s"),Src.c_str()); } @@ -1004,6 +1032,9 @@ static bool DoBuildDep(CommandLine &CmdL) pkgSourceList *List = Cache.GetSourceList(); // Create the text record parsers +#if APT_PKG_ABI < 413 + pkgRecords Recs(Cache); +#endif pkgSrcRecords SrcRecs(*List); if (_error->PendingError() == true) return false; @@ -1050,7 +1081,11 @@ static bool DoBuildDep(CommandLine &CmdL) Last = Type->CreateSrcPkgParser(*I); } else { // normal case, search the cache for the source file - Last = FindSrc(*I,SrcRecs,Src,Cache); +#if APT_PKG_ABI >= 413 + Last = FindSrc(*I,SrcRecs,Src,Cache); +#else + Last = FindSrc(*I,Recs,SrcRecs,Src,Cache); +#endif } if (Last == 0) @@ -1407,9 +1442,18 @@ static string GetChangelogPath(CacheFile &Cache, pkgRecords Recs(Cache); pkgRecords::Parser &rec=Recs.Lookup(Ver.FileList()); string path = flNotFile(rec.FileName()); +#if APT_PKG_ABI >= 413 path.append(Ver.SourcePkgName()); path.append("_"); path.append(StripEpoch(Ver.SourceVerStr())); +#else + string srcpkg = rec.SourcePkg().empty() ? Ver.ParentPkg().Name() : rec.SourcePkg(); + string ver = Ver.VerStr(); + // if there is a source version it always wins + if (rec.SourceVer() != "") + ver = rec.SourceVer(); + path += srcpkg + "_" + StripEpoch(ver); +#endif return path; } /*}}}*/ -- cgit v1.2.3-70-g09d2 From 32ab4bd05cb298f6bf1f9574f5b20570beaae429 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 7 Nov 2014 19:18:21 +0100 Subject: guard pkg/grp hashtable creation changes The change itself is no problem ABI wise, but the remove of the old undynamic hashtables is, so we bring it back for older abis and happily use the now available free space to backport more recent additions like the dynamic hashtable itself. Git-Dch: Ignore --- apt-pkg/pkgcache.cc | 39 ++++++++++++--------- apt-pkg/pkgcache.h | 93 ++++++++++++++++++++++++++++++++++++++++++-------- apt-pkg/pkgcachegen.cc | 18 +++++----- cmdline/apt-cache.cc | 13 ++++--- 4 files changed, 118 insertions(+), 45 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 572685ba5..8d6e242b1 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -82,10 +82,13 @@ pkgCache::Header::Header() MaxDescFileSize = 0; FileList = 0; +#if APT_PKG_ABI < 413 + APT_IGNORE_DEPRECATED(StringList = 0;) +#endif VerSysName = 0; Architecture = 0; - Architectures = 0; - HashTableSize = _config->FindI("APT::Cache-HashTableSize", 10 * 1048); + SetArchitectures(0); + SetHashTableSize(_config->FindI("APT::Cache-HashTableSize", 10 * 1048)); memset(Pools,0,sizeof(Pools)); CacheFileSize = 0; @@ -114,6 +117,7 @@ bool pkgCache::Header::CheckSizes(Header &Against) const // Cache::pkgCache - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ +APT_IGNORE_DEPRECATED_PUSH pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map) { // call getArchitectures() with cached=false to ensure that the @@ -123,6 +127,7 @@ pkgCache::pkgCache(MMap *Map, bool DoMap) : Map(*Map) if (DoMap == true) ReMap(); } +APT_IGNORE_DEPRECATED_POP /*}}}*/ // Cache::ReMap - Reopen the cache file /*{{{*/ // --------------------------------------------------------------------- @@ -162,7 +167,7 @@ bool pkgCache::ReMap(bool const &Errorchecks) if (Map.Size() < HeaderP->CacheFileSize) return _error->Error(_("The package cache file is corrupted, it is too small")); - if (HeaderP->VerSysName == 0 || HeaderP->Architecture == 0 || HeaderP->Architectures == 0) + if (HeaderP->VerSysName == 0 || HeaderP->Architecture == 0 || HeaderP->GetArchitectures() == 0) return _error->Error(_("The package cache file is corrupted")); // Locate our VS.. @@ -176,8 +181,8 @@ bool pkgCache::ReMap(bool const &Errorchecks) for (++a; a != archs.end(); ++a) list.append(",").append(*a); if (_config->Find("APT::Architecture") != StrP + HeaderP->Architecture || - list != StrP + HeaderP->Architectures) - return _error->Error(_("The package cache was built for different architectures: %s vs %s"), StrP + HeaderP->Architectures, list.c_str()); + list != StrP + HeaderP->GetArchitectures()) + return _error->Error(_("The package cache was built for different architectures: %s vs %s"), StrP + HeaderP->GetArchitectures(), list.c_str()); return true; } @@ -192,7 +197,7 @@ map_id_t pkgCache::sHash(const string &Str) const unsigned long Hash = 0; for (string::const_iterator I = Str.begin(); I != Str.end(); ++I) Hash = 41 * Hash + tolower_ascii(*I); - return Hash % HeaderP->HashTableSize; + return Hash % HeaderP->GetHashTableSize(); } map_id_t pkgCache::sHash(const char *Str) const @@ -200,7 +205,7 @@ map_id_t pkgCache::sHash(const char *Str) const unsigned long Hash = tolower_ascii(*Str); for (const char *I = Str + 1; *I != 0; ++I) Hash = 41 * Hash + tolower_ascii(*I); - return Hash % HeaderP->HashTableSize; + return Hash % HeaderP->GetHashTableSize(); } /*}}}*/ // Cache::SingleArchFindPkg - Locate a package by name /*{{{*/ @@ -211,8 +216,8 @@ map_id_t pkgCache::sHash(const char *Str) const pkgCache::PkgIterator pkgCache::SingleArchFindPkg(const string &Name) { // Look at the hash bucket - Package *Pkg = PkgP + HeaderP->PkgHashTable()[Hash(Name)]; - for (; Pkg != PkgP; Pkg = PkgP + Pkg->Next) + Package *Pkg = PkgP + HeaderP->PkgHashTableP()[Hash(Name)]; + for (; Pkg != PkgP; Pkg = PkgP + Pkg->NextPackage) { int const cmp = strcmp(Name.c_str(), StrP + (GrpP + Pkg->Group)->Name); if (cmp == 0) @@ -273,7 +278,7 @@ pkgCache::GrpIterator pkgCache::FindGrp(const string &Name) { return GrpIterator(*this,0); // Look at the hash bucket for the group - Group *Grp = GrpP + HeaderP->GrpHashTable()[sHash(Name)]; + Group *Grp = GrpP + HeaderP->GrpHashTableP()[sHash(Name)]; for (; Grp != GrpP; Grp = GrpP + Grp->Next) { int const cmp = strcmp(Name.c_str(), StrP + Grp->Name); if (cmp == 0) @@ -359,7 +364,7 @@ pkgCache::PkgIterator pkgCache::GrpIterator::FindPkg(string Arch) const { // Iterate over the list to find the matching arch for (pkgCache::Package *Pkg = PackageList(); Pkg != Owner->PkgP; - Pkg = Owner->PkgP + Pkg->Next) { + Pkg = Owner->PkgP + Pkg->NextPackage) { if (stringcmp(Arch, Owner->StrP + Pkg->Arch) == 0) return PkgIterator(*Owner, Pkg); if ((Owner->PkgP + S->LastPackage) == Pkg) @@ -407,7 +412,7 @@ pkgCache::PkgIterator pkgCache::GrpIterator::NextPkg(pkgCache::PkgIterator const if (S->LastPackage == LastPkg.Index()) return PkgIterator(*Owner, 0); - return PkgIterator(*Owner, Owner->PkgP + LastPkg->Next); + return PkgIterator(*Owner, Owner->PkgP + LastPkg->NextPackage); } /*}}}*/ // GrpIterator::operator ++ - Postfix incr /*{{{*/ @@ -420,10 +425,10 @@ void pkgCache::GrpIterator::operator ++(int) S = Owner->GrpP + S->Next; // Follow the hash table - while (S == Owner->GrpP && (HashIndex+1) < (signed)Owner->HeaderP->HashTableSize) + while (S == Owner->GrpP && (HashIndex+1) < (signed)Owner->HeaderP->GetHashTableSize()) { HashIndex++; - S = Owner->GrpP + Owner->HeaderP->GrpHashTable()[HashIndex]; + S = Owner->GrpP + Owner->HeaderP->GrpHashTableP()[HashIndex]; } } /*}}}*/ @@ -434,13 +439,13 @@ void pkgCache::PkgIterator::operator ++(int) { // Follow the current links if (S != Owner->PkgP) - S = Owner->PkgP + S->Next; + S = Owner->PkgP + S->NextPackage; // Follow the hash table - while (S == Owner->PkgP && (HashIndex+1) < (signed)Owner->HeaderP->HashTableSize) + while (S == Owner->PkgP && (HashIndex+1) < (signed)Owner->HeaderP->GetHashTableSize()) { HashIndex++; - S = Owner->PkgP + Owner->HeaderP->PkgHashTable()[HashIndex]; + S = Owner->PkgP + Owner->HeaderP->PkgHashTableP()[HashIndex]; } } /*}}}*/ diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index a7b7fa539..2ba23c5c0 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -85,16 +85,45 @@ using std::string; #endif +#if APT_PKG_ABI >= 413 // storing file sizes of indexes, which are way below 4 GB for now typedef uint32_t map_filesize_t; +typedef map_filesize_t should_be_map_filesize_t; +#else +typedef unsigned long map_filesize_t; +typedef unsigned int should_be_map_filesize_t; +#endif +#if APT_PKG_ABI >= 413 // each package/group/dependency gets an id typedef uint32_t map_id_t; +typedef map_id_t should_be_map_id_t; +#else +typedef unsigned long map_id_t; +typedef unsigned int should_be_map_id_t; +#endif +#if APT_PKG_ABI >= 413 // some files get an id, too, but in far less absolute numbers typedef uint16_t map_fileid_t; +typedef map_fileid_t should_be_map_fileid_t; +#else +typedef unsigned long map_fileid_t; +typedef unsigned int should_be_map_fileid_t; +#endif +#if APT_PKG_ABI >= 413 // relative pointer from cache start typedef uint32_t map_pointer_t; +#else +typedef unsigned int map_pointer_t; +#endif // same as the previous, but documented to be to a string item typedef map_pointer_t map_stringitem_t; +#if APT_PKG_ABI >= 413 +typedef uint64_t should_be_uint64_t; +typedef uint64_t should_be_uint64_small_t; +#else +typedef unsigned long long should_be_uint64_t; +typedef unsigned long should_be_uint64_small_t; +#endif class pkgVersioningSystem; class pkgCache /*{{{*/ @@ -109,6 +138,7 @@ class pkgCache /*{{{*/ struct Description; struct Provides; struct Dependency; + struct StringItem; struct VerFile; struct DescFile; @@ -185,6 +215,7 @@ class pkgCache /*{{{*/ Description *DescP; Provides *ProvideP; Dependency *DepP; + APT_DEPRECATED StringItem *StringItemP; char *StrP; virtual bool ReMap(bool const &Errorchecks = true); @@ -288,12 +319,17 @@ struct pkgCache::Header The PackageFile structures are singly linked lists that represent all package files that have been merged into the cache. */ map_pointer_t FileList; +#if APT_PKG_ABI < 413 + APT_DEPRECATED map_pointer_t StringList; +#endif /** \brief String representing the version system used */ map_pointer_t VerSysName; /** \brief native architecture the cache was built against */ map_pointer_t Architecture; +#if APT_PKG_ABI >= 413 /** \brief all architectures the cache was built against */ map_pointer_t Architectures; +#endif /** \brief The maximum size of a raw entry from the original Package file */ map_filesize_t MaxVerFileSize; /** \brief The maximum size of a raw entry from the original Translation file */ @@ -319,12 +355,26 @@ struct pkgCache::Header In the PkgHashTable is it possible that multiple packages have the same name - these packages are stored as a sequence in the list. The size of both tables is the same. */ +#if APT_PKG_ABI >= 413 unsigned int HashTableSize; - map_pointer_t * PkgHashTable() const { return (map_pointer_t*) (this + 1); } - map_pointer_t * GrpHashTable() const { return PkgHashTable() + HashTableSize; } + unsigned int GetHashTableSize() const { return HashTableSize; } + void SetHashTableSize(unsigned int const sz) { HashTableSize = sz; } + map_pointer_t GetArchitectures() const { return Architectures; } + void SetArchitectures(map_pointer_t const idx) { Architectures = idx; } +#else + // BEWARE: these tables are pretty much empty and just here for abi compat + map_ptrloc PkgHashTable[2*1048]; + map_ptrloc GrpHashTable[2*1048]; + unsigned int GetHashTableSize() const { return PkgHashTable[0]; } + void SetHashTableSize(unsigned int const sz) { PkgHashTable[0] = sz; } + map_pointer_t GetArchitectures() const { return PkgHashTable[1]; } + void SetArchitectures(map_pointer_t const idx) { PkgHashTable[1] = idx; } +#endif + map_pointer_t * PkgHashTableP() const { return (map_pointer_t*) (this + 1); } + map_pointer_t * GrpHashTableP() const { return PkgHashTableP() + GetHashTableSize(); } /** \brief Size of the complete cache file */ - unsigned long long CacheFileSize; + should_be_uint64_small_t CacheFileSize; bool CheckSizes(Header &Against) const APT_PURE; Header(); @@ -350,7 +400,7 @@ struct pkgCache::Group /** \brief Link to the next Group */ map_pointer_t Next; // Group /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; }; /*}}}*/ @@ -387,12 +437,18 @@ struct pkgCache::Package map_pointer_t VersionList; // Version /** \brief index to the installed version */ map_pointer_t CurrentVer; // Version + /** \brief indicates nothing (consistently) + This field used to contain ONE section the package belongs to, + if those differs between versions it is a RANDOM one. + The Section() method tries to reproduce it, but the only sane + thing to do is use the Section field from the version! */ + APT_DEPRECATED map_ptrloc Section; // StringItem /** \brief index of the group this package belongs to */ map_pointer_t Group; // Group the Package belongs to // Linked list /** \brief Link to the next package in the same bucket */ - map_pointer_t Next; // Package + map_pointer_t NextPackage; // Package /** \brief List of all dependencies on this package */ map_pointer_t RevDepends; // Dependency /** \brief List of all "packages" this package provide */ @@ -416,7 +472,7 @@ struct pkgCache::Package This allows clients to create an array of size PackageCount and use it to store state information for the package map. For instance the status file emitter uses this to track which packages have been emitted already. */ - map_id_t ID; + should_be_map_id_t ID; /** \brief some useful indicators of the package's state */ unsigned long Flags; }; @@ -464,7 +520,7 @@ struct pkgCache::PackageFile /** \brief Link to the next PackageFile in the Cache */ map_pointer_t NextFile; // PackageFile /** \brief unique sequel ID */ - map_fileid_t ID; + should_be_map_fileid_t ID; }; /*}}}*/ // VerFile structure /*{{{*/ @@ -479,7 +535,7 @@ struct pkgCache::VerFile /** \brief next step in the linked list */ map_pointer_t NextFile; // PkgVerFile /** \brief position in the package file */ - map_filesize_t Offset; // File offset + should_be_map_filesize_t Offset; // File offset /** @TODO document pkgCache::VerFile::Size */ map_filesize_t Size; }; @@ -493,7 +549,7 @@ struct pkgCache::DescFile /** \brief next step in the linked list */ map_pointer_t NextFile; // PkgVerFile /** \brief position in the file */ - map_filesize_t Offset; // File offset + should_be_map_filesize_t Offset; // File offset /** @TODO document pkgCache::DescFile::Size */ map_filesize_t Size; }; @@ -556,16 +612,16 @@ struct pkgCache::Version /** \brief archive size for this version For Debian this is the size of the .deb file. */ - uint64_t Size; // These are the .deb size + should_be_uint64_t Size; // These are the .deb size /** \brief uncompressed size for this version */ - uint64_t InstalledSize; + should_be_uint64_t InstalledSize; /** \brief characteristic value representing this version No two packages in existence should have the same VerStr and Hash with different contents. */ unsigned short Hash; /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; /** \brief parsed priority value */ unsigned char Priority; }; @@ -593,7 +649,7 @@ struct pkgCache::Description map_pointer_t ParentPkg; // Package /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; }; /*}}}*/ // Dependency structure /*{{{*/ @@ -620,7 +676,7 @@ struct pkgCache::Dependency map_pointer_t ParentVer; // Version /** \brief unique sequel ID */ - map_id_t ID; + should_be_map_id_t ID; /** \brief Dependency type - Depends, Recommends, Conflicts, etc */ unsigned char Type; /** \brief comparison operator specified on the depends line @@ -656,6 +712,15 @@ struct pkgCache::Provides map_pointer_t NextPkgProv; // Provides }; /*}}}*/ +// UNUSED StringItem structure /*{{{*/ +struct APT_DEPRECATED pkgCache::StringItem +{ + /** \brief string this refers to */ + map_ptrloc String; // StringItem + /** \brief Next link in the chain */ + map_ptrloc NextItem; // StringItem +}; + /*}}}*/ inline char const * pkgCache::NativeArch() { return StrP + HeaderP->Architecture; } diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index a2c1c8760..ba454f057 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -74,7 +74,7 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : *Cache.HeaderP = pkgCache::Header(); // make room for the hashtables for packages and groups - if (Map.RawAllocate(2 * (Cache.HeaderP->HashTableSize * sizeof(map_pointer_t))) == 0) + if (Map.RawAllocate(2 * (Cache.HeaderP->GetHashTableSize() * sizeof(map_pointer_t))) == 0) return; map_stringitem_t const idxVerSysName = WriteStringInMap(_system->VS->Label); @@ -96,10 +96,10 @@ pkgCacheGenerator::pkgCacheGenerator(DynamicMMap *pMap,OpProgress *Prog) : map_stringitem_t const idxArchitectures = WriteStringInMap(list); if (unlikely(idxArchitectures == 0)) return; - Cache.HeaderP->Architectures = idxArchitectures; + Cache.HeaderP->SetArchitectures(idxArchitectures); } else - Cache.HeaderP->Architectures = idxArchitecture; + Cache.HeaderP->SetArchitectures(idxArchitecture); Cache.ReMap(); } @@ -616,7 +616,7 @@ bool pkgCacheGenerator::NewGroup(pkgCache::GrpIterator &Grp, const string &Name) // Insert it into the hash table unsigned long const Hash = Cache.Hash(Name); - map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTable()[Hash]; + map_pointer_t *insertAt = &Cache.HeaderP->GrpHashTableP()[Hash]; while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + *insertAt)->Name) > 0) insertAt = &(Cache.GrpP + *insertAt)->Next; Grp->Next = *insertAt; @@ -652,18 +652,18 @@ bool pkgCacheGenerator::NewPackage(pkgCache::PkgIterator &Pkg,const string &Name Grp->FirstPackage = Package; // Insert it into the hash table map_id_t const Hash = Cache.Hash(Name); - map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTable()[Hash]; + map_pointer_t *insertAt = &Cache.HeaderP->PkgHashTableP()[Hash]; while (*insertAt != 0 && strcasecmp(Name.c_str(), Cache.StrP + (Cache.GrpP + (Cache.PkgP + *insertAt)->Group)->Name) > 0) - insertAt = &(Cache.PkgP + *insertAt)->Next; - Pkg->Next = *insertAt; + insertAt = &(Cache.PkgP + *insertAt)->NextPackage; + Pkg->NextPackage = *insertAt; *insertAt = Package; } else // Group the Packages together { // this package is the new last package pkgCache::PkgIterator LastPkg(Cache, Cache.PkgP + Grp->LastPackage); - Pkg->Next = LastPkg->Next; - LastPkg->Next = Package; + Pkg->NextPackage = LastPkg->NextPackage; + LastPkg->NextPackage = Package; } Grp->LastPackage = Package; diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 1bd75dfba..9bac45029 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -267,11 +267,14 @@ static bool DumpPackage(CommandLine &CmdL) // ShowHashTableStats - Show stats about a hashtable /*{{{*/ // --------------------------------------------------------------------- /* */ +static map_pointer_t PackageNext(pkgCache::Package const * const P) { return P->NextPackage; } +static map_pointer_t GroupNext(pkgCache::Group const * const G) { return G->Next; } template static void ShowHashTableStats(std::string Type, T *StartP, map_pointer_t *Hashtable, - unsigned long Size) + unsigned long Size, + map_pointer_t(*Next)(T const * const)) { // hashtable stats for the HashTable unsigned long NumBuckets = Size; @@ -290,7 +293,7 @@ static void ShowHashTableStats(std::string Type, } ++UsedBuckets; unsigned long ThisBucketSize = 0; - for (; P != StartP; P = StartP + P->Next) + for (; P != StartP; P = StartP + Next(P)) ++ThisBucketSize; Entries += ThisBucketSize; LongestBucket = std::max(ThisBucketSize, LongestBucket); @@ -447,13 +450,13 @@ static bool Stats(CommandLine &) APT_CACHESIZE(VerFileCount, VerFileSz) + APT_CACHESIZE(DescFileCount, DescFileSz) + APT_CACHESIZE(ProvidesCount, ProvidesSz) + - (2 * Cache->Head().HashTableSize * sizeof(map_id_t)); + (2 * Cache->Head().GetHashTableSize() * sizeof(map_id_t)); cout << _("Total space accounted for: ") << SizeToStr(Total) << endl; #undef APT_CACHESIZE // hashtable stats - ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->Head().PkgHashTable(), Cache->Head().HashTableSize); - ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->Head().GrpHashTable(), Cache->Head().HashTableSize); + ShowHashTableStats("PkgHashTable", Cache->PkgP, Cache->Head().PkgHashTableP(), Cache->Head().GetHashTableSize(), PackageNext); + ShowHashTableStats("GrpHashTable", Cache->GrpP, Cache->Head().GrpHashTableP(), Cache->Head().GetHashTableSize(), GroupNext); return true; } -- cgit v1.2.3-70-g09d2 From ad7e0941b376d792911f240377094a2e78ca8756 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 18:14:46 +0100 Subject: streamline display of --help in all tools By convention, if I run a tool with --help or --version I expect it to exit successfully with the usage, while if I do call it wrong (like without any parameters) I expect the usage message shown with a non-zero exit. --- apt-pkg/contrib/cmndline.cc | 3 ++ apt-pkg/contrib/cmndline.h | 1 + apt-private/private-cmndline.cc | 33 ++++++++++++++++++ apt-private/private-cmndline.h | 6 ++++ cmdline/apt-cache.cc | 18 ++-------- cmdline/apt-cdrom.cc | 15 ++------- cmdline/apt-config.cc | 15 ++------- cmdline/apt-extracttemplates.cc | 32 +++++++----------- cmdline/apt-get.cc | 22 ++---------- cmdline/apt-helper.cc | 21 ++---------- cmdline/apt-internal-solver.cc | 17 +++------- cmdline/apt-mark.cc | 22 ++---------- cmdline/apt-sortpkgs.cc | 24 +++++-------- cmdline/apt.cc | 20 ++--------- cmdline/makefile | 8 ++--- debian/tests/run-tests | 1 + ftparchive/apt-ftparchive.cc | 19 +++-------- ftparchive/makefile | 4 +-- prepare-release | 4 +-- test/integration/framework | 2 ++ test/integration/test-00-commands-have-help | 52 +++++++++++++++++++++++++++++ 21 files changed, 149 insertions(+), 190 deletions(-) create mode 100755 test/integration/test-00-commands-have-help (limited to 'cmdline') diff --git a/apt-pkg/contrib/cmndline.cc b/apt-pkg/contrib/cmndline.cc index 93c1f4664..ff8b09ebc 100644 --- a/apt-pkg/contrib/cmndline.cc +++ b/apt-pkg/contrib/cmndline.cc @@ -33,6 +33,9 @@ using namespace std; CommandLine::CommandLine(Args *AList,Configuration *Conf) : ArgList(AList), Conf(Conf), FileList(0) { +} +CommandLine::CommandLine() : ArgList(NULL), Conf(NULL), FileList(0) +{ } /*}}}*/ // CommandLine::~CommandLine - Destructor /*{{{*/ diff --git a/apt-pkg/contrib/cmndline.h b/apt-pkg/contrib/cmndline.h index 143df58b2..58cbaa8c3 100644 --- a/apt-pkg/contrib/cmndline.h +++ b/apt-pkg/contrib/cmndline.h @@ -91,6 +91,7 @@ class CommandLine static CommandLine::Args MakeArgs(char ShortOpt, char const *LongOpt, char const *ConfName, unsigned long Flags) APT_CONST; + CommandLine(); CommandLine(Args *AList,Configuration *Conf); ~CommandLine(); }; diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc index aa01be757..bb9a00803 100644 --- a/apt-private/private-cmndline.cc +++ b/apt-private/private-cmndline.cc @@ -2,12 +2,17 @@ #include #include +#include +#include +#include +#include #include #include #include #include +#include #include /*}}}*/ @@ -287,3 +292,31 @@ std::vector getCommandArgs(char const * const Program, char c /*}}}*/ #undef CmdMatches #undef addArg +void ParseCommandLine(CommandLine &CmdL, CommandLine::Dispatch * const Cmds, CommandLine::Args * const Args,/*{{{*/ + Configuration * const * const Cnf, pkgSystem ** const Sys, int const argc, const char *argv[], bool(*ShowHelp)(CommandLine &CmdL)) +{ + CmdL = CommandLine(Args,_config); + if ((Cnf != NULL && pkgInitConfig(**Cnf) == false) || + CmdL.Parse(argc,argv) == false || + (Sys != NULL && pkgInitSystem(*_config, *Sys) == false)) + { + if (_config->FindB("version") == true) + ShowHelp(CmdL); + + _error->DumpErrors(); + exit(100); + } + + // See if the help should be shown + if (_config->FindB("help") == true || _config->FindB("version") == true) + { + ShowHelp(CmdL); + exit(0); + } + if (Cmds != NULL && CmdL.FileSize() == 0) + { + ShowHelp(CmdL); + exit(1); + } +} + /*}}}*/ diff --git a/apt-private/private-cmndline.h b/apt-private/private-cmndline.h index d0af16782..7b468456b 100644 --- a/apt-private/private-cmndline.h +++ b/apt-private/private-cmndline.h @@ -6,6 +6,12 @@ #include +class Configuration; +class pkgSystem; + APT_PUBLIC std::vector getCommandArgs(char const * const Program, char const * const Cmd); +APT_PUBLIC void ParseCommandLine(CommandLine &CmdL, CommandLine::Dispatch * const Cmds, CommandLine::Args * const Args, + Configuration * const * const Cnf, pkgSystem ** const Sys, int const argc, const char * argv[], + bool(*ShowHelp)(CommandLine &CmdL)); #endif diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 9bac45029..12ed4f719 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1896,23 +1896,9 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - // Deal with stdout not being a tty if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) _config->Set("quiet","1"); diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc index 53efe65b8..8ac73fd7e 100644 --- a/cmdline/apt-cdrom.cc +++ b/cmdline/apt-cdrom.cc @@ -249,19 +249,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || _config->FindB("version") == true || - CmdL.FileSize() == 0) - return ShowHelp(CmdL); + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // Deal with stdout not being a tty if (isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) diff --git a/cmdline/apt-config.cc b/cmdline/apt-config.cc index 40ba468eb..e0b8a624e 100644 --- a/cmdline/apt-config.cc +++ b/cmdline/apt-config.cc @@ -115,19 +115,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(CmdL); + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); std::vector const langs = APT::Configuration::getLanguages(true); _config->Clear("Acquire::Languages"); diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc index f95b9e5ba..5211ee027 100644 --- a/cmdline/apt-extracttemplates.cc +++ b/cmdline/apt-extracttemplates.cc @@ -33,6 +33,8 @@ #include #include +#include + #include #include #include @@ -215,15 +217,15 @@ bool DebFile::ParseInfo() // ShowHelp - show a short help text /*{{{*/ // --------------------------------------------------------------------- /* */ -static int ShowHelp(void) +static bool ShowHelp(CommandLine &) { - ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, + ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, COMMON_ARCH,__DATE__,__TIME__); - if (_config->FindB("version") == true) - return 0; + if (_config->FindB("version") == true) + return true; - cout << + cout << _("Usage: apt-extracttemplates file1 [file2 ...]\n" "\n" "apt-extracttemplates is a tool to extract config and template info\n" @@ -234,7 +236,7 @@ static int ShowHelp(void) " -t Set the temp dir\n" " -c=? Read this configuration file\n" " -o=? Set an arbitrary configuration option, eg -o dir::cache=/tmp\n"); - return 0; + return true; } /*}}}*/ // WriteFile - write the contents of the passed string to a file /*{{{*/ @@ -356,20 +358,10 @@ int main(int argc, const char **argv) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(); - + CommandLine::Dispatch Cmds[] = {{NULL, NULL}}; + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args, &_config, &_system, argc, argv, ShowHelp); + Go(CmdL); // Print any errors or warnings found during operation diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index eca4a723b..a9053bdfd 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -1746,26 +1746,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // see if we are in simulate mode CheckSimulateMode(CmdL); diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index 27abb2013..1b832f165 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -108,25 +108,8 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); InitOutput(); diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc index 92a4429e5..4fabeb02f 100644 --- a/cmdline/apt-internal-solver.cc +++ b/cmdline/apt-internal-solver.cc @@ -24,7 +24,9 @@ #include #include #include + #include +#include #include #include @@ -79,19 +81,8 @@ int main(int argc,const char *argv[]) /*{{{*/ // we really don't need anything DropPrivileges(); - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false) { - _error->DumpErrors(); - return 2; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true) { - ShowHelp(CmdL); - return 1; - } + CommandLine CmdL; + ParseCommandLine(CmdL, NULL, Args, &_config, NULL, argc, argv, ShowHelp); if (CmdL.FileList[0] != 0 && strcmp(CmdL.FileList[0], "scenario") == 0) { diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index ed348358a..487f3d8a1 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -441,26 +441,8 @@ int main(int argc,const char *argv[]) /*{{{*/ setlocale(LC_ALL,""); textdomain(PACKAGE); - // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(),_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - if (_config->FindB("version") == true) - ShowHelp(CmdL); - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); // Deal with stdout not being a tty if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc index c2b11890a..9b66ad4db 100644 --- a/cmdline/apt-sortpkgs.cc +++ b/cmdline/apt-sortpkgs.cc @@ -23,6 +23,8 @@ #include #include +#include + #include #include #include @@ -142,12 +144,12 @@ static bool DoIt(string InFile) // ShowHelp - Show the help text /*{{{*/ // --------------------------------------------------------------------- /* */ -static int ShowHelp() +static bool ShowHelp(CommandLine &) { ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION, COMMON_ARCH,__DATE__,__TIME__); if (_config->FindB("version") == true) - return 0; + return true; cout << _("Usage: apt-sortpkgs [options] file1 [file2 ...]\n" @@ -161,7 +163,7 @@ static int ShowHelp() " -c=? Read this configuration file\n" " -o=? Set an arbitrary configuration option, eg -o dir::cache=/tmp\n"); - return 0; + return true; } /*}}}*/ int main(int argc,const char *argv[]) /*{{{*/ @@ -179,19 +181,9 @@ int main(int argc,const char *argv[]) /*{{{*/ textdomain(PACKAGE); // Parse the command line and initialize the package library - CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || - CmdL.Parse(argc,argv) == false || - pkgInitSystem(*_config,_system) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - CmdL.FileSize() == 0) - return ShowHelp(); + CommandLine::Dispatch Cmds[] = {{NULL, NULL}}; + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args, &_config, &_system, argc, argv, ShowHelp); // Match the operation for (unsigned int I = 0; I != CmdL.FileSize(); I++) diff --git a/cmdline/apt.cc b/cmdline/apt.cc index 2cfdf8e8e..056cd213f 100644 --- a/cmdline/apt.cc +++ b/cmdline/apt.cc @@ -119,15 +119,10 @@ int main(int argc, const char *argv[]) /*{{{*/ _config->CndSet("APT::Cmd::Show-Update-Stats", true); // Parse the command line and initialize the package library - CommandLine CmdL(Args.data(), _config); - if (CmdL.Parse(argc, argv) == false || - pkgInitSystem(*_config, _system) == false) - { - _error->DumpErrors(); - return 100; - } + CommandLine CmdL; + ParseCommandLine(CmdL, Cmds, Args.data(), NULL, &_system, argc, argv, ShowHelp); - if(!isatty(STDOUT_FILENO) && + if(!isatty(STDOUT_FILENO) && _config->FindB("Apt::Cmd::Disable-Script-Warning", false) == false) { std::cerr << std::endl @@ -138,15 +133,6 @@ int main(int argc, const char *argv[]) /*{{{*/ << std::endl; } - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - // see if we are in simulate mode CheckSimulateMode(CmdL); diff --git a/cmdline/makefile b/cmdline/makefile index b7c35ddd1..816038c3b 100644 --- a/cmdline/makefile +++ b/cmdline/makefile @@ -67,15 +67,15 @@ APT_DOMAIN:=apt-utils # The apt-sortpkgs program PROGRAM=apt-sortpkgs -SLIBS = -lapt-pkg $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile +SLIBS = -lapt-pkg -lapt-private $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-private/makefile SOURCE = apt-sortpkgs.cc include $(PROGRAM_H) # The apt-extracttemplates program PROGRAM=apt-extracttemplates -SLIBS = -lapt-pkg -lapt-inst $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile apt-inst/makefile +SLIBS = -lapt-pkg -lapt-inst -lapt-private $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-inst/makefile apt-private/makefile SOURCE = apt-extracttemplates.cc include $(PROGRAM_H) diff --git a/debian/tests/run-tests b/debian/tests/run-tests index 9d7c484fc..7cc37f618 100644 --- a/debian/tests/run-tests +++ b/debian/tests/run-tests @@ -15,6 +15,7 @@ APT_INTEGRATION_TESTS_WEBSERVER_BIN_DIR=$(pwd)/build/bin \ APT_INTEGRATION_TESTS_METHODS_DIR=/usr/lib/apt/methods \ APT_INTEGRATION_TESTS_LIBEXEC_DIR=/usr/lib/apt/ \ APT_INTEGRATION_TESTS_INTERNAL_SOLVER=/usr/lib/apt/solvers/apt \ +APT_INTEGRATION_TESTS_DUMP_SOLVER=/usr/lib/apt/solvers/dump \ APT_INTEGRATION_TESTS_BUILD_DIR=/usr/bin \ APT_INTEGRATION_TESTS_LIBRARY_PATH=/dev/null/does/not/exist \ ./test/integration/run-tests diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index ebf99a8f8..7d9af4178 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -19,6 +19,8 @@ #include #include +#include + #include #include #include @@ -1060,21 +1062,8 @@ int main(int argc, const char *argv[]) // Parse the command line and initialize the package library CommandLine CmdL(Args,_config); - if (pkgInitConfig(*_config) == false || CmdL.Parse(argc,argv) == false) - { - _error->DumpErrors(); - return 100; - } - - // See if the help should be shown - if (_config->FindB("help") == true || - _config->FindB("version") == true || - CmdL.FileSize() == 0) - { - ShowHelp(CmdL); - return 0; - } - + ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp); + // Setup the output streams c0out.rdbuf(clog.rdbuf()); c1out.rdbuf(clog.rdbuf()); diff --git a/ftparchive/makefile b/ftparchive/makefile index d1ffe182a..e67272e1e 100644 --- a/ftparchive/makefile +++ b/ftparchive/makefile @@ -9,8 +9,8 @@ include ../buildlib/defaults.mak ifdef BDBLIB APT_DOMAIN:=apt-utils PROGRAM=apt-ftparchive -SLIBS = -lapt-pkg -lapt-inst $(BDBLIB) $(INTLLIBS) -LIB_MAKES = apt-pkg/makefile apt-inst/makefile +SLIBS = -lapt-pkg -lapt-inst -lapt-private $(BDBLIB) $(INTLLIBS) +LIB_MAKES = apt-pkg/makefile apt-inst/makefile apt-private/makefile SOURCE = apt-ftparchive.cc cachedb.cc writer.cc contents.cc override.cc \ multicompress.cc sources.cc include $(PROGRAM_H) diff --git a/prepare-release b/prepare-release index e61266eef..67f4638d6 100755 --- a/prepare-release +++ b/prepare-release @@ -130,8 +130,8 @@ elif [ "$1" = 'coverage' ]; then LCOVRC='--rc geninfo_checksum=1 --rc lcov_branch_coverage=1' mkdir "$DIR" lcov --no-external --directory . --capture --initial --output-file "${DIR}/apt.coverage.init" ${LCOVRC} - make test - ./test/integration/run-tests -q + make test || true + ./test/integration/run-tests -q || true lcov --no-external --directory . --capture --output-file "${DIR}/apt.coverage.run" ${LCOVRC} lcov -a "${DIR}/apt.coverage.init" -a "${DIR}/apt.coverage.run" -o "${DIR}/apt.coverage.total" ${LCOVRC} cp "${DIR}/apt.coverage.total" "${DIR}/apt.coverage.fixed" diff --git a/test/integration/framework b/test/integration/framework index 153c5bb25..dd66f2a0c 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -128,6 +128,7 @@ aptwebserver() { runapt "${APTWEBSERVERBINDIR}/aptwebserver" "$@"; } aptitude() { runapt aptitude "$@"; } aptextracttemplates() { runapt apt-extracttemplates "$@"; } aptinternalsolver() { runapt "${APTINTERNALSOLVER}" "$@"; } +aptdumpsolver() { runapt "${APTDUMPSOLVER}" "$@"; } dpkg() { command dpkg --root=${TMPWORKINGDIRECTORY}/rootdir --force-not-root --force-bad-path --log=${TMPWORKINGDIRECTORY}/rootdir/var/log/dpkg.log "$@" @@ -194,6 +195,7 @@ setupenvironment() { APTHELPERBINDIR=${APT_INTEGRATION_TESTS_LIBEXEC_DIR:-"${BUILDDIRECTORY}"} APTWEBSERVERBINDIR=${APT_INTEGRATION_TESTS_WEBSERVER_BIN_DIR:-"${BUILDDIRECTORY}"} APTINTERNALSOLVER=${APT_INTEGRATION_TESTS_INTERNAL_SOLVER:-"${BUILDDIRECTORY}/apt-internal-solver"} + APTDUMPSOLVER=${APT_INTEGRATION_TESTS_DUMP_SOLVER:-"${BUILDDIRECTORY}/apt-dump-solver"} test -x "${BUILDDIRECTORY}/apt-get" || msgdie "You need to build tree first" # ----- diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help new file mode 100755 index 000000000..ebf8b8cfa --- /dev/null +++ b/test/integration/test-00-commands-have-help @@ -0,0 +1,52 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' + +# this test does double duty: The obvious is checking for --help and co, +# but it also checks if the binary can find all methods in the library. +# The later is quite handy for manual testing of non-abibreaking changes +export LD_BIND_NOW=1 + +checkversionmessage() { + testsuccess grep '^apt .* compiled on ' ${1}-help.output +} + +checkhelpmessage() { + checkversionmessage "$1" + testsuccess grep '^Usage:' ${1}-help.output +} + +checkoptions() { + testsuccess $1 --help + cp -f rootdir/tmp/testsuccess.output ${1}-help.output + checkhelpmessage "$1" + + testsuccess $1 --version + cp -f rootdir/tmp/testsuccess.output ${1}-help.output + checkversionmessage "$1" +} + +for CMD in 'apt-cache' 'apt-cdrom' 'apt-config' \ + 'apt-extracttemplates' 'apt-get' 'apt-helper' \ + 'apt-mark' 'apt-sortpkgs' 'apt' 'apt-ftparchive'; do + cmd="$(echo "$CMD" | tr -d '-')" + msgtest 'Test for failure with no parameters calling' "$CMD" + if $cmd > ${cmd}-help.output 2>&1; then + echo + cat ${cmd}-help.output + msgfail 'zero exit' + else + msgpass + fi + checkhelpmessage "$cmd" + checkoptions "$cmd" +done + +for CMD in 'apt-dump-solver' 'apt-internal-solver'; do + checkoptions "$(echo "$CMD" | tr -d '-')" +done -- cgit v1.2.3-70-g09d2 From d9e518c6f7dc0ad464495b586d1b8e115d54d41a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 8 Nov 2014 20:44:44 +0100 Subject: use the same code to detect quiet setting in all tools Git-Dch: Ignore --- apt-private/private-output.cc | 8 ++-- apt-private/private-output.h | 3 +- cmdline/apt-cache.cc | 4 +- cmdline/apt-cdrom.cc | 7 ++- cmdline/apt-mark.cc | 18 +------ ftparchive/apt-ftparchive.cc | 19 ++------ test/integration/test-apt-ftparchive-cachedb | 30 +++++++----- test/integration/test-apt-ftparchive-src-cachedb | 60 +++++++++++------------- 8 files changed, 63 insertions(+), 86 deletions(-) (limited to 'cmdline') diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc index fc76a05bc..4e18030ab 100644 --- a/apt-private/private-output.cc +++ b/apt-private/private-output.cc @@ -52,14 +52,14 @@ static void SigWinch(int) #endif } /*}}}*/ -bool InitOutput() /*{{{*/ +bool InitOutput(std::basic_streambuf * const out) /*{{{*/ { if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) _config->Set("quiet","1"); - c0out.rdbuf(cout.rdbuf()); - c1out.rdbuf(cout.rdbuf()); - c2out.rdbuf(cout.rdbuf()); + c0out.rdbuf(out); + c1out.rdbuf(out); + c2out.rdbuf(out); if (_config->FindI("quiet",0) > 0) c0out.rdbuf(devnull.rdbuf()); if (_config->FindI("quiet",0) > 1) diff --git a/apt-private/private-output.h b/apt-private/private-output.h index e0dc9bf62..d5b57adec 100644 --- a/apt-private/private-output.h +++ b/apt-private/private-output.h @@ -6,6 +6,7 @@ #include #include +#include // forward declaration class pkgCacheFile; @@ -20,7 +21,7 @@ APT_PUBLIC extern std::ostream c2out; APT_PUBLIC extern std::ofstream devnull; APT_PUBLIC extern unsigned int ScreenWidth; -APT_PUBLIC bool InitOutput(); +APT_PUBLIC bool InitOutput(std::basic_streambuf * const out = std::cout.rdbuf()); void ListSingleVersion(pkgCacheFile &CacheFile, pkgRecords &records, pkgCache::VerIterator const &V, std::ostream &out, diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 12ed4f719..342ad1858 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1899,9 +1899,7 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); + InitOutput(); if (_config->Exists("APT::Cache::Generate") == true) _config->Set("pkgCacheFile::Generate", _config->FindB("APT::Cache::Generate", true)); diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc index 8ac73fd7e..327039e00 100644 --- a/cmdline/apt-cdrom.cc +++ b/cmdline/apt-cdrom.cc @@ -31,6 +31,7 @@ #include #include +#include #include /*}}}*/ @@ -252,10 +253,8 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); - + InitOutput(); + // Match the operation bool returned = CmdL.DispatchArg(Cmds); diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 487f3d8a1..2702dbbd3 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -41,10 +42,6 @@ /*}}}*/ using namespace std; -ostream c0out(0); -ostream c1out(0); -ostream c2out(0); -ofstream devnull("/dev/null"); /* DoAuto - mark packages as automatically/manually installed {{{*/ static bool DoAuto(CommandLine &CmdL) { @@ -444,18 +441,7 @@ int main(int argc,const char *argv[]) /*{{{*/ CommandLine CmdL; ParseCommandLine(CmdL, Cmds, Args.data(), &_config, &_system, argc, argv, ShowHelp); - // Deal with stdout not being a tty - if (!isatty(STDOUT_FILENO) && _config->FindI("quiet", -1) == -1) - _config->Set("quiet","1"); - - // Setup the output streams - c0out.rdbuf(cout.rdbuf()); - c1out.rdbuf(cout.rdbuf()); - c2out.rdbuf(cout.rdbuf()); - if (_config->FindI("quiet",0) > 0) - c0out.rdbuf(devnull.rdbuf()); - if (_config->FindI("quiet",0) > 1) - c1out.rdbuf(devnull.rdbuf()); + InitOutput(); // Match the operation CmdL.DispatchArg(Cmds); diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc index 7d9af4178..adf1b6d73 100644 --- a/ftparchive/apt-ftparchive.cc +++ b/ftparchive/apt-ftparchive.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -42,11 +43,7 @@ #include /*}}}*/ -using namespace std; -ostream c0out(0); -ostream c1out(0); -ostream c2out(0); -ofstream devnull("/dev/null"); +using namespace std; unsigned Quiet = 0; // struct PackageMap - List of all package files in the config file /*{{{*/ @@ -1064,16 +1061,10 @@ int main(int argc, const char *argv[]) CommandLine CmdL(Args,_config); ParseCommandLine(CmdL, Cmds, Args, &_config, NULL, argc, argv, ShowHelp); - // Setup the output streams - c0out.rdbuf(clog.rdbuf()); - c1out.rdbuf(clog.rdbuf()); - c2out.rdbuf(clog.rdbuf()); + _config->CndSet("quiet",0); Quiet = _config->FindI("quiet",0); - if (Quiet > 0) - c0out.rdbuf(devnull.rdbuf()); - if (Quiet > 1) - c1out.rdbuf(devnull.rdbuf()); - + InitOutput(clog.rdbuf()); + // Match the operation CmdL.DispatchArg(Cmds); diff --git a/test/integration/test-apt-ftparchive-cachedb b/test/integration/test-apt-ftparchive-cachedb index 0e1986bcd..866e5a469 100755 --- a/test/integration/test-apt-ftparchive-cachedb +++ b/test/integration/test-apt-ftparchive-cachedb @@ -69,32 +69,38 @@ buildsimplenativepackage 'foo' 'i386' '1' 'test' mv incoming/* aptarchive/pool/main/ # generate (empty cachedb) -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 2 - dists/test/Contents-i386: New 402 B Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 2 + dists/test/Contents-i386: New 402 B Misses in Cache: 0' # generate again -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 0 - dists/test/Contents-i386: Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 0 + dists/test/Contents-i386: Misses in Cache: 0' # and again (with removing the Packages file) rm -f ./aptarchive/dists/test/main/binary-i386/* rm -f ./aptarchive/dists/test/Contents-i386 -aptftparchive generate ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt +testsuccess aptftparchive generate ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt ensure_correct_packages_file ensure_correct_contents_file -testequal " Misses in Cache: 0 - dists/test/Contents-i386: New 402 B Misses in Cache: 0" grep Misses stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' ' Misses in Cache: 0 + dists/test/Contents-i386: New 402 B Misses in Cache: 0' # and clean rm -rf aptarchive/pool/main/* -testequal "packages-main-i386.db" aptftparchive clean ftparchive.conf -aptftparchive clean ftparchive.conf -o Debug::APT::FTPArchive::Clean=1 > clean-out.txt 2>&1 +testequal "packages-main-i386.db" aptftparchive clean ftparchive.conf -q=0 +testsuccess aptftparchive clean ftparchive.conf -q=0 -o Debug::APT::FTPArchive::Clean=1 +cp rootdir/tmp/testsuccess.output clean-out.txt testequal "0 Number of unique keys in the tree" grep unique clean-out.txt testequal "packages-main-i386.db" grep packages-main-i386.db clean-out.txt - diff --git a/test/integration/test-apt-ftparchive-src-cachedb b/test/integration/test-apt-ftparchive-src-cachedb index e7b148530..6e857c0cb 100755 --- a/test/integration/test-apt-ftparchive-src-cachedb +++ b/test/integration/test-apt-ftparchive-src-cachedb @@ -106,28 +106,22 @@ mkdir -p aptarchive/dists/test/main/source/ mkdir aptarchive-overrides mkdir aptarchive-cache - - -# generate with --db option -(cd aptarchive && aptftparchive --db ./test.db sources pool/main/ \ - -o APT::FTPArchive::ShowCacheMisses=1 \ - > dists/test/main/source/Sources \ - 2> stats-out.txt - testequal " Misses in Cache: 2" grep Misses stats-out.txt -) +msgtest 'generate with --db option' +cd aptarchive +aptftparchive --db ./test.db sources pool/main/ -q=0 -o APT::FTPArchive::ShowCacheMisses=1 > dists/test/main/source/Sources 2>stats-out.txt && msgpass || msgfail +testsuccess grep Misses stats-out.txt +testfileequal '../rootdir/tmp/testsuccess.output' ' Misses in Cache: 2' +cd .. assert_correct_sources_file -# generate with --db option (again to ensure its in the cache) -(cd aptarchive && aptftparchive --db ./test.db sources pool/main/ \ - -o APT::FTPArchive::ShowCacheMisses=1 \ - > dists/test/main/source/Sources \ - 2> stats-out.txt - testequal " Misses in Cache: 0" grep Misses stats-out.txt -) +msgtest 'generate with --db option (again to ensure its in the cache)' +cd aptarchive +aptftparchive --db ./test.db sources pool/main/ -q=0 -o APT::FTPArchive::ShowCacheMisses=1 > dists/test/main/source/Sources 2>stats-out.txt && msgpass || msgfail +testsuccess grep Misses stats-out.txt +testfileequal '../rootdir/tmp/testsuccess.output' ' Misses in Cache: 0' +cd .. assert_correct_sources_file - - # get ready for the "apt-ftparchive generate" command cat > apt-ftparchive.conf <<"EOF" Dir { @@ -159,20 +153,21 @@ Tree "dists/test" { }; EOF -# generate (empty cachedb) -aptftparchive generate apt-ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt -testequal " Misses in Cache: 2" grep Misses stats-out.txt +msgtest 'generate (empty cachedb)' +testsuccess aptftparchive generate apt-ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal rootdir/tmp/testsuccess.output ' Misses in Cache: 2' assert_correct_sources_file - -# generate again out of the cache +msgtest 'generate again out of the cache' rm -f ./aptarchive/dists/test/main/source/Sources -aptftparchive generate apt-ftparchive.conf -o APT::FTPArchive::ShowCacheMisses=1 2> stats-out.txt -testequal " Misses in Cache: 0" grep Misses stats-out.txt +testsuccess aptftparchive generate apt-ftparchive.conf -q=0 -o APT::FTPArchive::ShowCacheMisses=1 +cp rootdir/tmp/testsuccess.output stats-out.txt +testsuccess grep Misses stats-out.txt +testfileequal rootdir/tmp/testsuccess.output ' Misses in Cache: 0' assert_correct_sources_file - - # generate invalid files mkdir aptarchive/pool/invalid printf "meep" > aptarchive/pool/invalid/invalid_1.0.dsc @@ -191,8 +186,9 @@ E: DSC file 'aptarchive/pool/invalid/toobig_1.0.dsc' is too large!" aptftparchiv # ensure clean works rm -f aptarchive/pool/main/* -aptftparchive clean apt-ftparchive.conf -o Debug::APT::FTPArchive::Clean=1 > clean-out.txt 2>&1 -testequal "0 Number of unique keys in the tree" grep unique clean-out.txt -testequal "sources-main.db" grep sources-main.db clean-out.txt - - +testsuccess aptftparchive clean apt-ftparchive.conf -q=0 -o Debug::APT::FTPArchive::Clean=1 +cp rootdir/tmp/testsuccess.output clean-out.txt +testsuccess grep unique clean-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' "0 Number of unique keys in the tree" +testsuccess grep sources-main.db clean-out.txt +testfileequal 'rootdir/tmp/testsuccess.output' "sources-main.db" -- cgit v1.2.3-70-g09d2 From 374f8492e6f109e8427816a8f513e5e8feda9049 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 15:40:19 +0100 Subject: allow uninstalled packages to be put on hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dpkg wants to know about a package before it can be put on hold, so we have to at least hint about its existance in the available file it "maintaince" to know about such stuff. The simple thing would probably be to just feed all Packages files into dpkg as well, but what would be the point really? Exactly, so we take a shortcut here and just create dummies in the available file if we need to which isn't going to be that common as usually you are holding packages back and not off. Who would have thought that a simple feature like setting a package on hold requires more than 200 lines of code… at least with the testcase it is now explicitly tested code. --- cmdline/apt-mark.cc | 63 ++++++++++++++++++++++++++++++++---- test/integration/test-apt-mark | 72 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 6 deletions(-) create mode 100755 test/integration/test-apt-mark (limited to 'cmdline') diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 2702dbbd3..860982f9e 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -275,10 +275,64 @@ static bool DoHold(CommandLine &CmdL) } Args.erase(Args.begin() + BaseArgs, Args.end()); - Args.push_back("--set-selections"); + Args.push_back("--merge-avail"); + Args.push_back("-"); Args.push_back(NULL); int external[2] = {-1, -1}; + if (pipe(external) != 0) + return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); + + pid_t dpkgMergeAvail = ExecFork(); + if (dpkgMergeAvail == 0) + { + close(external[1]); + std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); + if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) + _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); + dup2(external[0], STDIN_FILENO); + int const nullfd = open("/dev/null", O_RDONLY); + dup2(nullfd, STDOUT_FILENO); + execvp(Args[0], (char**) &Args[0]); + _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); + _exit(2); + } + + FILE* dpkg = fdopen(external[1], "w"); + for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) + { + if (Pkg->CurrentVer != 0) + continue; + char const * Arch; + if (Pkg->VersionList != 0) + Arch = Pkg.VersionList().Arch(); + else + Arch = Pkg.Arch(); + fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" + "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + } + fclose(dpkg); + + if (dpkgMergeAvail > 0) + { + int Status = 0; + while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + { + if (errno == EINTR) + continue; + _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); + break; + } + if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) + return _error->Error(_("Executing dpkg failed. Are you root?")); + } + + Args.erase(Args.begin() + BaseArgs, Args.end()); + Args.push_back("--set-selections"); + Args.push_back(NULL); + + external[0] = -1; + external[1] = -1; if (pipe(external) != 0) return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --set-selections"); @@ -289,16 +343,13 @@ static bool DoHold(CommandLine &CmdL) std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --set-selections", chrootDir.c_str()); - int const nullfd = open("/dev/null", O_RDONLY); dup2(external[0], STDIN_FILENO); - dup2(nullfd, STDOUT_FILENO); - dup2(nullfd, STDERR_FILENO); execvp(Args[0], (char**) &Args[0]); - _error->WarningE("dpkgGo", "Can't detect if dpkg supports multi-arch!"); + _error->WarningE("dpkgGo", "Can't get dpkg --set-selections running!"); _exit(2); } - FILE* dpkg = fdopen(external[1], "w"); + dpkg = fdopen(external[1], "w"); for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (dpkgMultiArch == false) diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark new file mode 100755 index 000000000..69e0f933d --- /dev/null +++ b/test/integration/test-apt-mark @@ -0,0 +1,72 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework +setupenvironment +configarchitecture 'amd64' 'i386' + +insertpackage 'unstable' 'bar' 'amd64,i386' '1' +insertpackage 'unstable' 'uninstalled' 'all' '1' +insertpackage 'unstable' 'uninstalled-native' 'amd64' '1' + +insertinstalledpackage 'foo' 'all' '1' +insertinstalledpackage 'bar' 'amd64' '1' + +setupaptarchive + +# dpkg is "installed" by our test framework +testdpkginstalled dpkg + +testnoautopkg() { + testempty aptmark showauto + testequal 'bar +dpkg +foo' aptmark showmanual + testequal 'bar +foo' aptmark showmanual bar foo uninstalled +} +testmarkonpkgasauto() { + testsuccess aptmark $1 foo + testequal 'foo' aptmark showauto + testequal 'foo' aptmark showauto foo + testequal 'bar +dpkg' aptmark showmanual + testequal 'bar' aptmark showmanual bar + + testsuccess aptmark $2 foo + testnoautopkg +} + +testequal 'E: No packages found' aptmark auto +testequal 'E: No packages found' aptmark manual + +testnoautopkg +testmarkonpkgasauto 'auto' 'manual' +testmarkonpkgasauto 'markauto' 'unmarkauto' + +testnoholdpkg() { + testempty aptmark showhold + testempty aptmark showholds # typical "typo" + testempty aptmark showhold dpkg + testempty aptmark showholds dpkg +} +testmarkonepkgashold() { + testsuccess aptmark hold $1 + testequal "$1" aptmark showhold + testequal "$1" aptmark showholds + testsuccess aptmark unhold $1 + testnoholdpkg +} + +testequal 'E: No packages found' aptmark hold +testequal 'E: No packages found' aptmark unhold + +testnoholdpkg +testmarkonepkgashold 'foo' +testmarkonepkgashold 'bar' +testmarkonepkgashold 'uninstalled' +testmarkonepkgashold 'uninstalled-native' + +testequal 'uninstalled set on hold.' aptmark hold uninstalled +testequal 'uninstalled-native set on hold.' aptmark hold uninstalled-native -- cgit v1.2.3-70-g09d2 From 7e04a6bf23d857db60afd2ec3d0f4a8271b1c597 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 19:04:54 +0100 Subject: use pkgAcquire::GetLock instead of own code Do the same with less code in apt-get. This especially ensures that the lock file (and the parent directories) exist before we are trying to lock. It also means that clean now creates the directories if they are missing so we returned to a proper clean state now. Git-Dch: Ignore --- apt-pkg/acquire.cc | 2 ++ cmdline/apt-get.cc | 22 ++-------------------- 2 files changed, 4 insertions(+), 20 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 2c89c2dea..07c4646f5 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -152,6 +152,8 @@ bool pkgAcquire::GetLock(std::string const &Lock) return true; // Lock the directory this acquire object will work in + if (LockFD != -1) + close(LockFD); LockFD = ::GetLock(flCombine(Lock, "lock")); if (LockFD == -1) return _error->Error(_("Unable to lock directory %s"), Lock.c_str()); diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index a9053bdfd..eab792264 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -558,30 +558,12 @@ static bool DoClean(CommandLine &) return true; } - bool const NoLocking = _config->FindB("Debug::NoLocking",false); - // Lock the archive directory - FileFd Lock; - if (NoLocking == false) - { - int lock_fd = GetLock(archivedir + "lock"); - if (lock_fd < 0) - return _error->Error(_("Unable to lock directory %s"), archivedir.c_str()); - Lock.Fd(lock_fd); - } - pkgAcquire Fetcher; + Fetcher.GetLock(archivedir); Fetcher.Clean(archivedir); Fetcher.Clean(archivedir + "partial/"); - if (NoLocking == false) - { - Lock.Close(); - int lock_fd = GetLock(listsdir + "lock"); - if (lock_fd < 0) - return _error->Error(_("Unable to lock directory %s"), listsdir.c_str()); - Lock.Fd(lock_fd); - } - + Fetcher.GetLock(listsdir); Fetcher.Clean(listsdir + "partial/"); pkgCacheFile::RemoveCaches(); -- cgit v1.2.3-70-g09d2 From 3a2b39ee602dd5a98b8fdaee2f1c8e0b13a276e2 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 18 Aug 2013 23:27:24 +0200 Subject: use 'best' hash for source authentication Collect all hashes we can get from the source record and put them into a HashStringList so that 'apt-get source' can use it instead of using always the MD5sum. We therefore also deprecate the MD5 struct member in favor of the list. While at it, the parsing of the Files is enhanced so that records which miss "Files" (aka MD5 checksums) are still searched for other checksums as they include just as much data, just not with a nice and catchy name. This is a cherry-pick of 1262d35 with some dirty tricks to preserve ABI. LP: 1098738 --- apt-pkg/deb/debsrcrecords.cc | 162 +++++++++---- apt-pkg/deb/debsrcrecords.h | 1 + apt-pkg/srcrecords.cc | 31 ++- apt-pkg/srcrecords.h | 21 +- cmdline/apt-get.cc | 28 ++- debian/libapt-pkg4.12.symbols | 2 + .../test-ubuntu-bug-1098738-apt-get-source-md5sum | 260 +++++++++++++++++++++ 7 files changed, 445 insertions(+), 60 deletions(-) create mode 100755 test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum (limited to 'cmdline') diff --git a/apt-pkg/deb/debsrcrecords.cc b/apt-pkg/deb/debsrcrecords.cc index a444cbe4d..49a348dd4 100644 --- a/apt-pkg/deb/debsrcrecords.cc +++ b/apt-pkg/deb/debsrcrecords.cc @@ -118,13 +118,32 @@ bool debSrcRecordParser::BuildDepends(std::vector &List) +bool debSrcRecordParser::Files(std::vector &F) { - List.erase(List.begin(),List.end()); - - string Files = Sect.FindS("Files"); - if (Files.empty() == true) + std::vector F2; + if (Files2(F2) == false) return false; + for (std::vector::const_iterator f2 = F2.begin(); f2 != F2.end(); ++f2) + { + pkgSrcRecords::File2 f; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + f.MD5Hash = f2->MD5Hash; + f.Size = f2->Size; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + f.Path = f2->Path; + f.Type = f2->Type; + F.push_back(f); + } + return true; +} +bool debSrcRecordParser::Files2(std::vector &List) +{ + List.clear(); // Stash the / terminated directory prefix string Base = Sect.FindS("Directory"); @@ -133,51 +152,106 @@ bool debSrcRecordParser::Files(std::vector &List) std::vector const compExts = APT::Configuration::getCompressorExtensions(); - // Iterate over the entire list grabbing each triplet - const char *C = Files.c_str(); - while (*C != 0) - { - pkgSrcRecords::File F; - string Size; - - // Parse each of the elements - if (ParseQuoteWord(C,F.MD5Hash) == false || - ParseQuoteWord(C,Size) == false || - ParseQuoteWord(C,F.Path) == false) - return _error->Error("Error parsing file record"); - - // Parse the size and append the directory - F.Size = atoi(Size.c_str()); - F.Path = Base + F.Path; - - // Try to guess what sort of file it is we are getting. - string::size_type Pos = F.Path.length()-1; - while (1) + for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type) + { + // derive field from checksum type + std::string checksumField("Checksums-"); + if (strcmp(*type, "MD5Sum") == 0) + checksumField = "Files"; // historic name for MD5 checksums + else + checksumField.append(*type); + + string const Files = Sect.FindS(checksumField.c_str()); + if (Files.empty() == true) + continue; + + // Iterate over the entire list grabbing each triplet + const char *C = Files.c_str(); + while (*C != 0) { - string::size_type Tmp = F.Path.rfind('.',Pos); - if (Tmp == string::npos) - break; - if (F.Type == "tar") { - // source v3 has extension 'debian.tar.*' instead of 'diff.*' - if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") - F.Type = "diff"; - break; - } - F.Type = string(F.Path,Tmp+1,Pos-Tmp); - - if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || - F.Type == "tar") + string hash, size, path; + + // Parse each of the elements + if (ParseQuoteWord(C, hash) == false || + ParseQuoteWord(C, size) == false || + ParseQuoteWord(C, path) == false) + return _error->Error("Error parsing file record in %s of source package %s", checksumField.c_str(), Package().c_str()); + + HashString const hashString(*type, hash); + if (Base.empty() == false) + path = Base + path; + + // look if we have a record for this file already + std::vector::iterator file = List.begin(); + for (; file != List.end(); ++file) + if (file->Path == path) + break; + + // we have it already, store the new hash and be done + if (file != List.end()) { - Pos = Tmp-1; +#if __GNUC__ >= 4 + // set for compatibility only, so warn users not us + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + if (checksumField == "Files") + file->MD5Hash = hash; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + // an error here indicates that we have two different hashes for the same file + if (file->Hashes.push_back(hashString) == false) + return _error->Error("Error parsing checksum in %s of source package %s", checksumField.c_str(), Package().c_str()); continue; } - - break; + + // we haven't seen this file yet + pkgSrcRecords::File2 F; + F.Path = path; + F.FileSize = strtoull(size.c_str(), NULL, 10); + F.Hashes.push_back(hashString); + +#if __GNUC__ >= 4 + // set for compatibility only, so warn users not us + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + F.Size = F.FileSize; + if (checksumField == "Files") + F.MD5Hash = hash; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + + // Try to guess what sort of file it is we are getting. + string::size_type Pos = F.Path.length()-1; + while (1) + { + string::size_type Tmp = F.Path.rfind('.',Pos); + if (Tmp == string::npos) + break; + if (F.Type == "tar") { + // source v3 has extension 'debian.tar.*' instead of 'diff.*' + if (string(F.Path, Tmp+1, Pos-Tmp) == "debian") + F.Type = "diff"; + break; + } + F.Type = string(F.Path,Tmp+1,Pos-Tmp); + + if (std::find(compExts.begin(), compExts.end(), std::string(".").append(F.Type)) != compExts.end() || + F.Type == "tar") + { + Pos = Tmp-1; + continue; + } + + break; + } + List.push_back(F); } - - List.push_back(F); } - + return true; } /*}}}*/ diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h index b65d1480b..2a3fc86c9 100644 --- a/apt-pkg/deb/debsrcrecords.h +++ b/apt-pkg/deb/debsrcrecords.h @@ -53,6 +53,7 @@ class debSrcRecordParser : public pkgSrcRecords::Parser return std::string(Start,Stop); }; virtual bool Files(std::vector &F); + bool Files2(std::vector &F); debSrcRecordParser(std::string const &File,pkgIndexFile const *Index) : Parser(Index), Fd(File,FileFd::ReadOnly, FileFd::Extension), Tags(&Fd,102400), diff --git a/apt-pkg/srcrecords.cc b/apt-pkg/srcrecords.cc index 81b1c545d..3175ee75f 100644 --- a/apt-pkg/srcrecords.cc +++ b/apt-pkg/srcrecords.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -147,5 +148,33 @@ const char *pkgSrcRecords::Parser::BuildDepType(unsigned char const &Type) return fields[Type]; } /*}}}*/ +bool pkgSrcRecords::Parser::Files2(std::vector &F2)/*{{{*/ +{ + debSrcRecordParser * const deb = dynamic_cast(this); + if (deb != NULL) + return deb->Files2(F2); - + std::vector F; + if (Files(F) == false) + return false; + for (std::vector::const_iterator f = F.begin(); f != F.end(); ++f) + { + pkgSrcRecords::File2 f2; +#if __GNUC__ >= 4 + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + f2.MD5Hash = f->MD5Hash; + f2.Size = f->Size; + f2.Hashes.push_back(HashString("MD5Sum", f->MD5Hash)); + f2.FileSize = f->Size; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + f2.Path = f->Path; + f2.Type = f->Type; + F2.push_back(f2); + } + return true; +} + /*}}}*/ diff --git a/apt-pkg/srcrecords.h b/apt-pkg/srcrecords.h index e000e176a..dde22bd65 100644 --- a/apt-pkg/srcrecords.h +++ b/apt-pkg/srcrecords.h @@ -14,6 +14,7 @@ #define PKGLIB_SRCRECORDS_H #include +#include #include #include @@ -29,15 +30,28 @@ class pkgSrcRecords { public: +#if __GNUC__ >= 4 + // ensure that con- & de-structor don't trigger this warning + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif // Describes a single file struct File { - std::string MD5Hash; - unsigned long Size; + APT_DEPRECATED std::string MD5Hash; + APT_DEPRECATED unsigned long Size; std::string Path; std::string Type; }; - + struct File2 : public File + { + unsigned long long FileSize; + HashStringList Hashes; + }; +#if __GNUC__ >= 4 + #pragma GCC diagnostic pop +#endif + // Abstract parser for each source record class Parser { @@ -77,6 +91,7 @@ class pkgSrcRecords static const char *BuildDepType(unsigned char const &Type) APT_PURE; virtual bool Files(std::vector &F) = 0; + bool Files2(std::vector &F); Parser(const pkgIndexFile *Index) : iIndex(Index) {}; virtual ~Parser() {}; diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index cfa79339b..a28537712 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -797,13 +797,13 @@ static bool DoSource(CommandLine &CmdL) } // Back track - vector Lst; - if (Last->Files(Lst) == false) { + vector Lst; + if (Last->Files2(Lst) == false) { return false; } // Load them into the fetcher - for (vector::const_iterator I = Lst.begin(); + for (vector::const_iterator I = Lst.begin(); I != Lst.end(); ++I) { // Try to guess what sort of file it is we are getting. @@ -832,22 +832,26 @@ static bool DoSource(CommandLine &CmdL) queued.insert(Last->Index().ArchiveURI(I->Path)); // check if we have a file with that md5 sum already localy - if(!I->MD5Hash.empty() && FileExists(flNotDir(I->Path))) - { - FileFd Fd(flNotDir(I->Path), FileFd::ReadOnly); - MD5Summation sum; - sum.AddFD(Fd.Fd(), Fd.Size()); - Fd.Close(); - if((string)sum.Result() == I->MD5Hash) + std::string localFile = flNotDir(I->Path); + if (FileExists(localFile) == true) + if(I->Hashes.VerifyFile(localFile) == true) { ioprintf(c1out,_("Skipping already downloaded file '%s'\n"), - flNotDir(I->Path).c_str()); + localFile.c_str()); continue; } + + // see if we have a hash (Acquire::ForceHash is the only way to have none) + HashString const * const hs = I->Hashes.find(NULL); + if (hs == NULL && _config->FindB("APT::Get::AllowUnauthenticated",false) == false) + { + ioprintf(c1out, "Skipping download of file '%s' as requested hashsum is not available for authentication\n", + localFile.c_str()); + continue; } new pkgAcqFile(&Fetcher,Last->Index().ArchiveURI(I->Path), - I->MD5Hash,I->Size, + hs != NULL ? hs->toStr() : "", I->FileSize, Last->Index().SourceInfo(*Last,*I),Src); } } diff --git a/debian/libapt-pkg4.12.symbols b/debian/libapt-pkg4.12.symbols index d89f07bb5..d481e51ed 100644 --- a/debian/libapt-pkg4.12.symbols +++ b/debian/libapt-pkg4.12.symbols @@ -1587,6 +1587,8 @@ libapt-pkg.so.4.12 libapt-pkg4.12 #MINVER# (c++)"HashStringList::VerifyFile(std::basic_string, std::allocator >) const@Base" 1.0.9.4 (c++)"HashString::operator==(HashString const&) const@Base" 1.0.9.4 (c++)"HashString::operator!=(HashString const&) const@Base" 1.0.9.4 + (c++)"pkgSrcRecords::Parser::Files2(std::vector >&)@Base" 1.0.9.4 + (c++)"debSrcRecordParser::Files2(std::vector >&)@Base" 1.0.9.4 ### demangle strangeness - buildd report it as MISSING and as new… (c++)"pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire*, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::basic_string, std::allocator >, std::vector > const*, indexRecords*)@Base" 0.8.0 ### gcc-4.6 artefacts diff --git a/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum new file mode 100755 index 000000000..9bdc81264 --- /dev/null +++ b/test/integration/test-ubuntu-bug-1098738-apt-get-source-md5sum @@ -0,0 +1,260 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'native' + +cat > aptarchive/Sources < +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-ok_1.0.tar.gz + +Package: pkg-sha256-ok +Binary: pkg-sha256-ok +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-ok_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-ok_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-ok_1.0.tar.gz + +Package: pkg-sha256-bad +Binary: pkg-sha256-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-bad_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-bad_1.0.tar.gz +Checksums-Sha256: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-bad_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-bad_1.0.tar.gz + +Package: pkg-no-md5 +Binary: pkg-no-md5 +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-no-md5_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-no-md5_1.0.tar.gz + +Package: pkg-mixed-ok +Binary: pkg-mixed-ok +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-ok_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-ok_1.0.dsc + +Package: pkg-mixed-sha1-bad +Binary: pkg-mixed-sha1-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-mixed-sha1-bad_1.0.dsc +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-mixed-sha1-bad_1.0.tar.gz + +Package: pkg-mixed-sha2-bad +Binary: pkg-mixed-sha2-bad +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-mixed-sha2-bad_1.0.dsc +Checksums-Sha256: + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-mixed-sha2-bad_1.0.tar.gz + +Package: pkg-md5-disagree +Binary: pkg-md5-disagree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-disagree_1.0.tar.gz + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-md5-disagree_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-md5-disagree_1.0.tar.gz + +Package: pkg-md5-agree +Binary: pkg-md5-agree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.tar.gz + d41d8cd98f00b204e9800998ecf8427e 0 pkg-md5-agree_1.0.dsc + +Package: pkg-sha256-disagree +Binary: pkg-sha256-disagree +Version: 1.0 +Maintainer: Joe Sixpack +Architecture: all +Files: + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.dsc + d41d8cd98f00b204e9800998ecf8427e 0 pkg-sha256-disagree_1.0.tar.gz +Checksums-Sha1: + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.dsc + da39a3ee5e6b4b0d3255bfef95601890afd80709 0 pkg-sha256-disagree_1.0.tar.gz +Checksums-Sha256: + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.dsc + e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 0 pkg-sha256-disagree_1.0.tar.gz + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 0 pkg-sha256-disagree_1.0.dsc + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 0 pkg-sha256-disagree_1.0.tar.gz +EOF + +# create fetchable files +for x in 'pkg-md5-ok' 'pkg-sha256-ok' 'pkg-sha256-bad' 'pkg-no-md5' \ + 'pkg-mixed-ok' 'pkg-mixed-sha1-bad' 'pkg-mixed-sha2-bad' \ + 'pkg-md5-agree' 'pkg-md5-disagree' 'pkg-sha256-disagree'; do + touch aptarchive/${x}_1.0.dsc aptarchive/${x}_1.0.tar.gz +done + +setupaptarchive +changetowebserver +testsuccess aptget update + +testok() { + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +Download complete and in download only mode" aptget source -d "$@" + msgtest 'Files were successfully downloaded for' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz +} + +testkeep() { + touch ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Skipping already downloaded file '${1}_1.0.dsc' +Skipping already downloaded file '${1}_1.0.tar.gz' +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d "$@" + msgtest 'Files already downloaded are kept for' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz +} + +testmismatch() { + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +E: Failed to fetch http://localhost:8080/${1}_1.0.dsc Hash Sum mismatch + +E: Failed to fetch http://localhost:8080/${1}_1.0.tar.gz Hash Sum mismatch + +E: Failed to fetch some archives." aptget source -d "$@" + msgtest 'Files were not download as they have hashsum mismatches for' "$1" + testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Skipping download of file 'pkg-sha256-bad_1.0.dsc' as requested hashsum is not available for authentication +Skipping download of file 'pkg-sha256-bad_1.0.tar.gz' as requested hashsum is not available for authentication +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d "$@" -o Acquire::ForceHash=ROT26 + msgtest 'Files were not download as hash is unavailable for' "$1" + testfailure --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz + + rm -f ${1}_1.0.dsc ${1}_1.0.tar.gz + testequal "Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ $1 1.0 (dsc) +Get:2 http://localhost:8080/ $1 1.0 (tar) +Download complete and in download only mode" aptget source --allow-unauthenticated -d "$@" -o Acquire::ForceHash=ROT26 + msgtest 'Files were downloaded unauthenticated as user allowed it' "$1" + testsuccess --nomsg test -e ${1}_1.0.dsc -a -e ${1}_1.0.tar.gz +} + +testok pkg-md5-ok +testkeep pkg-md5-ok +testok pkg-sha256-ok +testkeep pkg-sha256-ok + +# pkg-sha256-bad has a bad SHA sum, but good MD5 sum. If apt is +# checking the best available hash (as it should), this will trigger +# a hash mismatch. +testmismatch pkg-sha256-bad +testmismatch pkg-sha256-bad +testok pkg-sha256-bad -o Acquire::ForceHash=MD5Sum + +# not having MD5 sum doesn't mean the file doesn't exist at all … +testok pkg-no-md5 +testok pkg-no-md5 -o Acquire::ForceHash=SHA256 +testequal "Reading package lists... +Building dependency tree... +Skipping download of file 'pkg-no-md5_1.0.dsc' as requested hashsum is not available for authentication +Skipping download of file 'pkg-no-md5_1.0.tar.gz' as requested hashsum is not available for authentication +Need to get 0 B of source archives. +Download complete and in download only mode" aptget source -d pkg-no-md5 -o Acquire::ForceHash=MD5Sum +msgtest 'Files were not download as MD5 is not available for this package' 'pkg-no-md5' +testfailure --nomsg test -e pkg-no-md5_1.0.dsc -a -e pkg-no-md5_1.0.tar.gz + +# deal with cases in which we haven't for all files the same checksum type +# mostly pathologic as this shouldn't happen, but just to be sure +testok pkg-mixed-ok +testequal 'Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (tar) +Get:2 http://localhost:8080/ pkg-mixed-sha1-bad 1.0 (dsc) +E: Failed to fetch http://localhost:8080/pkg-mixed-sha1-bad_1.0.dsc Hash Sum mismatch + +E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha1-bad +msgtest 'Only tar file is downloaded as the dsc has hashsum mismatch' 'pkg-mixed-sha1-bad' +testsuccess --nomsg test ! -e pkg-mixed-sha1-bad_1.0.dsc -a -e pkg-mixed-sha1-bad_1.0.tar.gz +testequal 'Reading package lists... +Building dependency tree... +Need to get 0 B of source archives. +Get:1 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (tar) +Get:2 http://localhost:8080/ pkg-mixed-sha2-bad 1.0 (dsc) +E: Failed to fetch http://localhost:8080/pkg-mixed-sha2-bad_1.0.tar.gz Hash Sum mismatch + +E: Failed to fetch some archives.' aptget source -d pkg-mixed-sha2-bad +msgtest 'Only dsc file is downloaded as the tar has hashsum mismatch' 'pkg-mixed-sha2-bad' +testsuccess --nomsg test -e pkg-mixed-sha2-bad_1.0.dsc -a ! -e pkg-mixed-sha2-bad_1.0.tar.gz + +# it gets even more pathologic: multiple entries for one file, some even disagreeing! +testok pkg-md5-agree +testequal 'Reading package lists... +Building dependency tree... +E: Error parsing checksum in Files of source package pkg-md5-disagree' aptget source -d pkg-md5-disagree +testequal 'Reading package lists... +Building dependency tree... +E: Error parsing checksum in Checksums-SHA256 of source package pkg-sha256-disagree' aptget source -d pkg-sha256-disagree -- cgit v1.2.3-70-g09d2 From 081c9d442a6d39fb9bc419fe3ce697cc791cb844 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 21:38:53 +0100 Subject: various small additional tests and testcases Usually they don't provide a lot in terms of what they test, but they help in covering many lines from strictly anecdotal commands (stats, moo) and error messages, so that stuff which really needs to be tested, but isn't is better visible in coverage reports. Git-Dch: Ignore --- cmdline/apt-cache.cc | 4 +- test/integration/framework | 2 +- test/integration/test-00-commands-have-help | 11 ++ test/integration/test-apt-cache | 124 +++++++++++++++++++++ test/integration/test-apt-cli-search | 3 + test/integration/test-apt-cli-show | 11 ++ test/integration/test-apt-cli-update | 7 ++ test/integration/test-apt-config | 36 ++++++ test/integration/test-apt-helper | 12 +- test/integration/test-apt-mark | 34 ++++-- test/integration/test-apt-update-stale | 6 +- .../test-external-dependency-solver-protocol | 32 +++++- 12 files changed, 263 insertions(+), 19 deletions(-) create mode 100755 test/integration/test-apt-cache create mode 100755 test/integration/test-apt-config (limited to 'cmdline') diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 342ad1858..a5024c581 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -116,7 +116,7 @@ static bool ShowUnMet(pkgCache::VerIterator const &V, bool const Important) continue; // Skip conflicts and replaces - if (End.IsNegative() == true) + if (End.IsNegative() == true || End->Type == pkgCache::Dep::Replaces) continue; // Verify the or group @@ -133,7 +133,7 @@ static bool ShowUnMet(pkgCache::VerIterator const &V, bool const Important) break; } delete [] VList; - + if (Start == End) break; ++Start; diff --git a/test/integration/framework b/test/integration/framework index d28085ee4..c9f62c141 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -300,7 +300,7 @@ getarchitecture() { } getarchitectures() { - echo "$(aptconfig dump | grep APT::Architecture | cut -d'"' -f 2 | sed '/^$/ d' | sort | uniq | tr '\n' ' ')" + aptconfig dump --no-empty --format '%v%n' APT::Architecture APT::Architectures | sort -u | tr '\n' ' ' } getarchitecturesfromcommalist() { diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help index ebf8b8cfa..bbd1475eb 100755 --- a/test/integration/test-00-commands-have-help +++ b/test/integration/test-00-commands-have-help @@ -50,3 +50,14 @@ done for CMD in 'apt-dump-solver' 'apt-internal-solver'; do checkoptions "$(echo "$CMD" | tr -d '-')" done + +# in times of need, we all look for super cow to save the day +testsuccess aptget moo +testsuccess aptget moo -q=2 +testsuccess aptget moo moo +testsuccess aptget moo moo -q=2 +testsuccess aptget moo moo --color +testsuccess aptget moo moo moo +testsuccess aptget moo moo moo -q=2 +testsuccess aptget moo moo moo moo +testsuccess aptget moo moo moo moo -q=2 diff --git a/test/integration/test-apt-cache b/test/integration/test-apt-cache new file mode 100755 index 000000000..f47c0e08b --- /dev/null +++ b/test/integration/test-apt-cache @@ -0,0 +1,124 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' 'i386' + +DESCR='Some description + That has multiple lines' +insertpackage 'unstable' 'fancy' 'all' '1' +insertpackage 'unstable,installed' 'foo' 'all' '1' 'Depends: bar +Conflicts: foobar +Recommends: cool (>= 2) | cooler (<< 5)' "$DESCR" +insertpackage 'unstable' 'bar' 'all' '1' 'Depends: bar +Breaks: foo (<< 1) +Replaces: foo (<< 1)' "$DESCR" + +setupaptarchive + +# dpkg is installed by our framework +testdpkginstalled 'dpkg' +testempty aptcache unmet dpkg + +# FIXME: Find some usecase for unmet as it seems kinda useless/broken +#testsuccess aptcache unmet +#testsuccess aptcache unmet foo + +# not too useful to test, but makes coverage green… +testsuccess aptcache stats +cp rootdir/tmp/testsuccess.output stats.output +testsuccess test -s stats.output +testsuccess aptcache xvcg foo +cp rootdir/tmp/testsuccess.output xvcg.output +testsuccess test -s xvcg.output +testsuccess aptcache dotty foo +cp rootdir/tmp/testsuccess.output dotty.output +testsuccess test -s dotty.output +# for this, even the sourcecode says it is useless (expect debugging) +testsuccess aptcache dump +cp rootdir/tmp/testsuccess.output dump.output +testsuccess test -s dump.output + +testequal 'dpkg +bar +fancy +foo' aptcache pkgnames +testequal 'bar' aptcache pkgnames bar +testequal 'fancy +foo' aptcache pkgnames f + +testequal " foo | 1 | file:$(readlink -f .)/aptarchive/ unstable/main amd64 Packages" aptcache madison foo + +### depends + +testequal 'foo + Depends: bar + |Recommends: + Recommends: + Conflicts: + Conflicts: ' aptcache depends foo +testequal 'foo + Depends: bar + Recommends: + Conflicts: + Conflicts: ' aptcache depends foo -o APT::Cache::ShowOnlyFirstOr=1 +testequal 'foo + Depends: bar + |Recommends: (>= 2) + Recommends: (<< 5) + Conflicts: + Conflicts: ' aptcache depends foo -o APT::Cache::ShowVersion=1 +testequal 'foo + Depends: bar + Conflicts: + Conflicts: ' aptcache depends foo --no-recommends +testequal 'foo + Depends: bar' aptcache depends foo --important +testequal 'foo + Conflicts: + Conflicts: ' aptcache depends foo --important --no-depends --conflicts +testequal 'foo + Depends: bar + |Recommends: + Recommends: + Conflicts: + Conflicts: +bar + Depends: bar + Breaks: foo + Breaks: + Replaces: foo + Replaces: + + + + +' aptcache depends foo --recurse +testequal 'foo + Depends: bar +bar + Depends: bar + Replaces: foo + Replaces: +' aptcache depends foo --recurse --important --replaces + +## rdpends + +testequal 'foo +Reverse Depends: + bar + bar' aptcache rdepends foo +testequal 'foo +Reverse Depends: + Replaces: bar + Breaks: bar' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 +testequal 'foo +Reverse Depends: + Replaces: bar (<< 1) + Breaks: bar (<< 1)' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 -o APT::Cache::ShowVersion=1 +testequal 'foo +Reverse Depends: + Breaks: bar (<< 1)' aptcache rdepends foo -o APT::Cache::ShowDependencyType=1 -o APT::Cache::ShowVersion=1 --important --breaks diff --git a/test/integration/test-apt-cli-search b/test/integration/test-apt-cli-search index 8f009d57c..1a28ba4da 100755 --- a/test/integration/test-apt-cli-search +++ b/test/integration/test-apt-cli-search @@ -25,6 +25,9 @@ setupaptarchive APTARCHIVE=$(readlink -f ./aptarchive) +testequal 'E: You must give at least one search pattern' aptcache search +testequal 'E: You must give at least one search pattern' apt search + # with OP progress testequal "Sorting... Full Text Search... diff --git a/test/integration/test-apt-cli-show b/test/integration/test-apt-cli-show index 4c8e134d6..930e591e0 100755 --- a/test/integration/test-apt-cli-show +++ b/test/integration/test-apt-cli-show @@ -36,3 +36,14 @@ APT-Sources: file:$APTARCHIVE/ unstable/main i386 Packages Description: Some description That has multiple lines " apt show foo + +# this is the default, but disabled by the testcases +testsuccess apt show foo -o Apt::Cmd::Disable-Script-Warning=0 +cp rootdir/tmp/testsuccess.output aptshow.output +testsuccess grep '^WARNING: ' aptshow.output + +if [ "$(id -u)" != '0' ]; then + testsuccess apt install foo -s -o APT::Get::Show-User-Simulation-Note=1 + cp rootdir/tmp/testsuccess.output aptshow.output + testsuccess grep '^NOTE: ' aptshow.output +fi diff --git a/test/integration/test-apt-cli-update b/test/integration/test-apt-cli-update index 987bb9adb..83cc94b93 100755 --- a/test/integration/test-apt-cli-update +++ b/test/integration/test-apt-cli-update @@ -8,10 +8,17 @@ setupenvironment configarchitecture "i386" insertpackage 'unstable' 'foo' 'all' '2.0' +cp rootdir/var/lib/dpkg/status dpkg.status insertinstalledpackage 'foo' 'all' '1.0' setupaptarchive --no-update APTARCHIVE=$(readlink -f ./aptarchive) +testequal 'E: The update command takes no arguments' apt update -q arguments + testequal "1 package can be upgraded. Run 'apt list --upgradable' to see it." apt update -q + +cp dpkg.status rootdir/var/lib/dpkg/status +insertinstalledpackage 'foo' 'all' '2.0' +testequal 'All packages are up to date.' apt update -q diff --git a/test/integration/test-apt-config b/test/integration/test-apt-config new file mode 100755 index 000000000..2f2ff9d38 --- /dev/null +++ b/test/integration/test-apt-config @@ -0,0 +1,36 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture 'amd64' + +testsuccess aptconfig dump +testequal 'APT::Architecture "amd64";' aptconfig dump APT::Architecture +testempty aptconfig dump config::which::does::not::exist + +testequal 'APT::Architectures ""; +APT::Architectures:: "amd64";' aptconfig dump APT::Architectures +testequal 'APT::Architectures:: "amd64";' aptconfig dump --no-empty APT::Architectures +testequal 'amd64' aptconfig dump --no-empty --format='%v%n' APT::Architectures + +testempty aptconfig shell +testequal 'E: Arguments not in pairs' aptconfig shell APT::Architecture +testempty aptconfig shell APT::Architecture ARCH # incorrect order +testequal "ARCH='amd64'" aptconfig shell ARCH APT::Architecture + +ROOTDIR="$(readlink -f rootdir)" +testequal "CONFIG='apt.conf'" aptconfig shell CONFIG Dir::Etc::main +testequal "CONFIG='${ROOTDIR}/etc/apt/apt.conf'" aptconfig shell CONFIG Dir::Etc::main/f +testequal "CONFIG='etc/apt/'" aptconfig shell CONFIG Dir::Etc +testequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/ # old style +testequal "CONFIG='${ROOTDIR}/etc/apt/'" aptconfig shell CONFIG Dir::Etc/d + +testempty aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles +export DEB_BUILD_PROFILES='nodoc stage1' +testequal 'nodoc +stage1' aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles +unset DEB_BUILD_PROFILES +testempty aptconfig dump --no-empty --format='%v%n' APT::Build-Profiles diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index 06e497ff7..3c1d393a6 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -53,7 +53,7 @@ echo "http://some-proxy" EOF chmod 755 apt-proxy-detect echo "Acquire::http::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect - + testequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com @@ -64,13 +64,15 @@ echo "https://https-proxy" EOF chmod 755 apt-proxy-detect echo "Acquire::https::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect - - testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com - - + testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com } test_apt_helper_download test_apt_helper_detect_proxy +# test failure modes +testequal 'E: Invalid operation download' apthelper download +testequal 'E: Must specify at least one pair url/filename' apthelper download-file +testequal 'E: Must specify at least one pair url/filename' apthelper download-file http://example.org/ +testequal 'E: Need one URL as argument' apthelper auto-detect-proxy diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark index 69e0f933d..0f62a12b4 100755 --- a/test/integration/test-apt-mark +++ b/test/integration/test-apt-mark @@ -20,21 +20,31 @@ testdpkginstalled dpkg testnoautopkg() { testempty aptmark showauto + testempty aptcache showauto testequal 'bar dpkg foo' aptmark showmanual testequal 'bar foo' aptmark showmanual bar foo uninstalled } -testmarkonpkgasauto() { - testsuccess aptmark $1 foo +testfooisauto() { testequal 'foo' aptmark showauto + testequal 'foo' aptcache showauto testequal 'foo' aptmark showauto foo + testequal 'foo' aptcache showauto foo testequal 'bar dpkg' aptmark showmanual testequal 'bar' aptmark showmanual bar +} +testmarkonpkgasauto() { + testsuccess $1 $2 foo + testfooisauto + testsuccess $1 $2 foo + testfooisauto - testsuccess aptmark $2 foo + testsuccess $1 $3 foo + testnoautopkg + testsuccess $1 $3 foo testnoautopkg } @@ -42,8 +52,9 @@ testequal 'E: No packages found' aptmark auto testequal 'E: No packages found' aptmark manual testnoautopkg -testmarkonpkgasauto 'auto' 'manual' -testmarkonpkgasauto 'markauto' 'unmarkauto' +testmarkonpkgasauto 'aptmark' 'auto' 'manual' +testmarkonpkgasauto 'aptmark' 'markauto' 'unmarkauto' +testmarkonpkgasauto 'aptget' 'markauto' 'unmarkauto' testnoholdpkg() { testempty aptmark showhold @@ -51,10 +62,19 @@ testnoholdpkg() { testempty aptmark showhold dpkg testempty aptmark showholds dpkg } -testmarkonepkgashold() { - testsuccess aptmark hold $1 +testpkgonhold() { testequal "$1" aptmark showhold testequal "$1" aptmark showholds + testequal "$1" aptmark showhold $1 + testequal "$1" aptmark showholds $1 +} +testmarkonepkgashold() { + testsuccess aptmark hold $1 + testpkgonhold $1 + testsuccess aptmark hold $1 + testpkgonhold $1 + testsuccess aptmark unhold $1 + testnoholdpkg testsuccess aptmark unhold $1 testnoholdpkg } diff --git a/test/integration/test-apt-update-stale b/test/integration/test-apt-update-stale index 52f94591f..277aa5b09 100755 --- a/test/integration/test-apt-update-stale +++ b/test/integration/test-apt-update-stale @@ -14,9 +14,11 @@ configarchitecture "i386" insertpackage 'unstable' 'foo' 'all' '1.0' -setupaptarchive +setupaptarchive --no-update changetowebserver -aptget update -qq + +echo "Acquire::Languages \"none\";" > rootdir/etc/apt/apt.conf.d/00nolanguages +testsuccess aptget update listcurrentlistsdirectory > lists.before # insert new version diff --git a/test/integration/test-external-dependency-solver-protocol b/test/integration/test-external-dependency-solver-protocol index 07d2441b6..fd68578c5 100755 --- a/test/integration/test-external-dependency-solver-protocol +++ b/test/integration/test-external-dependency-solver-protocol @@ -12,7 +12,10 @@ insertinstalledpackage 'stuff' 'all' '1' insertpackage 'unstable' 'cool' 'all' '2' 'Multi-Arch: foreign' insertpackage 'unstable' 'stuff' 'all' '2' 'Multi-Arch: foreign' insertpackage 'unstable' 'coolstuff' 'i386,amd64' '2' 'Depends: cool, stuff' -insertpackage 'unstable' 'awesome' 'all' '2' 'Multi-Arch: foreign' +insertpackage 'unstable' 'awesome' 'all' '2' 'Multi-Arch: foreign +Conflicts: badstuff' +insertpackage 'unstable' 'badstuff' 'all' '2' 'Multi-Arch: foreign +Conflicts: awesome' insertpackage 'unstable' 'awesomecoolstuff' 'i386' '2' 'Depends: coolstuff, awesome' insertpackage 'experimental' 'cool' 'all' '3' 'Multi-Arch: foreign' @@ -44,6 +47,14 @@ The following NEW packages will be installed: Inst coolstuff (3 experimental [amd64]) Conf coolstuff (3 experimental [amd64])' aptget install --solver apt coolstuff -s +testequal 'Reading package lists... +Building dependency tree... +Execute external solver... +The following packages will be REMOVED: + cool* +0 upgraded, 0 newly installed, 1 to remove and 1 not upgraded. +Purg cool [1]' aptget purge --solver apt cool -s + testsuccess aptget install awesomecoolstuff:i386 -s testsuccess aptget install --solver apt awesomecoolstuff:i386 -s @@ -57,9 +68,13 @@ testsuccess aptget dist-upgrade -s --solver apt testsuccess aptget upgrade -s testsuccess aptget upgrade -s --solver apt +testfailure aptget install awesome badstuff -s +testfailure aptget install awesome badstuff -s --solver apt +testsuccess grep 'ERR_UNSOLVABLE' rootdir/tmp/testfailure.output + configarchitecture 'armel' msgtest 'Test direct calling is okay for' 'apt-internal-solver' -cat /tmp/dump.edsp | aptinternalsolver > solver.result 2>&1 || true +cat /tmp/dump.edsp | aptinternalsolver -q=0 > solver.result 2>&1 || true if [ "$(tail -n2 solver.result | head -n1 )" = "Message: Done" ]; then msgpass else @@ -69,3 +84,16 @@ fi rm -f /tmp/dump.edsp testfailure aptget install --solver apt awesomecoolstuff:i386 -s + +testsuccess aptinternalsolver scenario +testequal 'Package: stuff +Source: stuff +Architecture: all +Version: 1 +Installed: yes +APT-ID: 2 +Priority: optional +Section: other +APT-Pin: 100 +APT-Candidate: yes +' aptinternalsolver scenario stuff -- cgit v1.2.3-70-g09d2 From 1d838084dc775c0a4184edb4f3b9138903ac27fb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 18 Nov 2014 13:41:18 +0100 Subject: use dpkg --merge-avail only if needed in apt-mark Only "recent" versions of dpkg support stdin for merge instead of a file, so as a quick fix we delay calling it until we really need it which fixes most of the problem already. Checking for a specific dpkg version here is deemed too much work, just like using a temporary file here and depends a too high requirement for this minor usecase. After all, it didn't work at all before, so we break nobody here and can fix it if someone complains (with a patch). --- cmdline/apt-mark.cc | 100 ++++++++++++++++++++++------------------- test/integration/test-apt-mark | 9 ++++ 2 files changed, 64 insertions(+), 45 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc index 860982f9e..8974397c9 100644 --- a/cmdline/apt-mark.cc +++ b/cmdline/apt-mark.cc @@ -274,65 +274,75 @@ static bool DoHold(CommandLine &CmdL) return true; } - Args.erase(Args.begin() + BaseArgs, Args.end()); - Args.push_back("--merge-avail"); - Args.push_back("-"); - Args.push_back(NULL); - - int external[2] = {-1, -1}; - if (pipe(external) != 0) - return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); - - pid_t dpkgMergeAvail = ExecFork(); - if (dpkgMergeAvail == 0) - { - close(external[1]); - std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); - if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) - _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); - dup2(external[0], STDIN_FILENO); - int const nullfd = open("/dev/null", O_RDONLY); - dup2(nullfd, STDOUT_FILENO); - execvp(Args[0], (char**) &Args[0]); - _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); - _exit(2); - } - - FILE* dpkg = fdopen(external[1], "w"); + APT::PackageList keepoffset; for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (Pkg->CurrentVer != 0) continue; - char const * Arch; - if (Pkg->VersionList != 0) - Arch = Pkg.VersionList().Arch(); - else - Arch = Pkg.Arch(); - fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" - "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + keepoffset.insert(Pkg); } - fclose(dpkg); - if (dpkgMergeAvail > 0) + if (keepoffset.empty() == false) { - int Status = 0; - while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + Args.erase(Args.begin() + BaseArgs, Args.end()); + Args.push_back("--merge-avail"); + // FIXME: supported only since 1.17.7 in dpkg + Args.push_back("-"); + Args.push_back(NULL); + + int external[2] = {-1, -1}; + if (pipe(external) != 0) + return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --merge-avail"); + + pid_t dpkgMergeAvail = ExecFork(); + if (dpkgMergeAvail == 0) { - if (errno == EINTR) - continue; - _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); - break; + close(external[1]); + std::string const chrootDir = _config->FindDir("DPkg::Chroot-Directory"); + if (chrootDir != "/" && chroot(chrootDir.c_str()) != 0 && chdir("/") != 0) + _error->WarningE("getArchitecture", "Couldn't chroot into %s for dpkg --merge-avail", chrootDir.c_str()); + dup2(external[0], STDIN_FILENO); + int const nullfd = open("/dev/null", O_RDONLY); + dup2(nullfd, STDOUT_FILENO); + execvp(Args[0], (char**) &Args[0]); + _error->WarningE("dpkgGo", "Can't get dpkg --merge-avail running!"); + _exit(2); + } + + FILE* dpkg = fdopen(external[1], "w"); + for (APT::PackageList::iterator Pkg = keepoffset.begin(); Pkg != keepoffset.end(); ++Pkg) + { + char const * Arch; + if (Pkg->VersionList != 0) + Arch = Pkg.VersionList().Arch(); + else + Arch = Pkg.Arch(); + fprintf(dpkg, "Package: %s\nVersion: 0~\nArchitecture: %s\nMaintainer: Dummy Example \n" + "Description: dummy package record\n A record is needed to put a package on hold, so here it is.\n\n", Pkg.Name(), Arch); + } + fclose(dpkg); + keepoffset.clear(); + + if (dpkgMergeAvail > 0) + { + int Status = 0; + while (waitpid(dpkgMergeAvail, &Status, 0) != dpkgMergeAvail) + { + if (errno == EINTR) + continue; + _error->WarningE("dpkgGo", _("Waited for %s but it wasn't there"), "dpkg --merge-avail"); + break; + } + if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) + return _error->Error(_("Executing dpkg failed. Are you root?")); } - if (WIFEXITED(Status) == false || WEXITSTATUS(Status) != 0) - return _error->Error(_("Executing dpkg failed. Are you root?")); } Args.erase(Args.begin() + BaseArgs, Args.end()); Args.push_back("--set-selections"); Args.push_back(NULL); - external[0] = -1; - external[1] = -1; + int external[2] = {-1, -1}; if (pipe(external) != 0) return _error->WarningE("DoHold", "Can't create IPC pipe for dpkg --set-selections"); @@ -349,7 +359,7 @@ static bool DoHold(CommandLine &CmdL) _exit(2); } - dpkg = fdopen(external[1], "w"); + FILE* dpkg = fdopen(external[1], "w"); for (APT::PackageList::iterator Pkg = pkgset.begin(); Pkg != pkgset.end(); ++Pkg) { if (dpkgMultiArch == false) diff --git a/test/integration/test-apt-mark b/test/integration/test-apt-mark index 0f62a12b4..5a3ae4b2f 100755 --- a/test/integration/test-apt-mark +++ b/test/integration/test-apt-mark @@ -85,6 +85,15 @@ testequal 'E: No packages found' aptmark unhold testnoholdpkg testmarkonepkgashold 'foo' testmarkonepkgashold 'bar' + +msgtest 'dpkg supports --merge-avail via' 'stdin' +if dpkg --merge-avail - < /dev/null >/dev/null 2>&1; then + msgpass +else + msgskip 'dpkg version too old' + exit 0 +fi + testmarkonepkgashold 'uninstalled' testmarkonepkgashold 'uninstalled-native' -- cgit v1.2.3-70-g09d2 From 69d8b8537af1dd52db5f2e0e785bdce3e52fdf8d Mon Sep 17 00:00:00 2001 From: James McCoy Date: Fri, 28 Nov 2014 14:21:06 +0100 Subject: support long keyids in "apt-key del" instead of ignoring them apt-key given a long keyid reports just "OK" all the time, but doesn't delete the mentioned key as it doesn't find the key. Note: In debian/experimental this was closed with 29f1b977100aeb6d6ebd38923eeb7a623e264ffe which just added the testcase as the rewrite of apt-key had fixed this as well. Closes: 754436 --- cmdline/apt-key.in | 2 +- test/integration/test-apt-key | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'cmdline') diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in index 0774cf4b7..b4e071000 100644 --- a/cmdline/apt-key.in +++ b/cmdline/apt-key.in @@ -180,7 +180,7 @@ update() { remove_key_from_keyring() { local GPG="$GPG_CMD --keyring $1" # check if the key is in this keyring: the key id is in the 5 column at the end - if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]\+$2:"; then + if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then return fi if [ ! -w "$1" ]; then diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key index 68b3f9710..47230cb55 100755 --- a/test/integration/test-apt-key +++ b/test/integration/test-apt-key @@ -105,3 +105,9 @@ testfileequal ./aptkey.list 'pub 2048R/528144E2 2011-01-16' testsuccess test ! -e rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg testsuccess cmp keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg~ testsuccess cmp keys/testcase-multikey.pub rootdir/etc/apt/trusted.gpg.d/multikey.gpg~ + +msgtest 'Test key removal with' '8 byte key ID' +cleanplate +cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg +testsuccess --nomsg aptkey --fakeroot del 5A90D141DBAC8DAE +testempty aptkey list -- cgit v1.2.3-70-g09d2 From ed793a19ec00b83254029509bc516e3ba911c75a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 29 Nov 2014 17:59:52 +0100 Subject: dispose http(s) 416 error page as non-content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real webservers (like apache) actually send an error page with a 416 response, but our client didn't expect it leaving the page on the socket to be parsed as response for the next request (http) or as file content (https), which isn't what we want at all… Symptom is a "Bad header line" as html usually doesn't parse that well to an http-header. This manifests itself e.g. if we have a complete file (or larger) in partial/ which isn't discarded by If-Range as the server doesn't support it (or it is just newer, think: mirror rotation). It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a, which removed the filesize - 1 trick, but this had its own problems… To properly test this our webserver gains the ability to reply with transfer-encoding: chunked as most real webservers will use it to send the dynamically generated error pages. Closes: 768797 --- cmdline/apt-helper.cc | 31 +++-- methods/http.cc | 2 + methods/https.cc | 15 ++- methods/server.cc | 26 ++-- methods/server.h | 5 +- test/integration/framework | 4 +- test/integration/test-partial-file-support | 62 ++++++++- test/interactive-helper/aptwebserver.cc | 193 ++++++++++++++++++----------- 8 files changed, 233 insertions(+), 105 deletions(-) (limited to 'cmdline') diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index 1b832f165..a05ae90a2 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -51,22 +51,33 @@ static bool DoDownloadFile(CommandLine &CmdL) AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); pkgAcquire Fetcher(&Stat); - std::string download_uri = CmdL.FileList[1]; - std::string targetfile = CmdL.FileList[2]; - std::string hash; - if (CmdL.FileSize() > 3) - hash = CmdL.FileList[3]; - // we use download_uri as descr and targetfile as short-descr - new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, - "dest-dir-ignored", targetfile); + size_t fileind = 0; + std::vector targetfiles; + while (fileind + 2 <= CmdL.FileSize()) + { + std::string download_uri = CmdL.FileList[fileind + 1]; + std::string targetfile = CmdL.FileList[fileind + 2]; + std::string hash; + if (CmdL.FileSize() > fileind + 3) + hash = CmdL.FileList[fileind + 3]; + // we use download_uri as descr and targetfile as short-descr + new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, + "dest-dir-ignored", targetfile); + targetfiles.push_back(targetfile); + fileind += 3; + } // Disable drop-privs if "_apt" can not write to the target dir CheckDropPrivsMustBeDisabled(Fetcher); bool Failed = false; - if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true || - FileExists(targetfile) == false) + if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true) return _error->Error(_("Download Failed")); + if (targetfiles.empty() == false) + for (std::vector::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f) + if (FileExists(*f) == false) + return _error->Error(_("Download Failed")); + return true; } diff --git a/methods/http.cc b/methods/http.cc index a5de13511..ad1347d36 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -444,6 +444,8 @@ bool HttpServerState::RunData(FileFd * const File) loss of the connection means we are done */ if (Encoding == Closes) In.Limit(-1); + else if (JunkSize != 0) + In.Limit(JunkSize); else In.Limit(Size - StartPos); diff --git a/methods/https.cc b/methods/https.cc index 366148e19..23b3a10d4 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -69,6 +69,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { me->Server->Result = 200; me->Server->StartPos = me->Server->Size; + // the actual size is not important for https as curl will deal with it + // by itself and e.g. doesn't bother us with transport-encoding… + me->Server->JunkSize = std::numeric_limits::max(); } else me->Server->StartPos = 0; @@ -86,19 +89,25 @@ size_t HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) { HttpsMethod *me = (HttpsMethod *)userp; + size_t buffer_size = size * nmemb; + // we don't need to count the junk here, just drop anything we get as + // we don't always know how long it would be, e.g. in chunked encoding. + if (me->Server->JunkSize != 0) + return buffer_size; if (me->Res.Size == 0) me->URIStart(me->Res); - if(me->File->Write(buffer, size*nmemb) != true) + if(me->File->Write(buffer, buffer_size) != true) return false; if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) { me->SetFailReason("MaximumSizeExceeded"); - return _error->Error("Writing more data than expected (%llu > %llu)", + _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->MaximumSize); + return 0; } - return size*nmemb; + return buffer_size; } int diff --git a/methods/server.cc b/methods/server.cc index c4689ff12..9b3d39cf2 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, Minor = 0; Result = 0; Size = 0; + JunkSize = 0; StartPos = 0; Encoding = Closes; HaveContent = false; @@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line) Encoding = Stream; HaveContent = true; - // The length is already set from the Content-Range header - if (StartPos != 0) - return true; + unsigned long long * SizePtr = &Size; + if (Result == 416) + SizePtr = &JunkSize; - Size = strtoull(Val.c_str(), NULL, 10); - if (Size >= std::numeric_limits::max()) + *SizePtr = strtoull(Val.c_str(), NULL, 10); + if (*SizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (Size == 0) + else if (*SizePtr == 0) HaveContent = false; return true; } @@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line) // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) - { - StartPos = 1; // ignore Content-Length, it would override Size - HaveContent = false; - } + ; // we got the expected filesize which is all we wanted else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); if ((unsigned long long)StartPos > Size) @@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res) if ((unsigned long long)SBuf.st_size == Server->Size) { // the file is completely downloaded, but was not moved + if (Server->HaveContent == true) + { + // Send to error page to dev/null + FileFd DevNull("/dev/null",FileFd::WriteExists); + Server->RunData(&DevNull); + } + Server->HaveContent = false; Server->StartPos = Server->Size; Server->Result = 200; - Server->HaveContent = false; } else if (unlink(Queue->DestFile.c_str()) == 0) { diff --git a/methods/server.h b/methods/server.h index 7d5198478..b974ec89a 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,7 +34,8 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; + unsigned long long Size; // size of the usable content (aka: the file) + unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; time_t Date; bool HaveContent; @@ -73,7 +74,7 @@ struct ServerState RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; diff --git a/test/integration/framework b/test/integration/framework index 930ab9367..f7f69f5d5 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1119,8 +1119,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom } downloadfile() { - local PROTO="$(echo "$1" | cut -d':' -f 1 )" - apthelper -o Debug::Acquire::${PROTO}=1 \ + local PROTO="${1%%:*}" + apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \ download-file "$1" "$2" 2>&1 || true # only if the file exists the download was successful if [ -r "$2" ]; then diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support index 98b2f242a..b6b305d25 100755 --- a/test/integration/test-partial-file-support +++ b/test/integration/test-partial-file-support @@ -24,13 +24,25 @@ testdownloadfile() { else msgpass fi - cat "$DOWNLOADLOG" | while read field hash; do + sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log + testsuccess test -s receivedhashes.log + local HASHES_OK=0 + local HASHES_BAD=0 + while read field hash; do local EXPECTED case "$field" in 'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";; + 'Checksum-FileSize-Hash:') + #filesize is too weak to check for != + if [ "$4" = '=' ]; then + EXPECTED="$(stat -c '%s' "$TESTFILE")" + else + continue + fi + ;; *) continue;; esac if [ "$4" = '=' ]; then @@ -40,15 +52,41 @@ testdownloadfile() { fi if [ "$EXPECTED" "$4" "$hash" ]; then msgpass + HASHES_OK=$((HASHES_OK+1)); else - cat >&2 "$DOWNLOADLOG" msgfail "expected: $EXPECTED ; got: $hash" + HASHES_BAD=$((HASHES_BAD+1)); fi - done + done < receivedhashes.log + msgtest 'At least one good hash and no bad ones' + if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then + cat >&2 "$DOWNLOADLOG" + msgfail + else + msgpass + fi } TESTFILE='aptarchive/testfile' cp -a ${TESTDIR}/framework $TESTFILE +cp -a ${TESTDIR}/framework "${TESTFILE}2" + +followuprequest() { + local DOWN='./downloaded/testfile' + + copysource $TESTFILE 1M $DOWN + testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' + testwebserverlaststatuscode '416' "$DOWNLOADLOG" + + copysource $TESTFILE 1M $DOWN + copysource "${TESTFILE}2" 20 "${DOWN}2" + msgtest 'Testing download of files with' 'completely downloaded file + partial file' + testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \ + download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2" + testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output' + testsuccess diff -u "$TESTFILE" "${DOWN}" + testsuccess diff -u "${DOWN}" "${DOWN}2" +} testrun() { webserverconfig 'aptwebserver::support::range' 'true' @@ -66,9 +104,11 @@ testrun() { testdownloadfile 'invalid partial data' "${1}/testfile" "$DOWN" '!=' testwebserverlaststatuscode '206' "$DOWNLOADLOG" - copysource $TESTFILE 1M $DOWN - testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' - testwebserverlaststatuscode '416' "$DOWNLOADLOG" + webserverconfig 'aptwebserver::closeOnError' 'false' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'true' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'false' copysource /dev/zero 1M $DOWN testdownloadfile 'too-big partial file' "${1}/testfile" "$DOWN" '=' @@ -86,8 +126,18 @@ testrun() { testwebserverlaststatuscode '200' "$DOWNLOADLOG" } +msgmsg 'http: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'http://localhost:8080' +msgmsg 'http: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'http://localhost:8080' changetohttpswebserver +msgmsg 'https: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'https://localhost:4433' +msgmsg 'https: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'https://localhost:4433' diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 00004a524..ca6f88c58 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -19,6 +19,8 @@ #include #include #include + +#include #include #include #include @@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode) /*{{{*/ return NULL; } /*}}}*/ +static bool chunkedTransferEncoding(std::list const &headers) { + if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end()) + return true; + if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true) + return true; + return false; +} static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/ { - std::ostringstream contentlength; - contentlength << "Content-Length: " << data.FileSize(); - headers.push_back(contentlength.str()); - + if (chunkedTransferEncoding(headers) == false) + { + std::ostringstream contentlength; + contentlength << "Content-Length: " << data.FileSize(); + headers.push_back(contentlength.str()); + } std::string lastmodified("Last-Modified: "); lastmodified.append(TimeRFC1123(data.ModificationTime())); headers.push_back(lastmodified); @@ -92,9 +103,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/ /*}}}*/ static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/ { - std::ostringstream contentlength; - contentlength << "Content-Length: " << data.size(); - headers.push_back(contentlength.str()); + if (chunkedTransferEncoding(headers) == false) + { + std::ostringstream contentlength; + contentlength << "Content-Length: " << data.size(); + headers.push_back(contentlength.str()); + } } /*}}}*/ static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/ @@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl; bool Success = true; for (std::list::const_iterator h = headers.begin(); @@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list const &headers, FileFd &data)/*{{{*/ { bool Success = true; + bool const chunked = chunkedTransferEncoding(headers); char buffer[500]; unsigned long long actual = 0; while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true) { if (actual == 0) break; - Success &= FileFd::Write(client, buffer, actual); + + if (chunked == true) + { + std::string size; + strprintf(size, "%llX\r\n", actual); + Success &= FileFd::Write(client, size.c_str(), size.size()); + Success &= FileFd::Write(client, buffer, actual); + Success &= FileFd::Write(client, "\r\n", strlen("\r\n")); + } + else + Success &= FileFd::Write(client, buffer, actual); + } + if (chunked == true) + { + char const * const finish = "0\r\n\r\n"; + Success &= FileFd::Write(client, finish, strlen(finish)); } if (Success == false) - std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl; + std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl; return Success; } /*}}}*/ -static bool sendData(int const client, std::string const &data) /*{{{*/ +static bool sendData(int const client, std::list const &headers, std::string const &data)/*{{{*/ { - if (FileFd::Write(client, data.c_str(), data.size()) == false) + if (chunkedTransferEncoding(headers) == true) + { + unsigned long long const ullsize = data.length(); + std::string size; + strprintf(size, "%llX\r\n", ullsize); + char const * const finish = "\r\n0\r\n\r\n"; + if (FileFd::Write(client, size.c_str(), size.length()) == false || + FileFd::Write(client, data.c_str(), ullsize) == false || + FileFd::Write(client, finish, strlen(finish)) == false) + { + std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl; + return false; + } + } + else if (FileFd::Write(client, data.c_str(), data.size()) == false) { std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl; return false; @@ -157,33 +204,38 @@ static bool sendData(int const client, std::string const &data) /*{{{*/ } /*}}}*/ static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/ - bool content, std::string const &error = "", std::list headers = std::list()) + bool const content, std::string const &error, std::list &headers) { std::string response(""); response.append(httpcodeToStr(httpcode)).append(""); response.append("

").append(httpcodeToStr(httpcode)).append("

"); if (httpcode != 200) - { - if (error.empty() == false) - response.append("

Error: ").append(error).append("

"); - response.append("This error is a result of the request: 
");
-   }
+      response.append("
Error: ");
+   else
+      response.append("
Success: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("
This error is a result of the request: 
");
    else
-   {
-      if (error.empty() == false)
-	 response.append("
Success: ").append(error).append("
");
       response.append("The successfully executed operation was requested by: 
");
-   }
    response.append(request).append("
");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -220,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -262,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list &headers)
 {
-   std::list headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -310,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string ¶ms, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -332,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -344,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -359,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -370,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
 
@@ -382,9 +433,9 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       if (authConf.empty() != auth.empty())
       {
 	 if (auth.empty())
-	    sendError(client, 407, request, sendContent, "Proxy requires authentication");
+	    sendError(client, 407, request, sendContent, "Proxy requires authentication", headers);
 	 else
-	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it");
+	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it", headers);
 	return false;
       }
       if (authConf.empty() == false)
@@ -395,7 +446,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 	    auth.erase(0, strlen(basic));
 	    if (auth != authConf)
 	    {
-	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match");
+	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match", headers);
 	       return false;
 	    }
 	 }
@@ -410,7 +461,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos && APT::String::Startswith(filename, "/_config/") == false)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -421,9 +472,9 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       if (authConf.empty() != auth.empty())
       {
 	 if (auth.empty())
-	    sendError(client, 401, request, sendContent, "Server requires authentication");
+	    sendError(client, 401, request, sendContent, "Server requires authentication", headers);
 	 else
-	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it");
+	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it", headers);
 	 return false;
       }
       if (authConf.empty() == false)
@@ -434,13 +485,12 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 	    auth.erase(0, strlen(basic));
 	    if (auth != authConf)
 	    {
-	       sendError(client, 401, request, sendContent, "Authentication doesn't match");
+	       sendError(client, 401, request, sendContent, "Authentication doesn't match", headers);
 	       return false;
 	    }
 	 }
 	 else
 	 {
-	    std::list headers;
 	    headers.push_back("WWW-Authenticate: Basic");
 	    sendError(client, 401, request, sendContent, "Unsupported Authentication Scheme", headers);
 	    return false;
@@ -463,7 +513,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -499,7 +550,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector parts, std::list &headers)
 {
    size_t const pcount = parts.size();
    for (size_t i = 0; i < pcount; ++i)
@@ -507,40 +559,38 @@ static bool handleOnTheFlyReconfiguration(int const client, std::string const &r
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -549,18 +599,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -569,7 +623,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -601,7 +655,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -620,7 +674,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -676,17 +730,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -695,22 +748,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From 92e8c1ff287ab829de825e00cdf94744e699ff97 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 29 Nov 2014 17:59:52 +0100
Subject: dispose http(s) 416 error page as non-content
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Real webservers (like apache) actually send an error page with a 416
response, but our client didn't expect it leaving the page on the socket
to be parsed as response for the next request (http) or as file content
(https), which isn't what we want at all… Symptom is a "Bad header line"
as html usually doesn't parse that well to an http-header.

This manifests itself e.g. if we have a complete file (or larger) in
partial/ which isn't discarded by If-Range as the server doesn't support
it (or it is just newer, think: mirror rotation).
It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a,
which removed the filesize - 1 trick, but this had its own problems…

To properly test this our webserver gains the ability to reply with
transfer-encoding: chunked as most real webservers will use it to send
the dynamically generated error pages.

(The tests and their binary helpers had to be slightly modified to
apply, but the patch to fix the issue itself is unchanged.)

Closes: 768797
---
 cmdline/apt-helper.cc                      |  35 ++++--
 methods/http.cc                            |   2 +
 methods/https.cc                           |  12 +-
 methods/server.cc                          |  26 +++--
 methods/server.h                           |   5 +-
 test/integration/framework                 |   6 +-
 test/integration/test-apt-helper           |  24 ++--
 test/integration/test-partial-file-support |  62 +++++++++-
 test/interactive-helper/aptwebserver.cc    | 181 ++++++++++++++++++-----------
 9 files changed, 241 insertions(+), 112 deletions(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index dd43ea1bc..63f70983c 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,23 +48,34 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
-
    pkgAcquire Fetcher;
    AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
    Fetcher.Setup(&Stat);
-   std::string download_uri = CmdL.FileList[1];
-   std::string targetfile = CmdL.FileList[2];
-   std::string hash;
-   if (CmdL.FileSize() > 3)
-      hash = CmdL.FileList[3];
-   // we use download_uri as descr and targetfile as short-descr
-   new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, 
-                  "dest-dir-ignored", targetfile);
-   Fetcher.Run();
+
+   size_t fileind = 0;
+   std::vector targetfiles;
+   while (fileind + 2 <= CmdL.FileSize())
+   {
+      std::string download_uri = CmdL.FileList[fileind + 1];
+      std::string targetfile = CmdL.FileList[fileind + 2];
+      std::string hash;
+      if (CmdL.FileSize() > fileind + 3)
+	 hash = CmdL.FileList[fileind + 3];
+      // we use download_uri as descr and targetfile as short-descr
+      new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile,
+	    "dest-dir-ignored", targetfile);
+      targetfiles.push_back(targetfile);
+      fileind += 3;
+   }
+
    bool Failed = false;
-   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true ||
-	 FileExists(targetfile) == false)
+   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true)
       return _error->Error(_("Download Failed"));
+   if (targetfiles.empty() == false)
+      for (std::vector::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f)
+	 if (FileExists(*f) == false)
+	    return _error->Error(_("Download Failed"));
+
    return true;
 }
 
diff --git a/methods/http.cc b/methods/http.cc
index f2a4a4db6..1b996db98 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -440,6 +440,8 @@ bool HttpServerState::RunData(FileFd * const File)
          loss of the connection means we are done */
       if (Encoding == Closes)
 	 In.Limit(-1);
+      else if (JunkSize != 0)
+	 In.Limit(JunkSize);
       else
 	 In.Limit(Size - StartPos);
       
diff --git a/methods/https.cc b/methods/https.cc
index 0499af0c5..65a744e2a 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -59,6 +59,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       {
          me->Server->Result = 200;
 	 me->Server->StartPos = me->Server->Size;
+	 // the actual size is not important for https as curl will deal with it
+	 // by itself and e.g. doesn't bother us with transport-encoding…
+	 me->Server->JunkSize = std::numeric_limits::max();
       }
       else
 	 me->Server->StartPos = 0;
@@ -76,13 +79,18 @@ size_t
 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    HttpsMethod *me = (HttpsMethod *)userp;
+   size_t buffer_size = size * nmemb;
+   // we don't need to count the junk here, just drop anything we get as
+   // we don't always know how long it would be, e.g. in chunked encoding.
+   if (me->Server->JunkSize != 0)
+      return buffer_size;
 
    if (me->Res.Size == 0)
       me->URIStart(me->Res);
-   if(me->File->Write(buffer, size*nmemb) != true)
+   if(me->File->Write(buffer, buffer_size) != true)
       return false;
 
-   return size*nmemb;
+   return buffer_size;
 }
 
 int
diff --git a/methods/server.cc b/methods/server.cc
index 92d94e638..cb0341d5f 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
    Minor = 0; 
    Result = 0; 
    Size = 0; 
+   JunkSize = 0;
    StartPos = 0;
    Encoding = Closes;
    HaveContent = false;
@@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line)
 	 Encoding = Stream;
       HaveContent = true;
 
-      // The length is already set from the Content-Range header
-      if (StartPos != 0)
-	 return true;
+      unsigned long long * SizePtr = &Size;
+      if (Result == 416)
+	 SizePtr = &JunkSize;
 
-      Size = strtoull(Val.c_str(), NULL, 10);
-      if (Size >= std::numeric_limits::max())
+      *SizePtr = strtoull(Val.c_str(), NULL, 10);
+      if (*SizePtr >= std::numeric_limits::max())
 	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
-      else if (Size == 0)
+      else if (*SizePtr == 0)
 	 HaveContent = false;
       return true;
    }
@@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line)
 
       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
-      {
-	 StartPos = 1; // ignore Content-Length, it would override Size
-	 HaveContent = false;
-      }
+	 ; // we got the expected filesize which is all we wanted
       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
@@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
 	 if ((unsigned long long)SBuf.st_size == Server->Size)
 	 {
 	    // the file is completely downloaded, but was not moved
+	    if (Server->HaveContent == true)
+	    {
+	       // Send to error page to dev/null
+	       FileFd DevNull("/dev/null",FileFd::WriteExists);
+	       Server->RunData(&DevNull);
+	    }
+	    Server->HaveContent = false;
 	    Server->StartPos = Server->Size;
 	    Server->Result = 200;
-	    Server->HaveContent = false;
 	 }
 	 else if (unlink(Queue->DestFile.c_str()) == 0)
 	 {
diff --git a/methods/server.h b/methods/server.h
index f5e68d902..1b81e3549 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,7 +34,8 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
-   unsigned long long Size;
+   unsigned long long Size; // size of the usable content (aka: the file)
+   unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
    time_t Date;
    bool HaveContent;
@@ -71,7 +72,7 @@ struct ServerState
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
-   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
+   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
 		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true;};
    virtual bool WriteResponse(std::string const &Data) = 0;
diff --git a/test/integration/framework b/test/integration/framework
index df1942ff9..ac482a7a0 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1064,8 +1064,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 }
 
 downloadfile() {
-	local PROTO="$(echo "$1" | cut -d':' -f 1 )"
-	apthelper -o Debug::Acquire::${PROTO}=1 \
+	local PROTO="${1%%:*}"
+	apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
 		download-file "$1" "$2" 2>&1 || true
 	# only if the file exists the download was successful
 	if [ -e "$2" ]; then
@@ -1221,7 +1221,7 @@ testsuccess() {
 		msgtest 'Test for successful execution of' "$*"
 	fi
 	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
-	if $@ >${OUTPUT} 2>&1; then
+	if "$@" >${OUTPUT} 2>&1; then
 		msgpass
 	else
 		echo >&2
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index c749224ca..31e471677 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -10,34 +10,36 @@ configarchitecture "i386"
 changetohttpswebserver
 
 test_apt_helper_download() {
-    echo "foo" > aptarchive/foo
+    echo 'foo' > aptarchive/foo
+    echo 'bar' > aptarchive/foo2
 
     msgtest 'apt-file download-file md5sum'
-    apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00
     testfileequal foo2 'foo'
 
     msgtest 'apt-file download-file sha1'
-    apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
     testfileequal foo1 'foo'
 
     msgtest 'apt-file download-file sha256'
-    apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c
     testfileequal foo3 'foo'
 
     msgtest 'apt-file download-file no-hash'
-    apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo4
     testfileequal foo4 'foo'
     
     msgtest 'apt-file download-file wrong hash'
-    if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then
-        msgpass
-    else
-        msgfail
-    fi
-    testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
+    testfailure --nomsg apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc
+    testfileequal rootdir/tmp/testfailure.output 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
 
 E: Download Failed'
     testfileequal foo5.FAILED 'foo'
+
+    msgtest 'apt-file download-file md5sum sha1'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo6 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 http://localhost:8080/foo2 foo7 SHA1:e242ed3bffccdf271b7fbaf34ed72d089537b42f
+    testfileequal foo6 'foo'
+    testfileequal foo7 'bar'
 }
 
 test_apt_helper_detect_proxy() {
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 5ab326def..160d451b6 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -24,13 +24,25 @@ testdownloadfile() {
 	else
 		msgpass
 	fi
-	cat "$DOWNLOADLOG" | while read field hash; do
+	sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log
+	testsuccess test -s receivedhashes.log
+	local HASHES_OK=0
+	local HASHES_BAD=0
+	while read field hash; do
 		local EXPECTED
 		case "$field" in
 		'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";;
+		'Checksum-FileSize-Hash:')
+			#filesize is too weak to check for !=
+			if [ "$4" = '=' ]; then
+				EXPECTED="$(stat -c '%s' "$TESTFILE")"
+			else
+				continue
+			fi
+			;;
 		*) continue;;
 		esac
 		if [ "$4" = '=' ]; then
@@ -40,15 +52,41 @@ testdownloadfile() {
 		fi
 		if [ "$EXPECTED" "$4" "$hash" ]; then
 			msgpass
+			HASHES_OK=$((HASHES_OK+1));
 		else
-			cat >&2 "$DOWNLOADLOG"
 			msgfail "expected: $EXPECTED ; got: $hash"
+			HASHES_BAD=$((HASHES_BAD+1));
 		fi
-	done
+	done < receivedhashes.log
+	msgtest 'At least one good hash and no bad ones'
+	if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then
+		cat >&2 "$DOWNLOADLOG"
+		msgfail
+	else
+		msgpass
+	fi
 }
 
 TESTFILE='aptarchive/testfile'
 cp -a ${TESTDIR}/framework $TESTFILE
+cp -a ${TESTDIR}/framework "${TESTFILE}2"
+
+followuprequest() {
+	local DOWN='./testfile'
+
+	copysource $TESTFILE 1M $DOWN
+	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
+	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+
+	copysource $TESTFILE 1M $DOWN
+	copysource "${TESTFILE}2" 20 "${DOWN}2"
+	msgtest 'Testing download of files with' 'completely downloaded file + partial file'
+	testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \
+		download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2"
+	testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output'
+	testsuccess diff -u "$TESTFILE" "${DOWN}"
+	testsuccess diff -u "${DOWN}" "${DOWN}2"
+}
 
 testrun() {
 	webserverconfig 'aptwebserver::support::range' 'true'
@@ -65,9 +103,11 @@ testrun() {
 	testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
 	testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 
-	copysource $TESTFILE 1M ./testfile
-	testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '='
-	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'true'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
 
 	copysource /dev/zero 1M ./testfile
 	testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='
@@ -85,8 +125,18 @@ testrun() {
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 
+msgmsg 'http: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'http://localhost:8080'
+msgmsg 'http: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'http://localhost:8080'
 
 changetohttpswebserver
 
+msgmsg 'https: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'https://localhost:4433'
+msgmsg 'https: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'https://localhost:4433'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 34476e1af..cd52da692 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -19,6 +19,8 @@
 #include 
 #include 
 #include 
+
+#include 
 #include 
 #include 
 #include 
@@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode)		/*{{{*/
    return NULL;
 }
 									/*}}}*/
+static bool chunkedTransferEncoding(std::list const &headers) {
+   if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end())
+      return true;
+   if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true)
+      return true;
+   return false;
+}
 static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.FileSize();
-   headers.push_back(contentlength.str());
-
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.FileSize();
+      headers.push_back(contentlength.str());
+   }
    std::string lastmodified("Last-Modified: ");
    lastmodified.append(TimeRFC1123(data.ModificationTime()));
    headers.push_back(lastmodified);
@@ -92,9 +103,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
 									/*}}}*/
 static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.size();
-   headers.push_back(contentlength.str());
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.size();
+      headers.push_back(contentlength.str());
+   }
 }
 									/*}}}*/
 static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/
@@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl;
    bool Success = true;
    for (std::list::const_iterator h = headers.begin();
@@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list const &headers, FileFd &data)/*{{{*/
 {
    bool Success = true;
+   bool const chunked = chunkedTransferEncoding(headers);
    char buffer[500];
    unsigned long long actual = 0;
    while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true)
    {
       if (actual == 0)
 	 break;
-      Success &= FileFd::Write(client, buffer, actual);
+
+      if (chunked == true)
+      {
+	 std::string size;
+	 strprintf(size, "%llX\r\n", actual);
+	 Success &= FileFd::Write(client, size.c_str(), size.size());
+	 Success &= FileFd::Write(client, buffer, actual);
+	 Success &= FileFd::Write(client, "\r\n", strlen("\r\n"));
+      }
+      else
+	 Success &= FileFd::Write(client, buffer, actual);
+   }
+   if (chunked == true)
+   {
+      char const * const finish = "0\r\n\r\n";
+      Success &= FileFd::Write(client, finish, strlen(finish));
    }
    if (Success == false)
-      std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl;
+      std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl;
    return Success;
 }
 									/*}}}*/
-static bool sendData(int const client, std::string const &data)		/*{{{*/
+static bool sendData(int const client, std::list const &headers, std::string const &data)/*{{{*/
 {
-   if (FileFd::Write(client, data.c_str(), data.size()) == false)
+   if (chunkedTransferEncoding(headers) == true)
+   {
+      unsigned long long const ullsize = data.length();
+      std::string size;
+      strprintf(size, "%llX\r\n", ullsize);
+      char const * const finish = "\r\n0\r\n\r\n";
+      if (FileFd::Write(client, size.c_str(), size.length()) == false ||
+	    FileFd::Write(client, data.c_str(), ullsize) == false ||
+	    FileFd::Write(client, finish, strlen(finish)) == false)
+      {
+	 std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl;
+	 return false;
+      }
+   }
+   else if (FileFd::Write(client, data.c_str(), data.size()) == false)
    {
       std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl;
       return false;
@@ -157,34 +204,38 @@ static bool sendData(int const client, std::string const &data)		/*{{{*/
 }
 									/*}}}*/
 static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   std::list headers;
    std::string response("");
    response.append(httpcodeToStr(httpcode)).append("");
    response.append("
").append(httpcodeToStr(httpcode)).append("
");
    if (httpcode != 200)
-   {
-      if (error.empty() == false)
-	 response.append("
Error: ").append(error).append("
");
-      response.append("This error is a result of the request: 
");
-   }
+      response.append("
Error: ");
+   else
+      response.append("
Success: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("
This error is a result of the request: 
");
    else
-   {
-      if (error.empty() == false)
-	 response.append("
Success: ").append(error).append("
");
       response.append("The successfully executed operation was requested by: 
");
-   }
    response.append(request).append("
");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -221,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -263,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list &headers)
 {
-   std::list headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -311,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string ¶ms, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -333,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -345,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -360,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -371,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
       // strip the host from the request to make it an absolute path
@@ -379,7 +429,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -398,7 +448,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -434,46 +485,45 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector const &parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector parts, std::list &headers)
 {
    size_t const pcount = parts.size();
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -482,18 +532,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -502,7 +556,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -534,7 +588,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -553,7 +607,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -609,17 +663,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -628,22 +681,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From 3bf4c93ac241e270c374d1a49be11fb4d673abc1 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 9 Mar 2015 16:00:23 +0100
Subject: (error) Same iterator is used with different containers

cppcheck reports this error, its not really a problem for us as the API
can actually deal with it via implicit conversion, but being explicit
can't hurt and the less reported errors the better.

Git-Dch: Ignore
---
 cmdline/apt-mark.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc
index 8974397c9..47777009b 100644
--- a/cmdline/apt-mark.cc
+++ b/cmdline/apt-mark.cc
@@ -279,7 +279,7 @@ static bool DoHold(CommandLine &CmdL)
    {
       if (Pkg->CurrentVer != 0)
 	 continue;
-      keepoffset.insert(Pkg);
+      keepoffset.insert(*Pkg);
    }
 
    if (keepoffset.empty() == false)
-- 
cgit v1.2.3-70-g09d2


From 249aec3b7397662a678ea0014f94392085477b09 Mon Sep 17 00:00:00 2001
From: Jérémy Bobbio 
Date: Tue, 10 Mar 2015 10:09:44 +0100
Subject: stop displaying time of build in online help
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

As part of the “reproducible builds” effort [1], we have noticed that
apt could not be built reproducibly.

One issue is that it uses the __DATE__ and __TIME__ macros of the C
preprocessor to display the time of build in the online help. We believe
this information not to be really useful to users as they can always
look at the package data and metadata to figure it out.

The attached patch simply removes this information. All
non-documentation packages can then be built reproducibly with our
current experimental framework.

[David: changed the string slightly to be untranslateable as well]

Closes: 774342
---
 cmdline/apt-cache.cc                        | 5 ++---
 cmdline/apt-cdrom.cc                        | 4 ++--
 cmdline/apt-config.cc                       | 5 ++---
 cmdline/apt-dump-solver.cc                  | 3 +--
 cmdline/apt-extracttemplates.cc             | 3 +--
 cmdline/apt-get.cc                          | 9 ++++-----
 cmdline/apt-helper.cc                       | 3 +--
 cmdline/apt-internal-solver.cc              | 3 +--
 cmdline/apt-mark.cc                         | 3 +--
 cmdline/apt-sortpkgs.cc                     | 3 +--
 cmdline/apt.cc                              | 3 +--
 ftparchive/apt-ftparchive.cc                | 3 +--
 test/integration/test-00-commands-have-help | 2 +-
 13 files changed, 19 insertions(+), 30 deletions(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index a5024c581..e2cf7e8b7 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -1824,9 +1824,8 @@ static bool GenCaches(CommandLine &)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
-   
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
      return true;
 
diff --git a/cmdline/apt-cdrom.cc b/cmdline/apt-cdrom.cc
index 327039e00..d95c169cd 100644
--- a/cmdline/apt-cdrom.cc
+++ b/cmdline/apt-cdrom.cc
@@ -205,8 +205,8 @@ static bool DoIdent(CommandLine &)
 // ShowHelp - Show the help screen					/*{{{*/
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
       return true;
    
diff --git a/cmdline/apt-config.cc b/cmdline/apt-config.cc
index e0b8a624e..4479b84a7 100644
--- a/cmdline/apt-config.cc
+++ b/cmdline/apt-config.cc
@@ -80,11 +80,10 @@ static bool DoDump(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
-   
+
    cout <<
     _("Usage: apt-config [options] command\n"
       "\n"
diff --git a/cmdline/apt-dump-solver.cc b/cmdline/apt-dump-solver.cc
index f765234c5..4729eac55 100644
--- a/cmdline/apt-dump-solver.cc
+++ b/cmdline/apt-dump-solver.cc
@@ -21,9 +21,8 @@
 // ---------------------------------------------------------------------
 /* */
 static bool ShowHelp() {
-
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 	std::cout <<
-		PACKAGE " " PACKAGE_VERSION " for " COMMON_ARCH " compiled on " __DATE__ " " __TIME__ << std::endl <<
 		"Usage: apt-dump-resolver\n"
 		"\n"
 		"apt-dump-resolver is a dummy solver who just dumps its input to the\n"
diff --git a/cmdline/apt-extracttemplates.cc b/cmdline/apt-extracttemplates.cc
index 5211ee027..3e4f89286 100644
--- a/cmdline/apt-extracttemplates.cc
+++ b/cmdline/apt-extracttemplates.cc
@@ -219,8 +219,7 @@ bool DebFile::ParseInfo()
 /* */
 static bool ShowHelp(CommandLine &)
 {
-	ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
 	if (_config->FindB("version") == true)
 		return true;
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index 33bbdd861..a0d78257c 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -1607,13 +1607,12 @@ static bool DoChangelog(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
-	    
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
+
    if (_config->FindB("version") == true)
    {
       cout << _("Supported modules:") << endl;
-      
+
       for (unsigned I = 0; I != pkgVersioningSystem::GlobalListLen; I++)
       {
 	 pkgVersioningSystem *VS = pkgVersioningSystem::GlobalList[I];
@@ -1622,7 +1621,7 @@ static bool ShowHelp(CommandLine &)
 	 else
 	    cout << ' ';
 	 cout << "Ver: " << VS->Label << endl;
-	 
+
 	 /* Print out all the packaging systems that will work with 
 	    this VS */
 	 for (unsigned J = 0; J != pkgSystem::GlobalListLen; J++)
diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index a05ae90a2..65db857ad 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -83,8 +83,7 @@ static bool DoDownloadFile(CommandLine &CmdL)
 
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(std::cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    if (_config->FindB("version") == true)
      return true;
diff --git a/cmdline/apt-internal-solver.cc b/cmdline/apt-internal-solver.cc
index 4fabeb02f..939061b93 100644
--- a/cmdline/apt-internal-solver.cc
+++ b/cmdline/apt-internal-solver.cc
@@ -44,8 +44,7 @@
 // ---------------------------------------------------------------------
 /* */
 static bool ShowHelp(CommandLine &) {
-	ioprintf(std::cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-		 COMMON_ARCH,__DATE__,__TIME__);
+	ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
 	std::cout <<
 		_("Usage: apt-internal-solver\n"
diff --git a/cmdline/apt-mark.cc b/cmdline/apt-mark.cc
index 47777009b..de1c80309 100644
--- a/cmdline/apt-mark.cc
+++ b/cmdline/apt-mark.cc
@@ -443,8 +443,7 @@ static bool ShowHold(CommandLine &CmdL)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    cout <<
     _("Usage: apt-mark [options] {auto|manual} pkg1 [pkg2 ...]\n"
diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc
index 9b66ad4db..971900e4f 100644
--- a/cmdline/apt-sortpkgs.cc
+++ b/cmdline/apt-sortpkgs.cc
@@ -146,8 +146,7 @@ static bool DoIt(string InFile)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(std::cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
    
diff --git a/cmdline/apt.cc b/cmdline/apt.cc
index 056cd213f..2f7eddb61 100644
--- a/cmdline/apt.cc
+++ b/cmdline/apt.cc
@@ -39,8 +39,7 @@
 
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(c1out,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(c1out, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
 
    // FIXME: generate from CommandLine
    c1out << 
diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index adf1b6d73..69b936dff 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -616,8 +616,7 @@ static void LoadBinDir(vector &PkgList,Configuration &Setup)
 /* */
 static bool ShowHelp(CommandLine &)
 {
-   ioprintf(cout,_("%s %s for %s compiled on %s %s\n"),PACKAGE,PACKAGE_VERSION,
-	    COMMON_ARCH,__DATE__,__TIME__);
+   ioprintf(cout, "%s %s (%s)\n", PACKAGE, PACKAGE_VERSION, COMMON_ARCH);
    if (_config->FindB("version") == true)
       return true;
 
diff --git a/test/integration/test-00-commands-have-help b/test/integration/test-00-commands-have-help
index bbd1475eb..f2317dbdf 100755
--- a/test/integration/test-00-commands-have-help
+++ b/test/integration/test-00-commands-have-help
@@ -13,7 +13,7 @@ configarchitecture 'amd64'
 export LD_BIND_NOW=1
 
 checkversionmessage() {
-	testsuccess grep '^apt .* compiled on ' ${1}-help.output
+	testsuccess grep '^apt .* (' ${1}-help.output
 }
 
 checkhelpmessage() {
-- 
cgit v1.2.3-70-g09d2


From d5cf8851753dde4f45bfd3b48fcdf34247a8752a Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 7 Apr 2015 22:34:34 +0200
Subject: keyids in "apt-key del" should be case-insensitive

gnupg is case-insensitive about keyids, so back then apt-key called it
directly any keyid was accepted, but now that we work more with the
keyid ourself we regressed to require uppercase keyids by accident.

This is also inconsistent with other apt-key commands which still use
gnupg directly. A single case-insensitive grep and we are fine again.

Closes: 781696
---
 cmdline/apt-key.in            | 2 +-
 test/integration/test-apt-key | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-key.in b/cmdline/apt-key.in
index b4e071000..1da311d35 100644
--- a/cmdline/apt-key.in
+++ b/cmdline/apt-key.in
@@ -180,7 +180,7 @@ update() {
 remove_key_from_keyring() {
     local GPG="$GPG_CMD --keyring $1"
     # check if the key is in this keyring: the key id is in the 5 column at the end
-    if ! $GPG --with-colons --list-keys 2>&1 | grep -q "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then
+    if ! $GPG --with-colons --list-keys 2>&1 | grep -iq "^pub:[^:]*:[^:]*:[^:]*:[0-9A-F]*$2:"; then
 	return
     fi
     if [ ! -w "$1" ]; then
diff --git a/test/integration/test-apt-key b/test/integration/test-apt-key
index 47230cb55..b4f823ef1 100755
--- a/test/integration/test-apt-key
+++ b/test/integration/test-apt-key
@@ -111,3 +111,9 @@ cleanplate
 cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
 testsuccess --nomsg aptkey --fakeroot del 5A90D141DBAC8DAE
 testempty aptkey list
+
+msgtest 'Test key removal with' 'lowercase key ID' #keylength somewher between 8byte and short
+cleanplate
+cp -a keys/joesixpack.pub rootdir/etc/apt/trusted.gpg.d/joesixpack.gpg
+testsuccess --nomsg aptkey --fakeroot del d141dbac8dae
+testempty aptkey list
-- 
cgit v1.2.3-70-g09d2


From dfad5beea77d75983f6ff8a1b8296b74dd48203e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sun, 15 Mar 2015 22:34:54 +0100
Subject: add a simple unit test for acquire progress

This isn't testing much of the 'complex' parts,
but its better than nothing for now.

Git-Dch: Ignore
---
 apt-private/acqprogress.cc      |  53 ++++++-------
 apt-private/acqprogress.h       |   4 +-
 apt-private/private-install.cc  |   2 +-
 apt-private/private-update.cc   |   4 +-
 cmdline/apt-get.cc              |   6 +-
 cmdline/apt-helper.cc           |   2 +-
 test/libapt/acqprogress_test.cc | 170 ++++++++++++++++++++++++++++++++++++++++
 7 files changed, 205 insertions(+), 36 deletions(-)
 create mode 100644 test/libapt/acqprogress_test.cc

(limited to 'cmdline')

diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index e834d7d6a..0c606e48e 100644
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -1,10 +1,9 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acqprogress.cc,v 1.24 2003/04/27 01:56:48 doogie Exp $
 /* ######################################################################
 
-   Acquire Progress - Command line progress meter 
-   
+   Acquire Progress - Command line progress meter
+
    ##################################################################### */
 									/*}}}*/
 // Include files							/*{{{*/
@@ -29,13 +28,11 @@
 #include 
 									/*}}}*/
 
-using namespace std;
-
 // AcqTextStatus::AcqTextStatus - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-AcqTextStatus::AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet) :
-    pkgAcquireStatus(), ScreenWidth(ScreenWidth), LastLineLength(0), ID(0), Quiet(Quiet)
+AcqTextStatus::AcqTextStatus(std::ostream &out, unsigned int &ScreenWidth,unsigned int const Quiet) :
+    pkgAcquireStatus(), out(out), ScreenWidth(ScreenWidth), LastLineLength(0), ID(0), Quiet(Quiet)
 {
    // testcases use it to disable pulses without disabling other user messages
    if (Quiet == 0 && _config->FindB("quiet::NoUpdate", false) == true)
@@ -62,8 +59,8 @@ void AcqTextStatus::IMSHit(pkgAcquire::ItemDesc &Itm)
 
    clearLastLine();
 
-   cout << _("Hit ") << Itm.Description;
-   cout << endl;
+   out << _("Hit ") << Itm.Description;
+   out << std::endl;
    Update = true;
 }
 									/*}}}*/
@@ -83,10 +80,10 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
 
    clearLastLine();
 
-   cout << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
+   out << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
    if (Itm.Owner->FileSize != 0)
-      cout << " [" << SizeToStr(Itm.Owner->FileSize) << "B]";
-   cout << endl;
+      out << " [" << SizeToStr(Itm.Owner->FileSize) << "B]";
+   out << std::endl;
 }
 									/*}}}*/
 // AcqTextStatus::Done - Completed a download				/*{{{*/
@@ -113,15 +110,15 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
 
    if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
    {
-      cout << _("Ign ") << Itm.Description << endl;
+      out << _("Ign ") << Itm.Description << std::endl;
       if (Itm.Owner->ErrorText.empty() == false &&
 	    _config->FindB("Acquire::Progress::Ignore::ShowErrorText", false) == true)
-	 cout << "  " << Itm.Owner->ErrorText << endl;
+	 out << "  " << Itm.Owner->ErrorText << std::endl;
    }
    else
    {
-      cout << _("Err ") << Itm.Description << endl;
-      cout << "  " << Itm.Owner->ErrorText << endl;
+      out << _("Err ") << Itm.Description << std::endl;
+      out << "  " << Itm.Owner->ErrorText << std::endl;
    }
 
    Update = true;
@@ -143,7 +140,7 @@ void AcqTextStatus::Stop()
       return;
 
    if (FetchedBytes != 0 && _error->PendingError() == false)
-      ioprintf(cout,_("Fetched %sB in %s (%sB/s)\n"),
+      ioprintf(out,_("Fetched %sB in %s (%sB/s)\n"),
 	       SizeToStr(FetchedBytes).c_str(),
 	       TimeToStr(ElapsedTime).c_str(),
 	       SizeToStr(CurrentCPS).c_str());
@@ -152,7 +149,7 @@ void AcqTextStatus::Stop()
 // AcqTextStatus::Pulse - Regular event pulse				/*{{{*/
 // ---------------------------------------------------------------------
 /* This draws the current progress. Each line has an overall percent
-   meter and a per active item status meter along with an overall 
+   meter and a per active item status meter along with an overall
    bandwidth and ETA indicator. */
 bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 {
@@ -250,14 +247,14 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 
    // Draw the current status
    if (_config->FindB("Apt::Color", false) == true)
-      cout << _config->Find("APT::Color::Yellow");
+      out << _config->Find("APT::Color::Yellow");
    if (LastLineLength > Line.length())
       clearLastLine();
    else
-      cout << '\r';
-   cout << Line << flush;
+      out << '\r';
+   out << Line << std::flush;
    if (_config->FindB("Apt::Color", false) == true)
-      cout << _config->Find("APT::Color::Neutral") << flush;
+      out << _config->Find("APT::Color::Neutral") << std::flush;
 
    LastLineLength = Line.length();
    Update = false;
@@ -268,7 +265,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 // AcqTextStatus::MediaChange - Media need to be swapped		/*{{{*/
 // ---------------------------------------------------------------------
 /* Prompt for a media swap */
-bool AcqTextStatus::MediaChange(string Media,string Drive)
+bool AcqTextStatus::MediaChange(std::string Media, std::string Drive)
 {
    // If we do not output on a terminal and one of the options to avoid user
    // interaction is given, we assume that no user is present who could react
@@ -281,7 +278,7 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
       return false;
 
    clearLastLine();
-   ioprintf(cout,_("Media change: please insert the disc labeled\n"
+   ioprintf(out,_("Media change: please insert the disc labeled\n"
 		   " '%s'\n"
 		   "in the drive '%s' and press enter\n"),
 	    Media.c_str(),Drive.c_str());
@@ -301,16 +298,16 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
 }
 									/*}}}*/
 void AcqTextStatus::clearLastLine() {					/*{{{*/
-   if (Quiet > 0)
+   if (Quiet > 0 || LastLineLength == 0)
       return;
 
    // do not try to clear more than the (now smaller) screen
    if (LastLineLength > ScreenWidth)
       LastLineLength = ScreenWidth;
 
-   std::cout << '\r';
+   out << '\r';
    for (size_t i = 0; i < LastLineLength; ++i)
-      std::cout << ' ';
-   std::cout << '\r' << std::flush;
+      out << ' ';
+   out << '\r' << std::flush;
 }
 									/*}}}*/
diff --git a/apt-private/acqprogress.h b/apt-private/acqprogress.h
index ab8170126..7cf990c65 100644
--- a/apt-private/acqprogress.h
+++ b/apt-private/acqprogress.h
@@ -13,9 +13,11 @@
 #include 
 
 #include 
+#include 
 
 class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
 {
+   std::ostream &out;
    unsigned int &ScreenWidth;
    size_t LastLineLength;
    unsigned long ID;
@@ -35,7 +37,7 @@ class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
 
    bool Pulse(pkgAcquire *Owner);
 
-   AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet);
+   AcqTextStatus(std::ostream &out, unsigned int &ScreenWidth,unsigned int const Quiet);
 };
 
 #endif
diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc
index 79a22edf4..acc6d42c2 100644
--- a/apt-private/private-install.cc
+++ b/apt-private/private-install.cc
@@ -116,7 +116,7 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
       return false;
 
    // Create the download object
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
    if (_config->FindB("APT::Get::Print-URIs", false) == true)
    {
diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc
index df77ac33a..73a82e988 100644
--- a/apt-private/private-update.cc
+++ b/apt-private/private-update.cc
@@ -38,8 +38,8 @@ bool DoUpdate(CommandLine &CmdL)
    pkgSourceList *List = Cache.GetSourceList();
 
    // Create the progress
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
-      
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
+
    // Just print out the uris an exit if the --print-uris flag was used
    if (_config->FindB("APT::Get::Print-URIs") == true)
    {
diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index a0d78257c..1a4f4c5a8 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -624,7 +624,7 @@ static bool DoDownload(CommandLine &CmdL)
    if (verset.empty() == true)
       return false;
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet", 0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    pkgRecords Recs(Cache);
@@ -729,7 +729,7 @@ static bool DoSource(CommandLine &CmdL)
       return false;
 
    // Create the download object
-   AcqTextStatus Stat(ScreenWidth,_config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    SPtrArray Dsc = new DscFile[CmdL.FileSize()];
@@ -1550,7 +1550,7 @@ static bool DoChangelog(CommandLine &CmdL)
       return Success;
    }
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    Fetcher.SetLog(&Stat);
 
    bool const downOnly = _config->FindB("APT::Get::Download-Only", false);
diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index 65db857ad..a97fc903f 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,7 +48,7 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
-   AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
+   AcqTextStatus Stat(std::cout, ScreenWidth,_config->FindI("quiet",0));
    pkgAcquire Fetcher(&Stat);
 
    size_t fileind = 0;
diff --git a/test/libapt/acqprogress_test.cc b/test/libapt/acqprogress_test.cc
new file mode 100644
index 000000000..288e05aca
--- /dev/null
+++ b/test/libapt/acqprogress_test.cc
@@ -0,0 +1,170 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+class TestItem: public pkgAcquire::Item
+{
+public:
+   TestItem(pkgAcquire * const Acq) : pkgAcquire::Item(Acq, "", NULL) {}
+
+   virtual std::string DescURI() { return ""; }
+
+};
+
+TEST(AcqProgress, IMSHit)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire::ItemDesc hit;
+   hit.URI = "http://example.org/file";
+   hit.Description = "Example File from example.org";
+   hit.ShortDesc = "Example File";
+   hit.Owner = NULL;
+
+   EXPECT_EQ("", out.str());
+   Stat.IMSHit(hit);
+   EXPECT_EQ("Hit Example File from example.org\n", out.str());
+   Stat.IMSHit(hit);
+   EXPECT_EQ("Hit Example File from example.org\n"
+	     "Hit Example File from example.org\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Hit Example File from example.org\n"
+	     "Hit Example File from example.org\n", out.str());
+}
+TEST(AcqProgress, FetchNoFileSize)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org\n", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org\n"
+	     "Get:2 Example File from example.org\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Get:1 Example File from example.org\n"
+	     "Get:2 Example File from example.org\n", out.str());
+}
+TEST(AcqProgress, FetchFileSize)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n", out.str());
+   fetchO.FileSize = 42;
+   Stat.Fetch(fetch);
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n"
+	     "Get:2 Example File from example.org [42 B]\n", out.str());
+   Stat.Stop();
+   EXPECT_EQ("Get:1 Example File from example.org [100 B]\n"
+	     "Get:2 Example File from example.org [42 B]\n", out.str());
+}
+TEST(AcqProgress, Fail)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   Stat.Start();
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetchO.Status = pkgAcquire::Item::StatIdle;
+   fetch.Owner = &fetchO;
+
+   EXPECT_EQ("", out.str());
+   Stat.Fail(fetch);
+   EXPECT_EQ("", out.str());
+   fetchO.Status = pkgAcquire::Item::StatDone;
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n", out.str());
+   fetchO.Status = pkgAcquire::Item::StatError;
+   fetchO.ErrorText = "An error test!";
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n", out.str());
+   _config->Set("Acquire::Progress::Ignore::ShowErrorText", true);
+   fetchO.Status = pkgAcquire::Item::StatDone;
+   Stat.Fail(fetch);
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n"
+	     "Ign Example File from example.org\n"
+	     "  An error test!\n", out.str());
+   _config->Set("Acquire::Progress::Ignore::ShowErrorText", true);
+   Stat.Stop();
+   EXPECT_EQ("Ign Example File from example.org\n"
+	     "Err Example File from example.org\n"
+	     "  An error test!\n"
+	     "Ign Example File from example.org\n"
+	     "  An error test!\n", out.str());
+}
+TEST(AcqProgress, Pulse)
+{
+   std::ostringstream out;
+   unsigned int width = 80;
+   AcqTextStatus Stat(out, width, 0);
+   _config->Set("APT::Sandbox::User", ""); // ensure we aren't sandboxing
+
+   pkgAcquire Acq(&Stat);
+   pkgAcquire::ItemDesc fetch;
+   fetch.URI = "http://example.org/file";
+   fetch.Description = "Example File from example.org";
+   fetch.ShortDesc = "Example File";
+   TestItem fetchO(&Acq);
+   fetchO.FileSize = 100;
+   fetchO.Status = pkgAcquire::Item::StatFetching;
+   fetch.Owner = &fetchO;
+
+   // make screen smaller and bigger again while running
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]", out.str());
+   width = 8;
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]"
+	     "\r        "
+	     "\r0% [Work", out.str());
+   width = 80;
+   EXPECT_TRUE(Stat.Pulse(&Acq));
+   EXPECT_EQ("\r0% [Working]"
+	     "\r        "
+	     "\r0% [Work"
+	     "\r0% [Working]", out.str());
+}
-- 
cgit v1.2.3-70-g09d2


From 353c135e45d3b76dbecc1ba1b2bd9266601181ee Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 9 May 2015 15:10:55 +0200
Subject: drop incorrect parameter implicitely converted to bool
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The helper expects to be told if it should generate messages, not where
these messages should be printed – as it isn't printing such messages,
but puts them in _error. apt-get uses in other methods a helper
specialisation which does also print stuff to a stream through, so this
is likely a copy&paste error.

Git-Dch: Ignore
---
 cmdline/apt-get.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'cmdline')

diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc
index 1a4f4c5a8..c1f78523c 100644
--- a/cmdline/apt-get.cc
+++ b/cmdline/apt-get.cc
@@ -617,7 +617,7 @@ static bool DoDownload(CommandLine &CmdL)
    if (Cache.ReadOnlyOpen() == false)
       return false;
 
-   APT::CacheSetHelper helper(c0out);
+   APT::CacheSetHelper helper;
    APT::VersionSet verset = APT::VersionSet::FromCommandLine(Cache,
 		CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper);
 
@@ -1534,7 +1534,7 @@ static bool DoChangelog(CommandLine &CmdL)
    if (Cache.ReadOnlyOpen() == false)
       return false;
    
-   APT::CacheSetHelper helper(c0out);
+   APT::CacheSetHelper helper;
    APT::VersionList verset = APT::VersionList::FromCommandLine(Cache,
 		CmdL.FileList + 1, APT::CacheSetHelper::CANDIDATE, helper);
    if (verset.empty() == true)
-- 
cgit v1.2.3-70-g09d2


From 88593886a42025d51d76051da5929b044e42efee Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 11 May 2015 15:08:08 +0200
Subject: rewrite all TFRewrite instances to use the new pkgTagSection::Write
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

While it is mostly busywork to rewrite all instances it actually fixes
bugs as the data storage used by the new method is std::string rather
than a char*, the later mostly created by c_str() from a std::string
which the caller has to ensure keeps in scope – something apt-ftparchive
actually didn't ensure and relied on copy-on-write behavior instead
which c++11 forbids and hence the new default gcc abi doesn't use it.
---
 apt-pkg/depcache.cc                       |  50 +++---
 apt-pkg/indexcopy.cc                      | 125 ++++++--------
 apt-pkg/indexcopy.h                       |   7 +-
 apt-private/private-show.cc               |  46 ++---
 cmdline/apt-cache.cc                      |  21 ++-
 cmdline/apt-sortpkgs.cc                   |  14 +-
 ftparchive/apt-ftparchive.cc              |  51 +++---
 ftparchive/contents.cc                    |  34 ++--
 ftparchive/contents.h                     |   7 +-
 ftparchive/multicompress.cc               |  19 +--
 ftparchive/multicompress.h                |   2 +-
 ftparchive/writer.cc                      | 204 +++++++++++-----------
 ftparchive/writer.h                       |  45 +++--
 test/integration/test-apt-cli-show        |   4 +-
 test/libapt/indexcopytosourcelist_test.cc |   2 +-
 test/libapt/tagsection_test.cc            | 270 ++++++++++++++++++++++++++++++
 16 files changed, 559 insertions(+), 342 deletions(-)
 create mode 100644 test/libapt/tagsection_test.cc

(limited to 'cmdline')

diff --git a/apt-pkg/depcache.cc b/apt-pkg/depcache.cc
index 73c5bb320..b73c336db 100644
--- a/apt-pkg/depcache.cc
+++ b/apt-pkg/depcache.cc
@@ -33,7 +33,6 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 
 #include 
@@ -252,17 +251,14 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
       return _error->Error(_("Failed to open StateFile %s"),
 			   state.c_str());
 
-   FILE *OutFile;
-   string const outfile = state + ".tmp";
-   if((OutFile = fopen(outfile.c_str(),"w")) == NULL)
-      return _error->Error(_("Failed to write temporary StateFile %s"),
-			   outfile.c_str());
+   FileFd OutFile(state, FileFd::ReadWrite | FileFd::Atomic);
+   if (OutFile.IsOpen() == false || OutFile.Failed() == true)
+      return _error->Error(_("Failed to write temporary StateFile %s"), state.c_str());
 
    // first merge with the existing sections
    pkgTagFile tagfile(&StateFile);
    pkgTagSection section;
    std::set pkgs_seen;
-   const char *nullreorderlist[] = {0};
    while(tagfile.Step(section)) {
 	 string const pkgname = section.FindS("Package");
 	 string pkgarch = section.FindS("Architecture");
@@ -271,7 +267,7 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	 // Silently ignore unknown packages and packages with no actual
 	 // version.
 	 pkgCache::PkgIterator pkg = Cache->FindPkg(pkgname, pkgarch);
-	 if(pkg.end() || pkg.VersionList().end()) 
+	 if(pkg.end() || pkg.VersionList().end())
 	    continue;
 	 StateCache const &P = PkgState[pkg->ID];
 	 bool newAuto = (P.Flags & Flag::Auto);
@@ -292,21 +288,17 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	 if(_config->FindB("Debug::pkgAutoRemove",false))
 	    std::clog << "Update existing AutoInstall info: " 
 		      << pkg.FullName() << std::endl;
-	 TFRewriteData rewrite[3];
-	 rewrite[0].Tag = "Architecture";
-	 rewrite[0].Rewrite = pkg.Arch();
-	 rewrite[0].NewTag = 0;
-	 rewrite[1].Tag = "Auto-Installed";
-	 rewrite[1].Rewrite = newAuto ? "1" : "0";
-	 rewrite[1].NewTag = 0;
-	 rewrite[2].Tag = 0;
-	 TFRewrite(OutFile, section, nullreorderlist, rewrite);
-	 fprintf(OutFile,"\n");
+
+	 std::vector rewrite;
+	 rewrite.push_back(pkgTagSection::Tag::Rewrite("Architecture", pkg.Arch()));
+	 rewrite.push_back(pkgTagSection::Tag::Rewrite("Auto-Installed", newAuto ? "1" : "0"));
+	 section.Write(OutFile, NULL, rewrite);
+	 if (OutFile.Write("\n", 1) == false)
+	    return false;
 	 pkgs_seen.insert(pkg.FullName());
    }
-   
+
    // then write the ones we have not seen yet
-   std::ostringstream ostr;
    for(pkgCache::PkgIterator pkg=Cache->PkgBegin(); !pkg.end(); ++pkg) {
       StateCache const &P = PkgState[pkg->ID];
       if(P.Flags & Flag::Auto) {
@@ -325,19 +317,17 @@ bool pkgDepCache::writeStateFile(OpProgress * /*prog*/, bool InstalledOnly)	/*{{
 	    continue;
 	 if(debug_autoremove)
 	    std::clog << "Writing new AutoInstall: " << pkg.FullName() << std::endl;
-	 ostr.str(string(""));
-	 ostr << "Package: " << pkg.Name()
-	      << "\nArchitecture: " << pkgarch
-	      << "\nAuto-Installed: 1\n\n";
-	 fprintf(OutFile,"%s",ostr.str().c_str());
+	 std::string stanza = "Package: ";
+	 stanza.append(pkg.Name())
+	      .append("\nArchitecture: ").append(pkgarch)
+	      .append("\nAuto-Installed: 1\n\n");
+	 if (OutFile.Write(stanza.c_str(), stanza.length()) == false)
+	    return false;
       }
    }
-   fclose(OutFile);
-
-   // move the outfile over the real file and set permissions
-   rename(outfile.c_str(), state.c_str());
+   if (OutFile.Close() == false)
+      return false;
    chmod(state.c_str(), 0644);
-
    return true;
 }
 									/*}}}*/
diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc
index 144c508be..461aa4217 100644
--- a/apt-pkg/indexcopy.cc
+++ b/apt-pkg/indexcopy.cc
@@ -108,10 +108,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
       }
       if (_error->PendingError() == true)
 	 return false;
-      FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
-      if (TargetFl == 0)
-	 return _error->Errno("fdopen","Failed to reopen fd");
-      
+
       // Setup the progress meter
       if(Progress)
 	 Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
@@ -132,14 +129,11 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	 string File;
 	 unsigned long long Size;
 	 if (GetFile(File,Size) == false)
-	 {
-	    fclose(TargetFl);
 	    return false;
-	 }
-	 
+
 	 if (Chop != 0)
 	    File = OrigPath + ChopDirs(File,Chop);
-	 
+
 	 // See if the file exists
 	 if (NoStat == false || Hits < 10)
 	 {
@@ -157,10 +151,10 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	       if (Chop != 0)
 		  File = OrigPath + ChopDirs(File,Chop);
 	    }
-	    
+
 	    // Get the size
 	    struct stat Buf;
-	    if (stat((CDROM + Prefix + File).c_str(),&Buf) != 0 || 
+	    if (stat((CDROM + Prefix + File).c_str(),&Buf) != 0 ||
 		Buf.st_size == 0)
 	    {
 	       bool Mangled = false;
@@ -173,7 +167,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 		  File.replace(Start,End-Start,"binary-all");
 		  Mangled = true;
 	       }
-	       
+
 	       if (Mangled == false ||
 		   stat((CDROM + Prefix + File).c_str(),&Buf) != 0)
 	       {
@@ -181,9 +175,9 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 		     clog << "Missed(2): " << OrigFile << endl;
 		  NotFound++;
 		  continue;
-	       }	       
-	    }	    
-	    			    	    
+	       }
+	    }
+
 	    // Size match
 	    if ((unsigned long long)Buf.st_size != Size)
 	    {
@@ -193,21 +187,17 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	       continue;
 	    }
 	 }
-	 
+
 	 Packages++;
 	 Hits++;
-	 
-	 if (RewriteEntry(TargetFl,File) == false)
-	 {
-	    fclose(TargetFl);
+
+	 if (RewriteEntry(Target, File) == false)
 	    return false;
-	 }
       }
-      fclose(TargetFl);
 
       if (Debug == true)
 	 cout << " Processed by using Prefix '" << Prefix << "' and chop " << Chop << endl;
-	 
+
       if (_config->FindB("APT::CDROM::NoAct",false) == false)
       {
 	 // Move out of the partial directory
@@ -218,39 +208,38 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector &List,
 	    return _error->Errno("rename","Failed to rename");
 	 ChangeOwnerAndPermissionOfFile("CopyPackages", FinalF.c_str(), "root", "root", 0644);
       }
-	 
+
       /* Mangle the source to be in the proper notation with
-       	 prefix dist [component] */ 
+	 prefix dist [component] */
       *I = string(*I,Prefix.length());
       ConvertToSourceList(CDROM,*I);
       *I = Prefix + ' ' + *I;
-      
+
       CurrentSize += FileSize;
-   }   
+   }
    if(Progress)
       Progress->Done();
-   
+
    // Some stats
    if(log) {
       stringstream msg;
       if(NotFound == 0 && WrongSize == 0)
 	 ioprintf(msg, _("Wrote %i records.\n"), Packages);
       else if (NotFound != 0 && WrongSize == 0)
-	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 		  Packages, NotFound);
       else if (NotFound == 0 && WrongSize != 0)
-	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 		  Packages, WrongSize);
       if (NotFound != 0 && WrongSize != 0)
 	 ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
    }
-   
+
    if (Packages == 0)
       _error->Warning("No valid records were found.");
 
    if (NotFound + WrongSize > 10)
       _error->Warning("A lot of entries were discarded, something may be wrong.\n");
-   
 
    return true;
 }
@@ -267,10 +256,10 @@ string IndexCopy::ChopDirs(string Path,unsigned int Depth)
       Depth--;
    }
    while (I != string::npos && Depth != 0);
-   
+
    if (I == string::npos)
       return string();
-   
+
    return string(Path,I+1);
 }
 									/*}}}*/
@@ -433,17 +422,15 @@ bool PackageCopy::GetFile(string &File,unsigned long long &Size)
 }
 									/*}}}*/
 // PackageCopy::RewriteEntry - Rewrite the entry with a new filename	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool PackageCopy::RewriteEntry(FILE *Target,string File)
+bool PackageCopy::RewriteEntry(FileFd &Target,string const &File)
 {
-   TFRewriteData Changes[] = {{ "Filename", File.c_str(), NULL },
-                              { NULL, NULL, NULL }};
-   
-   if (TFRewrite(Target,*Section,TFRewritePackageOrder,Changes) == false)
+   string const Dir(File,0,File.rfind('/'));
+   std::vector Changes;
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", File));
+
+   if (Section->Write(Target, TFRewritePackageOrder, Changes) == false)
       return false;
-   fputc('\n',Target);
-   return true;
+   return Target.Write("\n", 1);
 }
 									/*}}}*/
 // SourceCopy::GetFile - Get the file information from the section	/*{{{*/
@@ -478,23 +465,18 @@ bool SourceCopy::GetFile(string &File,unsigned long long &Size)
 }
 									/*}}}*/
 // SourceCopy::RewriteEntry - Rewrite the entry with a new filename	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool SourceCopy::RewriteEntry(FILE *Target,string File)
+bool SourceCopy::RewriteEntry(FileFd &Target, std::string const &File)
 {
-   string Dir(File,0,File.rfind('/'));
-   TFRewriteData Changes[] = {{ "Directory", Dir.c_str(), NULL },
-                              { NULL, NULL, NULL }};
-   
-   if (TFRewrite(Target,*Section,TFRewriteSourceOrder,Changes) == false)
+   string const Dir(File,0,File.rfind('/'));
+   std::vector Changes;
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Dir));
+
+   if (Section->Write(Target, TFRewriteSourceOrder, Changes) == false)
       return false;
-   fputc('\n',Target);
-   return true;
+   return Target.Write("\n", 1);
 }
 									/*}}}*/
-// SigVerify::Verify - Verify a files md5sum against its metaindex     	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
+// SigVerify::Verify - Verify a files md5sum against its metaindex	/*{{{*/
 bool SigVerify::Verify(string prefix, string file, indexRecords *MetaIndex)
 {
    const indexRecords::checkSum *Record = MetaIndex->Lookup(file);
@@ -702,7 +684,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
       pkgTagFile Parser(&Pkg);
       if (_error->PendingError() == true)
 	 return false;
-      
+
       // Open the output file
       char S[400];
       snprintf(S,sizeof(S),"cdrom:[%s]/%s",Name.c_str(),
@@ -719,10 +701,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
       }
       if (_error->PendingError() == true)
 	 return false;
-      FILE *TargetFl = fdopen(dup(Target.Fd()),"w");
-      if (TargetFl == 0)
-	 return _error->Errno("fdopen","Failed to reopen fd");
-      
+
       // Setup the progress meter
       if(Progress)
 	 Progress->OverallProgress(CurrentSize,TotalSize,FileSize,
@@ -740,20 +719,16 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
 	 if(Progress)
 	    Progress->Progress(Parser.Offset());
 
-	 const char *Start;
-	 const char *Stop;
-	 Section.GetSection(Start,Stop);
-	 fwrite(Start,Stop-Start, 1, TargetFl);
-	 fputc('\n',TargetFl);
+	 if (Section.Write(Target) == false || Target.Write("\n", 1) == false)
+	    return false;
 
 	 Packages++;
 	 Hits++;
       }
-      fclose(TargetFl);
 
       if (Debug == true)
 	 cout << " Processed by using Prefix '" << Prefix << "' and chop " << endl;
-	 
+
       if (_config->FindB("APT::CDROM::NoAct",false) == false)
       {
 	 // Move out of the partial directory
@@ -764,34 +739,32 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
 	    return _error->Errno("rename","Failed to rename");
 	 ChangeOwnerAndPermissionOfFile("CopyTranslations", FinalF.c_str(), "root", "root", 0644);
       }
-      
-      
+
       CurrentSize += FileSize;
-   }   
+   }
    if(Progress)
       Progress->Done();
-   
+
    // Some stats
    if(log) {
       stringstream msg;
       if(NotFound == 0 && WrongSize == 0)
 	 ioprintf(msg, _("Wrote %i records.\n"), Packages);
       else if (NotFound != 0 && WrongSize == 0)
-	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i missing files.\n"),
 		  Packages, NotFound);
       else if (NotFound == 0 && WrongSize != 0)
-	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"), 
+	 ioprintf(msg, _("Wrote %i records with %i mismatched files\n"),
 		  Packages, WrongSize);
       if (NotFound != 0 && WrongSize != 0)
 	 ioprintf(msg, _("Wrote %i records with %i missing files and %i mismatched files\n"), Packages, NotFound, WrongSize);
    }
-   
+
    if (Packages == 0)
       _error->Warning("No valid records were found.");
 
    if (NotFound + WrongSize > 10)
       _error->Warning("A lot of entries were discarded, something may be wrong.\n");
-   
 
    return true;
 }
diff --git a/apt-pkg/indexcopy.h b/apt-pkg/indexcopy.h
index 701beb075..729b0c8cb 100644
--- a/apt-pkg/indexcopy.h
+++ b/apt-pkg/indexcopy.h
@@ -28,6 +28,7 @@ using std::vector;
 class pkgTagSection;
 class indexRecords;
 class pkgCdromStatus;
+class FileFd;
 
 class IndexCopy								/*{{{*/
 {
@@ -45,7 +46,7 @@ class IndexCopy								/*{{{*/
    void ConvertToSourceList(std::string CD,std::string &Path);
    bool GrabFirst(std::string Path,std::string &To,unsigned int Depth);
    virtual bool GetFile(std::string &Filename,unsigned long long &Size) = 0;
-   virtual bool RewriteEntry(FILE *Target,std::string File) = 0;
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File) = 0;
    virtual const char *GetFileName() = 0;
    virtual const char *Type() = 0;
    
@@ -61,7 +62,7 @@ class PackageCopy : public IndexCopy					/*{{{*/
    protected:
    
    virtual bool GetFile(std::string &Filename,unsigned long long &Size);
-   virtual bool RewriteEntry(FILE *Target,std::string File);
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File);
    virtual const char *GetFileName() {return "Packages";};
    virtual const char *Type() {return "Package";};
    
@@ -72,7 +73,7 @@ class SourceCopy : public IndexCopy					/*{{{*/
    protected:
    
    virtual bool GetFile(std::string &Filename,unsigned long long &Size);
-   virtual bool RewriteEntry(FILE *Target,std::string File);
+   virtual bool RewriteEntry(FileFd &Target, std::string const &File);
    virtual const char *GetFileName() {return "Sources";};
    virtual const char *Type() {return "Source";};
    
diff --git a/apt-private/private-show.cc b/apt-private/private-show.cc
index 289f035a6..790bc0092 100644
--- a/apt-private/private-show.cc
+++ b/apt-private/private-show.cc
@@ -97,28 +97,30 @@ static bool DisplayRecord(pkgCacheFile &CacheFile, pkgCache::VerIterator V,
       manual_installed = 0;
 
    // FIXME: add verbose that does not do the removal of the tags?
-   TFRewriteData RW[] = {
-      // delete, apt-cache show has this info and most users do not care
-      {"MD5sum", NULL, NULL},
-      {"SHA1", NULL, NULL},
-      {"SHA256", NULL, NULL},
-      {"Filename", NULL, NULL},
-      {"Multi-Arch", NULL, NULL},
-      {"Architecture", NULL, NULL},
-      {"Conffiles", NULL, NULL},
-      // we use the translated description
-      {"Description", NULL, NULL},
-      {"Description-md5", NULL, NULL},
-      // improve
-      {"Installed-Size", installed_size.c_str(), NULL},
-      {"Size", package_size.c_str(), "Download-Size"},
-      // add
-      {"APT-Manual-Installed", manual_installed, NULL},
-      {"APT-Sources", source_index_file.c_str(), NULL},
-      {NULL, NULL, NULL}
-   };
-
-   if(TFRewrite(stdout, Tags, NULL, RW) == false)
+   std::vector RW;
+   // delete, apt-cache show has this info and most users do not care
+   RW.push_back(pkgTagSection::Tag::Remove("MD5sum"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA1"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA256"));
+   RW.push_back(pkgTagSection::Tag::Remove("SHA512"));
+   RW.push_back(pkgTagSection::Tag::Remove("Filename"));
+   RW.push_back(pkgTagSection::Tag::Remove("Multi-Arch"));
+   RW.push_back(pkgTagSection::Tag::Remove("Architecture"));
+   RW.push_back(pkgTagSection::Tag::Remove("Conffiles"));
+   // we use the translated description
+   RW.push_back(pkgTagSection::Tag::Remove("Description"));
+   RW.push_back(pkgTagSection::Tag::Remove("Description-md5"));
+   // improve
+   RW.push_back(pkgTagSection::Tag::Rewrite("Installed-Size", installed_size));
+   RW.push_back(pkgTagSection::Tag::Remove("Size"));
+   RW.push_back(pkgTagSection::Tag::Rewrite("Download-Size", package_size));
+   // add
+   RW.push_back(pkgTagSection::Tag::Rewrite("APT-Manual-Installed", manual_installed));
+   RW.push_back(pkgTagSection::Tag::Rewrite("APT-Sources", source_index_file));
+
+   FileFd stdoutfd;
+   if (stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false) == false ||
+	 Tags.Write(stdoutfd, TFRewritePackageOrder, RW) == false || stdoutfd.Close() == false)
       return _error->Error("Internal Error, Unable to parse a package record");
 
    // write the description
diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc
index e2cf7e8b7..690b03bcc 100644
--- a/cmdline/apt-cache.cc
+++ b/cmdline/apt-cache.cc
@@ -580,6 +580,12 @@ static bool DumpAvail(CommandLine &)
    
    LocalitySort(VFList,Count,sizeof(*VFList));
 
+   std::vector RW;
+   RW.push_back(pkgTagSection::Tag::Remove("Status"));
+   RW.push_back(pkgTagSection::Tag::Remove("Config-Version"));
+   FileFd stdoutfd;
+   stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
+
    // Iterate over all the package files and write them out.
    char *Buffer = new char[Cache->HeaderP->MaxVerFileSize+10];
    for (pkgCache::VerFile **J = VFList; *J != 0;)
@@ -620,35 +626,32 @@ static bool DumpAvail(CommandLine &)
 	 if (PkgF.Read(Buffer,VF.Size + Jitter) == false)
 	    break;
 	 Buffer[VF.Size + Jitter] = '\n';
-	 
+
 	 // See above..
 	 if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource)
 	 {
 	    pkgTagSection Tags;
-	    TFRewriteData RW[] = {{"Status", NULL, NULL},{"Config-Version", NULL, NULL},{NULL, NULL, NULL}};
-	    const char *Zero = 0;
 	    if (Tags.Scan(Buffer+Jitter,VF.Size+1) == false ||
-		TFRewrite(stdout,Tags,&Zero,RW) == false)
+		Tags.Write(stdoutfd, NULL, RW) == false ||
+		stdoutfd.Write("\n", 1) == false)
 	    {
 	       _error->Error("Internal Error, Unable to parse a package record");
 	       break;
 	    }
-	    fputc('\n',stdout);
 	 }
 	 else
 	 {
-	    if (fwrite(Buffer+Jitter,VF.Size+1,1,stdout) != 1)
+	    if (stdoutfd.Write(Buffer + Jitter, VF.Size + 1) == false)
 	       break;
 	 }
-	 
+
 	 Pos = VF.Offset + VF.Size;
       }
 
-      fflush(stdout);
       if (_error->PendingError() == true)
          break;
    }
-   
+
    delete [] Buffer;
    delete [] VFList;
    return !_error->PendingError();
diff --git a/cmdline/apt-sortpkgs.cc b/cmdline/apt-sortpkgs.cc
index 971900e4f..12ef8dda0 100644
--- a/cmdline/apt-sortpkgs.cc
+++ b/cmdline/apt-sortpkgs.cc
@@ -108,8 +108,10 @@ static bool DoIt(string InFile)
    const char **Order = TFRewritePackageOrder;
    if (Source == true)
       Order = TFRewriteSourceOrder;
-   
+
    // Emit
+   FileFd stdoutfd;
+   stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
    unsigned char *Buffer = new unsigned char[Largest+1];
    for (vector::iterator I = List.begin(); I != List.end(); ++I)
    {
@@ -119,8 +121,8 @@ static bool DoIt(string InFile)
 	 delete [] Buffer;
 	 return false;
       }
-      
-      Buffer[I->Length] = '\n';      
+
+      Buffer[I->Length] = '\n';
       if (Section.Scan((char *)Buffer,I->Length+1) == false)
       {
 	 delete [] Buffer;
@@ -128,15 +130,13 @@ static bool DoIt(string InFile)
       }
 
       // Sort the section
-      if (TFRewrite(stdout,Section,Order,0) == false)
+      if (Section.Write(stdoutfd, Order) == false || stdoutfd.Write("\n", 1) == false)
       {
 	 delete [] Buffer;
 	 return _error->Error("Internal error, failed to sort fields");
       }
-      
-      fputc('\n',stdout);      
    }
-   
+
    delete [] Buffer;
    return true;
 }
diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index 69b936dff..62108f7ca 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -178,7 +178,9 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    PkgDone = true;
    
    // Create a package writer object.
-   PackagesWriter Packages(flCombine(CacheDir,BinCacheDB),
+   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
+		      PkgCompress,Permissions);
+   PackagesWriter Packages(&Comp.Input, flCombine(CacheDir,BinCacheDB),
 			   flCombine(OverrideDir,BinOverride),
 			   flCombine(OverrideDir,ExtraOverride),
 			   Arch);
@@ -197,10 +199,6 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
    Packages.DeLinkLimit = DeLinkLimit;
 
-   // Create a compressor object
-   MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
-		      PkgCompress,Permissions);
-   Packages.Output = Comp.Input;
    if (_error->PendingError() == true)
       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
    
@@ -272,7 +270,9 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
    SrcDone = true;
    
    // Create a package writer object.
-   SourcesWriter Sources(flCombine(CacheDir, SrcCacheDB),
+   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
+		      SrcCompress,Permissions);
+   SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
 			 flCombine(OverrideDir,BinOverride),
 			 flCombine(OverrideDir,SrcOverride),
 			 flCombine(OverrideDir,SrcExtraOverride));
@@ -287,11 +287,7 @@ bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
 
    Sources.DeLinkLimit = DeLinkLimit;
    Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
-   
-   // Create a compressor object
-   MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
-		      SrcCompress,Permissions);
-   Sources.Output = Comp.Input;
+
    if (_error->PendingError() == true)
       return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
 
@@ -365,16 +361,15 @@ bool PackageMap::GenContents(Configuration &Setup,
    gettimeofday(&StartTime,0);   
    
    // Create a package writer object.
-   ContentsWriter Contents("", Arch);
+   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
+		      CntCompress,Permissions);
+   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
+   ContentsWriter Contents(&Comp.Input, "", Arch);
    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
       return false;
 
-   MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
-		      CntCompress,Permissions);
-   Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
-   Contents.Output = Comp.Input;
    if (_error->PendingError() == true)
       return false;
 
@@ -384,7 +379,7 @@ bool PackageMap::GenContents(Configuration &Setup,
       FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
       if (_error->PendingError() == true)
 	 return false;
-      
+
       unsigned long long Size = Head.Size();
       unsigned char Buf[4096];
       while (Size != 0)
@@ -392,17 +387,17 @@ bool PackageMap::GenContents(Configuration &Setup,
 	 unsigned long long ToRead = Size;
 	 if (Size > sizeof(Buf))
 	    ToRead = sizeof(Buf);
-	 
+
 	 if (Head.Read(Buf,ToRead) == false)
 	    return false;
-	 
-	 if (fwrite(Buf,1,ToRead,Comp.Input) != ToRead)
+
+	 if (Comp.Input.Write(Buf, ToRead) == false)
 	    return _error->Errno("fwrite",_("Error writing header to contents file"));
-	 
+
 	 Size -= ToRead;
-      }            
-   }  
-      
+      }
+   }
+
    /* Go over all the package file records and parse all the package
       files associated with this contents file into one great big honking
       memory structure, then dump the sorted version */
@@ -676,7 +671,7 @@ static bool SimpleGenPackages(CommandLine &CmdL)
       Override = CmdL.FileList[2];
    
    // Create a package writer object.
-   PackagesWriter Packages(_config->Find("APT::FTPArchive::DB"),
+   PackagesWriter Packages(NULL, _config->Find("APT::FTPArchive::DB"),
 			   Override, "", _config->Find("APT::FTPArchive::Architecture"));
    if (_error->PendingError() == true)
       return false;
@@ -704,7 +699,7 @@ static bool SimpleGenContents(CommandLine &CmdL)
       return ShowHelp(CmdL);
    
    // Create a package writer object.
-   ContentsWriter Contents(_config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
+   ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
    if (_error->PendingError() == true)
       return false;
    
@@ -737,7 +732,7 @@ static bool SimpleGenSources(CommandLine &CmdL)
 			     SOverride.c_str());
        
    // Create a package writer object.
-   SourcesWriter Sources(_config->Find("APT::FTPArchive::DB"),Override,SOverride);
+   SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
    if (_error->PendingError() == true)
       return false;
    
@@ -764,7 +759,7 @@ static bool SimpleGenRelease(CommandLine &CmdL)
 
    string Dir = CmdL.FileList[1];
 
-   ReleaseWriter Release("");
+   ReleaseWriter Release(NULL, "");
    Release.DirStrip = Dir;
 
    if (_error->PendingError() == true)
diff --git a/ftparchive/contents.cc b/ftparchive/contents.cc
index 8c4181eda..145f3910e 100644
--- a/ftparchive/contents.cc
+++ b/ftparchive/contents.cc
@@ -38,6 +38,7 @@
 #include 
 #include 
 #include 
+#include 
 
 #include 
 #include 
@@ -238,19 +239,19 @@ void GenContents::Add(const char *Dir,const char *Package)
 // GenContents::WriteSpace - Write a given number of white space chars	/*{{{*/
 // ---------------------------------------------------------------------
 /* We mod 8 it and write tabs where possible. */
-void GenContents::WriteSpace(FILE *Out,unsigned int Current,unsigned int Target)
+void GenContents::WriteSpace(std::string &out, size_t Current, size_t Target)
 {
    if (Target <= Current)
       Target = Current + 1;
-   
+
    /* Now we write tabs so long as the next tab stop would not pass
       the target */
    for (; (Current/8 + 1)*8 < Target; Current = (Current/8 + 1)*8)
-      fputc('\t',Out);
+      out.append("\t");
 
    // Fill the last bit with spaces
    for (; Current < Target; Current++)
-      fputc(' ',Out);
+      out.append(" ");
 }
 									/*}}}*/
 // GenContents::Print - Display the tree				/*{{{*/
@@ -259,13 +260,13 @@ void GenContents::WriteSpace(FILE *Out,unsigned int Current,unsigned int Target)
    calls itself and runs over each section of the tree printing out
    the pathname and the hit packages. We use Buf to build the pathname
    summed over all the directory parents of this node. */
-void GenContents::Print(FILE *Out)
+void GenContents::Print(FileFd &Out)
 {
    char Buffer[1024];
    Buffer[0] = 0;
    DoPrint(Out,&Root,Buffer);
 }
-void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf)
+void GenContents::DoPrint(FileFd &Out,GenContents::Node *Top, char *Buf)
 {
    if (Top == 0)
       return;
@@ -278,26 +279,27 @@ void GenContents::DoPrint(FILE *Out,GenContents::Node *Top, char *Buf)
    if (Top->Path != 0)
    {
       strcat(Buf,Top->Path);
-      
+
       // Do not show the item if it is a directory with dups
       if (Top->Path[strlen(Top->Path)-1] != '/' /*|| Top->Dups == 0*/)
       {
-	 fputs(Buf,Out);
-	 WriteSpace(Out,strlen(Buf),60);
+	 std::string out = Buf;
+	 WriteSpace(out, out.length(), 60);
 	 for (Node *I = Top; I != 0; I = I->Dups)
 	 {
 	    if (I != Top)
-	       fputc(',',Out);
-	    fputs(I->Package,Out);
+	       out.append(",");
+	    out.append(I->Package);
 	 }
-         fputc('\n',Out);
-      }      
-   }   
-   
+         out.append("\n");
+	 Out.Write(out.c_str(), out.length());
+      }
+   }
+
    // Go along the directory link
    DoPrint(Out,Top->DirDown,Buf);
    *OldEnd = 0;
-   
+
    // Go right
    DoPrint(Out,Top->BTreeRight,Buf);  
 }
diff --git a/ftparchive/contents.h b/ftparchive/contents.h
index f58e3278e..953d0d54b 100644
--- a/ftparchive/contents.h
+++ b/ftparchive/contents.h
@@ -17,6 +17,7 @@
 #include 
 
 class debDebFile;
+class FileFd;
 
 class GenContents
 {
@@ -54,14 +55,14 @@ class GenContents
    unsigned long NodeLeft;
    
    Node *Grab(Node *Top,const char *Name,const char *Package);
-   void WriteSpace(FILE *Out,unsigned int Current,unsigned int Target);
-   void DoPrint(FILE *Out,Node *Top, char *Buf);
+   void WriteSpace(std::string &out, size_t Current, size_t Target);
+   void DoPrint(FileFd &Out,Node *Top, char *Buf);
    
    public:
    
    char *Mystrdup(const char *From);
    void Add(const char *Dir,const char *Package);   
-   void Print(FILE *Out);
+   void Print(FileFd &Out);
 
    GenContents() : BlockList(0), StrPool(0), StrLeft(0), 
                    NodePool(0), NodeLeft(0) {};
diff --git a/ftparchive/multicompress.cc b/ftparchive/multicompress.cc
index f35d5304a..08a3cff5a 100644
--- a/ftparchive/multicompress.cc
+++ b/ftparchive/multicompress.cc
@@ -46,7 +46,6 @@ MultiCompress::MultiCompress(string const &Output,string const &Compress,
 {
    Outputs = 0;
    Outputter = -1;
-   Input = 0;
    UpdateMTime = 0;
 
    /* Parse the compression string, a space separated lists of compresison
@@ -187,12 +186,11 @@ bool MultiCompress::Start()
    };
 
    close(Pipe[0]);
-   Input = fdopen(Pipe[1],"w");
-   if (Input == 0)
-      return _error->Errno("fdopen",_("Failed to create FILE*"));
-   
+   if (Input.OpenDescriptor(Pipe[1], FileFd::WriteOnly, true) == false)
+      return false;
+
    if (Outputter == -1)
-      return _error->Errno("fork",_("Failed to fork"));   
+      return _error->Errno("fork",_("Failed to fork"));
    return true;
 }
 									/*}}}*/
@@ -201,11 +199,10 @@ bool MultiCompress::Start()
 /* */
 bool MultiCompress::Die()
 {
-   if (Input == 0)
+   if (Input.IsOpen() == false)
       return true;
-   
-   fclose(Input);
-   Input = 0;
+
+   Input.Close();
    bool Res = ExecWait(Outputter,_("Compress child"),false);
    Outputter = -1;
    return Res;
@@ -217,7 +214,7 @@ bool MultiCompress::Die()
 bool MultiCompress::Finalize(unsigned long long &OutSize)
 {
    OutSize = 0;
-   if (Input == 0 || Die() == false)
+   if (Input.IsOpen() == false || Die() == false)
       return false;
    
    time_t Now;
diff --git a/ftparchive/multicompress.h b/ftparchive/multicompress.h
index ddd1815a3..161716b86 100644
--- a/ftparchive/multicompress.h
+++ b/ftparchive/multicompress.h
@@ -48,7 +48,7 @@ class MultiCompress
    public:
    
    // The FD to write to for compression.
-   FILE *Input;
+   FileFd Input;
    unsigned long UpdateMTime;
    
    bool Finalize(unsigned long long &OutSize);
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index 855e0ef79..7cf7e6efc 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -51,20 +51,6 @@
 using namespace std;
 FTWScanner *FTWScanner::Owner;
 
-// SetTFRewriteData - Helper for setting rewrite lists			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-static inline TFRewriteData SetTFRewriteData(const char *tag,
-			     const char *rewrite,
-			     const char *newtag = 0)
-{
-   TFRewriteData tfrd;
-   tfrd.Tag = tag;
-   tfrd.Rewrite = rewrite;
-   tfrd.NewTag = newtag;
-   return tfrd;
-}
-									/*}}}*/
 // ConfigToDoHashes - which hashes to generate				/*{{{*/
 static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
 {
@@ -85,8 +71,15 @@ static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
 // FTWScanner::FTWScanner - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-FTWScanner::FTWScanner(string const &Arch): Arch(Arch), DoHashes(~0)
+FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch): Arch(Arch), DoHashes(~0)
 {
+   if (GivenOutput == NULL)
+   {
+      Output = new FileFd;
+      Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
+   }
+   else
+      Output = GivenOutput;
    ErrorPrinted = false;
    NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
    ConfigToDoHashes(DoHashes, "APT::FTPArchive");
@@ -331,11 +324,10 @@ bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
 // PackagesWriter::PackagesWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
+PackagesWriter::PackagesWriter(FileFd * const GivenOutput, string const &DB,string const &Overrides,string const &ExtOverrides,
 			       string const &Arch) :
-   FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
+   FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
 {
-   Output = stdout;
    SetExts(".deb .udeb");
    DeLinkLimit = 0;
 
@@ -461,32 +453,32 @@ bool PackagesWriter::DoPackage(string FileName)
    }
 
    // This lists all the changes to the fields we are going to make.
-   std::vector Changes;
+   std::vector Changes;
 
    std::string Size;
    strprintf(Size, "%llu", (unsigned long long) FileSize);
-   Changes.push_back(SetTFRewriteData("Size", Size.c_str()));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Size", Size));
 
    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
    {
       if (hs->HashType() == "MD5Sum")
-	 Changes.push_back(SetTFRewriteData("MD5sum", hs->HashValue().c_str()));
+	 Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue()));
       else if (hs->HashType() == "Checksum-FileSize")
 	 continue;
       else
-	 Changes.push_back(SetTFRewriteData(hs->HashType().c_str(), hs->HashValue().c_str()));
+	 Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue()));
    }
-   Changes.push_back(SetTFRewriteData("Filename", NewFileName.c_str()));
-   Changes.push_back(SetTFRewriteData("Priority", OverItem->Priority.c_str()));
-   Changes.push_back(SetTFRewriteData("Status", 0));
-   Changes.push_back(SetTFRewriteData("Optional", 0));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority));
+   Changes.push_back(pkgTagSection::Tag::Remove("Status"));
+   Changes.push_back(pkgTagSection::Tag::Remove("Optional"));
 
    string DescriptionMd5;
    if (LongDescription == false) {
       MD5Summation descmd5;
       descmd5.Add(desc.c_str());
       DescriptionMd5 = descmd5.Result().Value();
-      Changes.push_back(SetTFRewriteData("Description-md5", DescriptionMd5.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5));
       if (TransWriter != NULL)
 	 TransWriter->DoPackage(Package, desc, DescriptionMd5);
    }
@@ -505,7 +497,7 @@ bool PackagesWriter::DoPackage(string FileName)
    }
 
    if (NewMaint.empty() == false)
-      Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint));
 
    /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
       dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
@@ -517,19 +509,17 @@ bool PackagesWriter::DoPackage(string FileName)
    {
       if (Tags.FindS("Suggests").empty() == false)
 	 OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
-      Changes.push_back(SetTFRewriteData("Suggests", OptionalStr.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Suggests", OptionalStr));
    }
 
    for (map::const_iterator I = OverItem->FieldOverride.begin();
         I != OverItem->FieldOverride.end(); ++I)
-      Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
-
-   Changes.push_back(SetTFRewriteData( 0, 0));
+      Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 
    // Rewrite and store the fields.
-   if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes.data()) == false)
+   if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false ||
+	 Output->Write("\n", 1) == false)
       return false;
-   fprintf(Output,"\n");
 
    return Db.Finish();
 }
@@ -539,14 +529,13 @@ bool PackagesWriter::DoPackage(string FileName)
 // ---------------------------------------------------------------------
 /* Create a Translation-Master file for this Packages file */
 TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
-					mode_t const &Permissions) : Output(NULL),
-							RefCounter(0)
+					mode_t const &Permissions) : RefCounter(0)
 {
    if (File.empty() == true)
       return;
 
    Comp = new MultiCompress(File, TransCompress, Permissions);
-   Output = Comp->Input;
+   Output = &Comp->Input;
 }
 									/*}}}*/
 // TranslationWriter::DoPackage - Process a single package		/*{{{*/
@@ -565,8 +554,10 @@ bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
    if (Included.find(Record) != Included.end())
       return true;
 
-   fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
+   std::string out;
+   strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
 	   Pkg.c_str(), MD5.c_str(), Desc.c_str());
+   Output->Write(out.c_str(), out.length());
 
    Included.insert(Record);
    return true;
@@ -587,11 +578,10 @@ TranslationWriter::~TranslationWriter()
 // SourcesWriter::SourcesWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string const &SOverrides,
+SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides,
 			     string const &ExtOverrides) :
-   Db(DB), Stats(Db.Stats)
+   FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats)
 {
-   Output = stdout;
    AddPattern("*.dsc");
    DeLinkLimit = 0;
    Buffer = 0;
@@ -620,15 +610,16 @@ SourcesWriter::SourcesWriter(string const &DB, string const &BOverrides,string c
 }
 									/*}}}*/
 // SourcesWriter::DoPackage - Process a single package			/*{{{*/
-static std::ostream& addDscHash(std::ostream &out, unsigned int const DoHashes,
+static std::string getDscHash(unsigned int const DoHashes,
       Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, char const * const FieldName,
       HashString const * const Hash, unsigned long long Size, std::string FileName)
 {
    if ((DoHashes & DoIt) != DoIt || Tags.Exists(FieldName) == false || Hash == NULL)
-      return out;
+      return "";
+   std::ostringstream out;
    out << "\n " << Hash->HashValue() << " " << Size << " " << FileName
       << "\n " << Tags.FindS(FieldName);
-   return out;
+   return out.str();
 }
 bool SourcesWriter::DoPackage(string FileName)
 {
@@ -721,16 +712,10 @@ bool SourcesWriter::DoPackage(string FileName)
 
    // Add the dsc to the files hash list
    string const strippedName = flNotDir(FileName);
-   std::ostringstream ostreamFiles;
-   addDscHash(ostreamFiles, DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
-   string const Files = ostreamFiles.str();
-
-   std::ostringstream ostreamSha1;
-   addDscHash(ostreamSha1, DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
-   std::ostringstream ostreamSha256;
-   addDscHash(ostreamSha256, DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
-   std::ostringstream ostreamSha512;
-   addDscHash(ostreamSha512, DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
+   std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
+   std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
+   std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
+   std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
 
    // Strip the DirStrip prefix from the FileName and add the PathPrefix
    string NewFileName;
@@ -752,7 +737,7 @@ bool SourcesWriter::DoPackage(string FileName)
    char *RealPath = NULL;
    for (;isspace(*C); C++);
    while (*C != 0)
-   {   
+   {
       // Parse each of the elements
       if (ParseQuoteWord(C,ParseJnk) == false ||
 	  ParseQuoteWord(C,ParseJnk) == false ||
@@ -782,21 +767,21 @@ bool SourcesWriter::DoPackage(string FileName)
 	    if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
 	       continue;
 	    char const * fieldname;
-	    std::ostream * out;
+	    std::string * out;
 	    if (hs->HashType() == "SHA1")
 	    {
 	       fieldname = "Checksums-Sha1";
-	       out = &ostreamSha1;
+	       out = &ChecksumsSha1;
 	    }
 	    else if (hs->HashType() == "SHA256")
 	    {
 	       fieldname = "Checksums-Sha256";
-	       out = &ostreamSha256;
+	       out = &ChecksumsSha256;
 	    }
 	    else if (hs->HashType() == "SHA512")
 	    {
 	       fieldname = "Checksums-Sha512";
-	       out = &ostreamSha512;
+	       out = &ChecksumsSha512;
 	    }
 	    else
 	    {
@@ -805,10 +790,12 @@ bool SourcesWriter::DoPackage(string FileName)
 	    }
 	    if (Tags.Exists(fieldname) == true)
 	       continue;
-	    (*out) << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
+	    std::ostringstream streamout;
+	    streamout << "\n " << hs->HashValue() << " " << Db.GetFileSize() << " " << ParseJnk;
+	    out->append(streamout.str());
 	 }
 
-	 // write back the GetFileInfo() stats data 
+	 // write back the GetFileInfo() stats data
 	 Db.Finish();
       }
 
@@ -829,54 +816,48 @@ bool SourcesWriter::DoPackage(string FileName)
    if (Directory.length() > 2)
       Directory.erase(Directory.end()-1);
 
-   string const ChecksumsSha1 = ostreamSha1.str();
-   string const ChecksumsSha256 = ostreamSha256.str();
-   string const ChecksumsSha512 = ostreamSha512.str();
-
    // This lists all the changes to the fields we are going to make.
    // (5 hardcoded + checksums + maintainer + end marker)
-   std::vector Changes;
+   std::vector Changes;
 
-   Changes.push_back(SetTFRewriteData("Source", 0));
-   Changes.push_back(SetTFRewriteData("Package",Package.c_str()));
+   Changes.push_back(pkgTagSection::Tag::Remove("Source"));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package));
    if (Files.empty() == false)
-      Changes.push_back(SetTFRewriteData("Files",Files.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files));
    if (ChecksumsSha1.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha1",ChecksumsSha1.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1));
    if (ChecksumsSha256.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha256",ChecksumsSha256.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256));
    if (ChecksumsSha512.empty() == false)
-      Changes.push_back(SetTFRewriteData("Checksums-Sha512",ChecksumsSha512.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512));
    if (Directory != "./")
-      Changes.push_back(SetTFRewriteData("Directory",Directory.c_str()));
-   Changes.push_back(SetTFRewriteData("Priority",BestPrio.c_str()));
-   Changes.push_back(SetTFRewriteData("Status",0));
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory));
+   Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio));
+   Changes.push_back(pkgTagSection::Tag::Remove("Status"));
 
    // Rewrite the maintainer field if necessary
    bool MaintFailed;
-   string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
+   string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"), MaintFailed);
    if (MaintFailed == true)
    {
       if (NoOverride == false)
       {
-	 NewLine(1);	 
+	 NewLine(1);
 	 ioprintf(c1out, _("  %s maintainer is %s not %s\n"), Package.c_str(),
 	       Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
-      }      
+      }
    }
    if (NewMaint.empty() == false)
-      Changes.push_back(SetTFRewriteData("Maintainer", NewMaint.c_str()));
-   
-   for (map::const_iterator I = SOverItem->FieldOverride.begin(); 
+      Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str()));
+
+   for (map::const_iterator I = SOverItem->FieldOverride.begin();
         I != SOverItem->FieldOverride.end(); ++I)
-      Changes.push_back(SetTFRewriteData(I->first.c_str(),I->second.c_str()));
+      Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
 
-   Changes.push_back(SetTFRewriteData(0, 0));
-      
    // Rewrite and store the fields.
-   if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes.data()) == false)
+   if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false ||
+	 Output->Write("\n", 1) == false)
       return false;
-   fprintf(Output,"\n");
 
    Stats.Packages++;
    
@@ -887,12 +868,11 @@ bool SourcesWriter::DoPackage(string FileName)
 // ContentsWriter::ContentsWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
-		    FTWScanner(Arch), Db(DB), Stats(Db.Stats)
+ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB, string const &Arch) :
+		    FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats)
 
 {
    SetExts(".deb");
-   Output = stdout;
 }
 									/*}}}*/
 // ContentsWriter::DoPackage - Process a single package			/*{{{*/
@@ -974,7 +954,7 @@ bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompres
 // ReleaseWriter::ReleaseWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-ReleaseWriter::ReleaseWriter(string const &/*DB*/)
+ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput)
 {
    if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
    {
@@ -996,7 +976,6 @@ ReleaseWriter::ReleaseWriter(string const &/*DB*/)
    }
    AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
 
-   Output = stdout;
    time_t const now = time(NULL);
 
    setlocale(LC_TIME, "C");
@@ -1040,7 +1019,8 @@ ReleaseWriter::ReleaseWriter(string const &/*DB*/)
       if (Value == "")
          continue;
 
-      fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
+      std::string const out = I->first + ": " + Value + "\n";
+      Output->Write(out.c_str(), out.length());
    }
 
    ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
@@ -1087,29 +1067,35 @@ bool ReleaseWriter::DoPackage(string FileName)
 									/*}}}*/
 // ReleaseWriter::Finish - Output the checksums				/*{{{*/
 // ---------------------------------------------------------------------
-static void printChecksumTypeRecord(FILE * const Output, char const * const Type, map const &CheckSums)
+static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map const &CheckSums)
 {
-      fprintf(Output, "%s:\n", Type);
-      for(map::const_iterator I = CheckSums.begin();
-	  I != CheckSums.end(); ++I)
-      {
-	 HashString const * const hs = I->second.Hashes.find(Type);
-	 if (hs == NULL)
-	    continue;
-	 fprintf(Output, " %s %16llu %s\n",
-		 hs->HashValue().c_str(),
-		 (*I).second.size,
-		 (*I).first.c_str());
-      }
+   {
+      std::string out;
+      strprintf(out, "%s:\n", Type);
+      Output.Write(out.c_str(), out.length());
+   }
+   for(map::const_iterator I = CheckSums.begin();
+	 I != CheckSums.end(); ++I)
+   {
+      HashString const * const hs = I->second.Hashes.find(Type);
+      if (hs == NULL)
+	 continue;
+      std::string out;
+      strprintf(out, " %s %16llu %s\n",
+	    hs->HashValue().c_str(),
+	    (*I).second.size,
+	    (*I).first.c_str());
+      Output.Write(out.c_str(), out.length());
+   }
 }
 void ReleaseWriter::Finish()
 {
    if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
-      printChecksumTypeRecord(Output, "MD5Sum", CheckSums);
+      printChecksumTypeRecord(*Output, "MD5Sum", CheckSums);
    if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
-      printChecksumTypeRecord(Output, "SHA1", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA1", CheckSums);
    if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
-      printChecksumTypeRecord(Output, "SHA256", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA256", CheckSums);
    if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
-      printChecksumTypeRecord(Output, "SHA512", CheckSums);
+      printChecksumTypeRecord(*Output, "SHA512", CheckSums);
 }
diff --git a/ftparchive/writer.h b/ftparchive/writer.h
index 226996475..0ba60db5e 100644
--- a/ftparchive/writer.h
+++ b/ftparchive/writer.h
@@ -42,10 +42,10 @@ class FTWScanner
    string Arch;
    const char *OriginalPath;
    bool ErrorPrinted;
-   
+
    // Stuff for the delinker
    bool NoLinkAct;
-   
+
    static FTWScanner *Owner;
    static int ScannerFTW(const char *File,const struct stat *sb,int Flag);
    static int ScannerFile(const char *File, bool const &ReadLink);
@@ -59,10 +59,11 @@ class FTWScanner
       {
 	 c1out << endl;
 	 ErrorPrinted = true;
-      }	 
+      }
    }
-   
+
    public:
+   FileFd *Output;
    unsigned int DoHashes;
 
    unsigned long DeLinkLimit;
@@ -76,8 +77,8 @@ class FTWScanner
    void AddPattern(char const *Pattern) { Patterns.push_back(Pattern); };
    void AddPatterns(std::vector const &patterns) { Patterns.insert(Patterns.end(), patterns.begin(), patterns.end()); };
    bool SetExts(string const &Vals);
-      
-   FTWScanner(string const &Arch = string());
+
+   FTWScanner(FileFd * const Output, string const &Arch = string());
    virtual ~FTWScanner() {};
 };
 
@@ -86,9 +87,9 @@ class MultiCompress;
 class TranslationWriter
 {
    MultiCompress *Comp;
-   FILE *Output;
    std::set Included;
    unsigned short RefCounter;
+   FileFd *Output;
 
    public:
    void IncreaseRefCounter() { ++RefCounter; };
@@ -97,7 +98,7 @@ class TranslationWriter
    bool DoPackage(string const &Pkg, string const &Desc, string const &MD5);
 
    TranslationWriter(string const &File, string const &TransCompress, mode_t const &Permissions);
-   TranslationWriter() : Comp(NULL), Output(NULL), RefCounter(0) {};
+   TranslationWriter() : Comp(NULL), RefCounter(0) {};
    ~TranslationWriter();
 };
 
@@ -105,7 +106,7 @@ class PackagesWriter : public FTWScanner
 {
    Override Over;
    CacheDB Db;
-      
+
    public:
 
    // Some flags
@@ -117,7 +118,6 @@ class PackagesWriter : public FTWScanner
    // General options
    string PathPrefix;
    string DirStrip;
-   FILE *Output;
    struct CacheDB::Stats &Stats;
    TranslationWriter *TransWriter;
 
@@ -126,7 +126,7 @@ class PackagesWriter : public FTWScanner
       {return Over.ReadExtraOverride(File);};
    virtual bool DoPackage(string FileName);
 
-   PackagesWriter(string const &DB,
+   PackagesWriter(FileFd * const Output, string const &DB,
                   string const &Overrides,
                   string const &ExtOverrides = "",
 		  string const &Arch = "");
@@ -136,25 +136,24 @@ class PackagesWriter : public FTWScanner
 class ContentsWriter : public FTWScanner
 {
    CacheDB Db;
-   
+
    GenContents Gen;
-   
+
    public:
 
    // General options
-   FILE *Output;
    struct CacheDB::Stats &Stats;
    string Prefix;
-   
+
    bool DoPackage(string FileName,string Package);
    virtual bool DoPackage(string FileName) 
              {return DoPackage(FileName,string());};
    bool ReadFromPkgs(string const &PkgFile,string const &PkgCompress);
 
-   void Finish() {Gen.Print(Output);};
+   void Finish() {Gen.Print(*Output);};
    inline bool ReadyDB(string const &DB) {return Db.ReadyDB(DB);};
-   
-   ContentsWriter(string const &DB, string const &Arch = string());
+
+   ContentsWriter(FileFd * const Output, string const &DB, string const &Arch = string());
    virtual ~ContentsWriter() {};
 };
 
@@ -165,21 +164,20 @@ class SourcesWriter : public FTWScanner
    Override SOver;
    char *Buffer;
    unsigned long long BufSize;
-   
+
    public:
 
    bool NoOverride;
    bool DoAlwaysStat;
-   
+
    // General options
    string PathPrefix;
    string DirStrip;
-   FILE *Output;
    struct CacheDB::Stats &Stats;
 
    virtual bool DoPackage(string FileName);
 
-   SourcesWriter(string const &DB,string const &BOverrides,string const &SOverrides,
+   SourcesWriter(FileFd * const Output, string const &DB,string const &BOverrides,string const &SOverrides,
 		 string const &ExtOverrides=string());
    virtual ~SourcesWriter() {free(Buffer);};
 };
@@ -187,11 +185,10 @@ class SourcesWriter : public FTWScanner
 class ReleaseWriter : public FTWScanner
 {
 public:
-   ReleaseWriter(string const &DB);
+   ReleaseWriter(FileFd * const Output, string const &DB);
    virtual bool DoPackage(string FileName);
    void Finish();
 
-   FILE *Output;
    // General options
    string PathPrefix;
    string DirStrip;
diff --git a/test/integration/test-apt-cli-show b/test/integration/test-apt-cli-show
index 5604620fd..43072cf03 100755
--- a/test/integration/test-apt-cli-show
+++ b/test/integration/test-apt-cli-show
@@ -25,11 +25,11 @@ APTARCHIVE=$(readlink -f ./aptarchive)
 # note that we do not display Description-md5 with the "apt" cmd
 # and also show some additional fields that are calculated
 testsuccessequal "Package: foo
+Version: 1.0
 Priority: optional
 Section: other
-Installed-Size: 43.0 kB
 Maintainer: Joe Sixpack 
-Version: 1.0
+Installed-Size: 43.0 kB
 Download-Size: unknown
 APT-Manual-Installed: yes
 APT-Sources: file:$APTARCHIVE/ unstable/main i386 Packages
diff --git a/test/libapt/indexcopytosourcelist_test.cc b/test/libapt/indexcopytosourcelist_test.cc
index bec87601f..1b0427564 100644
--- a/test/libapt/indexcopytosourcelist_test.cc
+++ b/test/libapt/indexcopytosourcelist_test.cc
@@ -16,7 +16,7 @@ class NoCopy : public IndexCopy {
 	 return Path;
       }
       bool GetFile(std::string &/*Filename*/, unsigned long long &/*Size*/) { return false; }
-      bool RewriteEntry(FILE * /*Target*/, std::string /*File*/) { return false; }
+      bool RewriteEntry(FileFd & /*Target*/, std::string const &/*File*/) { return false; }
       const char *GetFileName() { return NULL; }
       const char *Type() { return NULL; }
 
diff --git a/test/libapt/tagsection_test.cc b/test/libapt/tagsection_test.cc
new file mode 100644
index 000000000..f250177af
--- /dev/null
+++ b/test/libapt/tagsection_test.cc
@@ -0,0 +1,270 @@
+#include 
+
+#include 
+#include 
+
+#include 
+#include 
+
+#include 
+
+#include "file-helpers.h"
+
+std::string packageValue = "aaaa";
+std::string typoValue = "aa\n"
+   " .\n"
+   " cc";
+std::string typoRawValue = "\n " + typoValue;
+std::string overrideValue = "1";
+/*
+   std::cerr << "FILECONTENT: »";
+   char buffer[3000];
+   while (fd.ReadLine(buffer, sizeof(buffer)))
+      std::cerr << buffer;
+   std::cerr << "«" << std::endl;;
+*/
+
+void setupTestcaseStart(FileFd &fd, pkgTagSection §ion, std::string &content)
+{
+   createTemporaryFile("writesection", fd, NULL, NULL);
+   content = "Package: " + packageValue + "\n"
+      "TypoA:\n " + typoValue + "\n"
+      "Override: " + overrideValue + "\n"
+      "Override-Backup: " + overrideValue + "\n"
+      "\n";
+   EXPECT_TRUE(section.Scan(content.c_str(), content.length(), true));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(typoRawValue, section.FindRawS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodified)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   EXPECT_TRUE(section.Write(fd));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrder)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrderReversed)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Override", "TypoA", "Package", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteUnmodifiedOrderNotAll)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   char const * const order[] = { "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoA"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRename)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rename("TypoA", "TypoB"));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoB"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRemove)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Remove("TypoA"));
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", ""));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_FALSE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(2, section.Count());
+}
+TEST(TagSectionTest,WriteNoOrderRewrite)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", "42"));
+   EXPECT_TRUE(section.Write(fd, NULL, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(42, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRename)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rename("TypoA", "TypoB"));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_TRUE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(typoValue, section.FindS("TypoB"));
+   EXPECT_EQ(1, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRemove)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Remove("TypoA"));
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", ""));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_FALSE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_FALSE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(2, section.Count());
+}
+TEST(TagSectionTest,WriteOrderRewrite)
+{
+   FileFd fd;
+   pkgTagSection section;
+   std::string content;
+   setupTestcaseStart(fd, section, content);
+   std::vector rewrite;
+   rewrite.push_back(pkgTagSection::Tag::Rewrite("Override", "42"));
+   char const * const order[] = { "Package", "TypoA", "Override", NULL };
+   EXPECT_TRUE(section.Write(fd, order, rewrite));
+   EXPECT_TRUE(fd.Seek(0));
+   pkgTagFile tfile(&fd);
+   ASSERT_TRUE(tfile.Step(section));
+   EXPECT_TRUE(section.Exists("Package"));
+   EXPECT_TRUE(section.Exists("TypoA"));
+   EXPECT_FALSE(section.Exists("TypoB"));
+   EXPECT_TRUE(section.Exists("Override"));
+   EXPECT_TRUE(section.Exists("Override-Backup"));
+   EXPECT_EQ(packageValue, section.FindS("Package"));
+   EXPECT_EQ(42, section.FindI("Override"));
+   EXPECT_EQ(1, section.FindI("Override-Backup"));
+   EXPECT_EQ(4, section.Count());
+}
-- 
cgit v1.2.3-70-g09d2