From 895417ef99bb1371d8970da1afe87c6d64382f67 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Mon, 28 Apr 2014 10:02:27 +0200 Subject: reenable pipelining via hashsum reordering support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that methods have the expected hashes available they can check if the response from the server is what they expected. Pipelining is one of those areas in which servers can mess up by not supporting it properly, which forced us to disable it for the time being. Now, we check if we got a response out of order, which we can not only use to disable pipelining automatically for the next requests, but we can fix it up just like the server responded in proper order for the current requests. To ensure that this little trick works pipelining is only attempt if we have hashsums for all the files in the chain which in theory reduces the use of pipelining usage even on the many servers which work properly, but in practice only the InRelease file (or similar such) will be requested without a hashsum – and as it is the only file requested in that stage it can't be pipelined even if we wanted to. Some minor annoyances remain: The display of the progress we have doesn't reflect this change, so it looks like the same package gets downloaded multiple times while others aren't at all. Further more, partial files are not supported in this recovery as the received data was appended to the wrong file, so the hashsum doesn't match. Both seem to be minor enough to reenable pipelining by default until further notice through to test if it really solves the problem. This therefore reverts commit 8221431757c775ee875a061b184b5f6f2330f928. --- methods/server.cc | 46 ++++++++++++++++++++++++++++++++++++++++++---- methods/server.h | 2 +- 2 files changed, 43 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index 5a13f18a7..c91d3b218 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -392,9 +392,16 @@ bool ServerMethod::Fetch(FetchItem *) for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++) { - // If pipelining is disabled, we only queue 1 request - if (Server->Pipeline == false && Depth >= 0) - break; + if (Depth >= 0) + { + // If pipelining is disabled, we only queue 1 request + if (Server->Pipeline == false) + break; + // if we have no hashes, do at most one such request + // as we can't fixup pipeling misbehaviors otherwise + else if (I->ExpectedHashes.usable() == false) + break; + } // Make sure we stick with the same server if (Server->Comp(I->Uri) == false) @@ -546,7 +553,38 @@ int ServerMethod::Loop() // Send status to APT if (Result == true) { - Res.TakeHashes(*Server->GetHashes()); + Hashes * const resultHashes = Server->GetHashes(); + HashStringList const hashList = resultHashes->GetHashStringList(); + if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList) + { + // we did not get the expected hash… mhhh: + // could it be that server/proxy messed up pipelining? + FetchItem * BeforeI = Queue; + for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + { + if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList) + { + // yes, he did! Disable pipelining and rewrite queue + if (Server->Pipeline == true) + { + // FIXME: fake a warning message as we have no proper way of communicating here + std::string out; + strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth"); + std::cerr << "W: " << out << std::endl; + Server->Pipeline = false; + // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well + } + Rename(Res.Filename, I->DestFile); + Res.Filename = I->DestFile; + BeforeI->Next = I->Next; + I->Next = Queue; + Queue = I; + break; + } + BeforeI = I; + } + } + Res.TakeHashes(*resultHashes); URIDone(Res); } else diff --git a/methods/server.h b/methods/server.h index 0f45ab994..5299b3954 100644 --- a/methods/server.h +++ b/methods/server.h @@ -140,7 +140,7 @@ class ServerMethod : public pkgAcqMethod virtual ServerState * CreateServerState(URI uri) = 0; virtual void RotateDNS() = 0; - ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), Server(NULL), File(NULL), PipelineDepth(0), AllowRedirect(false), Debug(false) {}; + ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), Server(NULL), File(NULL), PipelineDepth(10), AllowRedirect(false), Debug(false) {}; virtual ~ServerMethod() {}; }; -- cgit v1.2.3-70-g09d2 From 7b734b09f6bd9356e4622aee64bd2e5e43554570 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 24 Jun 2014 15:45:09 +0200 Subject: methods/http.cc: use Req.str() in debug output --- methods/http.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/http.cc b/methods/http.cc index c734d3799..7c7949eac 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -744,7 +744,7 @@ void HttpMethod::SendReq(FetchItem *Itm) Req << "\r\n"; if (Debug == true) - cerr << Req << endl; + cerr << Req.str() << endl; Server->WriteResponse(Req.str()); } -- cgit v1.2.3-70-g09d2 From 2737f28a1cb2d03c66d2a7edd04215566903dbf1 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 21 Jul 2014 11:19:37 +0200 Subject: Download Release first, then Release.gpg The old way of handling this was that pkgAcqMetaIndex was responsible to check/move both Release and Release.gpg in place. This breaks the assumption of the transaction that each pkgAcquire::Item has a single File that its responsible for. --- apt-pkg/acquire-item.cc | 93 ++++++++++++++------------- apt-pkg/acquire-item.h | 35 +++++----- apt-pkg/deb/debmetaindex.cc | 16 ++--- methods/gpgv.cc | 2 +- test/integration/test-apt-update-transactions | 1 + 5 files changed, 79 insertions(+), 68 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 81afdf4b1..fa41bca7a 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -88,7 +88,8 @@ pkgAcquire::Item::~Item() void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { Status = StatIdle; - ErrorText = LookupTag(Message,"Message"); + if(ErrorText == "") + ErrorText = LookupTag(Message,"Message"); UsedMirror = LookupTag(Message,"UsedMirror"); if (QueueCounter <= 1) { @@ -1314,19 +1315,20 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Item::Failed(Message,Cnf); } /*}}}*/ -pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ +pkgAcqMetaSig::pkgAcqMetaSig(pkgAcqMetaIndex *MetaOwner, /*{{{*/ string URI,string URIDesc,string ShortDesc, - string MetaIndexURI, string MetaIndexURIDesc, - string MetaIndexShortDesc, + string MetaIndexFile, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList()), RealURI(URI), MetaIndexURI(MetaIndexURI), - MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), - MetaIndexParser(MetaIndexParser), IndexTargets(IndexTargets) + Item(MetaOwner->GetOwner(), HashStringList()), RealURI(URI), + MetaIndexParser(MetaIndexParser), MetaIndexFile(MetaIndexFile), + IndexTargets(IndexTargets), AuthPass(false) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); + TransactionID = (unsigned long)MetaOwner; + // remove any partial downloaded sig-file in partial/. // it may confuse proxies and is too small to warrant a // partial download anyway @@ -1337,7 +1339,8 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ Desc.Owner = this; Desc.ShortDesc = ShortDesc; Desc.URI = URI; - + +#if 0 string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); if (RealFileExists(Final) == true) @@ -1350,9 +1353,9 @@ pkgAcqMetaSig::pkgAcqMetaSig(pkgAcquire *Owner, /*{{{*/ LastGoodSig = DestFile+".reverify"; Rename(Final,LastGoodSig); } - +#endif // we expect the indextargets + one additional Release file - ExpectedAdditionalItems = IndexTargets->size() + 1; + //ExpectedAdditionalItems = IndexTargets->size() + 1; QueueURI(Desc); } @@ -1404,10 +1407,17 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList return; } - Complete = true; + // queue for verify + if(AuthPass == false) + { + AuthPass = true; + Desc.URI = "gpgv:" + DestFile; + DestFile = MetaIndexFile; + QueueURI(Desc); + return; + } - // at this point pkgAcqMetaIndex takes over - ExpectedAdditionalItems = 0; + Complete = true; // put the last known good file back on i-m-s hit (it will // be re-verified again) @@ -1416,10 +1426,13 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList Rename(LastGoodSig, DestFile); // queue for copy - PartialFile = DestFile; + PartialFile = _config->FindDir("Dir::State::lists") + "partial/"; + PartialFile += URItoFileName(RealURI); + DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI); +#if 0 // queue a pkgAcqMetaIndex to be verified against the sig we just retrieved pkgAcqMetaIndex *metaindex = new pkgAcqMetaIndex( Owner, MetaIndexURI, MetaIndexURIDesc, @@ -1427,15 +1440,12 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size, HashStringList MetaIndexParser); TransactionID = (unsigned long)metaindex; +#endif } /*}}}*/ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ { string Final = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI); - - // at this point pkgAcqMetaIndex takes over - ExpectedAdditionalItems = 0; - // if we get a network error we fail gracefully if(Status == StatTransientNetworkError) { @@ -1451,11 +1461,11 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ // Delete any existing sigfile when the acquire failed unlink(Final.c_str()); - +#if 0 // queue a pkgAcqMetaIndex with no sigfile new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, - "", IndexTargets, MetaIndexParser); - + "", IndexTargets, MetaIndexParser); +#endif if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { @@ -1471,11 +1481,13 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/ /*}}}*/ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ string URI,string URIDesc,string ShortDesc, - string SigFile, + string MetaIndexSigURI,string MetaIndexSigURIDesc, string MetaIndexSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - Item(Owner, HashStringList()), RealURI(URI), SigFile(SigFile), IndexTargets(IndexTargets), - MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false) + Item(Owner, HashStringList()), RealURI(URI), IndexTargets(IndexTargets), + MetaIndexParser(MetaIndexParser), AuthPass(false), IMSHit(false), + MetaIndexSigURI(MetaIndexSigURI), MetaIndexSigURIDesc(MetaIndexSigURIDesc), + MetaIndexSigShortDesc(MetaIndexSigShortDesc) { DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); @@ -1490,7 +1502,6 @@ pkgAcqMetaIndex::pkgAcqMetaIndex(pkgAcquire *Owner, /*{{{*/ // we expect more item ExpectedAdditionalItems = IndexTargets->size(); - QueueURI(Desc); } /*}}}*/ @@ -1534,10 +1545,9 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,HashStringList if (SigFile == "") { - // There was no signature file, so we are finished. Download - // the indexes and do only hashsum verification if possible + // load indexes, the signature will downloaded afterwards MetaIndexParser->Load(DestFile); - QueueIndexes(false); + QueueIndexes(true); } else { @@ -1615,6 +1625,13 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ } DestFile = FinalFile; } + + // queue a signature + if(SigFile != DestFile) + new pkgAcqMetaSig(this, MetaIndexSigURI, MetaIndexSigURIDesc, + MetaIndexSigShortDesc, DestFile, IndexTargets, + MetaIndexParser); + Complete = true; } /*}}}*/ @@ -1659,16 +1676,6 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ /*}}}*/ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ { -#if 0 - /* Reject invalid, existing Release files (LP: #346386) (Closes: #627642) - * FIXME: Disabled; it breaks unsigned repositories without hashes */ - if (!verify && FileExists(DestFile) && !MetaIndexParser->Load(DestFile)) - { - Status = StatError; - ErrorText = MetaIndexParser->ErrorText; - return; - } -#endif bool transInRelease = false; { std::vector const keys = MetaIndexParser->MetaKeys(); @@ -1919,9 +1926,9 @@ pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, const vector* IndexTargets, indexRecords* MetaIndexParser) : - pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser), - MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), - MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) + pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, MetaSigURI, MetaSigURIDesc,MetaSigShortDesc, IndexTargets, MetaIndexParser), + MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), + MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) { SigFile = DestFile; @@ -1983,9 +1990,9 @@ void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /* if (FileExists(FinalFile)) unlink(FinalFile.c_str()); - new pkgAcqMetaSig(Owner, - MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, + new pkgAcqMetaIndex(Owner, MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, + MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, IndexTargets, MetaIndexParser); if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message, "Transient-Failure"), false) == false) diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 6c9fec695..6235c353b 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -450,6 +450,15 @@ class pkgAcqMetaIndex : public pkgAcquire::Item * no hashsum checking will be performed. */ void QueueIndexes(bool verify); + + /** \brief The URI of the meta-index file for the detached signature */ + std::string MetaIndexSigURI; + + /** \brief A "URI-style" description of the meta-index file */ + std::string MetaIndexSigURIDesc; + + /** \brief A brief description of the meta-index file */ + std::string MetaIndexSigShortDesc; public: @@ -464,7 +473,7 @@ class pkgAcqMetaIndex : public pkgAcquire::Item /** \brief Create a new pkgAcqMetaIndex. */ pkgAcqMetaIndex(pkgAcquire *Owner, std::string URI,std::string URIDesc, std::string ShortDesc, - std::string SigFile, + std::string MetaIndexSigURI, std::string MetaIndexSigURIDesc, std::string MetaIndexSigShortDesc, const std::vector* IndexTargets, indexRecords* MetaIndexParser); }; @@ -1009,22 +1018,12 @@ class pkgAcqMetaSig : public pkgAcquire::Item */ std::string RealURI; - /** \brief The URI of the meta-index file to be fetched after the signature. */ - std::string MetaIndexURI; - - /** \brief A "URI-style" description of the meta-index file to be - * fetched after the signature. - */ - std::string MetaIndexURIDesc; - - /** \brief A brief description of the meta-index file to be fetched - * after the signature. - */ - std::string MetaIndexShortDesc; - /** \brief A package-system-specific parser for the meta-index file. */ indexRecords* MetaIndexParser; + /** \brief The file we need to verify */ + std::string MetaIndexFile; + /** \brief The index files which should be looked up in the meta-index * and then downloaded. * @@ -1032,6 +1031,9 @@ class pkgAcqMetaSig : public pkgAcquire::Item */ const std::vector* IndexTargets; + /** \brief if we are in fetching or download state */ + bool AuthPass; + public: // Specialized action members @@ -1042,8 +1044,9 @@ class pkgAcqMetaSig : public pkgAcquire::Item virtual std::string DescURI() const {return RealURI; }; /** \brief Create a new pkgAcqMetaSig. */ - pkgAcqMetaSig(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc, - std::string MetaIndexURI, std::string MetaIndexURIDesc, std::string MetaIndexShortDesc, + pkgAcqMetaSig(pkgAcqMetaIndex *MetaOwner, + std::string URI,std::string URIDesc, std::string ShortDesc, + std::string MetaIndexFile, const std::vector* IndexTargets, indexRecords* MetaIndexParser); virtual ~pkgAcqMetaSig(); diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 73010e867..98f99e888 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -267,24 +267,24 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const if (tryInRelease == false) new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", - MetaIndexURI("Release.gpg"), + MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", ComputeIndexTargets(), new indexRecords (Dist)); } if (tryInRelease == true) - new pkgAcqMetaClearSig(Owner, MetaIndexURI("InRelease"), - MetaIndexInfo("InRelease"), "InRelease", + new pkgAcqMetaClearSig(Owner, + MetaIndexURI("InRelease"), MetaIndexInfo("InRelease"), "InRelease", MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", ComputeIndexTargets(), new indexRecords (Dist)); else - new pkgAcqMetaSig(Owner, MetaIndexURI("Release.gpg"), - MetaIndexInfo("Release.gpg"), "Release.gpg", - MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", - ComputeIndexTargets(), - new indexRecords (Dist)); + new pkgAcqMetaIndex(Owner, + MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", + MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", + ComputeIndexTargets(), + new indexRecords (Dist)); return true; } diff --git a/methods/gpgv.cc b/methods/gpgv.cc index ae521a2ed..30fd217bd 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -159,7 +159,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, waitpid(pid, &status, 0); if (Debug == true) { - std::clog << "gpgv exited\n"; + ioprintf(std::clog, "gpgv exited with status %i\n", WEXITSTATUS(status)); } if (WEXITSTATUS(status) == 0) diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions index ee8d20dbf..247334991 100755 --- a/test/integration/test-apt-update-transactions +++ b/test/integration/test-apt-update-transactions @@ -21,3 +21,4 @@ compressfile aptarchive/dists/unstable/main/binary-i386/Packages '+1hour' # ensure that a update will only succeed entirely or not at all testfailure aptget update testequal "partial" ls rootdir/var/lib/apt/lists + -- cgit v1.2.3-70-g09d2 From 5f6c6c6e085bcba021a29d88859e70b1e03ff153 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 24 Aug 2014 16:00:36 -0700 Subject: make compressed-indexes test pass again --- apt-pkg/acquire-item.cc | 38 ++++++++++++++++---------------------- methods/copy.cc | 32 +++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 27 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 30953cc7d..d5cce8c49 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1162,7 +1162,6 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con Item::Done(Message,Size,Hashes,Cfg); std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (Decompression == true) { if (ExpectedHashes.usable() && ExpectedHashes != Hashes) @@ -1179,7 +1178,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con /* Always verify the index file for correctness (all indexes must * have a Package field) (LP: #346386) (Closes: #627642) */ - FileFd fd(DestFile, FileFd::ReadOnly); + FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); // Only test for correctness if the content of the file is not empty // (empty is ok) if (fd.Size() > 0) @@ -1208,19 +1207,19 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con DestFile = GetFinalFilename(RealURI, compExt); return; - } else { - // FIXME: use the same method to find - // check the compressed hash too - if(MetaKey != "" && Hashes.size() > 0) + } + + // FIXME: use the same method to find + // check the compressed hash too + if(MetaKey != "" && Hashes.size() > 0) + { + indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); + if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) { - indexRecords::checkSum *Record = MetaIndexParser->Lookup(MetaKey); - if(Record && Record->Hashes.usable() && Hashes != Record->Hashes) - { - RenameOnError(HashSumMismatch); - printHashSumComparision(RealURI, Record->Hashes, Hashes); - Failed(Message, Cfg); - return; - } + RenameOnError(HashSumMismatch); + printHashSumComparision(RealURI, Record->Hashes, Hashes); + Failed(Message, Cfg); + return; } } @@ -1269,14 +1268,9 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,HashStringList con // If we enable compressed indexes and already have gzip, keep it if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI) + ".gz"; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - - // Update DestFile for .gz suffix so that the clean operation keeps it - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; - DestFile += URItoFileName(RealURI) + ".gz"; + // Done, queue for rename on transaction finished + PartialFile = DestFile; + DestFile = GetFinalFilename(RealURI, compExt); return; } diff --git a/methods/copy.cc b/methods/copy.cc index d59f032ff..8c797ff1f 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -27,12 +28,28 @@ class CopyMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); + void CalculateHashes(FetchResult &Res); public: - CopyMethod() : pkgAcqMethod("1.0",SingleInstance) {}; + CopyMethod() : pkgAcqMethod("1.0",SingleInstance|SendConfig) {}; }; +void CopyMethod::CalculateHashes(FetchResult &Res) +{ + // For gzip indexes we need to look inside the gzip for the hash + // We can not use the extension here as its not used in partial + // on a IMS hit + FileFd::OpenMode OpenMode = FileFd::ReadOnly; + if (_config->FindB("Acquire::GzipIndexes", false) == true) + OpenMode = FileFd::ReadOnlyGzip; + + Hashes Hash; + FileFd Fd(Res.Filename, OpenMode); + Hash.AddFD(Fd); + Res.TakeHashes(Hash); +} + // CopyMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -53,6 +70,14 @@ bool CopyMethod::Fetch(FetchItem *Itm) Res.LastModified = Buf.st_mtime; Res.IMSHit = false; URIStart(Res); + + // when the files are identical, just compute the hashes + if(File == Itm->DestFile) + { + CalculateHashes(Res); + URIDone(Res); + return true; + } // See if the file exists FileFd From(File,FileFd::ReadOnly); @@ -82,10 +107,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) if (utimes(Res.Filename.c_str(), times) != 0) return _error->Errno("utimes",_("Failed to set modification time")); - Hashes Hash; - FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd); - Res.TakeHashes(Hash); + CalculateHashes(Res); URIDone(Res); return true; -- cgit v1.2.3-70-g09d2 From dcd5856b11c685ca6d4629212d2978ce196ea65c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 26 Aug 2014 19:08:37 -0700 Subject: Pass ExpectedSize to tthe backend method This ensures that we can stop downloading if the server send too much data by accident (or by a malicious attempt) --- apt-pkg/acquire-method.cc | 2 ++ apt-pkg/acquire-method.h | 1 + apt-pkg/acquire-worker.cc | 4 ++++ methods/http.cc | 10 +++++++++- methods/http.h | 4 +++- methods/server.cc | 5 +++++ methods/server.h | 4 +++- 7 files changed, 27 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index e4a937d1d..9fc176747 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -360,6 +360,8 @@ int pkgAcqMethod::Run(bool Single) if (hash.empty() == false) Tmp->ExpectedHashes.push_back(HashString(*t, hash)); } + char *End; + Tmp->ExpectedSize = strtoll(LookupTag(Message, "Expected-Size", "0").c_str(), &End, 10); Tmp->Next = 0; // Append it to the list diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index cbf79f860..8a680335e 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -48,6 +48,7 @@ class pkgAcqMethod bool IndexFile; bool FailIgnore; HashStringList ExpectedHashes; + unsigned long long ExpectedSize; }; struct FetchResult diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 54be8e99f..8bd1618f4 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -526,6 +526,9 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) if (OutFd == -1) return false; + string ExpectedSize; + strprintf(ExpectedSize, "%llu", Item->Owner->FileSize); + string Message = "600 URI Acquire\n"; Message.reserve(300); Message += "URI: " + Item->URI; @@ -533,6 +536,7 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) HashStringList const hsl = Item->Owner->HashSums(); for (HashStringList::const_iterator hs = hsl.begin(); hs != hsl.end(); ++hs) Message += "\nExpected-" + hs->HashType() + ": " + hs->HashValue(); + Message += "\nExpected-Size: " + ExpectedSize; Message += Item->Owner->Custom600Headers(); Message += "\n\n"; diff --git a/methods/http.cc b/methods/http.cc index c734d3799..916fa464f 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -63,7 +63,8 @@ const unsigned int CircleBuf::BW_HZ=10; // CircleBuf::CircleBuf - Circular input buffer /*{{{*/ // --------------------------------------------------------------------- /* */ -CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0) +CircleBuf::CircleBuf(unsigned long long Size) + : Size(Size), Hash(0), TotalWriten(0) { Buf = new unsigned char[Size]; Reset(); @@ -79,6 +80,7 @@ void CircleBuf::Reset() InP = 0; OutP = 0; StrPos = 0; + TotalWriten = 0; MaxGet = (unsigned long long)-1; OutQueue = string(); if (Hash != 0) @@ -216,6 +218,8 @@ bool CircleBuf::Write(int Fd) return false; } + + TotalWriten += Res; if (Hash != 0) Hash->Add(Buf + (OutP%Size),Res); @@ -649,6 +653,10 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) return _error->Errno("write",_("Error writing to output file")); } + if (ExpectedSize > 0 && In.TotalWriten > ExpectedSize) + return _error->Error("Writing more data than expected (%llu > %llu)", + In.TotalWriten, ExpectedSize); + // Handle commands from APT if (FD_ISSET(STDIN_FILENO,&rfds)) { diff --git a/methods/http.h b/methods/http.h index 5406ce4a7..c98fe8e5f 100644 --- a/methods/http.h +++ b/methods/http.h @@ -63,6 +63,8 @@ class CircleBuf public: Hashes *Hash; + // total amount of data that got written so far + unsigned long long TotalWriten; // Read data in bool Read(int Fd); @@ -81,8 +83,8 @@ class CircleBuf bool ReadSpace() const {return Size - (InP - OutP) > 0;}; bool WriteSpace() const {return InP - OutP > 0;}; - // Dump everything void Reset(); + // Dump everything void Stats(); CircleBuf(unsigned long long Size); diff --git a/methods/server.cc b/methods/server.cc index c91d3b218..1b6511c59 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -531,6 +531,11 @@ int ServerMethod::Loop() // Run the data bool Result = true; + + // ensure we don't fetch too much + if (Queue->ExpectedSize > 0) + Server->ExpectedSize = Queue->ExpectedSize; + if (Server->HaveContent) Result = Server->RunData(File); diff --git a/methods/server.h b/methods/server.h index 5299b3954..0d7333140 100644 --- a/methods/server.h +++ b/methods/server.h @@ -49,6 +49,8 @@ struct ServerState URI Proxy; unsigned long TimeOut; + unsigned long long ExpectedSize; + protected: ServerMethod *Owner; @@ -73,7 +75,7 @@ struct ServerState bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; - State = Header; Persistent = false; Pipeline = true;}; + State = Header; Persistent = false; Pipeline = true; ExpectedSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; /** \brief Transfer the data from the socket */ -- cgit v1.2.3-70-g09d2 From ffd2dd93a640b47663ebdccc4fda00b426b3db71 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 26 Aug 2014 19:20:04 -0700 Subject: make https honor ExpectedSize as well --- methods/https.cc | 6 ++++++ methods/https.h | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index e0348ab58..cacd8a6bc 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -81,6 +81,12 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if(me->File->Write(buffer, size*nmemb) != true) return false; + me->TotalWritten += size*nmemb; + if(me->TotalWritten > me->Queue->ExpectedSize) + return _error->Error("Writing more data than expected (%llu > %llu)", + me->TotalWritten, me->Queue->ExpectedSize); + + return size*nmemb; } diff --git a/methods/https.h b/methods/https.h index faac8a3cd..2335559fb 100644 --- a/methods/https.h +++ b/methods/https.h @@ -66,11 +66,12 @@ class HttpsMethod : public pkgAcqMethod CURL *curl; FetchResult Res; HttpsServerState *Server; + unsigned long long TotalWritten; public: FileFd *File; - HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), File(NULL) + HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), TotalWritten(0), File(NULL) { File = 0; curl = curl_easy_init(); -- cgit v1.2.3-70-g09d2 From c6ee61eab54edf6cc3fbe118d304d72a860e1451 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 2 Sep 2014 15:50:19 +0200 Subject: Make Proxy-Auto-Detect check for each host When doing Acquire::http{,s}::Proxy-Auto-Detect, run the auto-detect command for each host instead of only once. This should make using "proxy" from libproxy-tools feasible which can then be used for PAC style or other proxy configurations. Closes: #759264 --- apt-pkg/contrib/proxy.cc | 82 ++++++++++++++++++++++++++++++++++++++++ apt-pkg/contrib/proxy.h | 16 ++++++++ cmdline/apt-helper.cc | 16 ++++++++ methods/http.cc | 62 +----------------------------- methods/http.h | 3 -- methods/https.cc | 4 ++ test/integration/test-apt-helper | 80 ++++++++++++++++++++++++++++----------- 7 files changed, 178 insertions(+), 85 deletions(-) create mode 100644 apt-pkg/contrib/proxy.cc create mode 100644 apt-pkg/contrib/proxy.h (limited to 'methods') diff --git a/apt-pkg/contrib/proxy.cc b/apt-pkg/contrib/proxy.cc new file mode 100644 index 000000000..b58db8478 --- /dev/null +++ b/apt-pkg/contrib/proxy.cc @@ -0,0 +1,82 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Proxy - Proxy releated functions + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include +#include +#include +#include + +#include +#include + +#include "proxy.h" + + +// AutoDetectProxy - auto detect proxy /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool AutoDetectProxy(URI &URL) +{ + // we support both http/https debug options + bool Debug = _config->FindB("Debug::Acquire::"+URL.Access,false); + + // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old + // name without the dash ("-") + std::string AutoDetectProxyCmd = _config->Find("Acquire::"+URL.Access+"::Proxy-Auto-Detect", + _config->Find("Acquire::"+URL.Access+"::ProxyAutoDetect")); + + if (AutoDetectProxyCmd.empty()) + return true; + + if (Debug) + std::clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << std::endl; + + int Pipes[2] = {-1,-1}; + if (pipe(Pipes) != 0) + return _error->Errno("pipe", "Failed to create Pipe"); + + pid_t Process = ExecFork(); + if (Process == 0) + { + close(Pipes[0]); + dup2(Pipes[1],STDOUT_FILENO); + SetCloseExec(STDOUT_FILENO,false); + + std::string foo = URL; + const char *Args[4]; + Args[0] = AutoDetectProxyCmd.c_str(); + Args[1] = foo.c_str(); + Args[2] = 0; + execv(Args[0],(char **)Args); + std::cerr << "Failed to exec method " << Args[0] << std::endl; + _exit(100); + } + char buf[512]; + int InFd = Pipes[0]; + close(Pipes[1]); + int res = read(InFd, buf, sizeof(buf)-1); + ExecWait(Process, "ProxyAutoDetect", true); + + if (res < 0) + return _error->Errno("read", "Failed to read"); + if (res == 0) + return _error->Warning("ProxyAutoDetect returned no data"); + + // add trailing \0 + buf[res] = 0; + + if (Debug) + std::clog << "auto detect command returned: '" << buf << "'" << std::endl; + + if (strstr(buf, URL.Access.c_str()) == buf) + _config->Set("Acquire::"+URL.Access+"::proxy::"+URL.Host, _strstrip(buf)); + + return true; +} + /*}}}*/ diff --git a/apt-pkg/contrib/proxy.h b/apt-pkg/contrib/proxy.h new file mode 100644 index 000000000..2cbcd07b4 --- /dev/null +++ b/apt-pkg/contrib/proxy.h @@ -0,0 +1,16 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Proxy - Proxy operations + + ##################################################################### */ + /*}}}*/ +#ifndef PKGLIB_PROXY_H +#define PKGLIB_PROXY_H + +class URI; +bool AutoDetectProxy(URI &URL); + + +#endif diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index b0edafcbd..dd43ea1bc 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,19 @@ #include /*}}}*/ +static bool DoAutoDetectProxy(CommandLine &CmdL) +{ + if (CmdL.FileSize() != 2) + return _error->Error(_("Need one URL as argument")); + URI ServerURL(CmdL.FileList[1]); + AutoDetectProxy(ServerURL); + std::string SpecificProxy = _config->Find("Acquire::"+ServerURL.Access+"::Proxy::" + ServerURL.Host); + ioprintf(std::cout, "Using proxy '%s' for URL '%s'\n", + SpecificProxy.c_str(), std::string(ServerURL).c_str()); + + return true; +} + static bool DoDownloadFile(CommandLine &CmdL) { if (CmdL.FileSize() <= 2) @@ -70,6 +84,7 @@ static bool ShowHelp(CommandLine &) "\n" "Commands:\n" " download-file - download the given uri to the target-path\n" + " auto-detect-proxy - detect proxy using apt.conf\n" "\n" " This APT helper has Super Meep Powers.\n"); return true; @@ -80,6 +95,7 @@ int main(int argc,const char *argv[]) /*{{{*/ { CommandLine::Dispatch Cmds[] = {{"help",&ShowHelp}, {"download-file", &DoDownloadFile}, + {"auto-detect-proxy", &DoAutoDetectProxy}, {0,0}}; std::vector Args = getCommandArgs( diff --git a/methods/http.cc b/methods/http.cc index 7c7949eac..f2a4a4db6 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -304,6 +305,7 @@ bool HttpServerState::Open() Persistent = true; // Determine the proxy setting + AutoDetectProxy(ServerName); string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host); if (!SpecificProxy.empty()) { @@ -762,66 +764,6 @@ bool HttpMethod::Configuration(string Message) PipelineDepth); Debug = _config->FindB("Debug::Acquire::http",false); - // Get the proxy to use - AutoDetectProxy(); - - return true; -} - /*}}}*/ -// HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/ -// --------------------------------------------------------------------- -/* */ -bool HttpMethod::AutoDetectProxy() -{ - // option is "Acquire::http::Proxy-Auto-Detect" but we allow the old - // name without the dash ("-") - AutoDetectProxyCmd = _config->Find("Acquire::http::Proxy-Auto-Detect", - _config->Find("Acquire::http::ProxyAutoDetect")); - - if (AutoDetectProxyCmd.empty()) - return true; - - if (Debug) - clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl; - - int Pipes[2] = {-1,-1}; - if (pipe(Pipes) != 0) - return _error->Errno("pipe", "Failed to create Pipe"); - - pid_t Process = ExecFork(); - if (Process == 0) - { - close(Pipes[0]); - dup2(Pipes[1],STDOUT_FILENO); - SetCloseExec(STDOUT_FILENO,false); - - const char *Args[2]; - Args[0] = AutoDetectProxyCmd.c_str(); - Args[1] = 0; - execv(Args[0],(char **)Args); - cerr << "Failed to exec method " << Args[0] << endl; - _exit(100); - } - char buf[512]; - int InFd = Pipes[0]; - close(Pipes[1]); - int res = read(InFd, buf, sizeof(buf)-1); - ExecWait(Process, "ProxyAutoDetect", true); - - if (res < 0) - return _error->Errno("read", "Failed to read"); - if (res == 0) - return _error->Warning("ProxyAutoDetect returned no data"); - - // add trailing \0 - buf[res] = 0; - - if (Debug) - clog << "auto detect command returned: '" << buf << "'" << endl; - - if (strstr(buf, "http://") == buf) - _config->Set("Acquire::http::proxy", _strstrip(buf)); - return true; } /*}}}*/ diff --git a/methods/http.h b/methods/http.h index 5406ce4a7..1df9fa07d 100644 --- a/methods/http.h +++ b/methods/http.h @@ -124,9 +124,6 @@ class HttpMethod : public ServerMethod public: virtual void SendReq(FetchItem *Itm); - /** \brief Try to AutoDetect the proxy */ - bool AutoDetectProxy(); - virtual bool Configuration(std::string Message); virtual ServerState * CreateServerState(URI uri); diff --git a/methods/https.cc b/methods/https.cc index e0348ab58..0499af0c5 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -107,6 +108,9 @@ void HttpsMethod::SetupProxy() /*{{{*/ { URI ServerName = Queue->Uri; + // Determine the proxy setting + AutoDetectProxy(ServerName); + // Curl should never read proxy settings from the environment, as // we determine which proxy to use. Do this for consistency among // methods and prevent an environment variable overriding a diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper index 6505b5956..c749224ca 100755 --- a/test/integration/test-apt-helper +++ b/test/integration/test-apt-helper @@ -9,31 +9,67 @@ configarchitecture "i386" changetohttpswebserver -echo "foo" > aptarchive/foo +test_apt_helper_download() { + echo "foo" > aptarchive/foo -msgtest 'apt-file download-file md5sum' -apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail -testfileequal foo2 'foo' + msgtest 'apt-file download-file md5sum' + apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail + testfileequal foo2 'foo' -msgtest 'apt-file download-file sha1' -apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail -testfileequal foo1 'foo' + msgtest 'apt-file download-file sha1' + apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail + testfileequal foo1 'foo' -msgtest 'apt-file download-file sha256' -apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail -testfileequal foo3 'foo' + msgtest 'apt-file download-file sha256' + apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail + testfileequal foo3 'foo' -msgtest 'apt-file download-file no-hash' -apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail -testfileequal foo4 'foo' - -msgtest 'apt-file download-file wrong hash' -if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then - msgpass -else - msgfail -fi -testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo Hash Sum mismatch + msgtest 'apt-file download-file no-hash' + apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail + testfileequal foo4 'foo' + + msgtest 'apt-file download-file wrong hash' + if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then + msgpass + else + msgfail + fi + testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo Hash Sum mismatch E: Download Failed' -testfileequal foo5.FAILED 'foo' + testfileequal foo5.FAILED 'foo' +} + +test_apt_helper_detect_proxy() { + # no proxy + testequal "Using proxy '' for URL 'http://example.com/'" apthelper auto-detect-proxy http://example.com/ + + + # http auto detect proxy script + cat > apt-proxy-detect <<'EOF' +#!/bin/sh -e +echo "http://some-proxy" +EOF + chmod 755 apt-proxy-detect + echo "Acquire::http::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect + + testequal "Using proxy 'http://some-proxy' for URL 'http://www.example.com/'" apthelper auto-detect-proxy http://www.example.com + + + # https auto detect proxy script + cat > apt-proxy-detect <<'EOF' +#!/bin/sh -e +echo "https://https-proxy" +EOF + chmod 755 apt-proxy-detect + echo "Acquire::https::Proxy-Auto-Detect \"$(pwd)/apt-proxy-detect\";" > rootdir/etc/apt/apt.conf.d/02proxy-detect + + testequal "Using proxy 'https://https-proxy' for URL 'https://ssl.example.com/'" apthelper auto-detect-proxy https://ssl.example.com + + + +} + +test_apt_helper_download +test_apt_helper_detect_proxy + -- cgit v1.2.3-70-g09d2 From 9622b2111095c3fc705ec0615d27fe403e18c3b8 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 5 Sep 2014 16:24:32 +0200 Subject: Improve Debug::Acquire::http debug output Prefix all answers with the URL that the answer is for. This helps when debugging and pipeline is enabled. --- methods/server.cc | 7 ++++--- methods/server.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index 5a13f18a7..92d94e638 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -44,7 +44,8 @@ time_t ServerMethod::FailTime = 0; // --------------------------------------------------------------------- /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header parse error occurred */ -ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) +ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, + const std::string &Uri) { State = Header; @@ -66,7 +67,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) continue; if (Owner->Debug == true) - clog << Data; + clog << "Answer for: " << Uri << endl << Data; for (string::const_iterator I = Data.begin(); I < Data.end(); ++I) { @@ -478,7 +479,7 @@ int ServerMethod::Loop() Fetch(0); // Fetch the next URL header data from the server. - switch (Server->RunHeaders(File)) + switch (Server->RunHeaders(File, Queue->Uri)) { case ServerState::RUN_HEADERS_OK: break; diff --git a/methods/server.h b/methods/server.h index 0f45ab994..f5e68d902 100644 --- a/methods/server.h +++ b/methods/server.h @@ -68,7 +68,7 @@ struct ServerState RUN_HEADERS_PARSE_ERROR }; /** \brief Get the headers before the data */ - RunHeadersResult RunHeaders(FileFd * const File); + RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; -- cgit v1.2.3-70-g09d2 From ca7fd76c2f30c100dcf1c12e717ce397cccd690b Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 16 Sep 2014 20:23:43 +0200 Subject: SECURITY UPDATE for CVE-2014-{0488,0487,0489} incorrect invalidating of unauthenticated data (CVE-2014-0488) incorect verification of 304 reply (CVE-2014-0487) incorrect verification of Acquire::Gzip indexes (CVE-2014-0489) --- apt-pkg/acquire-item.cc | 97 +++++++++++++++++++++++------- apt-pkg/acquire-item.h | 8 +++ apt-pkg/contrib/fileutl.h | 2 + methods/copy.cc | 32 ++++++++-- test/integration/test-apt-update-stale | 46 ++++++++++++++ test/integration/test-apt-update-unauth | 48 +++++++++++++++ test/integration/test-hashsum-verification | 14 ++++- 7 files changed, 217 insertions(+), 30 deletions(-) create mode 100755 test/integration/test-apt-update-stale create mode 100755 test/integration/test-apt-update-unauth (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 6cb9b012a..058b8bf74 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1021,6 +1021,31 @@ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ Item::Failed(Message,Cnf); } /*}}}*/ +// pkgAcqIndex::GetFinalFilename - Return the full final file path /*{{{*/ +std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, + std::string const &compExt) +{ + std::string FinalFile = _config->FindDir("Dir::State::lists"); + FinalFile += URItoFileName(URI); + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + FinalFile += ".gz"; + return FinalFile; +} + /*}}}*/ +// AcqIndex::ReverifyAfterIMS - Reverify index after an ims-hit /*{{{*/ +void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) +{ + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + DestFile += ".gz"; + + string FinalFile = GetFinalFilename(RealURI, compExt); + Rename(FinalFile, FileName); + Decompression = true; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); +} + /*}}}*/ // AcqIndex::Done - Finished a fetch /*{{{*/ // --------------------------------------------------------------------- /* This goes through a number of states.. On the initial fetch the @@ -1032,6 +1057,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, pkgAcquire::MethodConfig *Cfg) { Item::Done(Message,Size,Hash,Cfg); + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); if (Decompression == true) { @@ -1043,6 +1069,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, if (!ExpectedHash.empty() && ExpectedHash.toStr() != Hash) { + Desc.URI = RealURI; RenameOnError(HashSumMismatch); return; } @@ -1053,9 +1080,9 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, /* Always verify the index file for correctness (all indexes must * have a Package field) (LP: #346386) (Closes: #627642) */ - FileFd fd(DestFile, FileFd::ReadOnly); + FileFd fd(DestFile, FileFd::ReadOnlyGzip); // Only test for correctness if the file is not empty (empty is ok) - if (fd.FileSize() > 0) + if (fd.Size() > 0) { pkgTagSection sec; pkgTagFile tag(&fd); @@ -1069,8 +1096,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, } // Done, move it into position - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI); + string FinalFile = GetFinalFilename(RealURI, compExt); Rename(DestFile,FinalFile); chmod(FinalFile.c_str(),0644); @@ -1078,7 +1104,9 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, will work OK */ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(RealURI); - + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") + DestFile += ".gz"; + // Remove the compressed version. if (Erase == true) unlink(DestFile.c_str()); @@ -1094,7 +1122,10 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, { // The files timestamp matches if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true) - return; + { + ReverifyAfterIMS(FileName); + return; + } Decompression = true; Local = true; DestFile += ".decomp"; @@ -1111,15 +1142,12 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, ErrorText = "Method gave a blank filename"; } - std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - // The files timestamp matches - if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - // Update DestFile for .gz suffix so that the clean operation keeps it - DestFile += ".gz"; + if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) + { + ReverifyAfterIMS(FileName); return; - } + } if (FileName == DestFile) Erase = true; @@ -1128,16 +1156,16 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, string decompProg; - // If we enable compressed indexes and already have gzip, keep it - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) { - string FinalFile = _config->FindDir("Dir::State::lists"); - FinalFile += URItoFileName(RealURI) + ".gz"; - Rename(DestFile,FinalFile); - chmod(FinalFile.c_str(),0644); - - // Update DestFile for .gz suffix so that the clean operation keeps it - DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + // If we enable compressed indexes, queue for hash verification + if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) + { + DestFile = _config->FindDir("Dir::State::lists"); DestFile += URItoFileName(RealURI) + ".gz"; + + Decompression = true; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); + return; } @@ -1181,6 +1209,9 @@ string pkgAcqIndexTrans::Custom600Headers() string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); + if (_config->FindB("Acquire::GzipIndexes",false)) + Final += ".gz"; + struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) return "\nFail-Ignore: true\nIndex-File: true"; @@ -1510,6 +1541,28 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ std::cerr << "Signature verification succeeded: " << DestFile << std::endl; + // do not trust any previously unverified content that we may have + string LastGoodSigFile = _config->FindDir("Dir::State::lists").append("partial/").append(URItoFileName(RealURI)); + if (DestFile != SigFile) + LastGoodSigFile.append(".gpg"); + LastGoodSigFile.append(".reverify"); + if(IMSHit == false && RealFileExists(LastGoodSigFile) == false) + { + for (vector ::const_iterator Target = IndexTargets->begin(); + Target != IndexTargets->end(); + ++Target) + { + // remove old indexes + std::string index = _config->FindDir("Dir::State::lists") + + URItoFileName((*Target)->URI); + unlink(index.c_str()); + // and also old gzipindexes + index += ".gz"; + unlink(index.c_str()); + } + } + + // Download further indexes with verification QueueIndexes(true); diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 06537bf2c..384c5ee2b 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -706,6 +706,14 @@ class pkgAcqIndex : public pkgAcquire::Item */ std::string CompressionExtension; + /** \brief Get the full pathname of the final file for the given URI + */ + std::string GetFinalFilename(std::string const &URI, + std::string const &compExt); + + /** \brief Schedule file for verification after a IMS hit */ + void ReverifyAfterIMS(std::string const &FileName); + public: // Specialized action members diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index cc1a98eae..667057067 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -85,7 +85,9 @@ class FileFd bool Skip(unsigned long long To); bool Truncate(unsigned long long To); unsigned long long Tell(); + // the size of the file content (compressed files will be uncompressed first) unsigned long long Size(); + // the size of the file itself unsigned long long FileSize(); time_t ModificationTime(); diff --git a/methods/copy.cc b/methods/copy.cc index d59f032ff..5570f31c8 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -27,12 +28,28 @@ class CopyMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); + void CalculateHashes(FetchResult &Res); public: - CopyMethod() : pkgAcqMethod("1.0",SingleInstance) {}; + CopyMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {}; }; +void CopyMethod::CalculateHashes(FetchResult &Res) +{ + // For gzip indexes we need to look inside the gzip for the hash + // We can not use the extension here as its not used in partial + // on a IMS hit + FileFd::OpenMode OpenMode = FileFd::ReadOnly; + if (_config->FindB("Acquire::GzipIndexes", false) == true) + OpenMode = FileFd::ReadOnlyGzip; + + Hashes Hash; + FileFd Fd(Res.Filename, OpenMode); + Hash.AddFD(Fd); + Res.TakeHashes(Hash); +} + // CopyMethod::Fetch - Fetch a file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -54,6 +71,14 @@ bool CopyMethod::Fetch(FetchItem *Itm) Res.IMSHit = false; URIStart(Res); + // just calc the hashes if the source and destination are identical + if (File == Itm->DestFile) + { + CalculateHashes(Res); + URIDone(Res); + return true; + } + // See if the file exists FileFd From(File,FileFd::ReadOnly); FileFd To(Itm->DestFile,FileFd::WriteAtomic); @@ -82,10 +107,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) if (utimes(Res.Filename.c_str(), times) != 0) return _error->Errno("utimes",_("Failed to set modification time")); - Hashes Hash; - FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd); - Res.TakeHashes(Hash); + CalculateHashes(Res); URIDone(Res); return true; diff --git a/test/integration/test-apt-update-stale b/test/integration/test-apt-update-stale new file mode 100755 index 000000000..780ff79af --- /dev/null +++ b/test/integration/test-apt-update-stale @@ -0,0 +1,46 @@ +#!/bin/sh +# +# Ensure that a MITM can not stale the Packages/Sources without +# raising a error message. Note that the Release file is protected +# via the "Valid-Until" header +# +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +insertpackage 'unstable' 'foo' 'all' '1.0' + +setupaptarchive +changetowebserver +aptget update -qq + +# insert new version +mkdir aptarchive/dists/unstable/main/binary-i386/saved +cp -p aptarchive/dists/unstable/main/binary-i386/Packages* \ + aptarchive/dists/unstable/main/binary-i386/saved +insertpackage 'unstable' 'foo' 'all' '2.0' + +# not using compressfile for compat with older apt releases +gzip -c aptarchive/dists/unstable/main/binary-i386/Packages > \ + aptarchive/dists/unstable/main/binary-i386/Packages.gz +generatereleasefiles +signreleasefiles + +# ensure that we do not get a I-M-S hit for the Release file +touch -d "+1hour" aptarchive/dists/unstable/*Release* + +# but now only deliver the previous Packages file instead of the new one +# (simulating a stale attack) +cp -p aptarchive/dists/unstable/main/binary-i386/saved/Packages* \ + aptarchive/dists/unstable/main/binary-i386/ + +# ensure this raises a error +testequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages Hash Sum mismatch + +E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq + + diff --git a/test/integration/test-apt-update-unauth b/test/integration/test-apt-update-unauth new file mode 100755 index 000000000..13487603c --- /dev/null +++ b/test/integration/test-apt-update-unauth @@ -0,0 +1,48 @@ +#!/bin/sh +# +# Ensure that when going from unauthenticated to authenticated all +# files are checked again +# +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +insertpackage 'unstable' 'foo' 'all' '1.0' +insertsource 'unstable' 'foo' 'all' '1.0' + +setupaptarchive +changetowebserver + +runtest() { + # start unauthenticated + find rootdir/var/lib/apt/lists/ -type f | xargs rm -f + rm -f aptarchive/dists/unstable/*Release* + aptget update -qq + + # become authenticated + generatereleasefiles + signreleasefiles + + # and ensure we do download the data again + msgtest "Check that the data is check when going to authenticated" + if aptget update |grep -q Hit; then + msgfail + else + msgpass + fi +} + +for COMPRESSEDINDEXES in 'false' 'true'; do + echo "Acquire::GzipIndexes \"$COMPRESSEDINDEXES\";" > rootdir/etc/apt/apt.conf.d/compressindexes + if $COMPRESSEDINDEXES; then + msgmsg 'Run tests with GzipIndexes enabled' + else + msgmsg 'Run tests with GzipIndexes disabled' + fi + + runtest +done diff --git a/test/integration/test-hashsum-verification b/test/integration/test-hashsum-verification index e77efb46e..2a400dcb4 100755 --- a/test/integration/test-hashsum-verification +++ b/test/integration/test-hashsum-verification @@ -64,7 +64,7 @@ runtest() { msgtest 'No package from the source available' [ "$(aptcache show apt 2>&1)" = "E: No packages found" ] && msgpass || msgfail msgtest 'No Packages file in /var/lib/apt/lists' - [ "$(ls rootdir/var/lib/apt/lists/*Package* 2>/dev/null)" = "" ] && msgpass || msgfail + [ "$(ls rootdir/var/lib/apt/lists/*Package* 2>/dev/null | grep -v FAILED 2>/dev/null)" = "" ] && msgpass || msgfail # now with the unsigned Release file rm -rf rootdir/var/lib/apt/lists @@ -75,5 +75,13 @@ runtest() { } -runtest - +for COMPRESSEDINDEXES in 'false' 'true'; do + echo "Acquire::GzipIndexes \"$COMPRESSEDINDEXES\";" > rootdir/etc/apt/apt.conf.d/compressindexes + if $COMPRESSEDINDEXES; then + msgmsg 'Run tests with GzipIndexes enabled' + else + msgmsg 'Run tests with GzipIndexes disabled' + fi + + runtest +done -- cgit v1.2.3-70-g09d2 From 9da539c5aff025aab99537be1c75e8c6a853fd83 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 19 Sep 2014 16:41:55 +0200 Subject: Fix regression when copy: is used for a relative path When we do a ReverifyAfterIMS() we use the copy: method to verify the hashes again. If the user uses -o Dir=./something/relative this fails because we use the URI class in copy.cc that strips away the leading relative part. By not using URI this is fixed. Closes: #762160 --- methods/copy.cc | 4 ++-- test/integration/test-bug-762160-relpath | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index 5570f31c8..b78053d36 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -55,8 +55,8 @@ void CopyMethod::CalculateHashes(FetchResult &Res) /* */ bool CopyMethod::Fetch(FetchItem *Itm) { - URI Get = Itm->Uri; - std::string File = Get.Path; + // this ensures that relative paths work in copy + std::string File = Itm->Uri.substr(Itm->Uri.find(':')+1); // Stat the file and send a start message struct stat Buf; diff --git a/test/integration/test-bug-762160-relpath b/test/integration/test-bug-762160-relpath index 0af71f57b..204587727 100755 --- a/test/integration/test-bug-762160-relpath +++ b/test/integration/test-bug-762160-relpath @@ -1,9 +1,9 @@ #!/bin/sh +# regresion test for bug #762160 where apt-get update fails when a +# relative directory is given +# set -e -# dpkg implements versioned provides in commit 5bb02fe80e9f40dcad9703a72f67cf615ff217b5 -# but previous versions seem to allow parsing, working and ignoring it. - TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment @@ -11,5 +11,7 @@ configarchitecture 'amd64' insertpackage 'unstable' 'foo' 'all' '1' setupaptarchive +changetowebserver -aptget update -o Dir=./apt +testsuccess aptget update -o Dir=./rootdir +testsuccess aptget update -o Dir=./rootdir \ No newline at end of file -- cgit v1.2.3-70-g09d2 From b0f4b486e6850c5f98520ccf19da71d0ed748ae4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Sun, 21 Sep 2014 10:18:03 +0200 Subject: generalize Acquire::GzipIndex --- apt-pkg/acquire-item.cc | 44 ++++++++++------ apt-pkg/deb/debindexfile.cc | 60 ++++++++++++++-------- apt-private/private-update.cc | 9 +++- cmdline/apt-get.cc | 2 +- methods/copy.cc | 11 ++-- .../test-bug-595691-empty-and-broken-archive-files | 2 +- test/integration/test-compressed-indexes | 20 ++++---- 7 files changed, 90 insertions(+), 58 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 5df43726b..da57f8d3b 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -969,8 +969,10 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); if (comprExt == "uncompressed") Desc.URI = URI; - else + else { Desc.URI = URI + '.' + comprExt; + DestFile = DestFile + '.' + comprExt; + } Desc.Description = URIDesc; Desc.Owner = this; @@ -984,10 +986,11 @@ void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &S /* The only header we use is the last-modified header. */ string pkgAcqIndex::Custom600Headers() { + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + Final += compExt; string msg = "\nIndex-File: true"; // FIXME: this really should use "IndexTarget::IsOptional()" but that @@ -1027,8 +1030,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, { std::string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(URI); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - FinalFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false) == true) + FinalFile += '.' + compExt; return FinalFile; } /*}}}*/ @@ -1036,8 +1039,8 @@ std::string pkgAcqIndex::GetFinalFilename(std::string const &URI, void pkgAcqIndex::ReverifyAfterIMS(std::string const &FileName) { std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - DestFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false) == true) + DestFile += compExt; string FinalFile = GetFinalFilename(RealURI, compExt); Rename(FinalFile, FileName); @@ -1080,7 +1083,7 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, /* Always verify the index file for correctness (all indexes must * have a Package field) (LP: #346386) (Closes: #627642) */ - FileFd fd(DestFile, FileFd::ReadOnlyGzip); + FileFd fd(DestFile, FileFd::ReadOnly, FileFd::Extension); // Only test for correctness if the file is not empty (empty is ok) if (fd.Size() > 0) { @@ -1104,8 +1107,8 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, will work OK */ DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(RealURI); - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz") - DestFile += ".gz"; + if (_config->FindB("Acquire::GzipIndexes",false)) + DestFile += '.' + compExt; // Remove the compressed version. if (Erase == true) @@ -1146,16 +1149,23 @@ void pkgAcqIndex::Done(string Message,unsigned long long Size,string Hash, // matching the Release file if (!Local && StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { + // set destfile to the final destfile + if(_config->FindB("Acquire::GzipIndexes",false) == false) + { + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(RealURI); + } + ReverifyAfterIMS(FileName); return; } string decompProg; // If we enable compressed indexes, queue for hash verification - if (_config->FindB("Acquire::GzipIndexes",false) && compExt == "gz" && !Local) + if (_config->FindB("Acquire::GzipIndexes",false)) { DestFile = _config->FindDir("Dir::State::lists"); - DestFile += URItoFileName(RealURI) + ".gz"; + DestFile += URItoFileName(RealURI) + '.' + compExt; Decompression = true; Desc.URI = "copy:" + FileName; @@ -1201,11 +1211,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, // --------------------------------------------------------------------- string pkgAcqIndexTrans::Custom600Headers() { + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); string Final = _config->FindDir("Dir::State::lists"); Final += URItoFileName(RealURI); - if (_config->FindB("Acquire::GzipIndexes",false)) - Final += ".gz"; + Final += compExt; struct stat Buf; if (stat(Final.c_str(),&Buf) != 0) @@ -1552,8 +1562,12 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ URItoFileName((*Target)->URI); unlink(index.c_str()); // and also old gzipindexes - index += ".gz"; - unlink(index.c_str()); + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + index += '.' + (*t); + unlink(index.c_str()); + } } } diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index a0dd15cd8..5b4289e92 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -80,14 +80,18 @@ pkgSrcRecords::Parser *debSourcesIndex::CreateSrcParser() const { string SourcesURI = _config->FindDir("Dir::State::lists") + URItoFileName(IndexURI("Sources")); - string SourcesURIgzip = SourcesURI + ".gz"; - if (!FileExists(SourcesURI) && !FileExists(SourcesURIgzip)) - return NULL; - else if (!FileExists(SourcesURI) && FileExists(SourcesURIgzip)) - SourcesURI = SourcesURIgzip; - - return new debSrcRecordParser(SourcesURI,this); + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p; + p = SourcesURI + '.' + *t; + if (FileExists(p)) + return new debSrcRecordParser(p, this); + } + if (FileExists(SourcesURI)) + return new debSrcRecordParser(SourcesURI, this); + return NULL; } /*}}}*/ // SourcesIndex::Describe - Give a descriptive path to the index /*{{{*/ @@ -129,11 +133,15 @@ string debSourcesIndex::Info(const char *Type) const inline string debSourcesIndex::IndexFile(const char *Type) const { string s = URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debSourcesIndex::IndexURI(const char *Type) const @@ -259,11 +267,15 @@ string debPackagesIndex::Info(const char *Type) const inline string debPackagesIndex::IndexFile(const char *Type) const { string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debPackagesIndex::IndexURI(const char *Type) const { @@ -411,11 +423,15 @@ debTranslationsIndex::debTranslationsIndex(string URI,string Dist,string Section inline string debTranslationsIndex::IndexFile(const char *Type) const { string s =_config->FindDir("Dir::State::lists") + URItoFileName(IndexURI(Type)); - string sgzip = s + ".gz"; - if (!FileExists(s) && FileExists(sgzip)) - return sgzip; - else - return s; + + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + { + string p = s + '.' + *t; + if (FileExists(p)) + return p; + } + return s; } string debTranslationsIndex::IndexURI(const char *Type) const { diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc index 860d84b86..1cf3012ed 100644 --- a/apt-private/private-update.cc +++ b/apt-private/private-update.cc @@ -56,10 +56,17 @@ bool DoUpdate(CommandLine &CmdL) if (List->GetIndexes(&Fetcher,true) == false) return false; + std::string compExt = APT::Configuration::getCompressionTypes()[0]; pkgAcquire::UriIterator I = Fetcher.UriBegin(); for (; I != Fetcher.UriEnd(); ++I) - c1out << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' << + { + std::string FileName = flNotDir(I->Owner->DestFile); + if(compExt.empty() == false && + APT::String::Endswith(FileName, compExt)) + FileName = FileName.substr(0, FileName.size() - compExt.size() - 1); + c1out << '\'' << I->URI << "' " << FileName << ' ' << I->Owner->FileSize << ' ' << I->Owner->HashSum() << std::endl; + } return true; } diff --git a/cmdline/apt-get.cc b/cmdline/apt-get.cc index 845d67d2b..2e283da5a 100644 --- a/cmdline/apt-get.cc +++ b/cmdline/apt-get.cc @@ -665,7 +665,7 @@ static bool DoDownload(CommandLine &CmdL) { pkgAcquire::UriIterator I = Fetcher.UriBegin(); for (; I != Fetcher.UriEnd(); ++I) - cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' << + cout << '\'' << I->URI << "' " << flNotDir(I->Owner->DestFile) << ' ' << I->Owner->FileSize << ' ' << I->Owner->HashSum() << endl; return true; } diff --git a/methods/copy.cc b/methods/copy.cc index b78053d36..40f8f85ec 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -37,15 +37,12 @@ class CopyMethod : public pkgAcqMethod void CopyMethod::CalculateHashes(FetchResult &Res) { - // For gzip indexes we need to look inside the gzip for the hash - // We can not use the extension here as its not used in partial - // on a IMS hit - FileFd::OpenMode OpenMode = FileFd::ReadOnly; + Hashes Hash; + FileFd::CompressMode CompressMode = FileFd::None; if (_config->FindB("Acquire::GzipIndexes", false) == true) - OpenMode = FileFd::ReadOnlyGzip; + CompressMode = FileFd::Extension; - Hashes Hash; - FileFd Fd(Res.Filename, OpenMode); + FileFd Fd(Res.Filename, FileFd::ReadOnly, CompressMode); Hash.AddFD(Fd); Res.TakeHashes(Hash); } diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index a05ed5fa6..23a638801 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -121,7 +121,7 @@ Reading package lists..." "empty archive Packages.$COMPRESS over http" testaptgetupdate "Get: http://localhost:8080 Packages Err http://localhost:8080 Packages Empty files can't be valid archives -W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives +W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages.${COMPRESS}) Empty files can't be valid archives E: Some index files failed to download. They have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http" } diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes index 6671dd75a..819cbd35e 100755 --- a/test/integration/test-compressed-indexes +++ b/test/integration/test-compressed-indexes @@ -5,7 +5,7 @@ TESTDIR=$(readlink -f $(dirname $0)) . $TESTDIR/framework setupenvironment -configcompression '.' 'gz' # only gz is supported for this, so ensure it is used +configcompression '.' 'xz' 'gz' configarchitecture "i386" buildsimplenativepackage "testpkg" "i386" "1.0" @@ -32,13 +32,13 @@ testrun() { if [ "$1" = "compressed" ]; then ! test -e rootdir/var/lib/apt/lists/*_Packages || F=1 ! test -e rootdir/var/lib/apt/lists/*_Sources || F=1 - test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1 - test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1 + test -e rootdir/var/lib/apt/lists/*_Packages.xz || F=1 + test -e rootdir/var/lib/apt/lists/*_Sources.xz || F=1 else test -e rootdir/var/lib/apt/lists/*_Packages || F=1 test -e rootdir/var/lib/apt/lists/*_Sources || F=1 - ! test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1 - ! test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1 + ! test -e rootdir/var/lib/apt/lists/*_Packages.xz || F=1 + ! test -e rootdir/var/lib/apt/lists/*_Sources.xz || F=1 fi if [ -n "$F" ]; then ls -laR rootdir/var/lib/apt/lists/ @@ -84,10 +84,9 @@ msgmsg "File: Test with uncompressed indexes (update unchanged without pdiffs)" testrun rm -rf rootdir/var/lib/apt/lists -echo 'Acquire::CompressionTypes::Order:: "gz"; -Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex +echo 'Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex -testsuccess aptget update +testsuccess aptget update -o Debug::pkgAcquire::worker=1 msgmsg "File: Test with compressed indexes" testrun "compressed" @@ -118,14 +117,13 @@ msgmsg "HTTP: Test with uncompressed indexes (update unchanged without pdiffs)" testrun rm -rf rootdir/var/lib/apt/lists -echo 'Acquire::CompressionTypes::Order:: "gz"; -Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex +echo 'Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex testsuccess aptget update msgmsg "HTTP: Test with compressed indexes" testrun "compressed" -testsuccess aptget update -o Acquire::Pdiffs=1 +testsuccess aptget update -o Acquire::Pdiffs=1 -o debug::pkgAcquire::Worker=1 -o debug::pkgAcquire::Auth=1 msgmsg "HTTP: Test with compressed indexes (update unchanged with pdiffs)" testrun "compressed" -- cgit v1.2.3-70-g09d2 From 3927c6da48c206b6b251661f44680d9883b4f6b4 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Sep 2014 16:22:05 +0200 Subject: Drop Privileges to "Debian-apt" in most acquire methods Add a new "Debian-apt" user that owns the /var/lib/apt/lists and /var/cache/apt/archive directories. The methods http, https, ftp, gpgv, gzip switch to this user when they start. Thanks to Julian and "ioerror" and tors "switch_id()" code. --- apt-pkg/contrib/fileutl.cc | 34 +++++++++++++++++++++++++++++++++- debian/apt.postinst | 7 +++++++ methods/copy.cc | 2 ++ methods/ftp.cc | 3 +++ methods/gpgv.cc | 3 +++ methods/gzip.cc | 2 ++ methods/http_main.cc | 4 +++- methods/https.cc | 2 ++ 8 files changed, 55 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 6b8f04dea..de67a94b9 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -64,6 +65,10 @@ #include #include +#if __gnu_linux__ +#include +#endif + #include /*}}}*/ @@ -2173,14 +2178,41 @@ bool DropPrivs() if (getuid() != 0) return true; - const std::string nobody = _config->Find("APT::User::Nobody", "nobody"); + if(_config->FindB("Debug::NoDropPrivs", false) == true) + return true; + + const std::string nobody = _config->Find("APT::User::Nobody", "Debian-apt"); struct passwd *pw = getpwnam(nobody.c_str()); if (pw == NULL) return _error->Warning("No user %s, can not drop rights", nobody.c_str()); + +#if __gnu_linux__ + // see prctl(2), needs linux3.5 + int ret = prctl(PR_SET_NO_NEW_PRIVS, 1, 0,0, 0); + if(ret < 0) + _error->Warning("PR_SET_NO_NEW_PRIVS failed with %i", ret); +#endif + + if (setgroups(1, &pw->pw_gid) != 1) + return _error->Errno("setgroups", "Failed to setgroups"); + + if (setegid(pw->pw_gid) != 0) + return _error->Errno("setgid", "Failed to setgid"); if (setgid(pw->pw_gid) != 0) return _error->Errno("setgid", "Failed to setgid"); + if (setuid(pw->pw_uid) != 0) return _error->Errno("setuid", "Failed to setuid"); + // the seteuid() is probably uneeded (at least thats what the linux + // man-page says about setuid(2)) but we cargo culted it anyway + if (setuid(pw->pw_uid) != 0) + return _error->Errno("setuid", "Failed to setuid"); + + // be defensive + if(setgid(0) != -1 || setegid(0) != -1) + return _error->Error("Could restore gid to root, privilege dropping does not work"); + if(setuid(0) != -1 || seteuid(0) != -1) + return _error->Error("Could restore uid to root, privilege dropping does not work"); return true; } diff --git a/debian/apt.postinst b/debian/apt.postinst index fd3e273bb..08dc60f9c 100644 --- a/debian/apt.postinst +++ b/debian/apt.postinst @@ -26,6 +26,13 @@ case "$1" in fi fi + # add unprivileged user for the apt methods + adduser --force-badname --system --no-create-home \ + --quiet Debian-apt || true + chown -R Debian-apt:root \ + /var/lib/apt/lists \ + /var/cache/apt/archives + # ensure tighter permissons on the logs, see LP: #975199 if dpkg --compare-versions "$2" lt-nl 0.9.7.7; then # ensure permissions are right diff --git a/methods/copy.cc b/methods/copy.cc index b78053d36..18d70e153 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -118,6 +118,8 @@ int main() { setlocale(LC_ALL, ""); + DropPrivs(); + CopyMethod Mth; return Mth.Run(); } diff --git a/methods/ftp.cc b/methods/ftp.cc index 66787a7be..9d58aa3b9 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -1107,6 +1107,9 @@ int main(int, const char *argv[]) { setlocale(LC_ALL, ""); + // no more active ftp, sorry + DropPrivs(); + /* See if we should be come the http client - we do this for http proxy urls */ if (getenv("ftp_proxy") != 0) diff --git a/methods/gpgv.cc b/methods/gpgv.cc index ae521a2ed..159417883 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -262,6 +263,8 @@ int main() { setlocale(LC_ALL, ""); + DropPrivs(); + GPGVMethod Mth; return Mth.Run(); diff --git a/methods/gzip.cc b/methods/gzip.cc index df3f8828f..518e58f82 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -135,6 +135,8 @@ int main(int, char *argv[]) { setlocale(LC_ALL, ""); + DropPrivs(); + Prog = strrchr(argv[0],'/'); ++Prog; diff --git a/methods/http_main.cc b/methods/http_main.cc index 3b346a514..788582632 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -1,5 +1,5 @@ #include - +#include #include #include "http.h" @@ -12,6 +12,8 @@ int main() // closes the connection (this is dealt with via ServerDie()) signal(SIGPIPE, SIG_IGN); + DropPrivs(); + HttpMethod Mth; return Mth.Loop(); } diff --git a/methods/https.cc b/methods/https.cc index 0499af0c5..a40f37710 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -443,6 +443,8 @@ int main() { setlocale(LC_ALL, ""); + DropPrivs(); + HttpsMethod Mth; curl_global_init(CURL_GLOBAL_SSL) ; -- cgit v1.2.3-70-g09d2 From 7b18d5592fd5e0bb173e193d1e6693a66065f971 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 24 Sep 2014 21:49:19 +0200 Subject: methods: Fail if we cannot drop privileges --- apt-pkg/acquire-method.cc | 12 ++++++++++++ apt-pkg/acquire-method.h | 2 +- methods/copy.cc | 4 ++-- methods/ftp.cc | 6 +++--- methods/gpgv.cc | 4 ++-- methods/gzip.cc | 5 +++-- methods/http_main.cc | 4 ++-- methods/https.cc | 4 ++-- 8 files changed, 27 insertions(+), 14 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index e4a937d1d..82f2fb3ce 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -118,6 +118,18 @@ void pkgAcqMethod::Fail(string Err,bool Transient) std::cout << "\n" << std::flush; } + /*}}}*/ +// AcqMethod::DropPrivsOrDie - Drop privileges or die /*{{{*/ +// --------------------------------------------------------------------- +/* */ +void pkgAcqMethod::DropPrivsOrDie() +{ + if (!DropPrivs()) { + Fail(false); + exit(112); /* call the european emergency number */ + } +} + /*}}}*/ // AcqMethod::URIStart - Indicate a download is starting /*{{{*/ // --------------------------------------------------------------------- diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index cbf79f860..cdeecc9a7 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -105,7 +105,7 @@ class pkgAcqMethod pkgAcqMethod(const char *Ver,unsigned long Flags = 0); virtual ~pkgAcqMethod() {}; - + void DropPrivsOrDie(); private: APT_HIDDEN void Dequeue(); }; diff --git a/methods/copy.cc b/methods/copy.cc index 18d70e153..3883c822b 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -118,8 +118,8 @@ int main() { setlocale(LC_ALL, ""); - DropPrivs(); - CopyMethod Mth; + + Mth.DropPrivsOrDie(); return Mth.Run(); } diff --git a/methods/ftp.cc b/methods/ftp.cc index 9d58aa3b9..a658b5657 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -1107,9 +1107,6 @@ int main(int, const char *argv[]) { setlocale(LC_ALL, ""); - // no more active ftp, sorry - DropPrivs(); - /* See if we should be come the http client - we do this for http proxy urls */ if (getenv("ftp_proxy") != 0) @@ -1134,6 +1131,9 @@ int main(int, const char *argv[]) } FtpMethod Mth; + + // no more active ftp, sorry + Mth.DropPrivsOrDie(); return Mth.Run(); } diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 159417883..4071cbac6 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -262,10 +262,10 @@ bool GPGVMethod::Fetch(FetchItem *Itm) int main() { setlocale(LC_ALL, ""); - - DropPrivs(); GPGVMethod Mth; + Mth.DropPrivsOrDie(); + return Mth.Run(); } diff --git a/methods/gzip.cc b/methods/gzip.cc index 518e58f82..7ffcda60f 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -135,11 +135,12 @@ int main(int, char *argv[]) { setlocale(LC_ALL, ""); - DropPrivs(); - Prog = strrchr(argv[0],'/'); ++Prog; GzipMethod Mth; + + Mth.DropPrivsOrDie(); + return Mth.Run(); } diff --git a/methods/http_main.cc b/methods/http_main.cc index 788582632..d7724701a 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -12,8 +12,8 @@ int main() // closes the connection (this is dealt with via ServerDie()) signal(SIGPIPE, SIG_IGN); - DropPrivs(); - HttpMethod Mth; + + Mth.DropPrivsOrDie(); return Mth.Loop(); } diff --git a/methods/https.cc b/methods/https.cc index a40f37710..a74d2a38b 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -443,11 +443,11 @@ int main() { setlocale(LC_ALL, ""); - DropPrivs(); - HttpsMethod Mth; curl_global_init(CURL_GLOBAL_SSL) ; + Mth.DropPrivsOrDie(); + return Mth.Run(); } -- cgit v1.2.3-70-g09d2 From 47d278dc7184606f751d015689e0c49eccde4547 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 24 Sep 2014 20:14:55 +0200 Subject: releasing package apt version 1.1~exp3 --- debian/changelog | 19 +++++++++++++++++++ methods/http_main.cc | 1 + 2 files changed, 20 insertions(+) (limited to 'methods') diff --git a/debian/changelog b/debian/changelog index 32447d5e1..acbe7ddba 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,22 @@ +apt (1.1~exp3) experimental; urgency=medium + + [ Michael Vogt ] + * merged changes from debian/sid up to 1.0.9.1 + * Make /var/lib/apt/lists and /var/cache/apt/archives owned + by the new _apt user + * Drop Privileges in the following acquire methods: + copy, http, https, ftp, gpgv, gzip/bzip2/lzma/xz + * DropPrivs: Improvements based on feedback from error@debian.org + + [ Julian Andres Klode ] + * DropPriv: Really call seteuid and not setuid, and add more checks + * Use _apt as our unprivileged user name + * DropPrivs: Also check for saved set-user-ID and set-group-ID + * methods: Fail if we cannot drop privileges + * DropPrivs: Also check for saved set-user-ID and set-group-ID + + -- Michael Vogt Wed, 24 Sep 2014 22:30:09 +0200 + apt (1.1~exp2) experimental; urgency=medium [ Guillem Jover ] diff --git a/methods/http_main.cc b/methods/http_main.cc index d7724701a..f21a5709c 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -1,5 +1,6 @@ #include #include +#include #include #include "http.h" -- cgit v1.2.3-70-g09d2 From d36b27305dae21f9b3b6de056853ecd8bbd157e6 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 26 Sep 2014 17:50:18 +0200 Subject: Disable Mth.DropPrivsOrDie() in copy.cc for now Dch-Ignore: true --- methods/copy.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'methods') diff --git a/methods/copy.cc b/methods/copy.cc index 3883c822b..b65bc4dd5 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -120,6 +120,5 @@ int main() CopyMethod Mth; - Mth.DropPrivsOrDie(); return Mth.Run(); } -- cgit v1.2.3-70-g09d2 From 25613a61f6f3b9e54d5229af7e2278d0fa54bdd9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 26 Sep 2014 22:16:26 +0200 Subject: fix: Member variable 'X' is not initialized in the constructor. Reported-By: cppcheck Git-Dch: Ignore --- apt-pkg/acquire.cc | 21 ++++++--------------- apt-pkg/acquire.h | 2 +- apt-pkg/cdrom.cc | 10 +++++++--- apt-pkg/contrib/fileutl.cc | 2 +- apt-pkg/install-progress.cc | 2 ++ apt-pkg/install-progress.h | 5 ++--- apt-pkg/metaindex.h | 2 +- apt-pkg/pkgcache.cc | 1 + apt-pkg/pkgcachegen.h | 2 +- apt-pkg/tagfile.h | 2 +- ftparchive/sources.h | 2 +- methods/ftp.cc | 5 +++-- methods/https.h | 5 ++--- 13 files changed, 29 insertions(+), 32 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 8467dab5b..f0a88a538 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -581,27 +581,18 @@ pkgAcquire::UriIterator pkgAcquire::UriEnd() // Acquire::MethodConfig::MethodConfig - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquire::MethodConfig::MethodConfig() +pkgAcquire::MethodConfig::MethodConfig() : d(NULL), Next(0), SingleInstance(false), + Pipeline(false), SendConfig(false), LocalOnly(false), NeedsCleanup(false), + Removable(false) { - SingleInstance = false; - Pipeline = false; - SendConfig = false; - LocalOnly = false; - Removable = false; - Next = 0; } /*}}}*/ // Queue::Queue - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : Name(Name), - Owner(Owner) +pkgAcquire::Queue::Queue(string Name,pkgAcquire *Owner) : d(NULL), Next(0), + Name(Name), Items(0), Workers(0), Owner(Owner), PipeDepth(0), MaxPipeDepth(1) { - Items = 0; - Next = 0; - Workers = 0; - MaxPipeDepth = 1; - PipeDepth = 0; } /*}}}*/ // Queue::~Queue - Destructor /*{{{*/ @@ -805,7 +796,7 @@ void pkgAcquire::Queue::Bump() // AcquireStatus::pkgAcquireStatus - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Update(true), MorePulses(false) +pkgAcquireStatus::pkgAcquireStatus() : d(NULL), Percent(0), Update(true), MorePulses(false) { Start(); } diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index 0113021b2..a92ecc24c 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -585,7 +585,7 @@ class pkgAcquire::UriIterator * * \param Q The queue over which this UriIterator should iterate. */ - UriIterator(pkgAcquire::Queue *Q) : CurQ(Q), CurItem(0) + UriIterator(pkgAcquire::Queue *Q) : d(NULL), CurQ(Q), CurItem(0) { while (CurItem == 0 && CurQ != 0) { diff --git a/apt-pkg/cdrom.cc b/apt-pkg/cdrom.cc index a5ad6a9ff..b97f7b036 100644 --- a/apt-pkg/cdrom.cc +++ b/apt-pkg/cdrom.cc @@ -913,10 +913,14 @@ bool pkgCdrom::Add(pkgCdromStatus *log) /*{{{*/ return true; } /*}}}*/ -pkgUdevCdromDevices::pkgUdevCdromDevices() /*{{{*/ - : libudev_handle(NULL) +pkgUdevCdromDevices::pkgUdevCdromDevices() /*{{{*/ +: libudev_handle(NULL), udev_new(NULL), udev_enumerate_add_match_property(NULL), + udev_enumerate_scan_devices(NULL), udev_enumerate_get_list_entry(NULL), + udev_device_new_from_syspath(NULL), udev_enumerate_get_udev(NULL), + udev_list_entry_get_name(NULL), udev_device_get_devnode(NULL), + udev_enumerate_new(NULL), udev_list_entry_get_next(NULL), + udev_device_get_property_value(NULL), udev_enumerate_add_match_sysattr(NULL) { - } /*}}}*/ diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index e81f32a52..9e7702063 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -896,7 +896,7 @@ class FileFdPrivate { /*{{{*/ bool eof; bool compressing; - LZMAFILE() : file(NULL), eof(false), compressing(false) {} + LZMAFILE() : file(NULL), eof(false), compressing(false) { buffer[0] = '\0'; } ~LZMAFILE() { if (compressing == true) { diff --git a/apt-pkg/install-progress.cc b/apt-pkg/install-progress.cc index cf6c85912..0d180d59b 100644 --- a/apt-pkg/install-progress.cc +++ b/apt-pkg/install-progress.cc @@ -21,6 +21,8 @@ namespace APT { namespace Progress { +PackageManager::PackageManager() : d(NULL), percentage(0.0), last_reported_progress(-1) {}; +PackageManager::~PackageManager() {}; /* Return a APT::Progress::PackageManager based on the global * apt configuration (i.e. APT::Status-Fd and APT::Status-deb822-Fd) diff --git a/apt-pkg/install-progress.h b/apt-pkg/install-progress.h index 5d1a20e9b..912700e8d 100644 --- a/apt-pkg/install-progress.h +++ b/apt-pkg/install-progress.h @@ -26,9 +26,8 @@ namespace Progress { int last_reported_progress; public: - PackageManager() - : percentage(0.0), last_reported_progress(-1) {}; - virtual ~PackageManager() {}; + PackageManager(); + virtual ~PackageManager(); /* Global Start/Stop */ virtual void Start(int /*child_pty*/=-1) {}; diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index ffabaadbf..7c4d0c1aa 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -53,7 +53,7 @@ class metaIndex metaIndex(std::string const &URI, std::string const &Dist, char const * const Type) - : Indexes(NULL), Type(Type), URI(URI), Dist(Dist) + : Indexes(NULL), Type(Type), URI(URI), Dist(Dist), Trusted(false) { /* nothing */ } diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc index 4ab9e2a4b..572685ba5 100644 --- a/apt-pkg/pkgcache.cc +++ b/apt-pkg/pkgcache.cc @@ -84,6 +84,7 @@ pkgCache::Header::Header() FileList = 0; VerSysName = 0; Architecture = 0; + Architectures = 0; HashTableSize = _config->FindI("APT::Cache-HashTableSize", 10 * 1048); memset(Pools,0,sizeof(Pools)); diff --git a/apt-pkg/pkgcachegen.h b/apt-pkg/pkgcachegen.h index 84777b912..e1d22b88f 100644 --- a/apt-pkg/pkgcachegen.h +++ b/apt-pkg/pkgcachegen.h @@ -193,7 +193,7 @@ class pkgCacheGenerator::ListParser virtual bool CollectFileProvides(pkgCache &/*Cache*/, pkgCache::VerIterator &/*Ver*/) {return true;}; - ListParser() : FoundFileDeps(false) {}; + ListParser() : Owner(NULL), OldDepLast(NULL), FoundFileDeps(false) {}; virtual ~ListParser() {}; }; /*}}}*/ diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index 3e84d8fd4..39ec94d86 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -43,7 +43,7 @@ class pkgTagSection unsigned int StartValue; unsigned int NextInBucket; - TagData(unsigned int const StartTag) : StartTag(StartTag), NextInBucket(0) {} + TagData(unsigned int const StartTag) : StartTag(StartTag), EndTag(0), StartValue(0), NextInBucket(0) {} }; std::vector Tags; unsigned int LookupTable[0x100]; diff --git a/ftparchive/sources.h b/ftparchive/sources.h index 91e0b1376..9ada15728 100644 --- a/ftparchive/sources.h +++ b/ftparchive/sources.h @@ -17,7 +17,7 @@ class DscExtract bool TakeDsc(const void *Data, unsigned long Size); bool Read(std::string FileName); - DscExtract() : Data(0), Length(0) { + DscExtract() : Data(0), Length(0), IsClearSigned(false) { Data = new char[maxSize]; }; ~DscExtract() { diff --git a/methods/ftp.cc b/methods/ftp.cc index a658b5657..ac76295f0 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -75,9 +75,10 @@ time_t FtpMethod::FailTime = 0; // FTPConn::FTPConn - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1), +FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1), DataListenFd(-1), ServerName(Srv), - ForceExtended(false), TryPassive(true) + ForceExtended(false), TryPassive(true), + PeerAddrLen(0), ServerAddrLen(0) { Debug = _config->FindB("Debug::Acquire::Ftp",false); PasvAddr = 0; diff --git a/methods/https.h b/methods/https.h index faac8a3cd..45d1f7f63 100644 --- a/methods/https.h +++ b/methods/https.h @@ -69,10 +69,9 @@ class HttpsMethod : public pkgAcqMethod public: FileFd *File; - - HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), File(NULL) + + HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), Server(NULL), File(NULL) { - File = 0; curl = curl_easy_init(); }; -- cgit v1.2.3-70-g09d2 From b39bb552f8de65cea13dc5f1fae6fbeb198605c6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 26 Jan 2014 17:23:50 +0100 Subject: correct the error messages to refer to apt-key instead of gpgv Git-Dch: Ignore --- methods/gpgv.cc | 12 ++++++------ po/apt-all.pot | 4 ++-- po/ar.po | 4 ++-- po/ast.po | 8 ++++---- po/bg.po | 10 +++++----- po/bs.po | 4 ++-- po/ca.po | 10 +++++----- po/cs.po | 8 ++++---- po/cy.po | 4 ++-- po/da.po | 8 ++++---- po/de.po | 8 ++++---- po/dz.po | 6 +++--- po/el.po | 8 ++++---- po/es.po | 8 ++++---- po/eu.po | 9 ++++----- po/fi.po | 9 ++++----- po/fr.po | 10 +++++----- po/gl.po | 10 +++++----- po/he.po | 4 ++-- po/hu.po | 8 ++++---- po/it.po | 8 ++++---- po/ja.po | 4 ++-- po/km.po | 8 ++++---- po/ko.po | 8 ++++---- po/ku.po | 6 +++--- po/lt.po | 6 +++--- po/mr.po | 8 ++++---- po/nb.po | 8 ++++---- po/ne.po | 9 ++++----- po/nl.po | 8 ++++---- po/nn.po | 4 ++-- po/pl.po | 8 ++++---- po/pt.po | 8 ++++---- po/pt_BR.po | 9 ++++----- po/ro.po | 9 ++++----- po/ru.po | 8 ++++---- po/sk.po | 8 ++++---- po/sl.po | 8 ++++---- po/sv.po | 8 ++++---- po/th.po | 8 ++++---- po/tl.po | 9 ++++----- po/tr.po | 8 ++++---- po/uk.po | 8 ++++---- po/vi.po | 8 ++++---- po/zh_CN.po | 8 ++++---- po/zh_TW.po | 9 ++++----- 46 files changed, 173 insertions(+), 180 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 4071cbac6..02fb8c356 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -75,7 +75,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, FILE *pipein = fdopen(fd[0], "r"); - // Loop over the output of gpgv, and check the signatures. + // Loop over the output of apt-key (which really is gnupg), and check the signatures. size_t buffersize = 64; char *buffer = (char *) malloc(buffersize); size_t bufferoff = 0; @@ -160,7 +160,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, waitpid(pid, &status, 0); if (Debug == true) { - std::clog << "gpgv exited\n"; + std::clog << "apt-key exited\n"; } if (WEXITSTATUS(status) == 0) @@ -172,7 +172,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, else if (WEXITSTATUS(status) == 1) return _("At least one invalid signature was encountered."); else if (WEXITSTATUS(status) == 111) - return _("Could not execute 'gpgv' to verify signature (is gpgv installed?)"); + return _("Could not execute 'apt-key' to verify signature (is gnupg installed?)"); else if (WEXITSTATUS(status) == 112) { // acquire system checks for "NODATA" to generate GPG errors (the others are only warnings) @@ -182,7 +182,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, return errmsg; } else - return _("Unknown error executing gpgv"); + return _("Unknown error executing apt-key"); } bool GPGVMethod::Fetch(FetchItem *Itm) @@ -200,7 +200,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) Res.Filename = Itm->DestFile; URIStart(Res); - // Run gpgv on file, extract contents and get the key ID of the signer + // Run apt-key on file, extract contents and get the key ID of the signer string msg = VerifyGetSigners(Path.c_str(), Itm->DestFile.c_str(), GoodSigners, BadSigners, WorthlessSigners, NoPubKeySigners); @@ -252,7 +252,7 @@ bool GPGVMethod::Fetch(FetchItem *Itm) if (_config->FindB("Debug::Acquire::gpgv", false)) { - std::clog << "gpgv succeeded\n"; + std::clog << "apt-key succeeded\n"; } return true; diff --git a/po/apt-all.pot b/po/apt-all.pot index d2229a936..664965900 100644 --- a/po/apt-all.pot +++ b/po/apt-all.pot @@ -901,7 +901,7 @@ msgid "At least one invalid signature was encountered." msgstr "" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -913,7 +913,7 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" +msgid "Unknown error executing apt-key" msgstr "" #: methods/gpgv.cc:217 methods/gpgv.cc:224 diff --git a/po/ar.po b/po/ar.po index 2ab648323..1bea20877 100644 --- a/po/ar.po +++ b/po/ar.po @@ -914,7 +914,7 @@ msgid "At least one invalid signature was encountered." msgstr "" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -926,7 +926,7 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" +msgid "Unknown error executing apt-key" msgstr "" #: methods/gpgv.cc:217 methods/gpgv.cc:224 diff --git a/po/ast.po b/po/ast.po index a2f5a7df6..9f7d7b2d0 100644 --- a/po/ast.po +++ b/po/ast.po @@ -1021,8 +1021,8 @@ msgid "At least one invalid signature was encountered." msgstr "Atopóse polo menos una robla mala." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "Nun pudo executase 'gpgv' pa verificar la robla (¿ta instaláu gpgv?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "Nun pudo executase 'apt-key' pa verificar la robla (¿ta instaláu gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1033,8 +1033,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Fallu desconocíu al executar gpgv" +msgid "Unknown error executing apt-key" +msgstr "Fallu desconocíu al executar apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/bg.po b/po/bg.po index ad174d0a8..e41b2bdd0 100644 --- a/po/bg.po +++ b/po/bg.po @@ -1051,10 +1051,10 @@ msgid "At least one invalid signature was encountered." msgstr "Намерен е поне един невалиден подпис." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Неуспех при изпълнение на „gpgv“ за проверка на подписа (инсталиран ли е " -"gpgv?)" +"Неуспех при изпълнение на „apt-key“ за проверка на подписа (инсталиран ли е " +"gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1065,8 +1065,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Неизвестна грешка при изпълнението на gpgv" +msgid "Unknown error executing apt-key" +msgstr "Неизвестна грешка при изпълнението на apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/bs.po b/po/bs.po index 5f03012fc..81294d8c8 100644 --- a/po/bs.po +++ b/po/bs.po @@ -920,7 +920,7 @@ msgid "At least one invalid signature was encountered." msgstr "" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -932,7 +932,7 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" +msgid "Unknown error executing apt-key" msgstr "" #: methods/gpgv.cc:217 methods/gpgv.cc:224 diff --git a/po/ca.po b/po/ca.po index ee9ee46d4..db6d12e77 100644 --- a/po/ca.po +++ b/po/ca.po @@ -1034,10 +1034,10 @@ msgid "At least one invalid signature was encountered." msgstr "S'ha trobat almenys una signatura invàlida." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"No s'ha pogut executar el «gpgv» per a verificar la signatura (està " -"instaŀlat el gpgv?)" +"No s'ha pogut executar el «apt-key» per a verificar la signatura (està " +"instaŀlat el gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1048,8 +1048,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "S'ha produït un error desconegut en executar el gpgv" +msgid "Unknown error executing apt-key" +msgstr "S'ha produït un error desconegut en executar el apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/cs.po b/po/cs.po index 4e43807b7..5d7625d6c 100644 --- a/po/cs.po +++ b/po/cs.po @@ -1062,8 +1062,8 @@ msgid "At least one invalid signature was encountered." msgstr "Byl zaznamenán nejméně jeden neplatný podpis. " #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "Nelze spustit „gpgv“ pro ověření podpisu (je gpgv nainstalováno?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "Nelze spustit „apt-key“ pro ověření podpisu (je gnupg nainstalováno?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1076,8 +1076,8 @@ msgstr "" "ověření?)" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Neznámá chyba při spouštění gpgv" +msgid "Unknown error executing apt-key" +msgstr "Neznámá chyba při spouštění apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/cy.po b/po/cy.po index 3175bfa57..dd6feb3d4 100644 --- a/po/cy.po +++ b/po/cy.po @@ -1042,7 +1042,7 @@ msgid "At least one invalid signature was encountered." msgstr "" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -1054,7 +1054,7 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" +msgid "Unknown error executing apt-key" msgstr "" #: methods/gpgv.cc:217 methods/gpgv.cc:224 diff --git a/po/da.po b/po/da.po index f46a851da..e6c853fd3 100644 --- a/po/da.po +++ b/po/da.po @@ -1075,9 +1075,9 @@ msgid "At least one invalid signature was encountered." msgstr "Stødte på mindst én ugyldig signatur." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Kunne ikke køre »gpgv« for at verificere signaturen (er gpgv installeret?)" +"Kunne ikke køre »apt-key« for at verificere signaturen (er gnupg installeret?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1090,8 +1090,8 @@ msgstr "" "autentificering?)" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Ukendt fejl ved kørsel af gpgv" +msgid "Unknown error executing apt-key" +msgstr "Ukendt fejl ved kørsel af apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/de.po b/po/de.po index aef7c8fc4..13b291043 100644 --- a/po/de.po +++ b/po/de.po @@ -1113,9 +1113,9 @@ msgid "At least one invalid signature was encountered." msgstr "Mindestens eine ungültige Signatur wurde entdeckt." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"»gpgv« konnte zur Überprüfung der Signatur nicht ausgeführt werden (ist gpgv " +"»apt-key« konnte zur Überprüfung der Signatur nicht ausgeführt werden (ist gnupg " "installiert?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -1129,8 +1129,8 @@ msgstr "" "das Netzwerk eine Authentifizierung?)" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Unbekannter Fehler beim Ausführen von gpgv" +msgid "Unknown error executing apt-key" +msgstr "Unbekannter Fehler beim Ausführen von apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/dz.po b/po/dz.po index 59ed4305e..8ee4d15da 100644 --- a/po/dz.po +++ b/po/dz.po @@ -1016,7 +1016,7 @@ msgstr "ཉུང་མཐའ་རང་ནུས་མེད་ཀྱི་མ #: methods/gpgv.cc:174 #, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" "མིང་རྟགས་བདེན་སྦྱོར་འབད་ནི་ལུ་'%s'འདི་ལག་ལེན་འཐབ་མ་ཚུགས། (gpgv་དེ་ཁཞི་བཙུགས་འབད་ཡོདཔ་ཨིན་ན།?)" @@ -1029,8 +1029,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "gpgv་ལག་ལེན་འཐབ་ནི་ལུ་མ་ཤེས་པའི་འཛོལ་བ་།" +msgid "Unknown error executing apt-key" +msgstr "apt-key་ལག་ལེན་འཐབ་ནི་ལུ་མ་ཤེས་པའི་འཛོལ་བ་།" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/el.po b/po/el.po index 86ef14179..1b19f54da 100644 --- a/po/el.po +++ b/po/el.po @@ -1028,10 +1028,10 @@ msgstr "Βρέθηκε τουλάχιστον μια μη έγκυρη υπογ #: methods/gpgv.cc:174 #, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" "Αδυναμία εκτέλεσης του '%s' για την επαλήθευση της υπογραφής (είναι " -"εγκατεστημένο το gpgv;)" +"εγκατεστημένο το gnupg;)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1042,8 +1042,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Άγνωστο σφάλμα κατά την εκτέλεση του gpgv" +msgid "Unknown error executing apt-key" +msgstr "Άγνωστο σφάλμα κατά την εκτέλεση του apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/es.po b/po/es.po index 77cee1d5d..f808761df 100644 --- a/po/es.po +++ b/po/es.po @@ -1086,9 +1086,9 @@ msgid "At least one invalid signature was encountered." msgstr "Se encontró al menos una firma inválida." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"No se pudo ejecutar «gpgv» para verificar la firma (¿está instalado gpgv?)" +"No se pudo ejecutar «apt-key» para verificar la firma (¿está instalado gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1099,8 +1099,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Error desconocido ejecutando gpgv" +msgid "Unknown error executing apt-key" +msgstr "Error desconocido ejecutando apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/eu.po b/po/eu.po index 88561d003..8b3dd9025 100644 --- a/po/eu.po +++ b/po/eu.po @@ -1016,9 +1016,8 @@ msgid "At least one invalid signature was encountered." msgstr "Beintza sinadura baliogabe bat aurkitu da." #: methods/gpgv.cc:174 -#, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "Ezin da %s abiarazi sinadura egiaztatzeko (gpgv instalaturik al dago?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "Ezin da apt-key abiarazi sinadura egiaztatzeko (gnupg instalaturik al dago?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1029,8 +1028,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Errore ezezaguna gpgv exekutatzean" +msgid "Unknown error executing apt-key" +msgstr "Errore ezezaguna apt-key exekutatzean" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/fi.po b/po/fi.po index b18648579..b4333d57f 100644 --- a/po/fi.po +++ b/po/fi.po @@ -1007,10 +1007,9 @@ msgid "At least one invalid signature was encountered." msgstr "LÖytyi ainakin yksi kelvoton allekirjoitus." #: methods/gpgv.cc:174 -#, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Ei käynnistynyt \"%s\" allekirjoitusta tarkistamaan (onko gpgv asennettu?)" +"Ei käynnistynyt \"apt-key\" allekirjoitusta tarkistamaan (onko gnupg asennettu?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1021,8 +1020,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Tapahtui tuntematon virhe suoritettaessa gpgv" +msgid "Unknown error executing apt-key" +msgstr "Tapahtui tuntematon virhe suoritettaessa apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/fr.po b/po/fr.po index b2197906e..2325fc0bf 100644 --- a/po/fr.po +++ b/po/fr.po @@ -1077,10 +1077,10 @@ msgid "At least one invalid signature was encountered." msgstr "Au moins une signature non valable a été rencontrée." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Impossible d'exécuter « gpgv » pour contrôler la signature (veuillez " -"vérifier si gpgv est installé)." +"Impossible d'exécuter « apt-key » pour contrôler la signature (veuillez " +"vérifier si gnupg est installé)." #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1093,8 +1093,8 @@ msgstr "" "Peut-être le réseau nécessite-t-il une authentification." #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Erreur inconnue à l'exécution de gpgv" +msgid "Unknown error executing apt-key" +msgstr "Erreur inconnue à l'exécution de apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/gl.po b/po/gl.po index 53a866bc1..23d0f1c6b 100644 --- a/po/gl.po +++ b/po/gl.po @@ -1032,10 +1032,10 @@ msgid "At least one invalid signature was encountered." msgstr "Atopouse polo menos unha sinatura incorrecta." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Non é posíbel executar «gpgv» para verificar a sinatura (Está instalado " -"gpgv?)" +"Non é posíbel executar «apt-key» para verificar a sinatura (Está instalado " +"gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1046,8 +1046,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Produciuse un erro descoñecido ao executar gpgv" +msgid "Unknown error executing apt-key" +msgstr "Produciuse un erro descoñecido ao executar apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/he.po b/po/he.po index 4a44bc72d..8175d0ebb 100644 --- a/po/he.po +++ b/po/he.po @@ -1810,11 +1810,11 @@ msgstr "" #: methods/gpgv.cc:232 #, c-format -msgid "Could not execute '%s' to verify signature (is gpgv installed?)" +msgid "Could not execute '%s' to verify signature (is gnupg installed?)" msgstr "" #: methods/gpgv.cc:237 -msgid "Unknown error executing gpgv" +msgid "Unknown error executing apt-key" msgstr "" #: methods/gpgv.cc:271 methods/gpgv.cc:278 diff --git a/po/hu.po b/po/hu.po index 26a1bfa5c..9acff5c86 100644 --- a/po/hu.po +++ b/po/hu.po @@ -1050,9 +1050,9 @@ msgid "At least one invalid signature was encountered." msgstr "Legalább egy aláírás érvénytelen." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Nem indítható el a „gpgv” az aláírás ellenőrzéséhez (telepítve van a gpgv?)" +"Nem indítható el a „apt-key” az aláírás ellenőrzéséhez (telepítve van a gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1063,8 +1063,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Ismeretlen gpgv futtatási hiba" +msgid "Unknown error executing apt-key" +msgstr "Ismeretlen apt-key futtatási hiba" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/it.po b/po/it.po index c0ad782a7..050b56111 100644 --- a/po/it.po +++ b/po/it.po @@ -1093,9 +1093,9 @@ msgid "At least one invalid signature was encountered." msgstr "È stata trovata almeno una firma non valida." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Impossibile eseguire \"gpgv\" per verificare la firma (forse gpgv non è " +"Impossibile eseguire \"apt-key\" per verificare la firma (forse gnupg non è " "installato)" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -1109,8 +1109,8 @@ msgstr "" "richiede autenticazione?)" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Errore sconosciuto durante l'esecuzione di gpgv" +msgid "Unknown error executing apt-key" +msgstr "Errore sconosciuto durante l'esecuzione di apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/ja.po b/po/ja.po index c62ff95f7..90e3e1757 100644 --- a/po/ja.po +++ b/po/ja.po @@ -1084,9 +1084,9 @@ msgid "At least one invalid signature was encountered." msgstr "少なくとも 1 つの不正な署名が発見されました。" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"署名を検証するための 'gpgv' の実行ができませんでした (gpgv はインストールされ" +"署名を検証するための 'apt-key' の実行ができませんでした (gnupg はインストールされ" "ていますか?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' diff --git a/po/km.po b/po/km.po index 0e95cd985..ae00856f0 100644 --- a/po/km.po +++ b/po/km.po @@ -1005,8 +1005,8 @@ msgstr "​បានជួប​ប្រទះ​​​​ហត្ថលេខ #: methods/gpgv.cc:174 #, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "មិន​អាច​ប្រតិបត្តិ '%s' ដើម្បី​ផ្ទៀងផ្ទាត់​ហត្ថលេខា (តើ gpgv ត្រូវ​បាន​ដំឡើង​ឬនៅ ?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "មិន​អាច​ប្រតិបត្តិ 'apt-key' ដើម្បី​ផ្ទៀងផ្ទាត់​ហត្ថលេខា (តើ gnupg ត្រូវ​បាន​ដំឡើង​ឬនៅ ?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1017,8 +1017,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "មិនស្គាល់កំហុស ក្នុងការប្រតិបត្តិ gpgv" +msgid "Unknown error executing apt-key" +msgstr "មិនស្គាល់កំហុស ក្នុងការប្រតិបត្តិ apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/ko.po b/po/ko.po index 7ec20a7e7..661da2bde 100644 --- a/po/ko.po +++ b/po/ko.po @@ -1011,9 +1011,9 @@ msgid "At least one invalid signature was encountered." msgstr "최소한 하나 이상의 서명이 잘못되었습니다." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"서명을 확인하는 'gpgv' 프로그램을 실행할 수 없습니다. (gpgv를 설치했습니까?)" +"서명을 확인하는 'apt-key' 프로그램을 실행할 수 없습니다. (gnupg를 설치했습니까?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1024,8 +1024,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "gpgv 실행 도중 알 수 없는 오류 발생" +msgid "Unknown error executing apt-key" +msgstr "apt-key 실행 도중 알 수 없는 오류 발생" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/ku.po b/po/ku.po index 95c84ae0d..0aaa66c33 100644 --- a/po/ku.po +++ b/po/ku.po @@ -923,7 +923,7 @@ msgid "At least one invalid signature was encountered." msgstr "" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -935,8 +935,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Di xebitandina gpgv de çewtiya nenas" +msgid "Unknown error executing apt-key" +msgstr "Di xebitandina apt-key de çewtiya nenas" #: methods/gpgv.cc:217 methods/gpgv.cc:224 #, fuzzy diff --git a/po/lt.po b/po/lt.po index 9c1c549a3..e66708cdc 100644 --- a/po/lt.po +++ b/po/lt.po @@ -928,7 +928,7 @@ msgid "At least one invalid signature was encountered." msgstr "" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -940,8 +940,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Nežinoma klaida kviečiant gpgv" +msgid "Unknown error executing apt-key" +msgstr "Nežinoma klaida kviečiant apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/mr.po b/po/mr.po index 6dd53b8a7..3031722b7 100644 --- a/po/mr.po +++ b/po/mr.po @@ -1002,9 +1002,9 @@ msgstr "किमान एक अवैध सही सापडली." #: methods/gpgv.cc:174 #, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"सहीची खात्री करण्यासाठी '%s' कार्यान्वित करू शकत नाही (gpgv संस्थापित केले आहे का?)" +"सहीची खात्री करण्यासाठी 'apt-key' कार्यान्वित करू शकत नाही (gnupg संस्थापित केले आहे का?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1015,8 +1015,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "gpgv कार्यान्वित होत असताना अपरिचित त्रुटी" +msgid "Unknown error executing apt-key" +msgstr "apt-key कार्यान्वित होत असताना अपरिचित त्रुटी" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/nb.po b/po/nb.po index bc1e296d7..12da02f47 100644 --- a/po/nb.po +++ b/po/nb.po @@ -1017,9 +1017,9 @@ msgid "At least one invalid signature was encountered." msgstr "Minst en ugyldig signatur ble funnet." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Klarte ikke kjøre «gpgv» for å verifisere signaturen (er gpgv installert?)" +"Klarte ikke kjøre «apt-key» for å verifisere signaturen (er gnupg installert?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1030,8 +1030,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Ukjent feil ved kjøring av gpgv" +msgid "Unknown error executing apt-key" +msgstr "Ukjent feil ved kjøring av apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/ne.po b/po/ne.po index 7c0d06837..0d61a52be 100644 --- a/po/ne.po +++ b/po/ne.po @@ -1002,9 +1002,8 @@ msgid "At least one invalid signature was encountered." msgstr "कम्तिमा एउटा अवैध हस्ताक्षर विरोध भयो ।" #: methods/gpgv.cc:174 -#, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "हस्ताक्षर रूजू गर्न '%s' कार्यन्वयन गर्न सकिएन (के gpgv स्थापना भयो?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "हस्ताक्षर रूजू गर्न 'apt-key' कार्यन्वयन गर्न सकिएन (के gnupg स्थापना भयो?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1015,8 +1014,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "gpgv कार्यन्वयन गर्दा अज्ञात त्रुटि" +msgid "Unknown error executing apt-key" +msgstr "apt-key कार्यन्वयन गर्दा अज्ञात त्रुटि" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/nl.po b/po/nl.po index 12e1602b9..c333ae9f9 100644 --- a/po/nl.po +++ b/po/nl.po @@ -1032,9 +1032,9 @@ msgid "At least one invalid signature was encountered." msgstr "Er is tenminste één ongeldige ondertekening gevonden." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Kon 'gpgv' niet uitvoeren om ondertekening te verifiëren (is gpgv " +"Kon 'apt-key' niet uitvoeren om ondertekening te verifiëren (is gnupg " "geïnstalleerd?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -1046,8 +1046,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Onbekende fout bij het uitvoeren van gpgv" +msgid "Unknown error executing apt-key" +msgstr "Onbekende fout bij het uitvoeren van apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/nn.po b/po/nn.po index aeec41940..40f77e8be 100644 --- a/po/nn.po +++ b/po/nn.po @@ -1012,7 +1012,7 @@ msgid "At least one invalid signature was encountered." msgstr "" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -1024,7 +1024,7 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" +msgid "Unknown error executing apt-key" msgstr "" #: methods/gpgv.cc:217 methods/gpgv.cc:224 diff --git a/po/pl.po b/po/pl.po index d3829cb6d..1646a47e8 100644 --- a/po/pl.po +++ b/po/pl.po @@ -1060,9 +1060,9 @@ msgid "At least one invalid signature was encountered." msgstr "Napotkano przynajmniej jeden nieprawidłowy podpis." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Nie udało się uruchomić gpgv by zweryfikować podpis (czy gpgv jest " +"Nie udało się uruchomić apt-key by zweryfikować podpis (czy gnupg jest " "zainstalowane?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -1074,8 +1074,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Nieznany błąd podczas uruchamiania gpgv" +msgid "Unknown error executing apt-key" +msgstr "Nieznany błąd podczas uruchamiania apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/pt.po b/po/pt.po index 589af3181..b17fb0380 100644 --- a/po/pt.po +++ b/po/pt.po @@ -1050,9 +1050,9 @@ msgid "At least one invalid signature was encountered." msgstr "Pelo menos uma assinatura inválida foi encontrada." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Não foi possível executar 'gpgv' para verificar a assinatura (o gpgv está " +"Não foi possível executar 'apt-key' para verificar a assinatura (o gnupg está " "instalado?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -1064,8 +1064,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Erro desconhecido ao executar gpgv" +msgid "Unknown error executing apt-key" +msgstr "Erro desconhecido ao executar apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/pt_BR.po b/po/pt_BR.po index 87575301a..0be44e8af 100644 --- a/po/pt_BR.po +++ b/po/pt_BR.po @@ -1022,10 +1022,9 @@ msgid "At least one invalid signature was encountered." msgstr "Ao menos uma assinatura inválida foi encontrada." #: methods/gpgv.cc:174 -#, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Não foi possível executar '%s' para verificar a assinatura (o gpgv está " +"Não foi possível executar 'apt-key' para verificar a assinatura (o gnupg está " "instalado?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' @@ -1037,8 +1036,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Erro desconhecido executando gpgv" +msgid "Unknown error executing apt-key" +msgstr "Erro desconhecido executando apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/ro.po b/po/ro.po index 43c5bce69..191cd5a44 100644 --- a/po/ro.po +++ b/po/ro.po @@ -1023,10 +1023,9 @@ msgid "At least one invalid signature was encountered." msgstr "Cel puțin o semnătură nevalidă a fost întâlnită." #: methods/gpgv.cc:174 -#, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Nu s-a putut executa „%s” pentru verificarea semnăturii (gpgv este instalat?)" +"Nu s-a putut executa „apt-key” pentru verificarea semnăturii (gnupg este instalat?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1037,8 +1036,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Eroare necunoscută în timp ce se execută gpgv" +msgid "Unknown error executing apt-key" +msgstr "Eroare necunoscută în timp ce se execută apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/ru.po b/po/ru.po index f24e513e6..35dc160e1 100644 --- a/po/ru.po +++ b/po/ru.po @@ -1059,8 +1059,8 @@ msgid "At least one invalid signature was encountered." msgstr "Найдена как минимум одна неправильная подпись." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "Не удалось выполнить «gpgv» для проверки подписи (gpgv установлена?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "Не удалось выполнить «apt-key» для проверки подписи (gnupg установлена?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1071,8 +1071,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Неизвестная ошибка при выполнении gpgv" +msgid "Unknown error executing apt-key" +msgstr "Неизвестная ошибка при выполнении apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/sk.po b/po/sk.po index a9f4d2695..855f309d2 100644 --- a/po/sk.po +++ b/po/sk.po @@ -1039,8 +1039,8 @@ msgid "At least one invalid signature was encountered." msgstr "Bola zistená aspoň jedna nesprávna signatúra." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "Nedá sa spustiť „gpgv“ kvôli overeniu podpisu (je nainštalované gpgv?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "Nedá sa spustiť „apt-key“ kvôli overeniu podpisu (je nainštalované gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1051,8 +1051,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Neznáma chyba pri spustení gpgv" +msgid "Unknown error executing apt-key" +msgstr "Neznáma chyba pri spustení apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/sl.po b/po/sl.po index dcca81d87..536d99465 100644 --- a/po/sl.po +++ b/po/sl.po @@ -1036,8 +1036,8 @@ msgid "At least one invalid signature was encountered." msgstr "Najden je bil vsaj en neveljaven podpis." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "Ni mogoče izvesti 'gpgv' za preverjanje podpisa (je gpgv nameščen?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "Ni mogoče izvesti 'apt-key' za preverjanje podpisa (je gnupg nameščen?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1048,8 +1048,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Neznana napaka med izvajanjem gpgv" +msgid "Unknown error executing apt-key" +msgstr "Neznana napaka med izvajanjem apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/sv.po b/po/sv.po index b08ceb29f..d761c5c38 100644 --- a/po/sv.po +++ b/po/sv.po @@ -1025,9 +1025,9 @@ msgid "At least one invalid signature was encountered." msgstr "Minst en ogiltig signatur träffades på." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Kunde inte köra \"gpgv\" för att verifiera signatur (är gpgv installerad?)" +"Kunde inte köra \"apt-key\" för att verifiera signatur (är gnupg installerad?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1038,8 +1038,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Okänt fel vid körning av gpgv" +msgid "Unknown error executing apt-key" +msgstr "Okänt fel vid körning av apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/th.po b/po/th.po index b6d436a9a..d241c1314 100644 --- a/po/th.po +++ b/po/th.po @@ -1052,8 +1052,8 @@ msgid "At least one invalid signature was encountered." msgstr "พบลายเซ็นที่ใช้การไม่ได้อย่างน้อยหนึ่งรายการ" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "ไม่สามารถเรียก 'gpgv' เพื่อตรวจสอบลายเซ็น (ได้ติดตั้ง gpgv ไว้หรือไม่?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "ไม่สามารถเรียก 'apt-key' เพื่อตรวจสอบลายเซ็น (ได้ติดตั้ง gnupg ไว้หรือไม่?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1066,8 +1066,8 @@ msgstr "" "'%s' (เครือข่ายต้องยืนยันตัวบุคคลหรือไม่?)" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "เกิดข้อผิดพลาดไม่ทราบสาเหตุขณะเรียก gpgv" +msgid "Unknown error executing apt-key" +msgstr "เกิดข้อผิดพลาดไม่ทราบสาเหตุขณะเรียก apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/tl.po b/po/tl.po index e240e377a..e1186c424 100644 --- a/po/tl.po +++ b/po/tl.po @@ -1017,10 +1017,9 @@ msgid "At least one invalid signature was encountered." msgstr "Hindi kukulang sa isang hindi tanggap na lagda ang na-enkwentro." #: methods/gpgv.cc:174 -#, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Hindi maitakbo ang '%s' upang maberipika ang lagda (nakaluklok ba ang gpgv?)" +"Hindi maitakbo ang 'apt-key' upang maberipika ang lagda (nakaluklok ba ang gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1031,8 +1030,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Hindi kilalang error sa pag-execute ng gpgv" +msgid "Unknown error executing apt-key" +msgstr "Hindi kilalang error sa pag-execute ng apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/tr.po b/po/tr.po index 3aec2d65f..b0a08c46d 100644 --- a/po/tr.po +++ b/po/tr.po @@ -1077,8 +1077,8 @@ msgid "At least one invalid signature was encountered." msgstr "En az bir geçersiz imza ile karşılaşıldı." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "İmza doğrulama için 'gpgv' çalıştırılamadı (gpgv kurulu mu?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "İmza doğrulama için 'apt-key' çalıştırılamadı (gnupg kurulu mu?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1091,8 +1091,8 @@ msgstr "" "gerektiriyor mu?)" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "gpgv çalıştırılırken bilinmeyen hata" +msgid "Unknown error executing apt-key" +msgstr "apt-key çalıştırılırken bilinmeyen hata" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/uk.po b/po/uk.po index a5c7d7cf8..910e48dbf 100644 --- a/po/uk.po +++ b/po/uk.po @@ -1057,8 +1057,8 @@ msgid "At least one invalid signature was encountered." msgstr "Знайдено як мінімум один невірний підпис." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "Неможливо виконати 'gpgv' для перевірки підпису (чи встановлено gpgv?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "Неможливо виконати 'apt-key' для перевірки підпису (чи встановлено gnupg?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1069,8 +1069,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Невідома помилка виконання gpgv" +msgid "Unknown error executing apt-key" +msgstr "Невідома помилка виконання apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/vi.po b/po/vi.po index 87ec3b722..2532d68bf 100644 --- a/po/vi.po +++ b/po/vi.po @@ -1092,9 +1092,9 @@ msgid "At least one invalid signature was encountered." msgstr "Gặp ít nhất một chữ ký không hợp lệ." #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" msgstr "" -"Không thể thực hiện “gpgv” để thẩm tra chữ ký (gpgv đã được cài đặt chưa?)" +"Không thể thực hiện “apt-key” để thẩm tra chữ ký (gnupg đã được cài đặt chưa?)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1107,8 +1107,8 @@ msgstr "" "không?)" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "Gặp lỗi không rõ khi thực hiện gpgv" +msgid "Unknown error executing apt-key" +msgstr "Gặp lỗi không rõ khi thực hiện apt-key" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/zh_CN.po b/po/zh_CN.po index c06970a01..79e433f3d 100644 --- a/po/zh_CN.po +++ b/po/zh_CN.po @@ -1006,8 +1006,8 @@ msgid "At least one invalid signature was encountered." msgstr "至少发现一个无效的签名。" #: methods/gpgv.cc:174 -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "无法运行 gpgv 以验证签名(您安装了 gpgv 吗？)" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "无法运行 apt-key 以验证签名(您安装了 gnupg 吗？)" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1018,8 +1018,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "运行 gpgv 时发生未知错误" +msgid "Unknown error executing apt-key" +msgstr "运行 apt-key 时发生未知错误" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" diff --git a/po/zh_TW.po b/po/zh_TW.po index 9d6b7d2ea..c39e039af 100644 --- a/po/zh_TW.po +++ b/po/zh_TW.po @@ -998,9 +998,8 @@ msgid "At least one invalid signature was encountered." msgstr "至少發現一個無效的簽章。" #: methods/gpgv.cc:174 -#, fuzzy -msgid "Could not execute 'gpgv' to verify signature (is gpgv installed?)" -msgstr "無法執行 '%s' 來驗證簽章（gpgv 是否安裝了？）" +msgid "Could not execute 'apt-key' to verify signature (is gnupg installed?)" +msgstr "無法執行 'apt-key' 來驗證簽章（gnupg 是否安裝了？）" #. TRANSLATORS: %s is a single techy word like 'NODATA' #: methods/gpgv.cc:180 @@ -1011,8 +1010,8 @@ msgid "" msgstr "" #: methods/gpgv.cc:184 -msgid "Unknown error executing gpgv" -msgstr "在執行 gpgv 時發生未知的錯誤" +msgid "Unknown error executing apt-key" +msgstr "在執行 apt-key 時發生未知的錯誤" #: methods/gpgv.cc:217 methods/gpgv.cc:224 msgid "The following signatures were invalid:\n" -- cgit v1.2.3-70-g09d2 From 84361defb573c52d9a5368069254d1e18105aa62 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 27 Sep 2014 01:00:14 +0200 Subject: fix: %i in format string (no. 1) requires 'int' but the argument type is 'unsigned int' Git-Dch: Ignore Reported-By: cppcheck --- methods/server.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index ff67eb09b..4a961f454 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -324,10 +324,10 @@ ServerMethod::DealWithHeaders(FetchResult &Res) failure */ if (Server->Result < 200 || Server->Result >= 300) { - char err[255]; - snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result); + std::string err; + strprintf(err, "HttpError%u", Server->Result); SetFailReason(err); - _error->Error("%u %s",Server->Result,Server->Code); + _error->Error("%u %s", Server->Result, Server->Code); if (Server->HaveContent == true) return ERROR_WITH_CONTENT_PAGE; return ERROR_UNRECOVERABLE; -- cgit v1.2.3-70-g09d2 From 62acbba8d1e8c9f395ccd1068e89bf14a93d4aa8 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 08:16:51 +0200 Subject: methods/https.cc: use File->Tell() here too --- methods/https.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index eec858417..f8e84a2ff 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -82,8 +82,7 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if(me->File->Write(buffer, size*nmemb) != true) return false; - me->TotalWritten += size*nmemb; - if(me->TotalWritten > me->Queue->ExpectedSize) + if(me->Queue->ExpectedSize > 0 && me->File->Tell() > me->Queue->ExpectedSize) return _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->ExpectedSize); -- cgit v1.2.3-70-g09d2 From 5b33fab8c9a8c26cbc8ce3bf5e1573279d7884b3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 08:43:46 +0200 Subject: add ftp expected size check --- methods/ftp.cc | 10 +++++++--- methods/ftp.h | 2 +- methods/https.cc | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/ftp.cc b/methods/ftp.cc index ac76295f0..75ace1c5a 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -849,7 +849,7 @@ bool FTPConn::Finalize() /* This opens a data connection, sends REST and RETR and then transfers the file over. */ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &Hash,bool &Missing) + Hashes &Hash,bool &Missing, unsigned long long ExpectedSize) { Missing = false; if (CreateDataFd() == false) @@ -922,7 +922,11 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, { Close(); return false; - } + } + + if (ExpectedSize > 0 && To.Tell() > ExpectedSize) + return _error->Error("Writing more data than expected (%llu > %llu)", + To.Tell(), ExpectedSize); } // All done @@ -1063,7 +1067,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) FailFd = Fd.Fd(); bool Missing; - if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false) + if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->ExpectedSize) == false) { Fd.Close(); diff --git a/methods/ftp.h b/methods/ftp.h index dd92f0086..416a91980 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -62,7 +62,7 @@ class FTPConn bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); bool Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &MD5,bool &Missing); + Hashes &MD5,bool &Missing, unsigned long long ExpectedSize); FTPConn(URI Srv); ~FTPConn(); diff --git a/methods/https.cc b/methods/https.cc index f8e84a2ff..a4794e705 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -86,7 +86,6 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) return _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->ExpectedSize); - return size*nmemb; } -- cgit v1.2.3-70-g09d2 From c48eea97b93920062ea26001081d4fdf7eb967e3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 17:47:30 +0200 Subject: make expected-size a maximum-size check as this is what we want at this point --- apt-pkg/acquire-method.cc | 2 +- apt-pkg/acquire-method.h | 5 ++++- apt-pkg/acquire-worker.cc | 10 ++++++---- methods/ftp.cc | 8 ++++---- methods/ftp.h | 2 +- methods/http.cc | 4 ++-- methods/https.cc | 4 ++-- methods/server.cc | 4 ++-- methods/server.h | 4 ++-- test/integration/test-apt-update-expected-size | 7 +++++++ 10 files changed, 31 insertions(+), 19 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 330854e75..9c0558223 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -373,7 +373,7 @@ int pkgAcqMethod::Run(bool Single) Tmp->ExpectedHashes.push_back(HashString(*t, hash)); } char *End; - Tmp->ExpectedSize = strtoll(LookupTag(Message, "Expected-Size", "0").c_str(), &End, 10); + Tmp->MaximumSize = strtoll(LookupTag(Message, "Maximum-Size", "0").c_str(), &End, 10); Tmp->Next = 0; // Append it to the list diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 2e4e8281a..675c4f844 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -48,7 +48,10 @@ class pkgAcqMethod bool IndexFile; bool FailIgnore; HashStringList ExpectedHashes; - unsigned long long ExpectedSize; + // a maximum size we will download, this can be the exact filesize + // for when we know it or a arbitrary limit when we don't know the + // filesize (like a InRelease file) + unsigned long long MaximumSize; }; struct FetchResult diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 8bd1618f4..ffa84eb68 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -526,9 +526,6 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) if (OutFd == -1) return false; - string ExpectedSize; - strprintf(ExpectedSize, "%llu", Item->Owner->FileSize); - string Message = "600 URI Acquire\n"; Message.reserve(300); Message += "URI: " + Item->URI; @@ -536,7 +533,12 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) HashStringList const hsl = Item->Owner->HashSums(); for (HashStringList::const_iterator hs = hsl.begin(); hs != hsl.end(); ++hs) Message += "\nExpected-" + hs->HashType() + ": " + hs->HashValue(); - Message += "\nExpected-Size: " + ExpectedSize; + if(Item->Owner->FileSize > 0) + { + string MaximumSize; + strprintf(MaximumSize, "%llu", Item->Owner->FileSize); + Message += "\nMaximum-Size: " + MaximumSize; + } Message += Item->Owner->Custom600Headers(); Message += "\n\n"; diff --git a/methods/ftp.cc b/methods/ftp.cc index 75ace1c5a..bc84dda7d 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -849,7 +849,7 @@ bool FTPConn::Finalize() /* This opens a data connection, sends REST and RETR and then transfers the file over. */ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &Hash,bool &Missing, unsigned long long ExpectedSize) + Hashes &Hash,bool &Missing, unsigned long long MaximumSize) { Missing = false; if (CreateDataFd() == false) @@ -924,9 +924,9 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, return false; } - if (ExpectedSize > 0 && To.Tell() > ExpectedSize) + if (MaximumSize > 0 && To.Tell() > MaximumSize) return _error->Error("Writing more data than expected (%llu > %llu)", - To.Tell(), ExpectedSize); + To.Tell(), MaximumSize); } // All done @@ -1067,7 +1067,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) FailFd = Fd.Fd(); bool Missing; - if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->ExpectedSize) == false) + if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize) == false) { Fd.Close(); diff --git a/methods/ftp.h b/methods/ftp.h index 416a91980..a31ebc999 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -62,7 +62,7 @@ class FTPConn bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); bool Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &MD5,bool &Missing, unsigned long long ExpectedSize); + Hashes &MD5,bool &Missing, unsigned long long MaximumSize); FTPConn(URI Srv); ~FTPConn(); diff --git a/methods/http.cc b/methods/http.cc index b076e59cc..f8faa0cf8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -655,10 +655,10 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) return _error->Errno("write",_("Error writing to output file")); } - if (ExpectedSize > 0 && File && File->Tell() > ExpectedSize) + if (MaximumSize > 0 && File && File->Tell() > MaximumSize) { return _error->Error("Writing more data than expected (%llu > %llu)", - File->Tell(), ExpectedSize); + File->Tell(), MaximumSize); } // Handle commands from APT diff --git a/methods/https.cc b/methods/https.cc index a4794e705..787e4a507 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -82,9 +82,9 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if(me->File->Write(buffer, size*nmemb) != true) return false; - if(me->Queue->ExpectedSize > 0 && me->File->Tell() > me->Queue->ExpectedSize) + if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) return _error->Error("Writing more data than expected (%llu > %llu)", - me->TotalWritten, me->Queue->ExpectedSize); + me->TotalWritten, me->Queue->MaximumSize); return size*nmemb; } diff --git a/methods/server.cc b/methods/server.cc index 223737901..82f9b4750 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -534,8 +534,8 @@ int ServerMethod::Loop() bool Result = true; // ensure we don't fetch too much - if (Queue->ExpectedSize > 0) - Server->ExpectedSize = Queue->ExpectedSize; + if (Queue->MaximumSize > 0) + Server->MaximumSize = Queue->MaximumSize; if (Server->HaveContent) Result = Server->RunData(File); diff --git a/methods/server.h b/methods/server.h index 0134a9538..3093e00c9 100644 --- a/methods/server.h +++ b/methods/server.h @@ -49,7 +49,7 @@ struct ServerState URI Proxy; unsigned long TimeOut; - unsigned long long ExpectedSize; + unsigned long long MaximumSize; protected: ServerMethod *Owner; @@ -75,7 +75,7 @@ struct ServerState bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; - State = Header; Persistent = false; Pipeline = true; ExpectedSize = 0;}; + State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; /** \brief Transfer the data from the socket */ diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 72812336d..c1eecc08a 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -15,6 +15,13 @@ changetowebserver # normal update works fine testsuccess aptget update +# make InRelease really big +mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good +dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 +touch -d '+1hour' aptarchive/dists/unstable/InRelease +aptget update -o acquire::MaxReleaseFileSize=$((1*1000*1000)) + + # append junk at the end of the Packages.gz/Packages SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages.gz -- cgit v1.2.3-70-g09d2 From ee27950632c149bb14c9c490e92147579ba4fc2a Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 22:36:09 +0200 Subject: Send "Fail-Reason: MaximumSizeExceeded" from the method Communicate the fail reason from the methods to the parent and Rename() failed files. --- apt-pkg/acquire-item.cc | 6 +++++- methods/ftp.cc | 8 ++++++-- methods/ftp.h | 3 ++- methods/http.cc | 1 + methods/https.cc | 4 +++- test/integration/test-apt-update-expected-size | 5 ++++- 6 files changed, 21 insertions(+), 6 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 1dcbde223..f630129b9 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -148,8 +148,12 @@ void pkgAcquire::Item::Failed(string Message,pkgAcquire::MethodConfig *Cnf) else Status = StatIdle; - // report mirror failure back to LP if we actually use a mirror + // check fail reason string FailReason = LookupTag(Message, "FailReason"); + if(FailReason == "MaximumSizeExceeded") + Rename(DestFile, DestFile+".FAILED"); + + // report mirror failure back to LP if we actually use a mirror if(FailReason.size() != 0) ReportMirrorFailure(FailReason); else diff --git a/methods/ftp.cc b/methods/ftp.cc index bc84dda7d..5b739ea06 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -849,7 +849,8 @@ bool FTPConn::Finalize() /* This opens a data connection, sends REST and RETR and then transfers the file over. */ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &Hash,bool &Missing, unsigned long long MaximumSize) + Hashes &Hash,bool &Missing, unsigned long long MaximumSize, + pkgAcqMethod *Owner) { Missing = false; if (CreateDataFd() == false) @@ -925,8 +926,11 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, } if (MaximumSize > 0 && To.Tell() > MaximumSize) + { + Owner->SetFailReason("MaximumSizeExceeded"); return _error->Error("Writing more data than expected (%llu > %llu)", To.Tell(), MaximumSize); + } } // All done @@ -1067,7 +1071,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) FailFd = Fd.Fd(); bool Missing; - if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize) == false) + if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize,this) == false) { Fd.Close(); diff --git a/methods/ftp.h b/methods/ftp.h index a31ebc999..2efd28ec6 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -62,7 +62,8 @@ class FTPConn bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); bool Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &MD5,bool &Missing, unsigned long long MaximumSize); + Hashes &MD5,bool &Missing, unsigned long long MaximumSize, + pkgAcqMethod *Owner); FTPConn(URI Srv); ~FTPConn(); diff --git a/methods/http.cc b/methods/http.cc index f8faa0cf8..c00b439b7 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -657,6 +657,7 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) if (MaximumSize > 0 && File && File->Tell() > MaximumSize) { + Owner->SetFailReason("MaximumSizeExceeded"); return _error->Error("Writing more data than expected (%llu > %llu)", File->Tell(), MaximumSize); } diff --git a/methods/https.cc b/methods/https.cc index 787e4a507..16d564b34 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -83,9 +83,11 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) return false; if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) + { + me->SetFailReason("MaximumSizeExceeded"); return _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->MaximumSize); - + } return size*nmemb; } diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index f8ec24dcc..58920f544 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -19,7 +19,7 @@ testsuccess aptget update mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null touch -d '+1hour' aptarchive/dists/unstable/InRelease -aptget update -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 > output.log +aptget update -o Apt::Get::List-Cleanup=0 -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0 > output.log msgtest 'Check that the max write warning is triggered' if grep -q "Writing more data than expected" output.log; then msgpass @@ -27,8 +27,11 @@ else cat output.log msgfail fi +# ensure the failed InRelease file got renamed +testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED mv aptarchive/dists/unstable/InRelease.good aptarchive/dists/unstable/InRelease + # append junk at the end of the Packages.gz/Packages SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages.gz -- cgit v1.2.3-70-g09d2 From f2b47ba290f3a178c584da83f007cf0f720baabb Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Oct 2014 08:32:42 +0200 Subject: Fix http pipeline messup detection The Maximum-Size protection breaks the http pipeline reorder code because it relies on that the object got fetched entirely so that it can compare the hash of the downloaded data. So instead of stopping when the Maximum-Size of the expected item is reached we only stop when the maximum size of the biggest item in the queue is reached. This way the pipeline reoder code keeps working. --- methods/server.cc | 16 ++++++++++++++-- methods/server.h | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index 82f9b4750..cef809738 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -534,8 +534,10 @@ int ServerMethod::Loop() bool Result = true; // ensure we don't fetch too much - if (Queue->MaximumSize > 0) - Server->MaximumSize = Queue->MaximumSize; + // we could do "Server->MaximumSize = Queue->MaximumSize" here + // but that would break the clever pipeline messup detection + // so instead we use the size of the biggest item in the queue + Server->MaximumSize = FindMaximumObjectSizeInQueue(); if (Server->HaveContent) Result = Server->RunData(File); @@ -706,5 +708,15 @@ int ServerMethod::Loop() } return 0; +} + /*}}}*/ + /*{{{*/ +unsigned long long +ServerMethod::FindMaximumObjectSizeInQueue() const +{ + unsigned long long MaxSizeInQueue = 0; + for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize); + return MaxSizeInQueue; } /*}}}*/ diff --git a/methods/server.h b/methods/server.h index 3093e00c9..7d5198478 100644 --- a/methods/server.h +++ b/methods/server.h @@ -106,6 +106,10 @@ class ServerMethod : public pkgAcqMethod unsigned long PipelineDepth; bool AllowRedirect; + // Find the biggest item in the fetch queue for the checking of the maximum + // size + unsigned long long FindMaximumObjectSizeInQueue() const APT_PURE; + public: bool Debug; -- cgit v1.2.3-70-g09d2 From 8bb043fcd1def35cf516d0961a2b5e6ee53f0fd7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Oct 2014 09:45:11 +0200 Subject: Fix ServerMethod::FindMaximumObjectSizeInQueue() Git-Dch: ignore --- methods/server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'methods') diff --git a/methods/server.cc b/methods/server.cc index cef809738..c4689ff12 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -715,7 +715,7 @@ unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const { unsigned long long MaxSizeInQueue = 0; - for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next) MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize); return MaxSizeInQueue; } -- cgit v1.2.3-70-g09d2 From 180b693262d71381d650d10c3f95a5a70553f40f Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Oct 2014 11:35:48 +0200 Subject: methods/rsh.cc: replace strcat with std::string Instead of using strcat use a C++ std::string to avoid overflowing this buffer. Thanks to David Garfield Closes: #76442 --- methods/rsh.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'methods') diff --git a/methods/rsh.cc b/methods/rsh.cc index bd46d2515..0e949160b 100644 --- a/methods/rsh.cc +++ b/methods/rsh.cc @@ -218,17 +218,20 @@ bool RSHConn::WriteMsg(std::string &Text,bool Sync,const char *Fmt,...) va_list args; va_start(args,Fmt); - // sprintf the description - char S[512]; - vsnprintf(S,sizeof(S) - 4,Fmt,args); + // sprintf into a buffer + char Tmp[1024]; + vsnprintf(Tmp,sizeof(Tmp),Fmt,args); va_end(args); + // concat to create the real msg + std::string Msg; if (Sync == true) - strcat(S," 2> /dev/null || echo\n"); + Msg = std::string(Tmp) + " 2> /dev/null || echo\n"; else - strcat(S," 2> /dev/null\n"); + Msg = std::string(Tmp) + " 2> /dev/null\n"; // Send it off + const char *S = Msg.c_str(); unsigned long Len = strlen(S); unsigned long Start = 0; while (Len != 0) -- cgit v1.2.3-70-g09d2 From 9983999d294887046abf386adc31190700d89b61 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 13 Oct 2014 10:57:30 +0200 Subject: Fix backward compatiblity of the new pkgAcquireMethod::DropPrivsOrDie() Do not drop privileges in the methods when using a older version of libapt that does not support the chown magic in partial/ yet. To do this DropPrivileges() now will ignore a empty Apt::Sandbox::User. Cleanup all hardcoded _apt along the way. --- apt-pkg/acquire-item.cc | 12 +++++++++--- apt-pkg/acquire.cc | 5 +++-- apt-pkg/contrib/fileutl.cc | 7 ++++++- apt-pkg/init.cc | 3 +++ methods/ftp.cc | 7 ++++--- methods/gpgv.cc | 14 +++++++++++--- methods/gzip.cc | 12 ++++++++++-- methods/http.cc | 2 ++ methods/http_main.cc | 1 - methods/https.cc | 12 ++++++++++-- methods/https.h | 2 ++ 11 files changed, 60 insertions(+), 17 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 97ff1bd18..44d45cedb 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -215,7 +215,11 @@ bool pkgAcquire::Item::Rename(string From,string To) void pkgAcquire::Item::QueueURI(ItemDesc &Item) { if (RealFileExists(DestFile)) - ChangeOwnerAndPermissionOfFile("GetPartialFileName", DestFile.c_str(), "_apt", "root", 0600); + { + std::string SandboxUser = _config->Find("APT::Sandbox::User"); + ChangeOwnerAndPermissionOfFile("GetPartialFileName", DestFile.c_str(), + SandboxUser.c_str(), "root", 0600); + } Owner->Enqueue(Item); } void pkgAcquire::Item::Dequeue() @@ -2486,7 +2490,8 @@ bool pkgAcqArchive::QueueNext() else { PartialSize = Buf.st_size; - ChangeOwnerAndPermissionOfFile("pkgAcqArchive::QueueNext", DestFile.c_str(), "_apt", "root", 0600); + std::string SandboxUser = _config->Find("APT::Sandbox::User"); + ChangeOwnerAndPermissionOfFile("pkgAcqArchive::QueueNext",DestFile.c_str(), SandboxUser.c_str(), "root", 0600); } } @@ -2654,7 +2659,8 @@ pkgAcqFile::pkgAcqFile(pkgAcquire *Owner,string URI, HashStringList const &Hashe else { PartialSize = Buf.st_size; - ChangeOwnerAndPermissionOfFile("pkgAcqFile", DestFile.c_str(), "_apt", "root", 0600); + std::string SandboxUser = _config->Find("APT::Sandbox::User"); + ChangeOwnerAndPermissionOfFile("pkgAcqFile", DestFile.c_str(), SandboxUser.c_str(), "root", 0600); } } diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 34c7f06de..3ef912600 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -83,10 +83,11 @@ static bool SetupAPTPartialDirectory(std::string const &grand, std::string const if (getuid() == 0) // if we aren't root, we can't chown, so don't try it { - struct passwd *pw = getpwnam("_apt"); + std::string SandboxUser = _config->Find("APT::Sandbox::User"); + struct passwd *pw = getpwnam(SandboxUser.c_str()); struct group *gr = getgrnam("root"); if (pw != NULL && gr != NULL && chown(partial.c_str(), pw->pw_uid, gr->gr_gid) != 0) - _error->WarningE("SetupAPTPartialDirectory", "chown to _apt:root of directory %s failed", partial.c_str()); + _error->WarningE("SetupAPTPartialDirectory", "chown to %s:root of directory %s failed", SandboxUser.c_str(), partial.c_str()); } if (chmod(partial.c_str(), 0700) != 0) _error->WarningE("SetupAPTPartialDirectory", "chmod 0700 of directory %s failed", partial.c_str()); diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 3e592d94f..f6351b7b5 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -2168,6 +2168,12 @@ bool DropPrivileges() /*{{{*/ if(_config->FindB("Debug::NoDropPrivs", false) == true) return true; + // empty setting disables DropPrivilidges - this also ensures + // backward compatibility, see bug #764506 + const std::string toUser = _config->Find("APT::Sandbox::User"); + if (toUser.empty()) + return true; + #if __gnu_linux__ #if defined(PR_SET_NO_NEW_PRIVS) && ( PR_SET_NO_NEW_PRIVS != 38 ) #error "PR_SET_NO_NEW_PRIVS is defined, but with a different value than expected!" @@ -2186,7 +2192,6 @@ bool DropPrivileges() /*{{{*/ if (old_uid != 0) return true; - const std::string toUser = _config->Find("APT::Sandbox::User", "_apt"); struct passwd *pw = getpwnam(toUser.c_str()); if (pw == NULL) return _error->Error("No user %s, can not drop rights", toUser.c_str()); diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc index 82dff4ee8..8d206e428 100644 --- a/apt-pkg/init.cc +++ b/apt-pkg/init.cc @@ -95,6 +95,9 @@ bool pkgInitConfig(Configuration &Cnf) // Default cdrom mount point Cnf.CndSet("Acquire::cdrom::mount", "/media/cdrom/"); + // The default user we drop to in the methods + Cnf.CndSet("APT::Sandbox::User", "_apt"); + bool Res = true; // Read an alternate config file diff --git a/methods/ftp.cc b/methods/ftp.cc index 5b739ea06..0504e5872 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -988,6 +988,10 @@ bool FtpMethod::Configuration(string Message) return false; TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut); + + // no more active ftp, sorry + DropPrivsOrDie(); + return true; } /*}}}*/ @@ -1141,8 +1145,5 @@ int main(int, const char *argv[]) FtpMethod Mth; - // no more active ftp, sorry - Mth.DropPrivsOrDie(); - return Mth.Run(); } diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 7e8500c51..488c16826 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -44,12 +44,22 @@ class GPGVMethod : public pkgAcqMethod protected: virtual bool Fetch(FetchItem *Itm); - + virtual bool Configuration(string Message); public: GPGVMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {}; }; +bool GPGVMethod::Configuration(string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + DropPrivsOrDie(); + + return true; +} + string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, vector &GoodSigners, vector &BadSigners, @@ -265,7 +275,5 @@ int main() GPGVMethod Mth; - Mth.DropPrivsOrDie(); - return Mth.Run(); } diff --git a/methods/gzip.cc b/methods/gzip.cc index 7ffcda60f..387c05f2e 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -33,12 +33,22 @@ const char *Prog; class GzipMethod : public pkgAcqMethod { virtual bool Fetch(FetchItem *Itm); + virtual bool Configuration(std::string Message); public: GzipMethod() : pkgAcqMethod("1.1",SingleInstance | SendConfig) {}; }; +bool GzipMethod::Configuration(std::string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + DropPrivsOrDie(); + + return true; +} // GzipMethod::Fetch - Decompress the passed URI /*{{{*/ // --------------------------------------------------------------------- @@ -140,7 +150,5 @@ int main(int, char *argv[]) GzipMethod Mth; - Mth.DropPrivsOrDie(); - return Mth.Run(); } diff --git a/methods/http.cc b/methods/http.cc index c00b439b7..a5de13511 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -770,6 +770,8 @@ bool HttpMethod::Configuration(string Message) if (ServerMethod::Configuration(Message) == false) return false; + DropPrivsOrDie(); + AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", PipelineDepth); diff --git a/methods/http_main.cc b/methods/http_main.cc index f21a5709c..cd52c42e8 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -15,6 +15,5 @@ int main() HttpMethod Mth; - Mth.DropPrivsOrDie(); return Mth.Loop(); } diff --git a/methods/https.cc b/methods/https.cc index 16d564b34..366148e19 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -37,6 +37,16 @@ /*}}}*/ using namespace std; +bool HttpsMethod::Configuration(std::string Message) +{ + if (pkgAcqMethod::Configuration(Message) == false) + return false; + + DropPrivsOrDie(); + + return true; +} + size_t HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { @@ -452,8 +462,6 @@ int main() HttpsMethod Mth; curl_global_init(CURL_GLOBAL_SSL) ; - Mth.DropPrivsOrDie(); - return Mth.Run(); } diff --git a/methods/https.h b/methods/https.h index 0387cb9b5..9df18e83a 100644 --- a/methods/https.h +++ b/methods/https.h @@ -58,6 +58,8 @@ class HttpsMethod : public pkgAcqMethod static const int DL_MIN_SPEED = 10; virtual bool Fetch(FetchItem *); + virtual bool Configuration(std::string Message); + static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp); static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp); static int progress_callback(void *clientp, double dltotal, double dlnow, -- cgit v1.2.3-70-g09d2 From 9dd940ed722e8235c615e79b7eb688eb427e9a23 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 5 Nov 2014 18:33:07 +0100 Subject: Assert statement calls a function which may have desired side effects: 'pos_is_okay' It does not have any desired sideeffect, so we just mark it as const to properly advertise this fact to developer, compiler and linter alike. Reported-By: cppcheck Git-Dch: Ignore --- methods/rred.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/rred.cc b/methods/rred.cc index cabb3c456..774b58a40 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -150,11 +150,11 @@ class FileChanges { std::list::iterator where; size_t pos; // line number is as far left of iterator as possible - bool pos_is_okay(void) + bool pos_is_okay(void) const { #ifdef POSDEBUG size_t cpos = 0; - std::list::iterator x; + std::list::const_iterator x; for (x = changes.begin(); x != where; ++x) { assert(x != changes.end()); cpos += x->offset + x->add_cnt; -- cgit v1.2.3-70-g09d2 From bf6ac7ca615922c23d1f3cf1963efc5be9c23e32 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 9 Nov 2014 15:57:43 +0100 Subject: use getline() instead of rolling our own We use it in other places already as well even though it is farly new addition to the POSIX family with 2008, but rolling our own here is really something which should be avoided in such a important method. Git-Dch: Ignore --- methods/gpgv.cc | 31 +++++-------------------------- 1 file changed, 5 insertions(+), 26 deletions(-) (limited to 'methods') diff --git a/methods/gpgv.cc b/methods/gpgv.cc index 488c16826..41f138be6 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -86,33 +86,12 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, FILE *pipein = fdopen(fd[0], "r"); // Loop over the output of apt-key (which really is gnupg), and check the signatures. - size_t buffersize = 64; - char *buffer = (char *) malloc(buffersize); - size_t bufferoff = 0; + size_t buffersize = 0; + char *buffer = NULL; while (1) { - int c; - - // Read a line. Sigh. - while ((c = getc(pipein)) != EOF && c != '\n') - { - if (bufferoff == buffersize) - { - char* newBuffer = (char *) realloc(buffer, buffersize *= 2); - if (newBuffer == NULL) - { - free(buffer); - return "Couldn't allocate a buffer big enough for reading"; - } - buffer = newBuffer; - } - *(buffer+bufferoff) = c; - bufferoff++; - } - if (bufferoff == 0 && c == EOF) - break; - *(buffer+bufferoff) = '\0'; - bufferoff = 0; + if (getline(&buffer, &buffersize, pipein) == -1) + break; if (Debug == true) std::clog << "Read: " << buffer << std::endl; @@ -126,7 +105,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, std::clog << "Got BADSIG! " << std::endl; BadSigners.push_back(string(buffer+sizeof(GNUPGPREFIX))); } - + if (strncmp(buffer, GNUPGNOPUBKEY, sizeof(GNUPGNOPUBKEY)-1) == 0) { if (Debug == true) -- cgit v1.2.3-70-g09d2 From ed793a19ec00b83254029509bc516e3ba911c75a Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 29 Nov 2014 17:59:52 +0100 Subject: dispose http(s) 416 error page as non-content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real webservers (like apache) actually send an error page with a 416 response, but our client didn't expect it leaving the page on the socket to be parsed as response for the next request (http) or as file content (https), which isn't what we want at all… Symptom is a "Bad header line" as html usually doesn't parse that well to an http-header. This manifests itself e.g. if we have a complete file (or larger) in partial/ which isn't discarded by If-Range as the server doesn't support it (or it is just newer, think: mirror rotation). It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a, which removed the filesize - 1 trick, but this had its own problems… To properly test this our webserver gains the ability to reply with transfer-encoding: chunked as most real webservers will use it to send the dynamically generated error pages. Closes: 768797 --- cmdline/apt-helper.cc | 31 +++-- methods/http.cc | 2 + methods/https.cc | 15 ++- methods/server.cc | 26 ++-- methods/server.h | 5 +- test/integration/framework | 4 +- test/integration/test-partial-file-support | 62 ++++++++- test/interactive-helper/aptwebserver.cc | 193 ++++++++++++++++++----------- 8 files changed, 233 insertions(+), 105 deletions(-) (limited to 'methods') diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc index 1b832f165..a05ae90a2 100644 --- a/cmdline/apt-helper.cc +++ b/cmdline/apt-helper.cc @@ -51,22 +51,33 @@ static bool DoDownloadFile(CommandLine &CmdL) AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0)); pkgAcquire Fetcher(&Stat); - std::string download_uri = CmdL.FileList[1]; - std::string targetfile = CmdL.FileList[2]; - std::string hash; - if (CmdL.FileSize() > 3) - hash = CmdL.FileList[3]; - // we use download_uri as descr and targetfile as short-descr - new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, - "dest-dir-ignored", targetfile); + size_t fileind = 0; + std::vector targetfiles; + while (fileind + 2 <= CmdL.FileSize()) + { + std::string download_uri = CmdL.FileList[fileind + 1]; + std::string targetfile = CmdL.FileList[fileind + 2]; + std::string hash; + if (CmdL.FileSize() > fileind + 3) + hash = CmdL.FileList[fileind + 3]; + // we use download_uri as descr and targetfile as short-descr + new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, + "dest-dir-ignored", targetfile); + targetfiles.push_back(targetfile); + fileind += 3; + } // Disable drop-privs if "_apt" can not write to the target dir CheckDropPrivsMustBeDisabled(Fetcher); bool Failed = false; - if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true || - FileExists(targetfile) == false) + if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true) return _error->Error(_("Download Failed")); + if (targetfiles.empty() == false) + for (std::vector::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f) + if (FileExists(*f) == false) + return _error->Error(_("Download Failed")); + return true; } diff --git a/methods/http.cc b/methods/http.cc index a5de13511..ad1347d36 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -444,6 +444,8 @@ bool HttpServerState::RunData(FileFd * const File) loss of the connection means we are done */ if (Encoding == Closes) In.Limit(-1); + else if (JunkSize != 0) + In.Limit(JunkSize); else In.Limit(Size - StartPos); diff --git a/methods/https.cc b/methods/https.cc index 366148e19..23b3a10d4 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -69,6 +69,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) { me->Server->Result = 200; me->Server->StartPos = me->Server->Size; + // the actual size is not important for https as curl will deal with it + // by itself and e.g. doesn't bother us with transport-encoding… + me->Server->JunkSize = std::numeric_limits::max(); } else me->Server->StartPos = 0; @@ -86,19 +89,25 @@ size_t HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) { HttpsMethod *me = (HttpsMethod *)userp; + size_t buffer_size = size * nmemb; + // we don't need to count the junk here, just drop anything we get as + // we don't always know how long it would be, e.g. in chunked encoding. + if (me->Server->JunkSize != 0) + return buffer_size; if (me->Res.Size == 0) me->URIStart(me->Res); - if(me->File->Write(buffer, size*nmemb) != true) + if(me->File->Write(buffer, buffer_size) != true) return false; if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) { me->SetFailReason("MaximumSizeExceeded"); - return _error->Error("Writing more data than expected (%llu > %llu)", + _error->Error("Writing more data than expected (%llu > %llu)", me->TotalWritten, me->Queue->MaximumSize); + return 0; } - return size*nmemb; + return buffer_size; } int diff --git a/methods/server.cc b/methods/server.cc index c4689ff12..9b3d39cf2 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, Minor = 0; Result = 0; Size = 0; + JunkSize = 0; StartPos = 0; Encoding = Closes; HaveContent = false; @@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line) Encoding = Stream; HaveContent = true; - // The length is already set from the Content-Range header - if (StartPos != 0) - return true; + unsigned long long * SizePtr = &Size; + if (Result == 416) + SizePtr = &JunkSize; - Size = strtoull(Val.c_str(), NULL, 10); - if (Size >= std::numeric_limits::max()) + *SizePtr = strtoull(Val.c_str(), NULL, 10); + if (*SizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (Size == 0) + else if (*SizePtr == 0) HaveContent = false; return true; } @@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line) // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) - { - StartPos = 1; // ignore Content-Length, it would override Size - HaveContent = false; - } + ; // we got the expected filesize which is all we wanted else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); if ((unsigned long long)StartPos > Size) @@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res) if ((unsigned long long)SBuf.st_size == Server->Size) { // the file is completely downloaded, but was not moved + if (Server->HaveContent == true) + { + // Send to error page to dev/null + FileFd DevNull("/dev/null",FileFd::WriteExists); + Server->RunData(&DevNull); + } + Server->HaveContent = false; Server->StartPos = Server->Size; Server->Result = 200; - Server->HaveContent = false; } else if (unlink(Queue->DestFile.c_str()) == 0) { diff --git a/methods/server.h b/methods/server.h index 7d5198478..b974ec89a 100644 --- a/methods/server.h +++ b/methods/server.h @@ -34,7 +34,8 @@ struct ServerState char Code[360]; // These are some statistics from the last parsed header lines - unsigned long long Size; + unsigned long long Size; // size of the usable content (aka: the file) + unsigned long long JunkSize; // size of junk content (aka: server error pages) unsigned long long StartPos; time_t Date; bool HaveContent; @@ -73,7 +74,7 @@ struct ServerState RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri); bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; - virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; diff --git a/test/integration/framework b/test/integration/framework index 930ab9367..f7f69f5d5 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -1119,8 +1119,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom } downloadfile() { - local PROTO="$(echo "$1" | cut -d':' -f 1 )" - apthelper -o Debug::Acquire::${PROTO}=1 \ + local PROTO="${1%%:*}" + apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \ download-file "$1" "$2" 2>&1 || true # only if the file exists the download was successful if [ -r "$2" ]; then diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support index 98b2f242a..b6b305d25 100755 --- a/test/integration/test-partial-file-support +++ b/test/integration/test-partial-file-support @@ -24,13 +24,25 @@ testdownloadfile() { else msgpass fi - cat "$DOWNLOADLOG" | while read field hash; do + sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log + testsuccess test -s receivedhashes.log + local HASHES_OK=0 + local HASHES_BAD=0 + while read field hash; do local EXPECTED case "$field" in 'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";; 'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";; + 'Checksum-FileSize-Hash:') + #filesize is too weak to check for != + if [ "$4" = '=' ]; then + EXPECTED="$(stat -c '%s' "$TESTFILE")" + else + continue + fi + ;; *) continue;; esac if [ "$4" = '=' ]; then @@ -40,15 +52,41 @@ testdownloadfile() { fi if [ "$EXPECTED" "$4" "$hash" ]; then msgpass + HASHES_OK=$((HASHES_OK+1)); else - cat >&2 "$DOWNLOADLOG" msgfail "expected: $EXPECTED ; got: $hash" + HASHES_BAD=$((HASHES_BAD+1)); fi - done + done < receivedhashes.log + msgtest 'At least one good hash and no bad ones' + if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then + cat >&2 "$DOWNLOADLOG" + msgfail + else + msgpass + fi } TESTFILE='aptarchive/testfile' cp -a ${TESTDIR}/framework $TESTFILE +cp -a ${TESTDIR}/framework "${TESTFILE}2" + +followuprequest() { + local DOWN='./downloaded/testfile' + + copysource $TESTFILE 1M $DOWN + testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' + testwebserverlaststatuscode '416' "$DOWNLOADLOG" + + copysource $TESTFILE 1M $DOWN + copysource "${TESTFILE}2" 20 "${DOWN}2" + msgtest 'Testing download of files with' 'completely downloaded file + partial file' + testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \ + download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2" + testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output' + testsuccess diff -u "$TESTFILE" "${DOWN}" + testsuccess diff -u "${DOWN}" "${DOWN}2" +} testrun() { webserverconfig 'aptwebserver::support::range' 'true' @@ -66,9 +104,11 @@ testrun() { testdownloadfile 'invalid partial data' "${1}/testfile" "$DOWN" '!=' testwebserverlaststatuscode '206' "$DOWNLOADLOG" - copysource $TESTFILE 1M $DOWN - testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' - testwebserverlaststatuscode '416' "$DOWNLOADLOG" + webserverconfig 'aptwebserver::closeOnError' 'false' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'true' + followuprequest "$1" + webserverconfig 'aptwebserver::closeOnError' 'false' copysource /dev/zero 1M $DOWN testdownloadfile 'too-big partial file' "${1}/testfile" "$DOWN" '=' @@ -86,8 +126,18 @@ testrun() { testwebserverlaststatuscode '200' "$DOWNLOADLOG" } +msgmsg 'http: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'http://localhost:8080' +msgmsg 'http: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'http://localhost:8080' changetohttpswebserver +msgmsg 'https: Test with Content-Length' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false' +testrun 'https://localhost:4433' +msgmsg 'https: Test with Transfer-Encoding: chunked' +webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true' testrun 'https://localhost:4433' diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc index 00004a524..ca6f88c58 100644 --- a/test/interactive-helper/aptwebserver.cc +++ b/test/interactive-helper/aptwebserver.cc @@ -19,6 +19,8 @@ #include #include #include + +#include #include #include #include @@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode) /*{{{*/ return NULL; } /*}}}*/ +static bool chunkedTransferEncoding(std::list const &headers) { + if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end()) + return true; + if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true) + return true; + return false; +} static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/ { - std::ostringstream contentlength; - contentlength << "Content-Length: " << data.FileSize(); - headers.push_back(contentlength.str()); - + if (chunkedTransferEncoding(headers) == false) + { + std::ostringstream contentlength; + contentlength << "Content-Length: " << data.FileSize(); + headers.push_back(contentlength.str()); + } std::string lastmodified("Last-Modified: "); lastmodified.append(TimeRFC1123(data.ModificationTime())); headers.push_back(lastmodified); @@ -92,9 +103,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/ /*}}}*/ static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/ { - std::ostringstream contentlength; - contentlength << "Content-Length: " << data.size(); - headers.push_back(contentlength.str()); + if (chunkedTransferEncoding(headers) == false) + { + std::ostringstream contentlength; + contentlength << "Content-Length: " << data.size(); + headers.push_back(contentlength.str()); + } } /*}}}*/ static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/ @@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl; bool Success = true; for (std::list::const_iterator h = headers.begin(); @@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list const &headers, FileFd &data)/*{{{*/ { bool Success = true; + bool const chunked = chunkedTransferEncoding(headers); char buffer[500]; unsigned long long actual = 0; while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true) { if (actual == 0) break; - Success &= FileFd::Write(client, buffer, actual); + + if (chunked == true) + { + std::string size; + strprintf(size, "%llX\r\n", actual); + Success &= FileFd::Write(client, size.c_str(), size.size()); + Success &= FileFd::Write(client, buffer, actual); + Success &= FileFd::Write(client, "\r\n", strlen("\r\n")); + } + else + Success &= FileFd::Write(client, buffer, actual); + } + if (chunked == true) + { + char const * const finish = "0\r\n\r\n"; + Success &= FileFd::Write(client, finish, strlen(finish)); } if (Success == false) - std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl; + std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl; return Success; } /*}}}*/ -static bool sendData(int const client, std::string const &data) /*{{{*/ +static bool sendData(int const client, std::list const &headers, std::string const &data)/*{{{*/ { - if (FileFd::Write(client, data.c_str(), data.size()) == false) + if (chunkedTransferEncoding(headers) == true) + { + unsigned long long const ullsize = data.length(); + std::string size; + strprintf(size, "%llX\r\n", ullsize); + char const * const finish = "\r\n0\r\n\r\n"; + if (FileFd::Write(client, size.c_str(), size.length()) == false || + FileFd::Write(client, data.c_str(), ullsize) == false || + FileFd::Write(client, finish, strlen(finish)) == false) + { + std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl; + return false; + } + } + else if (FileFd::Write(client, data.c_str(), data.size()) == false) { std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl; return false; @@ -157,33 +204,38 @@ static bool sendData(int const client, std::string const &data) /*{{{*/ } /*}}}*/ static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/ - bool content, std::string const &error = "", std::list headers = std::list()) + bool const content, std::string const &error, std::list &headers) { std::string response(""); response.append(httpcodeToStr(httpcode)).append(""); response.append("

").append(httpcodeToStr(httpcode)).append("

"); if (httpcode != 200) - { - if (error.empty() == false) - response.append("

Error: ").append(error).append("

"); - response.append("This error is a result of the request: 
");
-   }
+      response.append("
Error: ");
+   else
+      response.append("
Success: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("
This error is a result of the request: 
");
    else
-   {
-      if (error.empty() == false)
-	 response.append("
Success: ").append(error).append("
");
       response.append("The successfully executed operation was requested by: 
");
-   }
    response.append(request).append("
");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -220,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -262,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list &headers)
 {
-   std::list headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -310,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string ¶ms, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -332,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -344,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -359,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -370,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
 
@@ -382,9 +433,9 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       if (authConf.empty() != auth.empty())
       {
 	 if (auth.empty())
-	    sendError(client, 407, request, sendContent, "Proxy requires authentication");
+	    sendError(client, 407, request, sendContent, "Proxy requires authentication", headers);
 	 else
-	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it");
+	    sendError(client, 407, request, sendContent, "Client wants to authenticate to proxy, but proxy doesn't need it", headers);
 	return false;
       }
       if (authConf.empty() == false)
@@ -395,7 +446,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 	    auth.erase(0, strlen(basic));
 	    if (auth != authConf)
 	    {
-	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match");
+	       sendError(client, 407, request, sendContent, "Proxy-Authentication doesn't match", headers);
 	       return false;
 	    }
 	 }
@@ -410,7 +461,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos && APT::String::Startswith(filename, "/_config/") == false)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -421,9 +472,9 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       if (authConf.empty() != auth.empty())
       {
 	 if (auth.empty())
-	    sendError(client, 401, request, sendContent, "Server requires authentication");
+	    sendError(client, 401, request, sendContent, "Server requires authentication", headers);
 	 else
-	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it");
+	    sendError(client, 401, request, sendContent, "Client wants to authenticate to server, but server doesn't need it", headers);
 	 return false;
       }
       if (authConf.empty() == false)
@@ -434,13 +485,12 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 	    auth.erase(0, strlen(basic));
 	    if (auth != authConf)
 	    {
-	       sendError(client, 401, request, sendContent, "Authentication doesn't match");
+	       sendError(client, 401, request, sendContent, "Authentication doesn't match", headers);
 	       return false;
 	    }
 	 }
 	 else
 	 {
-	    std::list headers;
 	    headers.push_back("WWW-Authenticate: Basic");
 	    sendError(client, 401, request, sendContent, "Unsupported Authentication Scheme", headers);
 	    return false;
@@ -463,7 +513,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -499,7 +550,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector parts, std::list &headers)
 {
    size_t const pcount = parts.size();
    for (size_t i = 0; i < pcount; ++i)
@@ -507,40 +559,38 @@ static bool handleOnTheFlyReconfiguration(int const client, std::string const &r
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -549,18 +599,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -569,7 +623,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -601,7 +655,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -620,7 +674,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -676,17 +730,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -695,22 +748,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From 92e8c1ff287ab829de825e00cdf94744e699ff97 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 29 Nov 2014 17:59:52 +0100
Subject: dispose http(s) 416 error page as non-content
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Real webservers (like apache) actually send an error page with a 416
response, but our client didn't expect it leaving the page on the socket
to be parsed as response for the next request (http) or as file content
(https), which isn't what we want at all… Symptom is a "Bad header line"
as html usually doesn't parse that well to an http-header.

This manifests itself e.g. if we have a complete file (or larger) in
partial/ which isn't discarded by If-Range as the server doesn't support
it (or it is just newer, think: mirror rotation).
It is a sort-of regression of 78c72d0ce22e00b194251445aae306df357d5c1a,
which removed the filesize - 1 trick, but this had its own problems…

To properly test this our webserver gains the ability to reply with
transfer-encoding: chunked as most real webservers will use it to send
the dynamically generated error pages.

(The tests and their binary helpers had to be slightly modified to
apply, but the patch to fix the issue itself is unchanged.)

Closes: 768797
---
 cmdline/apt-helper.cc                      |  35 ++++--
 methods/http.cc                            |   2 +
 methods/https.cc                           |  12 +-
 methods/server.cc                          |  26 +++--
 methods/server.h                           |   5 +-
 test/integration/framework                 |   6 +-
 test/integration/test-apt-helper           |  24 ++--
 test/integration/test-partial-file-support |  62 +++++++++-
 test/interactive-helper/aptwebserver.cc    | 181 ++++++++++++++++++-----------
 9 files changed, 241 insertions(+), 112 deletions(-)

(limited to 'methods')

diff --git a/cmdline/apt-helper.cc b/cmdline/apt-helper.cc
index dd43ea1bc..63f70983c 100644
--- a/cmdline/apt-helper.cc
+++ b/cmdline/apt-helper.cc
@@ -48,23 +48,34 @@ static bool DoDownloadFile(CommandLine &CmdL)
    if (CmdL.FileSize() <= 2)
       return _error->Error(_("Must specify at least one pair url/filename"));
 
-
    pkgAcquire Fetcher;
    AcqTextStatus Stat(ScreenWidth, _config->FindI("quiet",0));
    Fetcher.Setup(&Stat);
-   std::string download_uri = CmdL.FileList[1];
-   std::string targetfile = CmdL.FileList[2];
-   std::string hash;
-   if (CmdL.FileSize() > 3)
-      hash = CmdL.FileList[3];
-   // we use download_uri as descr and targetfile as short-descr
-   new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile, 
-                  "dest-dir-ignored", targetfile);
-   Fetcher.Run();
+
+   size_t fileind = 0;
+   std::vector targetfiles;
+   while (fileind + 2 <= CmdL.FileSize())
+   {
+      std::string download_uri = CmdL.FileList[fileind + 1];
+      std::string targetfile = CmdL.FileList[fileind + 2];
+      std::string hash;
+      if (CmdL.FileSize() > fileind + 3)
+	 hash = CmdL.FileList[fileind + 3];
+      // we use download_uri as descr and targetfile as short-descr
+      new pkgAcqFile(&Fetcher, download_uri, hash, 0, download_uri, targetfile,
+	    "dest-dir-ignored", targetfile);
+      targetfiles.push_back(targetfile);
+      fileind += 3;
+   }
+
    bool Failed = false;
-   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true ||
-	 FileExists(targetfile) == false)
+   if (AcquireRun(Fetcher, 0, &Failed, NULL) == false || Failed == true)
       return _error->Error(_("Download Failed"));
+   if (targetfiles.empty() == false)
+      for (std::vector::const_iterator f = targetfiles.begin(); f != targetfiles.end(); ++f)
+	 if (FileExists(*f) == false)
+	    return _error->Error(_("Download Failed"));
+
    return true;
 }
 
diff --git a/methods/http.cc b/methods/http.cc
index f2a4a4db6..1b996db98 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -440,6 +440,8 @@ bool HttpServerState::RunData(FileFd * const File)
          loss of the connection means we are done */
       if (Encoding == Closes)
 	 In.Limit(-1);
+      else if (JunkSize != 0)
+	 In.Limit(JunkSize);
       else
 	 In.Limit(Size - StartPos);
       
diff --git a/methods/https.cc b/methods/https.cc
index 0499af0c5..65a744e2a 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -59,6 +59,9 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       {
          me->Server->Result = 200;
 	 me->Server->StartPos = me->Server->Size;
+	 // the actual size is not important for https as curl will deal with it
+	 // by itself and e.g. doesn't bother us with transport-encoding…
+	 me->Server->JunkSize = std::numeric_limits::max();
       }
       else
 	 me->Server->StartPos = 0;
@@ -76,13 +79,18 @@ size_t
 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    HttpsMethod *me = (HttpsMethod *)userp;
+   size_t buffer_size = size * nmemb;
+   // we don't need to count the junk here, just drop anything we get as
+   // we don't always know how long it would be, e.g. in chunked encoding.
+   if (me->Server->JunkSize != 0)
+      return buffer_size;
 
    if (me->Res.Size == 0)
       me->URIStart(me->Res);
-   if(me->File->Write(buffer, size*nmemb) != true)
+   if(me->File->Write(buffer, buffer_size) != true)
       return false;
 
-   return size*nmemb;
+   return buffer_size;
 }
 
 int
diff --git a/methods/server.cc b/methods/server.cc
index 92d94e638..cb0341d5f 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -55,6 +55,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
    Minor = 0; 
    Result = 0; 
    Size = 0; 
+   JunkSize = 0;
    StartPos = 0;
    Encoding = Closes;
    HaveContent = false;
@@ -163,14 +164,14 @@ bool ServerState::HeaderLine(string Line)
 	 Encoding = Stream;
       HaveContent = true;
 
-      // The length is already set from the Content-Range header
-      if (StartPos != 0)
-	 return true;
+      unsigned long long * SizePtr = &Size;
+      if (Result == 416)
+	 SizePtr = &JunkSize;
 
-      Size = strtoull(Val.c_str(), NULL, 10);
-      if (Size >= std::numeric_limits::max())
+      *SizePtr = strtoull(Val.c_str(), NULL, 10);
+      if (*SizePtr >= std::numeric_limits::max())
 	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
-      else if (Size == 0)
+      else if (*SizePtr == 0)
 	 HaveContent = false;
       return true;
    }
@@ -187,10 +188,7 @@ bool ServerState::HeaderLine(string Line)
 
       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
-      {
-	 StartPos = 1; // ignore Content-Length, it would override Size
-	 HaveContent = false;
-      }
+	 ; // we got the expected filesize which is all we wanted
       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
@@ -308,9 +306,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
 	 if ((unsigned long long)SBuf.st_size == Server->Size)
 	 {
 	    // the file is completely downloaded, but was not moved
+	    if (Server->HaveContent == true)
+	    {
+	       // Send to error page to dev/null
+	       FileFd DevNull("/dev/null",FileFd::WriteExists);
+	       Server->RunData(&DevNull);
+	    }
+	    Server->HaveContent = false;
 	    Server->StartPos = Server->Size;
 	    Server->Result = 200;
-	    Server->HaveContent = false;
 	 }
 	 else if (unlink(Queue->DestFile.c_str()) == 0)
 	 {
diff --git a/methods/server.h b/methods/server.h
index f5e68d902..1b81e3549 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,7 +34,8 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
-   unsigned long long Size;
+   unsigned long long Size; // size of the usable content (aka: the file)
+   unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
    time_t Date;
    bool HaveContent;
@@ -71,7 +72,7 @@ struct ServerState
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
-   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0;
+   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
 		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true;};
    virtual bool WriteResponse(std::string const &Data) = 0;
diff --git a/test/integration/framework b/test/integration/framework
index df1942ff9..ac482a7a0 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1064,8 +1064,8 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 }
 
 downloadfile() {
-	local PROTO="$(echo "$1" | cut -d':' -f 1 )"
-	apthelper -o Debug::Acquire::${PROTO}=1 \
+	local PROTO="${1%%:*}"
+	apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
 		download-file "$1" "$2" 2>&1 || true
 	# only if the file exists the download was successful
 	if [ -e "$2" ]; then
@@ -1221,7 +1221,7 @@ testsuccess() {
 		msgtest 'Test for successful execution of' "$*"
 	fi
 	local OUTPUT="${TMPWORKINGDIRECTORY}/rootdir/tmp/testsuccess.output"
-	if $@ >${OUTPUT} 2>&1; then
+	if "$@" >${OUTPUT} 2>&1; then
 		msgpass
 	else
 		echo >&2
diff --git a/test/integration/test-apt-helper b/test/integration/test-apt-helper
index c749224ca..31e471677 100755
--- a/test/integration/test-apt-helper
+++ b/test/integration/test-apt-helper
@@ -10,34 +10,36 @@ configarchitecture "i386"
 changetohttpswebserver
 
 test_apt_helper_download() {
-    echo "foo" > aptarchive/foo
+    echo 'foo' > aptarchive/foo
+    echo 'bar' > aptarchive/foo2
 
     msgtest 'apt-file download-file md5sum'
-    apthelper -qq download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo2 MD5Sum:d3b07384d113edec49eaa6238ad5ff00
     testfileequal foo2 'foo'
 
     msgtest 'apt-file download-file sha1'
-    apthelper -qq download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo1 SHA1:f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
     testfileequal foo1 'foo'
 
     msgtest 'apt-file download-file sha256'
-    apthelper -qq download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo3 SHA256:b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c
     testfileequal foo3 'foo'
 
     msgtest 'apt-file download-file no-hash'
-    apthelper -qq download-file http://localhost:8080/foo foo4 && msgpass || msgfail
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo4
     testfileequal foo4 'foo'
     
     msgtest 'apt-file download-file wrong hash'
-    if ! apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc 2>&1 2> download.stderr; then
-        msgpass
-    else
-        msgfail
-    fi
-    testfileequal download.stderr 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
+    testfailure --nomsg apthelper -qq download-file http://localhost:8080/foo foo5 MD5Sum:aabbcc
+    testfileequal rootdir/tmp/testfailure.output 'E: Failed to fetch http://localhost:8080/foo  Hash Sum mismatch
 
 E: Download Failed'
     testfileequal foo5.FAILED 'foo'
+
+    msgtest 'apt-file download-file md5sum sha1'
+    testsuccess --nomsg apthelper download-file http://localhost:8080/foo foo6 MD5Sum:d3b07384d113edec49eaa6238ad5ff00 http://localhost:8080/foo2 foo7 SHA1:e242ed3bffccdf271b7fbaf34ed72d089537b42f
+    testfileequal foo6 'foo'
+    testfileequal foo7 'bar'
 }
 
 test_apt_helper_detect_proxy() {
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 5ab326def..160d451b6 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -24,13 +24,25 @@ testdownloadfile() {
 	else
 		msgpass
 	fi
-	cat "$DOWNLOADLOG" | while read field hash; do
+	sed -e '/^ <- / s#%20# #g' -e '/^ <- / s#%0a#\n#g' "$DOWNLOADLOG" | grep '^.*-Hash: ' > receivedhashes.log
+	testsuccess test -s receivedhashes.log
+	local HASHES_OK=0
+	local HASHES_BAD=0
+	while read field hash; do
 		local EXPECTED
 		case "$field" in
 		'MD5Sum-Hash:') EXPECTED="$(md5sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA1-Hash:') EXPECTED="$(sha1sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA256-Hash:') EXPECTED="$(sha256sum "$TESTFILE" | cut -d' ' -f 1)";;
 		'SHA512-Hash:') EXPECTED="$(sha512sum "$TESTFILE" | cut -d' ' -f 1)";;
+		'Checksum-FileSize-Hash:')
+			#filesize is too weak to check for !=
+			if [ "$4" = '=' ]; then
+				EXPECTED="$(stat -c '%s' "$TESTFILE")"
+			else
+				continue
+			fi
+			;;
 		*) continue;;
 		esac
 		if [ "$4" = '=' ]; then
@@ -40,15 +52,41 @@ testdownloadfile() {
 		fi
 		if [ "$EXPECTED" "$4" "$hash" ]; then
 			msgpass
+			HASHES_OK=$((HASHES_OK+1));
 		else
-			cat >&2 "$DOWNLOADLOG"
 			msgfail "expected: $EXPECTED ; got: $hash"
+			HASHES_BAD=$((HASHES_BAD+1));
 		fi
-	done
+	done < receivedhashes.log
+	msgtest 'At least one good hash and no bad ones'
+	if [ $HASHES_OK -eq 0 ] || [ $HASHES_BAD -ne 0 ]; then
+		cat >&2 "$DOWNLOADLOG"
+		msgfail
+	else
+		msgpass
+	fi
 }
 
 TESTFILE='aptarchive/testfile'
 cp -a ${TESTDIR}/framework $TESTFILE
+cp -a ${TESTDIR}/framework "${TESTFILE}2"
+
+followuprequest() {
+	local DOWN='./testfile'
+
+	copysource $TESTFILE 1M $DOWN
+	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
+	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+
+	copysource $TESTFILE 1M $DOWN
+	copysource "${TESTFILE}2" 20 "${DOWN}2"
+	msgtest 'Testing download of files with' 'completely downloaded file + partial file'
+	testsuccess --nomsg apthelper -o Debug::Acquire::${1%%:*}=1 -o Debug::pkgAcquire::Worker=1 \
+		download-file "$1/testfile" "$DOWN" '' "$1/testfile2" "${DOWN}2"
+	testwebserverlaststatuscode '206' 'rootdir/tmp/testsuccess.output'
+	testsuccess diff -u "$TESTFILE" "${DOWN}"
+	testsuccess diff -u "${DOWN}" "${DOWN}2"
+}
 
 testrun() {
 	webserverconfig 'aptwebserver::support::range' 'true'
@@ -65,9 +103,11 @@ testrun() {
 	testdownloadfile 'invalid partial data' "${1}/testfile" './testfile' '!='
 	testwebserverlaststatuscode '206' "$DOWNLOADLOG"
 
-	copysource $TESTFILE 1M ./testfile
-	testdownloadfile 'completely downloaded file' "${1}/testfile" './testfile' '='
-	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'true'
+	followuprequest "$1"
+	webserverconfig 'aptwebserver::closeOnError' 'false'
 
 	copysource /dev/zero 1M ./testfile
 	testdownloadfile 'too-big partial file' "${1}/testfile" './testfile' '='
@@ -85,8 +125,18 @@ testrun() {
 	testwebserverlaststatuscode '200' "$DOWNLOADLOG"
 }
 
+msgmsg 'http: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'http://localhost:8080'
+msgmsg 'http: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'http://localhost:8080'
 
 changetohttpswebserver
 
+msgmsg 'https: Test with Content-Length'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'false'
+testrun 'https://localhost:4433'
+msgmsg 'https: Test with Transfer-Encoding: chunked'
+webserverconfig 'aptwebserver::chunked-transfer-encoding' 'true'
 testrun 'https://localhost:4433'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 34476e1af..cd52da692 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -19,6 +19,8 @@
 #include 
 #include 
 #include 
+
+#include 
 #include 
 #include 
 #include 
@@ -79,12 +81,21 @@ static char const * httpcodeToStr(int const httpcode)		/*{{{*/
    return NULL;
 }
 									/*}}}*/
+static bool chunkedTransferEncoding(std::list const &headers) {
+   if (std::find(headers.begin(), headers.end(), "Transfer-Encoding: chunked") != headers.end())
+      return true;
+   if (_config->FindB("aptwebserver::chunked-transfer-encoding", false) == true)
+      return true;
+   return false;
+}
 static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.FileSize();
-   headers.push_back(contentlength.str());
-
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.FileSize();
+      headers.push_back(contentlength.str());
+   }
    std::string lastmodified("Last-Modified: ");
    lastmodified.append(TimeRFC1123(data.ModificationTime()));
    headers.push_back(lastmodified);
@@ -92,9 +103,12 @@ static void addFileHeaders(std::list &headers, FileFd &data)/*{{{*/
 									/*}}}*/
 static void addDataHeaders(std::list &headers, std::string &data)/*{{{*/
 {
-   std::ostringstream contentlength;
-   contentlength << "Content-Length: " << data.size();
-   headers.push_back(contentlength.str());
+   if (chunkedTransferEncoding(headers) == false)
+   {
+      std::ostringstream contentlength;
+      contentlength << "Content-Length: " << data.size();
+      headers.push_back(contentlength.str());
+   }
 }
 									/*}}}*/
 static bool sendHead(int const client, int const httpcode, std::list &headers)/*{{{*/
@@ -114,6 +128,9 @@ static bool sendHead(int const client, int const httpcode, std::list>> RESPONSE to " << client << " >>>" << std::endl;
    bool Success = true;
    for (std::list::const_iterator h = headers.begin();
@@ -130,25 +147,55 @@ static bool sendHead(int const client, int const httpcode, std::list const &headers, FileFd &data)/*{{{*/
 {
    bool Success = true;
+   bool const chunked = chunkedTransferEncoding(headers);
    char buffer[500];
    unsigned long long actual = 0;
    while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true)
    {
       if (actual == 0)
 	 break;
-      Success &= FileFd::Write(client, buffer, actual);
+
+      if (chunked == true)
+      {
+	 std::string size;
+	 strprintf(size, "%llX\r\n", actual);
+	 Success &= FileFd::Write(client, size.c_str(), size.size());
+	 Success &= FileFd::Write(client, buffer, actual);
+	 Success &= FileFd::Write(client, "\r\n", strlen("\r\n"));
+      }
+      else
+	 Success &= FileFd::Write(client, buffer, actual);
+   }
+   if (chunked == true)
+   {
+      char const * const finish = "0\r\n\r\n";
+      Success &= FileFd::Write(client, finish, strlen(finish));
    }
    if (Success == false)
-      std::cerr << "SENDFILE: READ/WRITE ERROR to " << client << std::endl;
+      std::cerr << "SENDFILE:" << (chunked ? " CHUNKED" : "") << " READ/WRITE ERROR to " << client << std::endl;
    return Success;
 }
 									/*}}}*/
-static bool sendData(int const client, std::string const &data)		/*{{{*/
+static bool sendData(int const client, std::list const &headers, std::string const &data)/*{{{*/
 {
-   if (FileFd::Write(client, data.c_str(), data.size()) == false)
+   if (chunkedTransferEncoding(headers) == true)
+   {
+      unsigned long long const ullsize = data.length();
+      std::string size;
+      strprintf(size, "%llX\r\n", ullsize);
+      char const * const finish = "\r\n0\r\n\r\n";
+      if (FileFd::Write(client, size.c_str(), size.length()) == false ||
+	    FileFd::Write(client, data.c_str(), ullsize) == false ||
+	    FileFd::Write(client, finish, strlen(finish)) == false)
+      {
+	 std::cerr << "SENDDATA: CHUNK WRITE ERROR to " << client << std::endl;
+	 return false;
+      }
+   }
+   else if (FileFd::Write(client, data.c_str(), data.size()) == false)
    {
       std::cerr << "SENDDATA: WRITE ERROR to " << client << std::endl;
       return false;
@@ -157,34 +204,38 @@ static bool sendData(int const client, std::string const &data)		/*{{{*/
 }
 									/*}}}*/
 static void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   std::list headers;
    std::string response("");
    response.append(httpcodeToStr(httpcode)).append("");
    response.append("
").append(httpcodeToStr(httpcode)).append("
");
    if (httpcode != 200)
-   {
-      if (error.empty() == false)
-	 response.append("
Error: ").append(error).append("
");
-      response.append("This error is a result of the request: 
");
-   }
+      response.append("
Error: ");
+   else
+      response.append("
Success: ");
+   if (error.empty() == false)
+      response.append(error);
+   else
+      response.append(httpcodeToStr(httpcode));
+   if (httpcode != 200)
+      response.append("
This error is a result of the request: 
");
    else
-   {
-      if (error.empty() == false)
-	 response.append("
Success: ").append(error).append("
");
       response.append("The successfully executed operation was requested by: 
");
-   }
    response.append(request).append("
");
+   if (httpcode != 200)
+   {
+      if (_config->FindB("aptwebserver::closeOnError", false) == true)
+	 headers.push_back("Connection: close");
+   }
    addDataHeaders(headers, response);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 static void sendSuccess(int const client, std::string const &request,
-	       bool content, std::string const &error = "")
+	       bool const content, std::string const &error, std::list &headers)
 {
-   sendError(client, 200, request, content, error);
+   sendError(client, 200, request, content, error, headers);
 }
 									/*}}}*/
 static void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
@@ -221,7 +272,7 @@ static void sendRedirect(int const client, int const httpcode, std::string const
    headers.push_back(location);
    sendHead(client, httpcode, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static int filter_hidden_files(const struct dirent *a)			/*{{{*/
@@ -263,16 +314,15 @@ static int grouped_alpha_case_sort(const struct dirent **a, const struct dirent
 }
 									/*}}}*/
 static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*/
-			  std::string const &request, bool content)
+			  std::string const &request, bool content, std::list &headers)
 {
-   std::list headers;
    std::ostringstream listing;
 
    struct dirent **namelist;
    int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
    if (counter == -1)
    {
-      sendError(client, 500, request, content);
+      sendError(client, 500, request, content, "scandir failed", headers);
       return;
    }
 
@@ -311,18 +361,18 @@ static void sendDirectoryListing(int const client, std::string const &dir,/*{{{*
    addDataHeaders(headers, response);
    sendHead(client, 200, headers);
    if (content == true)
-      sendData(client, response);
+      sendData(client, headers, response);
 }
 									/*}}}*/
 static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
 		    std::string &filename, std::string ¶ms, bool &sendContent,
-		    bool &closeConnection)
+		    bool &closeConnection, std::list &headers)
 {
    if (strncmp(request.c_str(), "HEAD ", 5) == 0)
       sendContent = false;
    if (strncmp(request.c_str(), "GET ", 4) != 0)
    {
-      sendError(client, 501, request, true);
+      sendError(client, 501, request, true, "", headers);
       return false;
    }
 
@@ -333,7 +383,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (lineend == std::string::npos || filestart == std::string::npos ||
 	 fileend == std::string::npos || filestart == fileend)
    {
-      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      sendError(client, 500, request, sendContent, "Filename can't be extracted", headers);
       return false;
    }
 
@@ -345,14 +395,14 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
       closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
    else
    {
-      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request", headers);
       return false;
    }
 
    filename = request.substr(filestart, fileend - filestart);
    if (filename.find(' ') != std::string::npos)
    {
-      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space", headers);
       return false;
    }
 
@@ -360,7 +410,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    if (host.empty() == true)
    {
       // RFC 2616 §14.23 requires Host
-      sendError(client, 400, request, sendContent, "Host header is required");
+      sendError(client, 400, request, sendContent, "Host header is required", headers);
       return false;
    }
    host = "http://" + host;
@@ -371,7 +421,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    {
       if (absolute.find("uri") == std::string::npos)
       {
-	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that", headers);
 	 return false;
       }
       // strip the host from the request to make it an absolute path
@@ -379,7 +429,7 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    }
    else if (absolute.find("path") == std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that", headers);
       return false;
    }
 
@@ -398,7 +448,8 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
        filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
        filename.find("/../") != std::string::npos)
    {
-      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      std::list headers;
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)", headers);
       return false;
    }
 
@@ -434,46 +485,45 @@ static bool parseFirstLine(int const client, std::string const &request,/*{{{*/
    return true;
 }
 									/*}}}*/
-static bool handleOnTheFlyReconfiguration(int const client, std::string const &request, std::vector const &parts)/*{{{*/
+static bool handleOnTheFlyReconfiguration(int const client, std::string const &request,/*{{{*/
+      std::vector parts, std::list &headers)
 {
    size_t const pcount = parts.size();
    if (pcount == 4 && parts[1] == "set")
    {
       _config->Set(parts[2], parts[3]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was set to '" + parts[3] + "'!", headers);
       return true;
    }
    else if (pcount == 4 && parts[1] == "find")
    {
-      std::list headers;
       std::string response = _config->Find(parts[2], parts[3]);
       addDataHeaders(headers, response);
       sendHead(client, 200, headers);
-      sendData(client, response);
+      sendData(client, headers, response);
       return true;
    }
    else if (pcount == 3 && parts[1] == "find")
    {
-      std::list headers;
       if (_config->Exists(parts[2]) == true)
       {
 	 std::string response = _config->Find(parts[2]);
 	 addDataHeaders(headers, response);
 	 sendHead(client, 200, headers);
-	 sendData(client, response);
+	 sendData(client, headers, response);
 	 return true;
       }
-      sendError(client, 404, request, "Requested Configuration option doesn't exist.");
+      sendError(client, 404, request, true, "Requested Configuration option doesn't exist", headers);
       return false;
    }
    else if (pcount == 3 && parts[1] == "clear")
    {
       _config->Clear(parts[2]);
-      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.");
+      sendSuccess(client, request, true, "Option '" + parts[2] + "' was cleared.", headers);
       return true;
    }
 
-   sendError(client, 400, request, true, "Unknown on-the-fly configuration request");
+   sendError(client, 400, request, true, "Unknown on-the-fly configuration request", headers);
    return false;
 }
 									/*}}}*/
@@ -482,18 +532,22 @@ static void * handleClient(void * voidclient)				/*{{{*/
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
    std::vector messages;
-   while (ReadMessages(client, messages))
+   bool closeConnection = false;
+   std::list headers;
+   while (closeConnection == false && ReadMessages(client, messages))
    {
-      bool closeConnection = false;
+      // if we announced a closing, do the close
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 break;
+      headers.clear();
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
-	 std::list headers;
 	 std::string filename;
 	 std::string params;
 	 bool sendContent = true;
-	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection) == false)
+	 if (parseFirstLine(client, *m, filename, params, sendContent, closeConnection, headers) == false)
 	    continue;
 
 	 // special webserver command request
@@ -502,7 +556,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    std::vector parts = VectorizeString(filename, '/');
 	    if (parts[0] == "_config")
 	    {
-	       handleOnTheFlyReconfiguration(client, *m, parts);
+	       handleOnTheFlyReconfiguration(client, *m, parts, headers);
 	       continue;
 	    }
 	 }
@@ -534,7 +588,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	       {
 		  char error[300];
 		  regerror(res, pattern, error, sizeof(error));
-		  sendError(client, 500, *m, sendContent, error);
+		  sendError(client, 500, *m, sendContent, error, headers);
 		  continue;
 	       }
 	       if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
@@ -553,7 +607,7 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 if (_config->FindB("aptwebserver::support::http", true) == false &&
 	       LookupTag(*m, "Host").find(":4433") == std::string::npos)
 	 {
-	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS");
+	    sendError(client, 400, *m, sendContent, "HTTP disabled, all requests must be HTTPS", headers);
 	    continue;
 	 }
 	 else if (RealFileExists(filename) == true)
@@ -609,17 +663,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 			headers.push_back(contentrange.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
-			   sendFile(client, data);
+			   sendFile(client, headers, data);
 			continue;
 		     }
 		     else
 		     {
-			headers.push_back("Content-Length: 0");
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes */" << filesize;
 			headers.push_back(contentrange.str());
-			sendHead(client, 416, headers);
-			continue;
+			sendError(client, 416, *m, sendContent, "", headers);
+			break;
 		     }
 		  }
 	       }
@@ -628,22 +681,20 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	    addFileHeaders(headers, data);
 	    sendHead(client, 200, headers);
 	    if (sendContent == true)
-	       sendFile(client, data);
+	       sendFile(client, headers, data);
 	 }
 	 else if (DirectoryExists(filename) == true)
 	 {
 	    if (filename[filename.length()-1] == '/')
-	       sendDirectoryListing(client, filename, *m, sendContent);
+	       sendDirectoryListing(client, filename, *m, sendContent, headers);
 	    else
 	       sendRedirect(client, 301, filename.append("/"), *m, sendContent);
 	 }
 	 else
-	    sendError(client, 404, *m, sendContent);
+	    sendError(client, 404, *m, sendContent, "", headers);
       }
       _error->DumpErrors(std::cerr);
       messages.clear();
-      if (closeConnection == true)
-	 break;
    }
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
-- 
cgit v1.2.3-70-g09d2


From 9127d7aecf01f2999a2589e4b0503288518b2927 Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Mon, 5 Jan 2015 10:27:53 +0100
Subject: Fix missing URIStart() for https downloads

Add a explicit ReceivedData to HttpsMethod that indicates when
we got data from the connection so that we can send URISTart()
to the parent.

This is needed because URIStart got moved in f9b4f12d from
the progress_callback to write_data() and it only checks for
Res.Size. In the old code if progress_callback is called by
libcurl (and sets Res.Size) before write_data is called then
URIStart() is never send. Making this a explicit ReceivedData
variable fixes this issue.
---
 methods/https.cc | 9 +++++++--
 methods/https.h  | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index 65a744e2a..3a5981b58 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -85,8 +85,12 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
    if (me->Server->JunkSize != 0)
       return buffer_size;
 
-   if (me->Res.Size == 0)
+   if (me->ReceivedData == false)
+   {
       me->URIStart(me->Res);
+      me->ReceivedData = true;
+   }
+
    if(me->File->Write(buffer, buffer_size) != true)
       return false;
 
@@ -95,7 +99,7 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 
 int
 HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/,
-			      double /*ultotal*/, double /*ulnow*/)
+                             double /*ultotal*/, double /*ulnow*/)
 {
    HttpsMethod *me = (HttpsMethod *)clientp;
    if(dltotal > 0 && me->Res.Size == 0) {
@@ -179,6 +183,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    char curl_errorstr[CURL_ERROR_SIZE];
    URI Uri = Itm->Uri;
    string remotehost = Uri.Host;
+   ReceivedData = false;
 
    // TODO:
    //       - http::Pipeline-Depth
diff --git a/methods/https.h b/methods/https.h
index faac8a3cd..411b71440 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -66,6 +66,7 @@ class HttpsMethod : public pkgAcqMethod
    CURL *curl;
    FetchResult Res;
    HttpsServerState *Server;
+   bool ReceivedData;
 
    public:
    FileFd *File;
-- 
cgit v1.2.3-70-g09d2


From 0c2dc43d4fe1d026650b5e2920a021557f9534a6 Mon Sep 17 00:00:00 2001
From: Tomasz Buchert 
Date: Mon, 16 Feb 2015 00:57:29 +0100
Subject: Fix crash in the apt-transport-https when Owner is NULL

Do not crash in ServerState::HeaderLine if there is no Owner.

Closes: #778375
---
 methods/server.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'methods')

diff --git a/methods/server.cc b/methods/server.cc
index cb0341d5f..e321e0230 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -129,7 +129,7 @@ bool ServerState::HeaderLine(string Line)
 	 if (elements == 3)
 	 {
 	    Code[0] = '\0';
-	    if (Owner->Debug == true)
+	    if (Owner != NULL && Owner->Debug == true)
 	       clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
 	 }
 	 else if (elements != 4)
-- 
cgit v1.2.3-70-g09d2


From 905fba60a046646a26a56b4c5d4a5dc7d5906f0d Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 9 Mar 2015 01:54:46 +0100
Subject: derive more of https from http method

Bug #778375 uncovered that https wasn't properly integrated in the class
family tree of http as it was supposed to be leading to a NULL pointer
dereference. Fixing this 'properly' was deemed to much diff for
practically no gain that late in the release, so commit
0c2dc43d4fe1d026650b5e2920a021557f9534a6 just fixed the synptom, while
this commit here is fixing the cause plus adding a test.
---
 methods/http.cc                                    |  2 -
 methods/https.cc                                   | 37 +++++----
 methods/https.h                                    | 16 ++--
 methods/server.cc                                  |  7 +-
 methods/server.h                                   |  3 +-
 .../test-bug-778375-server-has-no-reason-phrase    | 40 ++++++++++
 test/interactive-helper/aptwebserver.cc            | 88 +++++++++++-----------
 7 files changed, 124 insertions(+), 69 deletions(-)
 create mode 100755 test/integration/test-bug-778375-server-has-no-reason-phrase

(limited to 'methods')

diff --git a/methods/http.cc b/methods/http.cc
index ad1347d36..021b284d0 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -772,8 +772,6 @@ bool HttpMethod::Configuration(string Message)
    if (ServerMethod::Configuration(Message) == false)
       return false;
 
-   DropPrivsOrDie();
-
    AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 				  PipelineDepth);
diff --git a/methods/https.cc b/methods/https.cc
index 37a8ff5fd..32de42e4b 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -37,16 +37,6 @@
 									/*}}}*/
 using namespace std;
 
-bool HttpsMethod::Configuration(std::string Message)
-{
-   if (pkgAcqMethod::Configuration(Message) == false)
-      return false;
-
-   DropPrivsOrDie();
-
-   return true;
-}
-
 size_t
 HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
 {
@@ -131,7 +121,7 @@ HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/,
 }
 
 // HttpsServerState::HttpsServerState - Constructor			/*{{{*/
-HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * /*Owner*/) : ServerState(Srv, NULL)
+HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner)
 {
    TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
    ReceivedData = false;
@@ -335,13 +325,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout);
 
    // set redirect options and default to 10 redirects
-   bool const AllowRedirect = _config->FindB("Acquire::https::AllowRedirect",
-	_config->FindB("Acquire::http::AllowRedirect",true));
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect);
    curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10);
 
    // debug
-   if(_config->FindB("Debug::Acquire::https", false))
+   if (Debug == true)
       curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
 
    // error handling
@@ -378,7 +366,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
 
    // go for it - if the file exists, append on it
    File = new FileFd(Itm->DestFile, FileFd::WriteAny);
-   Server = new HttpsServerState(Itm->Uri, this);
+   Server = CreateServerState(Itm->Uri);
 
    // keep apt updated
    Res.Filename = Itm->DestFile;
@@ -473,6 +461,25 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
 
    return true;
 }
+									/*}}}*/
+// HttpsMethod::Configuration - Handle a configuration message		/*{{{*/
+bool HttpsMethod::Configuration(string Message)
+{
+   if (ServerMethod::Configuration(Message) == false)
+      return false;
+
+   AllowRedirect = _config->FindB("Acquire::https::AllowRedirect",
+	_config->FindB("Acquire::http::AllowRedirect", true));
+   Debug = _config->FindB("Debug::Acquire::https",false);
+
+   return true;
+}
+									/*}}}*/
+ServerState * HttpsMethod::CreateServerState(URI uri)			/*{{{*/
+{
+   return new HttpsServerState(uri, this);
+}
+									/*}}}*/
 
 int main()
 {
diff --git a/methods/https.h b/methods/https.h
index 6917a6ff6..433a84680 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -50,17 +50,14 @@ class HttpsServerState : public ServerState
 
    HttpsServerState(URI Srv, HttpsMethod *Owner);
    virtual ~HttpsServerState() {Close();};
-
-   bool ReceivedData;
 };
 
-class HttpsMethod : public pkgAcqMethod
+class HttpsMethod : public ServerMethod
 {
    // minimum speed in bytes/se that triggers download timeout handling
    static const int DL_MIN_SPEED = 10;
 
    virtual bool Fetch(FetchItem *);
-   virtual bool Configuration(std::string Message);
 
    static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp);
    static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp);
@@ -69,12 +66,19 @@ class HttpsMethod : public pkgAcqMethod
    void SetupProxy();
    CURL *curl;
    FetchResult Res;
-   HttpsServerState *Server;
+   ServerState *Server;
+
+   // Used by ServerMethods unused by https
+   virtual void SendReq(FetchItem *) { exit(42); }
+   virtual void RotateDNS() { exit(42); }
 
    public:
    FileFd *File;
 
-   HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), Server(NULL), File(NULL)
+   virtual bool Configuration(std::string Message);
+   virtual ServerState * CreateServerState(URI uri);
+
+   HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL)
    {
       curl = curl_easy_init();
    };
diff --git a/methods/server.cc b/methods/server.cc
index c17f27f73..91ec824d1 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -238,7 +238,12 @@ ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOu
 
 bool ServerMethod::Configuration(string Message)			/*{{{*/
 {
-   return pkgAcqMethod::Configuration(Message);
+   if (pkgAcqMethod::Configuration(Message) == false)
+      return false;
+
+   DropPrivsOrDie();
+
+   return true;
 }
 									/*}}}*/
 
diff --git a/methods/server.h b/methods/server.h
index b974ec89a..3b232dcac 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -37,6 +37,7 @@ struct ServerState
    unsigned long long Size; // size of the usable content (aka: the file)
    unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
+   bool ReceivedData;
    time_t Date;
    bool HaveContent;
    enum {Chunked,Stream,Closes} Encoding;
@@ -75,7 +76,7 @@ struct ServerState
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
    virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
-		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
+		 StartPos = 0; ReceivedData = false; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;};
    virtual bool WriteResponse(std::string const &Data) = 0;
 
diff --git a/test/integration/test-bug-778375-server-has-no-reason-phrase b/test/integration/test-bug-778375-server-has-no-reason-phrase
new file mode 100755
index 000000000..23481ef88
--- /dev/null
+++ b/test/integration/test-bug-778375-server-has-no-reason-phrase
@@ -0,0 +1,40 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'native'
+
+echo 'found' > aptarchive/working
+changetohttpswebserver  -o 'aptwebserver::redirect::replace::/redirectme/=/' \
+	-o 'aptwebserver::httpcode::200=200' -o 'aptwebserver::httpcode::404=404' \
+	-o 'aptwebserver::httpcode::301=301'
+
+testdownload() {
+	rm -f downfile
+	msgtest "download of a $1 via" "${3%%:*}"
+	$2 --nomsg downloadfile "$3" downfile
+
+	cp rootdir/tmp/testsuccess.output download.log
+	#looking for "HTTP server doesn't give Reason-Phrase for 200"
+	testsuccess grep 'give Reason-Phrase for' download.log
+
+	if [ "$2" = 'testsuccess' ]; then
+		testfileequal downfile 'found'
+	else
+		testfailure test -e downfile
+	fi
+}
+
+runtest() {
+	testdownload 'file works' 'testsuccess' "$1/working"
+	testdownload 'file via redirect works' 'testsuccess' "$1/redirectme/working"
+
+	testdownload 'non-existent file fails' 'testfailure' "$1/failing"
+	testdownload 'non-existent file via redirect fails' 'testfailure' "$1/redirectme/failing"
+}
+
+runtest 'http://localhost:8080'
+runtest 'https://localhost:4433'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 3403bbdd2..644629a33 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -27,58 +27,58 @@
 #include 
 #include 
 
-static char const * httpcodeToStr(int const httpcode)		/*{{{*/
+static std::string httpcodeToStr(int const httpcode)			/*{{{*/
 {
    switch (httpcode)
    {
       // Informational 1xx
-      case 100: return "100 Continue";
-      case 101: return "101 Switching Protocols";
+      case 100: return _config->Find("aptwebserver::httpcode::100", "100 Continue");
+      case 101: return _config->Find("aptwebserver::httpcode::101", "101 Switching Protocols");
       // Successful 2xx
-      case 200: return "200 OK";
-      case 201: return "201 Created";
-      case 202: return "202 Accepted";
-      case 203: return "203 Non-Authoritative Information";
-      case 204: return "204 No Content";
-      case 205: return "205 Reset Content";
-      case 206: return "206 Partial Content";
+      case 200: return _config->Find("aptwebserver::httpcode::200", "200 OK");
+      case 201: return _config->Find("aptwebserver::httpcode::201", "201 Created");
+      case 202: return _config->Find("aptwebserver::httpcode::202", "202 Accepted");
+      case 203: return _config->Find("aptwebserver::httpcode::203", "203 Non-Authoritative Information");
+      case 204: return _config->Find("aptwebserver::httpcode::204", "204 No Content");
+      case 205: return _config->Find("aptwebserver::httpcode::205", "205 Reset Content");
+      case 206: return _config->Find("aptwebserver::httpcode::206", "206 Partial Content");
       // Redirections 3xx
-      case 300: return "300 Multiple Choices";
-      case 301: return "301 Moved Permanently";
-      case 302: return "302 Found";
-      case 303: return "303 See Other";
-      case 304: return "304 Not Modified";
-      case 305: return "304 Use Proxy";
-      case 307: return "307 Temporary Redirect";
+      case 300: return _config->Find("aptwebserver::httpcode::300", "300 Multiple Choices");
+      case 301: return _config->Find("aptwebserver::httpcode::301", "301 Moved Permanently");
+      case 302: return _config->Find("aptwebserver::httpcode::302", "302 Found");
+      case 303: return _config->Find("aptwebserver::httpcode::303", "303 See Other");
+      case 304: return _config->Find("aptwebserver::httpcode::304", "304 Not Modified");
+      case 305: return _config->Find("aptwebserver::httpcode::305", "305 Use Proxy");
+      case 307: return _config->Find("aptwebserver::httpcode::307", "307 Temporary Redirect");
       // Client errors 4xx
-      case 400: return "400 Bad Request";
-      case 401: return "401 Unauthorized";
-      case 402: return "402 Payment Required";
-      case 403: return "403 Forbidden";
-      case 404: return "404 Not Found";
-      case 405: return "405 Method Not Allowed";
-      case 406: return "406 Not Acceptable";
-      case 407: return "407 Proxy Authentication Required";
-      case 408: return "408 Request Time-out";
-      case 409: return "409 Conflict";
-      case 410: return "410 Gone";
-      case 411: return "411 Length Required";
-      case 412: return "412 Precondition Failed";
-      case 413: return "413 Request Entity Too Large";
-      case 414: return "414 Request-URI Too Large";
-      case 415: return "415 Unsupported Media Type";
-      case 416: return "416 Requested range not satisfiable";
-      case 417: return "417 Expectation Failed";
-      case 418: return "418 I'm a teapot";
+      case 400: return _config->Find("aptwebserver::httpcode::400", "400 Bad Request");
+      case 401: return _config->Find("aptwebserver::httpcode::401", "401 Unauthorized");
+      case 402: return _config->Find("aptwebserver::httpcode::402", "402 Payment Required");
+      case 403: return _config->Find("aptwebserver::httpcode::403", "403 Forbidden");
+      case 404: return _config->Find("aptwebserver::httpcode::404", "404 Not Found");
+      case 405: return _config->Find("aptwebserver::httpcode::405", "405 Method Not Allowed");
+      case 406: return _config->Find("aptwebserver::httpcode::406", "406 Not Acceptable");
+      case 407: return _config->Find("aptwebserver::httpcode::407", "407 Proxy Authentication Required");
+      case 408: return _config->Find("aptwebserver::httpcode::408", "408 Request Time-out");
+      case 409: return _config->Find("aptwebserver::httpcode::409", "409 Conflict");
+      case 410: return _config->Find("aptwebserver::httpcode::410", "410 Gone");
+      case 411: return _config->Find("aptwebserver::httpcode::411", "411 Length Required");
+      case 412: return _config->Find("aptwebserver::httpcode::412", "412 Precondition Failed");
+      case 413: return _config->Find("aptwebserver::httpcode::413", "413 Request Entity Too Large");
+      case 414: return _config->Find("aptwebserver::httpcode::414", "414 Request-URI Too Large");
+      case 415: return _config->Find("aptwebserver::httpcode::415", "415 Unsupported Media Type");
+      case 416: return _config->Find("aptwebserver::httpcode::416", "416 Requested range not satisfiable");
+      case 417: return _config->Find("aptwebserver::httpcode::417", "417 Expectation Failed");
+      case 418: return _config->Find("aptwebserver::httpcode::418", "418 I'm a teapot");
       // Server error 5xx
-      case 500: return "500 Internal Server Error";
-      case 501: return "501 Not Implemented";
-      case 502: return "502 Bad Gateway";
-      case 503: return "503 Service Unavailable";
-      case 504: return "504 Gateway Time-out";
-      case 505: return "505 HTTP Version not supported";
-   }
-   return NULL;
+      case 500: return _config->Find("aptwebserver::httpcode::500", "500 Internal Server Error");
+      case 501: return _config->Find("aptwebserver::httpcode::501", "501 Not Implemented");
+      case 502: return _config->Find("aptwebserver::httpcode::502", "502 Bad Gateway");
+      case 503: return _config->Find("aptwebserver::httpcode::503", "503 Service Unavailable");
+      case 504: return _config->Find("aptwebserver::httpcode::504", "504 Gateway Time-out");
+      case 505: return _config->Find("aptwebserver::httpcode::505", "505 HTTP Version not supported");
+   }
+   return "";
 }
 									/*}}}*/
 static bool chunkedTransferEncoding(std::list const &headers) {
-- 
cgit v1.2.3-70-g09d2


From 1296bc7c466181a7978c313c40a041b34ce3eaeb Mon Sep 17 00:00:00 2001
From: Robert Edmonds 
Date: Sun, 22 Mar 2015 00:12:45 -0400
Subject: HttpsMethod::Fetch(): Zero the FetchResult object when leaving due to
 404

---
 methods/https.cc | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index 3a5981b58..f2b00dd64 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -402,6 +402,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       _error->Error("%s", err);
       // unlink, no need keep 401/404 page content in partial/
       unlink(File->Name().c_str());
+      Res.Size = 0;
+      Res.LastModified = 0;
       return false;
    }
 
-- 
cgit v1.2.3-70-g09d2


From a2679f55b5b14092db88fab3799f06e6b68e439e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 7 Apr 2015 14:34:04 +0200
Subject: properly handle expected filesize in https

The worker expects that the methods tell him when they start or finish
downloading a file. Various information pieces are passed along in this report
including the (expected) filesize. https is using a "global" struct for
reporting which made it 'reuse' incorrect values in some cases like a
non-existent InRelease fallbacking to Release{,.gpg} resulting in an incorrect
size-mismatch warning scaring and desensitizing users as well as being subject
to a race between the write_data and progress callbacks generating incorrect
progress reporting and potentially the same error message.

Other branches as well as the bugreports contain 'better' fixes making the
struct local and other sensible changes, but are larger as a result, so in
this version we opted for short diff with minimal effect above else instead.

Closes: 777565, 781509
Thanks: Robert Edmonds and Anders Kaseorg for initial patchs
---
 methods/https.cc | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index 3a5981b58..cb11159bc 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -68,6 +68,8 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
 
       me->File->Truncate(me->Server->StartPos);
       me->File->Seek(me->Server->StartPos);
+
+      me->Res.Size = me->Server->Size;
    }
    else if (me->Server->HeaderLine(line) == false)
       return 0;
@@ -97,17 +99,6 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
    return buffer_size;
 }
 
-int
-HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/,
-                             double /*ultotal*/, double /*ulnow*/)
-{
-   HttpsMethod *me = (HttpsMethod *)clientp;
-   if(dltotal > 0 && me->Res.Size == 0) {
-      me->Res.Size = (unsigned long long)dltotal;
-   }
-   return 0;
-}
-
 // HttpsServerState::HttpsServerState - Constructor			/*{{{*/
 HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * /*Owner*/) : ServerState(Srv, NULL)
 {
@@ -201,10 +192,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
-   curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
-   curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
    // options
-   curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+   curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
    curl_easy_setopt(curl, CURLOPT_FILETIME, true);
    // only allow curl to handle https, not the other stuff it supports
    curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
@@ -357,6 +346,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    // go for it - if the file exists, append on it
    File = new FileFd(Itm->DestFile, FileFd::WriteAny);
    Server = new HttpsServerState(Itm->Uri, this);
+   Res = FetchResult();
 
    // keep apt updated
    Res.Filename = Itm->DestFile;
-- 
cgit v1.2.3-70-g09d2


From b8eba208daebe3e3f235983e44da9c398d6f7a57 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 10 Mar 2015 14:11:54 +0100
Subject: reimplement the last uses of sprintf

Working with strings c-style is complicated and error-prune,
so by converting to c++ style we gain some simplicity and
avoid buffer overflows by later extensions.

Git-Dch: Ignore
---
 apt-pkg/contrib/cdromutl.cc |  31 ++++----
 apt-pkg/contrib/strutl.cc   | 126 ++++++++++++++------------------
 apt-private/acqprogress.cc  | 173 ++++++++++++++++++++++----------------------
 apt-private/acqprogress.h   |   4 +-
 ftparchive/writer.cc        |   8 +-
 methods/ftp.cc              |  16 ++--
 test/libapt/strutil_test.cc |  22 ++++++
 test/libapt/uri_test.cc     |  12 +++
 8 files changed, 206 insertions(+), 186 deletions(-)

(limited to 'methods')

diff --git a/apt-pkg/contrib/cdromutl.cc b/apt-pkg/contrib/cdromutl.cc
index 936e377fb..6eb917457 100644
--- a/apt-pkg/contrib/cdromutl.cc
+++ b/apt-pkg/contrib/cdromutl.cc
@@ -207,7 +207,6 @@ bool IdentCdrom(string CD,string &Res,unsigned int Version)
    /* Run over the directory, we assume that the reader order will never
       change as the media is read-only. In theory if the kernel did
       some sort of wacked caching this might not be true.. */
-   char S[300];
    for (struct dirent *Dir = readdir(D); Dir != 0; Dir = readdir(D))
    {
       // Skip some files..
@@ -215,30 +214,32 @@ bool IdentCdrom(string CD,string &Res,unsigned int Version)
 	  strcmp(Dir->d_name,"..") == 0)
 	 continue;
 
+      std::string S;
       if (Version <= 1)
       {
-	 sprintf(S,"%lu",(unsigned long)Dir->d_ino);
+	 strprintf(S, "%lu", (unsigned long)Dir->d_ino);
       }
       else
       {
 	 struct stat Buf;
 	 if (stat(Dir->d_name,&Buf) != 0)
 	    continue;
-	 sprintf(S,"%lu",(unsigned long)Buf.st_mtime);
+	 strprintf(S, "%lu", (unsigned long)Buf.st_mtime);
       }
-      
-      Hash.Add(S);
+
+      Hash.Add(S.c_str());
       Hash.Add(Dir->d_name);
    };
-   
+
    if (chdir(StartDir.c_str()) != 0) {
       _error->Errno("chdir",_("Unable to change to %s"),StartDir.c_str());
       closedir(D);
       return false;
    }
    closedir(D);
-   
+
    // Some stats from the fsys
+   std::string S;
    if (_config->FindB("Debug::identcdrom",false) == false)
    {
       struct statvfs Buf;
@@ -248,19 +249,19 @@ bool IdentCdrom(string CD,string &Res,unsigned int Version)
       // We use a kilobyte block size to advoid overflow
       if (writable_media)
       {
-         sprintf(S,"%lu",(long)(Buf.f_blocks*(Buf.f_bsize/1024)));
+         strprintf(S, "%lu", (unsigned long)(Buf.f_blocks*(Buf.f_bsize/1024)));
       } else {
-         sprintf(S,"%lu %lu",(long)(Buf.f_blocks*(Buf.f_bsize/1024)),
-                 (long)(Buf.f_bfree*(Buf.f_bsize/1024)));
+         strprintf(S, "%lu %lu", (unsigned long)(Buf.f_blocks*(Buf.f_bsize/1024)),
+                 (unsigned long)(Buf.f_bfree*(Buf.f_bsize/1024)));
       }
-      Hash.Add(S);
-      sprintf(S,"-%u",Version);
+      Hash.Add(S.c_str());
+      strprintf(S, "-%u", Version);
    }
    else
-      sprintf(S,"-%u.debug",Version);
-   
+      strprintf(S, "-%u.debug", Version);
+
    Res = Hash.Result().Value() + S;
-   return true;   
+   return true;
 }
 									/*}}}*/
 // FindMountPointForDevice - Find mountpoint for the given device	/*{{{*/
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index 0ac587a9e..0db4c57b2 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -324,21 +324,19 @@ bool ParseCWord(const char *&String,string &Res)
 /* */
 string QuoteString(const string &Str, const char *Bad)
 {
-   string Res;
+   std::stringstream Res;
    for (string::const_iterator I = Str.begin(); I != Str.end(); ++I)
    {
-      if (strchr(Bad,*I) != 0 || isprint(*I) == 0 || 
+      if (strchr(Bad,*I) != 0 || isprint(*I) == 0 ||
 	  *I == 0x25 || // percent '%' char
 	  *I <= 0x20 || *I >= 0x7F) // control chars
       {
-	 char Buf[10];
-	 sprintf(Buf,"%%%02x",(int)*I);
-	 Res += Buf;
+	 ioprintf(Res,"%%%02x",(int)*I);
       }
       else
-	 Res += *I;
+	 Res << *I;
    }
-   return Res;
+   return Res.str();
 }
 									/*}}}*/
 // DeQuoteString - Convert a string from quoted from                    /*{{{*/
@@ -379,13 +377,12 @@ string DeQuoteString(string::const_iterator const &begin,
    YottaBytes (E24) */
 string SizeToStr(double Size)
 {
-   char S[300];
    double ASize;
    if (Size >= 0)
       ASize = Size;
    else
       ASize = -1*Size;
-   
+
    /* bytes, KiloBytes, MegaBytes, GigaBytes, TeraBytes, PetaBytes, 
       ExaBytes, ZettaBytes, YottaBytes */
    char Ext[] = {'\0','k','M','G','T','P','E','Z','Y'};
@@ -394,20 +391,21 @@ string SizeToStr(double Size)
    {
       if (ASize < 100 && I != 0)
       {
-         sprintf(S,"%'.1f %c",ASize,Ext[I]);
-	 break;
+	 std::string S;
+	 strprintf(S, "%'.1f %c", ASize, Ext[I]);
+	 return S;
       }
-      
+
       if (ASize < 10000)
       {
-         sprintf(S,"%'.0f %c",ASize,Ext[I]);
-	 break;
+	 std::string S;
+	 strprintf(S, "%'.0f %c", ASize, Ext[I]);
+	 return S;
       }
       ASize /= 1000.0;
       I++;
    }
-   
-   return S;
+   return "";
 }
 									/*}}}*/
 // TimeToStr - Convert the time into a string				/*{{{*/
@@ -415,36 +413,27 @@ string SizeToStr(double Size)
 /* Converts a number of seconds to a hms format */
 string TimeToStr(unsigned long Sec)
 {
-   char S[300];
-   
-   while (1)
+   std::string S;
+   if (Sec > 60*60*24)
    {
-      if (Sec > 60*60*24)
-      {
-	 //d means days, h means hours, min means minutes, s means seconds
-	 sprintf(S,_("%lid %lih %limin %lis"),Sec/60/60/24,(Sec/60/60) % 24,(Sec/60) % 60,Sec % 60);
-	 break;
-      }
-      
-      if (Sec > 60*60)
-      {
-	 //h means hours, min means minutes, s means seconds
-	 sprintf(S,_("%lih %limin %lis"),Sec/60/60,(Sec/60) % 60,Sec % 60);
-	 break;
-      }
-      
-      if (Sec > 60)
-      {
-	 //min means minutes, s means seconds
-	 sprintf(S,_("%limin %lis"),Sec/60,Sec % 60);
-	 break;
-      }
-
-      //s means seconds
-      sprintf(S,_("%lis"),Sec);
-      break;
+      //TRANSLATOR: d means days, h means hours, min means minutes, s means seconds
+      strprintf(S,_("%lid %lih %limin %lis"),Sec/60/60/24,(Sec/60/60) % 24,(Sec/60) % 60,Sec % 60);
+   }
+   else if (Sec > 60*60)
+   {
+      //TRANSLATOR: h means hours, min means minutes, s means seconds
+      strprintf(S,_("%lih %limin %lis"),Sec/60/60,(Sec/60) % 60,Sec % 60);
+   }
+   else if (Sec > 60)
+   {
+      //TRANSLATOR: min means minutes, s means seconds
+      strprintf(S,_("%limin %lis"),Sec/60,Sec % 60);
+   }
+   else
+   {
+      //TRANSLATOR: s means seconds
+      strprintf(S,_("%lis"),Sec);
    }
-   
    return S;
 }
 									/*}}}*/
@@ -1423,7 +1412,7 @@ size_t strv_length(const char **str_array)
       ;
    return i;
 }
-
+									/*}}}*/
 // DeEscapeString - unescape (\0XX and \xXX) from a string		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -1605,51 +1594,46 @@ void URI::CopyFrom(const string &U)
 /* */
 URI::operator string()
 {
-   string Res;
-   
+   std::stringstream Res;
+
    if (Access.empty() == false)
-      Res = Access + ':';
-   
+      Res << Access << ':';
+
    if (Host.empty() == false)
-   {	 
+   {
       if (Access.empty() == false)
-	 Res += "//";
-          
+	 Res << "//";
+
       if (User.empty() == false)
       {
 	 // FIXME: Technically userinfo is permitted even less
 	 // characters than these, but this is not conveniently
 	 // expressed with a blacklist.
-	 Res += QuoteString(User, ":/?#[]@");
+	 Res << QuoteString(User, ":/?#[]@");
 	 if (Password.empty() == false)
-	    Res += ":" + QuoteString(Password, ":/?#[]@");
-	 Res += "@";
+	    Res << ":" << QuoteString(Password, ":/?#[]@");
+	 Res << "@";
       }
-      
+
       // Add RFC 2732 escaping characters
-      if (Access.empty() == false &&
-	  (Host.find('/') != string::npos || Host.find(':') != string::npos))
-	 Res += '[' + Host + ']';
+      if (Access.empty() == false && Host.find_first_of("/:") != string::npos)
+	 Res << '[' << Host << ']';
       else
-	 Res += Host;
-      
+	 Res << Host;
+
       if (Port != 0)
-      {
-	 char S[30];
-	 sprintf(S,":%u",Port);
-	 Res += S;
-      }	 
+	 Res << ':' << Port;
    }
-   
+
    if (Path.empty() == false)
    {
       if (Path[0] != '/')
-	 Res += "/" + Path;
+	 Res << "/" << Path;
       else
-	 Res += Path;
+	 Res << Path;
    }
-   
-   return Res;
+
+   return Res.str();
 }
 									/*}}}*/
 // URI::SiteOnly - Return the schema and site for the URI		/*{{{*/
diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc
index 14a53eacb..e834d7d6a 100644
--- a/apt-private/acqprogress.cc
+++ b/apt-private/acqprogress.cc
@@ -23,6 +23,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #include 
@@ -34,9 +35,8 @@ using namespace std;
 // ---------------------------------------------------------------------
 /* */
 AcqTextStatus::AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet) :
-    pkgAcquireStatus(), ScreenWidth(ScreenWidth), ID(0), Quiet(Quiet)
+    pkgAcquireStatus(), ScreenWidth(ScreenWidth), LastLineLength(0), ID(0), Quiet(Quiet)
 {
-   BlankLine[0] = 0;
    // testcases use it to disable pulses without disabling other user messages
    if (Quiet == 0 && _config->FindB("quiet::NoUpdate", false) == true)
       this->Quiet = 1;
@@ -48,7 +48,7 @@ AcqTextStatus::AcqTextStatus(unsigned int &ScreenWidth,unsigned int const Quiet)
 void AcqTextStatus::Start()
 {
    pkgAcquireStatus::Start();
-   BlankLine[0] = 0;
+   LastLineLength = 0;
    ID = 1;
 }
 									/*}}}*/
@@ -60,8 +60,7 @@ void AcqTextStatus::IMSHit(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r';
+   clearLastLine();
 
    cout << _("Hit ") << Itm.Description;
    cout << endl;
@@ -82,8 +81,7 @@ void AcqTextStatus::Fetch(pkgAcquire::ItemDesc &Itm)
    if (Quiet > 1)
       return;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r';
+   clearLastLine();
 
    cout << _("Get:") << Itm.Owner->ID << ' ' << Itm.Description;
    if (Itm.Owner->FileSize != 0)
@@ -111,8 +109,7 @@ void AcqTextStatus::Fail(pkgAcquire::ItemDesc &Itm)
    if (Itm.Owner->Status == pkgAcquire::Item::StatIdle)
       return;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r';
+   clearLastLine();
 
    if (Itm.Owner->Status == pkgAcquire::Item::StatDone)
    {
@@ -140,8 +137,7 @@ void AcqTextStatus::Stop()
    if (Quiet > 1)
       return;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r' << flush;
+   clearLastLine();
 
    if (_config->FindB("quiet::NoStatistic", false) == true)
       return;
@@ -167,77 +163,66 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 
    enum {Long = 0,Medium,Short} Mode = Medium;
 
-   char Buffer[sizeof(BlankLine)];
-   char *End = Buffer + sizeof(Buffer);
-   char *S = Buffer;
-   if (ScreenWidth >= sizeof(Buffer))
-      ScreenWidth = sizeof(Buffer)-1;
-
-   // Put in the percent done
-   sprintf(S,"%.0f%%", Percent);
-
-   bool Shown = false;
-   for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
-	I = Owner->WorkerStep(I))
+   std::string Line;
    {
-      S += strlen(S);
-
-      // There is no item running
-      if (I->CurrentItem == 0)
+      std::stringstream S;
+      for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0;
+	    I = Owner->WorkerStep(I))
       {
-	 if (I->Status.empty() == false)
+	 // There is no item running
+	 if (I->CurrentItem == 0)
 	 {
-	    snprintf(S,End-S," [%s]",I->Status.c_str());
-	    Shown = true;
-	 }
-
-	 continue;
-      }
-
-      Shown = true;
+	    if (I->Status.empty() == false)
+	       S << " [" << I->Status << "]";
 
-      // Add in the short description
-      if (I->CurrentItem->Owner->ID != 0)
-	 snprintf(S,End-S," [%lu %s",I->CurrentItem->Owner->ID,
-		  I->CurrentItem->ShortDesc.c_str());
-      else
-	 snprintf(S,End-S," [%s",I->CurrentItem->ShortDesc.c_str());
-      S += strlen(S);
+	    continue;
+	 }
 
-      // Show the short mode string
-      if (I->CurrentItem->Owner->ActiveSubprocess.empty() == false)
-      {
-	 snprintf(S,End-S, " %s", I->CurrentItem->Owner->ActiveSubprocess.c_str());
-	 S += strlen(S);
-      }
+	 // Add in the short description
+	 S << " [";
+	 if (I->CurrentItem->Owner->ID != 0)
+	    S << I->CurrentItem->Owner->ID << " ";
+	 S << I->CurrentItem->ShortDesc;
 
-      // Add the current progress
-      if (Mode == Long)
-	 snprintf(S,End-S," %llu",I->CurrentSize);
-      else
-      {
-	 if (Mode == Medium || I->TotalSize == 0)
-	    snprintf(S,End-S," %sB",SizeToStr(I->CurrentSize).c_str());
-      }
-      S += strlen(S);
+	 // Show the short mode string
+	 if (I->CurrentItem->Owner->ActiveSubprocess.empty() == false)
+	    S << " " << I->CurrentItem->Owner->ActiveSubprocess;
 
-      // Add the total size and percent
-      if (I->TotalSize > 0 && I->CurrentItem->Owner->Complete == false)
-      {
-	 if (Mode == Short)
-	    snprintf(S,End-S," %.0f%%",
-		     (I->CurrentSize*100.0)/I->TotalSize);
+	 // Add the current progress
+	 if (Mode == Long)
+	    S << " " << I->CurrentSize;
 	 else
-	    snprintf(S,End-S,"/%sB %.0f%%",SizeToStr(I->TotalSize).c_str(),
+	 {
+	    if (Mode == Medium || I->TotalSize == 0)
+	       S << " " << SizeToStr(I->CurrentSize) << "B";
+	 }
+
+	 // Add the total size and percent
+	 if (I->TotalSize > 0 && I->CurrentItem->Owner->Complete == false)
+	 {
+	    if (Mode == Short)
+	       ioprintf(S, " %.0f%%", (I->CurrentSize*100.0)/I->TotalSize);
+	    else
+	       ioprintf(S, "/%sB %.0f%%", SizeToStr(I->TotalSize).c_str(),
 		     (I->CurrentSize*100.0)/I->TotalSize);
+	 }
+	 S << "]";
       }
-      S += strlen(S);
-      snprintf(S,End-S,"]");
-   }
 
-   // Show something..
-   if (Shown == false)
-      snprintf(S,End-S,_(" [Working]"));
+      // Show at least something
+      Line = S.str();
+      S.clear();
+      if (Line.empty() == true)
+	 Line = _(" [Working]");
+   }
+   // Put in the percent done
+   {
+      std::stringstream S;
+      ioprintf(S, "%.0f%%", Percent);
+      S << Line;
+      Line = S.str();
+      S.clear();
+   }
 
    /* Put in the ETA and cps meter, block off signals to prevent strangeness
       during resizing */
@@ -248,34 +233,33 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner)
 
    if (CurrentCPS != 0)
    {
-      char Tmp[300];
       unsigned long long ETA = (TotalBytes - CurrentBytes)/CurrentCPS;
-      sprintf(Tmp," %sB/s %s",SizeToStr(CurrentCPS).c_str(),TimeToStr(ETA).c_str());
-      unsigned int Len = strlen(Buffer);
-      unsigned int LenT = strlen(Tmp);
-      if (Len + LenT < ScreenWidth)
+      std::string Tmp = " " + SizeToStr(CurrentCPS) + "B/s " + TimeToStr(ETA);
+      size_t alignment = Line.length() + Tmp.length();
+      if (alignment < ScreenWidth)
       {
-	 memset(Buffer + Len,' ',ScreenWidth - Len);
-	 strcpy(Buffer + ScreenWidth - LenT,Tmp);
+	 alignment = ScreenWidth - alignment;
+	 for (size_t i = 0; i < alignment; ++i)
+	    Line.append(" ");
+	 Line.append(Tmp);
       }
    }
-   Buffer[ScreenWidth] = 0;
-   BlankLine[ScreenWidth] = 0;
+   if (Line.length() > ScreenWidth)
+      Line.erase(ScreenWidth);
    sigprocmask(SIG_SETMASK,&OldSigs,0);
 
    // Draw the current status
    if (_config->FindB("Apt::Color", false) == true)
       cout << _config->Find("APT::Color::Yellow");
-   if (strlen(Buffer) == strlen(BlankLine))
-      cout << '\r' << Buffer << flush;
+   if (LastLineLength > Line.length())
+      clearLastLine();
    else
-      cout << '\r' << BlankLine << '\r' << Buffer << flush;
+      cout << '\r';
+   cout << Line << flush;
    if (_config->FindB("Apt::Color", false) == true)
       cout << _config->Find("APT::Color::Neutral") << flush;
 
-   memset(BlankLine,' ',strlen(Buffer));
-   BlankLine[strlen(Buffer)] = 0;
-
+   LastLineLength = Line.length();
    Update = false;
 
    return true;
@@ -296,8 +280,7 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
 
       return false;
 
-   if (Quiet <= 0)
-      cout << '\r' << BlankLine << '\r';
+   clearLastLine();
    ioprintf(cout,_("Media change: please insert the disc labeled\n"
 		   " '%s'\n"
 		   "in the drive '%s' and press enter\n"),
@@ -317,3 +300,17 @@ bool AcqTextStatus::MediaChange(string Media,string Drive)
    return bStatus;
 }
 									/*}}}*/
+void AcqTextStatus::clearLastLine() {					/*{{{*/
+   if (Quiet > 0)
+      return;
+
+   // do not try to clear more than the (now smaller) screen
+   if (LastLineLength > ScreenWidth)
+      LastLineLength = ScreenWidth;
+
+   std::cout << '\r';
+   for (size_t i = 0; i < LastLineLength; ++i)
+      std::cout << ' ';
+   std::cout << '\r' << std::flush;
+}
+									/*}}}*/
diff --git a/apt-private/acqprogress.h b/apt-private/acqprogress.h
index 71a10d78a..ab8170126 100644
--- a/apt-private/acqprogress.h
+++ b/apt-private/acqprogress.h
@@ -17,10 +17,12 @@
 class APT_PUBLIC AcqTextStatus : public pkgAcquireStatus
 {
    unsigned int &ScreenWidth;
-   char BlankLine[1024];
+   size_t LastLineLength;
    unsigned long ID;
    unsigned long Quiet;
 
+   void clearLastLine();
+
    public:
 
    virtual bool MediaChange(std::string Media,std::string Drive);
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index a63d8846b..db68c21f0 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -440,9 +440,6 @@ bool PackagesWriter::DoPackage(string FileName)
       OverItem->Priority = Tags.FindS("Priority");
    }
 
-   char Size[40];
-   sprintf(Size,"%llu", (unsigned long long) FileSize);
-   
    // Strip the DirStrip prefix from the FileName and add the PathPrefix
    string NewFileName;
    if (DirStrip.empty() == false &&
@@ -466,7 +463,10 @@ bool PackagesWriter::DoPackage(string FileName)
    // This lists all the changes to the fields we are going to make.
    std::vector Changes;
 
-   Changes.push_back(SetTFRewriteData("Size", Size));
+   std::string Size;
+   strprintf(Size, "%llu", (unsigned long long) FileSize);
+   Changes.push_back(SetTFRewriteData("Size", Size.c_str()));
+
    for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
    {
       if (hs->HashType() == "MD5Sum")
diff --git a/methods/ftp.cc b/methods/ftp.cc
index 0504e5872..7764acf6a 100644
--- a/methods/ftp.cc
+++ b/methods/ftp.cc
@@ -259,19 +259,21 @@ bool FTPConn::Login()
       {
 	 if (Opts->Value.empty() == true)
 	    continue;
-	 
+
 	 // Substitute the variables into the command
-	 char SitePort[20];
-	 if (ServerName.Port != 0)
-	    sprintf(SitePort,"%u",ServerName.Port);
-	 else
-	    strcpy(SitePort,"21");
 	 string Tmp = Opts->Value;
 	 Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 	 Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 	 Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 	 Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
-	 Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
+	 if (ServerName.Port != 0)
+	 {
+	    std::string SitePort;
+	    strprintf(SitePort, "%u", ServerName.Port);
+	    Tmp = SubstVar(Tmp,"$(SITE_PORT)", SitePort);
+	 }
+	 else
+	    Tmp = SubstVar(Tmp,"$(SITE_PORT)", "21");
 	 Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 
 	 // Send the command
diff --git a/test/libapt/strutil_test.cc b/test/libapt/strutil_test.cc
index 9bc3c76fd..23dc08727 100644
--- a/test/libapt/strutil_test.cc
+++ b/test/libapt/strutil_test.cc
@@ -97,6 +97,28 @@ TEST(StrUtilTest,StartsWith)
    EXPECT_FALSE(Startswith("abcd", "x"));
    EXPECT_FALSE(Startswith("abcd", "abcndefg"));
 }
+TEST(StrUtilTest,TimeToStr)
+{
+   EXPECT_EQ("0s", TimeToStr(0));
+   EXPECT_EQ("42s", TimeToStr(42));
+   EXPECT_EQ("9min 21s", TimeToStr((9*60) + 21));
+   EXPECT_EQ("20min 42s", TimeToStr((20*60) + 42));
+   EXPECT_EQ("10h 42min 21s", TimeToStr((10*3600) + (42*60) + 21));
+   EXPECT_EQ("10h 42min 21s", TimeToStr((10*3600) + (42*60) + 21));
+   EXPECT_EQ("1988d 3h 29min 7s", TimeToStr((1988*86400) + (3*3600) + (29*60) + 7));
+
+   EXPECT_EQ("59s", TimeToStr(59));
+   EXPECT_EQ("60s", TimeToStr(60));
+   EXPECT_EQ("1min 1s", TimeToStr(61));
+   EXPECT_EQ("59min 59s", TimeToStr(3599));
+   EXPECT_EQ("60min 0s", TimeToStr(3600));
+   EXPECT_EQ("1h 0min 1s", TimeToStr(3601));
+   EXPECT_EQ("1h 1min 0s", TimeToStr(3660));
+   EXPECT_EQ("23h 59min 59s", TimeToStr(86399));
+   EXPECT_EQ("24h 0min 0s", TimeToStr(86400));
+   EXPECT_EQ("1d 0h 0min 1s", TimeToStr(86401));
+   EXPECT_EQ("1d 0h 1min 0s", TimeToStr(86460));
+}
 TEST(StrUtilTest,SubstVar)
 {
    EXPECT_EQ("", SubstVar("", "fails", "passes"));
diff --git a/test/libapt/uri_test.cc b/test/libapt/uri_test.cc
index 1662f51f0..5d5ae9679 100644
--- a/test/libapt/uri_test.cc
+++ b/test/libapt/uri_test.cc
@@ -12,6 +12,7 @@ TEST(URITest, BasicHTTP)
    EXPECT_EQ(90, U.Port);
    EXPECT_EQ("www.debian.org", U.Host);
    EXPECT_EQ("/temp/test", U.Path);
+   EXPECT_EQ("http://www.debian.org:90/temp/test", (std::string)U);
    // Login data
    U = URI("http://jgg:foo@ualberta.ca/blah");
    EXPECT_EQ("http", U.Access);
@@ -20,6 +21,7 @@ TEST(URITest, BasicHTTP)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("ualberta.ca", U.Host);
    EXPECT_EQ("/blah", U.Path);
+   EXPECT_EQ("http://jgg:foo@ualberta.ca/blah", (std::string)U);
 }
 TEST(URITest, SingeSlashFile)
 {
@@ -30,6 +32,7 @@ TEST(URITest, SingeSlashFile)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("", U.Host);
    EXPECT_EQ("/usr/bin/foo", U.Path);
+   EXPECT_EQ("file:/usr/bin/foo", (std::string)U);
 }
 TEST(URITest, BasicCDROM)
 {
@@ -40,6 +43,7 @@ TEST(URITest, BasicCDROM)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("Moo Cow Rom", U.Host);
    EXPECT_EQ("/debian", U.Path);
+   EXPECT_EQ("cdrom://Moo Cow Rom/debian", (std::string)U);
 }
 TEST(URITest, RelativeGzip)
 {
@@ -50,6 +54,7 @@ TEST(URITest, RelativeGzip)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ(".", U.Host);
    EXPECT_EQ("/bar/cow", U.Path);
+   EXPECT_EQ("gzip://./bar/cow", (std::string)U);
 }
 TEST(URITest, NoSlashFTP)
 {
@@ -60,6 +65,7 @@ TEST(URITest, NoSlashFTP)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("ftp.fr.debian.org", U.Host);
    EXPECT_EQ("/debian/pool/main/x/xtel/xtel_3.2.1-15_i386.deb", U.Path);
+   EXPECT_EQ("ftp://ftp.fr.debian.org/debian/pool/main/x/xtel/xtel_3.2.1-15_i386.deb", (std::string)U);
 }
 TEST(URITest, RFC2732)
 {
@@ -70,6 +76,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("1080::8:800:200C:417A", U.Host);
    EXPECT_EQ("/foo", U.Path);
+   EXPECT_EQ("http://[1080::8:800:200C:417A]/foo", (std::string)U);
    // with port
    U = URI("http://[::FFFF:129.144.52.38]:80/index.html");
    EXPECT_EQ("http", U.Access);
@@ -78,6 +85,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(80, U.Port);
    EXPECT_EQ("::FFFF:129.144.52.38", U.Host);
    EXPECT_EQ("/index.html", U.Path);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38]:80/index.html", (std::string)U);
    // extra colon
    U = URI("http://[::FFFF:129.144.52.38:]:80/index.html");
    EXPECT_EQ("http", U.Access);
@@ -86,6 +94,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(80, U.Port);
    EXPECT_EQ("::FFFF:129.144.52.38:", U.Host);
    EXPECT_EQ("/index.html", U.Path);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]:80/index.html", (std::string)U);
    // extra colon port
    U = URI("http://[::FFFF:129.144.52.38:]/index.html");
    EXPECT_EQ("http", U.Access);
@@ -94,6 +103,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("::FFFF:129.144.52.38:", U.Host);
    EXPECT_EQ("/index.html", U.Path);
+   EXPECT_EQ("http://[::FFFF:129.144.52.38:]/index.html", (std::string)U);
    // My Evil Corruption of RFC 2732 to handle CDROM names!
    // Fun for the whole family! */
    U = URI("cdrom:[The Debian 1.2 disk, 1/2 R1:6]/debian/");
@@ -103,6 +113,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("The Debian 1.2 disk, 1/2 R1:6", U.Host);
    EXPECT_EQ("/debian/", U.Path);
+   EXPECT_EQ("cdrom://[The Debian 1.2 disk, 1/2 R1:6]/debian/", (std::string)U);
    // no brackets
    U = URI("cdrom:Foo Bar Cow/debian/");
    EXPECT_EQ("cdrom", U.Access);
@@ -111,6 +122,7 @@ TEST(URITest, RFC2732)
    EXPECT_EQ(0, U.Port);
    EXPECT_EQ("Foo Bar Cow", U.Host);
    EXPECT_EQ("/debian/", U.Path);
+   EXPECT_EQ("cdrom://Foo Bar Cow/debian/", (std::string)U);
    // percent encoded
    U = URI("ftp://foo:b%40r@example.org");
    EXPECT_EQ("foo", U.User);
-- 
cgit v1.2.3-70-g09d2


From 3a53f6a1510d332e24c3330a69b987f2341d1a94 Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Mon, 13 Apr 2015 12:57:22 -0400
Subject: Revert "HttpsMethod::Fetch(): Zero the FetchResult object when
 leaving due to 404"

This reverts commit 1296bc7c466181a7978c313c40a041b34ce3eaeb.
---
 methods/https.cc | 2 --
 1 file changed, 2 deletions(-)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index f2b00dd64..3a5981b58 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -402,8 +402,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       _error->Error("%s", err);
       // unlink, no need keep 401/404 page content in partial/
       unlink(File->Name().c_str());
-      Res.Size = 0;
-      Res.LastModified = 0;
       return false;
    }
 
-- 
cgit v1.2.3-70-g09d2


From bb948ef562862e5cc9fcfb3d7b5e41c70382adeb Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 27 Mar 2015 11:14:44 +0100
Subject: do not unlink https file on general error

It might be quite interesting which file (content) made curl freak out
and other methods keep the file around as well.

Git-Dch: Ignore
---
 methods/https.cc | 1 -
 1 file changed, 1 deletion(-)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index 32de42e4b..c69e84d3a 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -386,7 +386,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    if (success != 0)
    {
       _error->Error("%s", curl_errorstr);
-      unlink(File->Name().c_str());
       return false;
    }
 
-- 
cgit v1.2.3-70-g09d2


From 27925d82dd0cbae74d48040363fe6f6c2bae5215 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 27 Mar 2015 15:53:43 +0100
Subject: improve https method queue progress reporting

The worker expects that the methods tell him when they start or finish
downloading a file. Various information pieces are passed along in this
report including the (expected) filesize. https was using a "global"
struct for reporting which made it 'reuse' incorrect values in some
cases like a non-existent InRelease fallbacking to Release{,.gpg}
resulting in a size-mismatch warning. Reducing the scope and redesigning
the setting of the values we can fix this and related issues.

Closes: 777565, 781509
Thanks: Robert Edmonds and Anders Kaseorg for initial patchs
---
 methods/https.cc                                  | 82 ++++++++++-----------
 methods/https.h                                   |  2 +-
 methods/server.h                                  |  3 +-
 test/integration/framework                        | 20 +++++-
 test/integration/test-apt-https-no-redirect       | 15 ++--
 test/integration/test-apt-update-expected-size    | 86 +++++++++++++++--------
 test/integration/test-bug-602412-dequote-redirect |  3 +-
 7 files changed, 121 insertions(+), 90 deletions(-)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index c69e84d3a..70f6a1046 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -37,11 +37,17 @@
 									/*}}}*/
 using namespace std;
 
+struct APT_HIDDEN CURLUserPointer {
+   HttpsMethod * const https;
+   HttpsMethod::FetchResult * const Res;
+   CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res) : https(https), Res(Res) {}
+};
+
 size_t
 HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
 {
    size_t len = size * nmemb;
-   HttpsMethod *me = (HttpsMethod *)userp;
+   CURLUserPointer *me = (CURLUserPointer *)userp;
    std::string line((char*) buffer, len);
    for (--len; len > 0; --len)
       if (isspace(line[len]) == 0)
@@ -53,23 +59,33 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
 
    if (line.empty() == true)
    {
-      if (me->Server->Result != 416 && me->Server->StartPos != 0)
+      if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0)
 	 ;
-      else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize())
+      else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize())
       {
-         me->Server->Result = 200;
-	 me->Server->StartPos = me->Server->Size;
+         me->https->Server->Result = 200;
+	 me->https->Server->StartPos = me->https->Server->Size;
 	 // the actual size is not important for https as curl will deal with it
 	 // by itself and e.g. doesn't bother us with transport-encoding…
-	 me->Server->JunkSize = std::numeric_limits::max();
+	 me->https->Server->JunkSize = std::numeric_limits::max();
       }
       else
-	 me->Server->StartPos = 0;
+	 me->https->Server->StartPos = 0;
+
+      me->https->File->Truncate(me->https->Server->StartPos);
+      me->https->File->Seek(me->https->Server->StartPos);
+
+      me->Res->LastModified = me->https->Server->Date;
+      me->Res->Size = me->https->Server->Size;
+      me->Res->ResumePoint = me->https->Server->StartPos;
 
-      me->File->Truncate(me->Server->StartPos);
-      me->File->Seek(me->Server->StartPos);
+      // we expect valid data, so tell our caller we get the file now
+      if (me->https->Server->Result >= 200 && me->https->Server->Result < 300 &&
+	    me->https->Server->JunkSize == 0 &&
+	    me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
+	 me->https->URIStart(*me->Res);
    }
-   else if (me->Server->HeaderLine(line) == false)
+   else if (me->https->Server->HeaderLine(line) == false)
       return 0;
 
    return size*nmemb;
@@ -85,12 +101,6 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
    if (me->Server->JunkSize != 0)
       return buffer_size;
 
-   if (me->Server->ReceivedData == false)
-   {
-      me->URIStart(me->Res);
-      me->Server->ReceivedData = true;
-   }
-
    if(me->File->Write(buffer, buffer_size) != true)
       return 0;
 
@@ -109,27 +119,15 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
    return buffer_size;
 }
 
-int
-HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/,
-			       double /*ultotal*/, double /*ulnow*/)
-{
-   HttpsMethod *me = (HttpsMethod *)clientp;
-   if(dltotal > 0 && me->Res.Size == 0) {
-      me->Res.Size = (unsigned long long)dltotal;
-   }
-   return 0;
-}
-
 // HttpsServerState::HttpsServerState - Constructor			/*{{{*/
 HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner)
 {
    TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
-   ReceivedData = false;
    Reset();
 }
 									/*}}}*/
 
-void HttpsMethod::SetupProxy()  					/*{{{*/
+void HttpsMethod::SetupProxy()						/*{{{*/
 {
    URI ServerName = Queue->Uri;
 
@@ -207,16 +205,16 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
 
    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 
+   FetchResult Res;
+   CURLUserPointer userp(this, &Res);
    // callbacks
    curl_easy_setopt(curl, CURLOPT_URL, static_cast(Uri).c_str());
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
-   curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this);
+   curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &userp);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
-   curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
-   curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
    // options
-   curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false);
+   curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
    curl_easy_setopt(curl, CURLOPT_FILETIME, true);
    // only allow curl to handle https, not the other stuff it supports
    curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
@@ -414,24 +412,23 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       return false;
    }
 
-   struct stat resultStat;
-   if (unlikely(stat(File->Name().c_str(), &resultStat) != 0))
-   {
-      _error->Errno("stat", "Unable to access file %s", File->Name().c_str());
-      return false;
-   }
-   Res.Size = resultStat.st_size;
-
    // invalid range-request
    if (Server->Result == 416)
    {
       unlink(File->Name().c_str());
-      Res.Size = 0;
       delete File;
       Redirect(Itm->Uri);
       return true;
    }
 
+   struct stat resultStat;
+   if (unlikely(stat(File->Name().c_str(), &resultStat) != 0))
+   {
+      _error->Errno("stat", "Unable to access file %s", File->Name().c_str());
+      return false;
+   }
+   Res.Size = resultStat.st_size;
+
    // Timestamp
    curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified);
    if (Res.LastModified != -1)
@@ -455,7 +452,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    URIDone(Res);
 
    // cleanup
-   Res.Size = 0;
    delete File;
 
    return true;
diff --git a/methods/https.h b/methods/https.h
index 433a84680..4cc48fc34 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -65,7 +65,6 @@ class HttpsMethod : public ServerMethod
 				 double ultotal, double ulnow);
    void SetupProxy();
    CURL *curl;
-   FetchResult Res;
    ServerState *Server;
 
    // Used by ServerMethods unused by https
@@ -77,6 +76,7 @@ class HttpsMethod : public ServerMethod
 
    virtual bool Configuration(std::string Message);
    virtual ServerState * CreateServerState(URI uri);
+   using pkgAcqMethod::FetchResult;
 
    HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL)
    {
diff --git a/methods/server.h b/methods/server.h
index 3b232dcac..b974ec89a 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -37,7 +37,6 @@ struct ServerState
    unsigned long long Size; // size of the usable content (aka: the file)
    unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
-   bool ReceivedData;
    time_t Date;
    bool HaveContent;
    enum {Chunked,Stream,Closes} Encoding;
@@ -76,7 +75,7 @@ struct ServerState
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
    virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
-		 StartPos = 0; ReceivedData = false; Encoding = Closes; time(&Date); HaveContent = false;
+		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;};
    virtual bool WriteResponse(std::string const &Data) = 0;
 
diff --git a/test/integration/framework b/test/integration/framework
index ec23e41e6..50c027a2c 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1125,8 +1125,10 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 
 downloadfile() {
 	local PROTO="${1%%:*}"
-	apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
-		download-file "$1" "$2" 2>&1 || true
+	if ! apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
+		download-file "$1" "$2" 2>&1 ; then
+		return 1
+	fi
 	# only if the file exists the download was successful
 	if [ -r "$2" ]; then
 		return 0
@@ -1407,6 +1409,20 @@ testfailureequal() {
 	testfileequal "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output" "$CMP"
 }
 
+testfailuremsg() {
+	local CMP="$1"
+	shift
+	testfailure "$@"
+	msgtest 'Check that the output of the previous failed command has expected' 'failures and warnings'
+	grep '^\(W\|E\):' "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output" > "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailureequal.output" 2>&1 || true
+	if echo "$CMP" | checkdiff - "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailureequal.output"; then
+		msgpass
+	else
+		echo '### Complete output ###'
+		cat "${TMPWORKINGDIRECTORY}/rootdir/tmp/testfailure.output"
+		msgfail
+	fi
+}
 
 testfilestats() {
 	msgtest "Test that file $1 has $2 $3" "$4"
diff --git a/test/integration/test-apt-https-no-redirect b/test/integration/test-apt-https-no-redirect
index bc744d6f2..c91c78916 100755
--- a/test/integration/test-apt-https-no-redirect
+++ b/test/integration/test-apt-https-no-redirect
@@ -14,22 +14,15 @@ echo 'alright' > aptarchive/working
 changetohttpswebserver  -o 'aptwebserver::redirect::replace::/redirectme/=http://localhost:8080/'
 
 msgtest 'download of a file works via' 'http'
-downloadfile 'http://localhost:8080/working' httpfile >/dev/null 2>&1 && msgpass || msgfail
+testsuccess --nomsg downloadfile 'http://localhost:8080/working' httpfile
 testfileequal httpfile 'alright'
 
 msgtest 'download of a file works via' 'https'
-downloadfile 'https://localhost:4433/working' httpsfile >/dev/null 2>&1 && msgpass || msgfail
+testsuccess --nomsg downloadfile 'https://localhost:4433/working' httpsfile
 testfileequal httpsfile 'alright'
 
 msgtest 'download of a file does not work if' 'https redirected to http'
-downloadfile 'https://localhost:4433/redirectme/working' redirectfile >curloutput 2>&1 && msgfail || msgpass
+testfailure --nomsg downloadfile 'https://localhost:4433/redirectme/working' redirectfile
 
 msgtest 'libcurl has forbidden access in last request to' 'http resource'
-if grep -q -E -- 'Protocol "?http"? not supported or disabled in libcurl' curloutput; then
-	msgpass
-else
-	cat curloutput
-	msgfail
-fi
-
-
+testsuccess --nomsg grep -q -E -- 'Protocol "?http"? not supported or disabled in libcurl' rootdir/tmp/testfailure.output
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 9711c293a..22de13ea5 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -10,35 +10,61 @@ configarchitecture "i386"
 insertpackage 'unstable' 'apt' 'all' '1.0'
 
 setupaptarchive --no-update
+cp -a aptarchive/dists aptarchive/dists.good
+
+test_inreleasetoobig() {
+	# make InRelease really big to trigger fallback
+	dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null
+	touch -d '+1hour' aptarchive/dists/unstable/InRelease
+	testsuccess aptget update -o Apt::Get::List-Cleanup=0  -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0
+	msgtest 'Check that the max write warning is triggered'
+	cp rootdir/tmp/testsuccess.output update.output
+	testsuccess --nomsg grep -q 'Writing more data than expected' update.output
+	rm -f update.output
+	# ensure the failed InRelease file got renamed
+	testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED
+}
+
+test_packagestoobig() {
+	# append junk at the end of the Packages.gz/Packages
+	SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
+	find aptarchive/dists -name 'Packages*' | while read pkg; do
+		echo "1234567890" >> "$pkg"
+		touch -d '+1hour' "$pkg"
+	done
+	NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
+	testfailuremsg "W: Failed to fetch ${1}/dists/unstable/main/binary-i386/Packages  Writing more data than expected ($NEW_SIZE > $SIZE)
+E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -o Debug::pkgAcquire::Worker=0
+}
+
+methodtest() {
+	msgmsg 'Test with' "$1" 'and clean start'
+	rm -rf rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
+	# normal update works fine
+	testsuccess aptget update
+	mv rootdir/var/lib/apt/lists rootdir/var/lib/apt/lists.good
+
+	# starting fresh works
+	test_inreleasetoobig "$1"
+	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists.good aptarchive/dists
+	test_packagestoobig "$1"
+	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists.good aptarchive/dists
+
+	msgmsg 'Test with' "$1" 'and existing old data'
+	cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists
+	test_inreleasetoobig "$1"
+	rm -rf aptarchive/dists rootdir/var/lib/apt/lists
+	cp -a rootdir/var/lib/apt/lists.good rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists.good aptarchive/dists
+	test_packagestoobig "$1"
+	rm -rf aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
+}
+
 changetowebserver
+methodtest 'http://localhost:8080'
 
-# normal update works fine
-testsuccess aptget update
-
-# make InRelease really big to trigger fallback
-mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good
-dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 2>/dev/null
-touch -d '+1hour' aptarchive/dists/unstable/InRelease
-testsuccess aptget update -o Apt::Get::List-Cleanup=0  -o acquire::MaxReleaseFileSize=$((1*1000*1000)) -o Debug::pkgAcquire::worker=0
-msgtest 'Check that the max write warning is triggered'
-if grep -q "Writing more data than expected" rootdir/tmp/testsuccess.output; then
-    msgpass
-else
-    cat rootdir/tmp/testsuccess.output
-    msgfail
-fi
-# ensure the failed InRelease file got renamed
-testsuccess ls rootdir/var/lib/apt/lists/partial/*InRelease.FAILED
-mv aptarchive/dists/unstable/InRelease.good aptarchive/dists/unstable/InRelease
-
-
-# append junk at the end of the Packages.gz/Packages
-SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
-find aptarchive -name 'Packages*' | while read pkg; do
-	echo "1234567890" >> "$pkg"
-done
-NEW_SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
-rm -f rootdir/var/lib/apt/lists/localhost*
-testfailureequal "W: Failed to fetch http://localhost:8080/dists/unstable/main/binary-i386/Packages  Writing more data than expected ($NEW_SIZE > $SIZE)
-
-E: Some index files failed to download. They have been ignored, or old ones used instead." aptget update -qq
+changetohttpswebserver
+methodtest 'https://localhost:4433'
diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect
index d3573a79a..384c8b113 100755
--- a/test/integration/test-bug-602412-dequote-redirect
+++ b/test/integration/test-bug-602412-dequote-redirect
@@ -8,7 +8,7 @@ configarchitecture 'amd64'
 
 buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
 
-setupaptarchive
+setupaptarchive --no-update
 changetowebserver -o aptwebserver::redirect::replace::/pool/=/newpool/ \
 		  -o aptwebserver::redirect::replace::/dists/=/newdists/
 
@@ -16,6 +16,7 @@ mv aptarchive/pool aptarchive/newpool
 mv aptarchive/dists aptarchive/newdists
 
 testrun() {
+	msgmsg 'Test redirection works in method boundaries' "$1"
 	msgtest 'Test redirection works in' 'apt-get update'
 	testsuccess --nomsg aptget update
 
-- 
cgit v1.2.3-70-g09d2


From 936d5613e4b6145798c5a1d0c484158115576fa8 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 27 Mar 2015 18:56:44 +0100
Subject: remove duplicated check for same file copy

Git-Dch: Ignore
---
 methods/copy.cc | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

(limited to 'methods')

diff --git a/methods/copy.cc b/methods/copy.cc
index a23c0316c..f3cd84215 100644
--- a/methods/copy.cc
+++ b/methods/copy.cc
@@ -65,17 +65,9 @@ bool CopyMethod::Fetch(FetchItem *Itm)
    Res.Size = Buf.st_size;
    Res.Filename = Itm->DestFile;
    Res.LastModified = Buf.st_mtime;
-   Res.IMSHit = false;      
+   Res.IMSHit = false;
    URIStart(Res);
 
-   // when the files are identical, just compute the hashes
-   if(File == Itm->DestFile)
-   {
-      CalculateHashes(Res);
-      URIDone(Res);
-      return true;
-   }
-   
    // just calc the hashes if the source and destination are identical
    if (File == Itm->DestFile)
    {
-- 
cgit v1.2.3-70-g09d2


From a09f6eb8fc67cd2d836019f448f18580396185e5 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Fri, 27 Mar 2015 19:59:44 +0100
Subject: send Alt-* info for uncompressed based on any compressions

file sends information about the uncompressed file if it can find it as
well as for the compressed file. This was done only for gzip so far, but
we support more compression types. That this information isn't used a
lot is a different story.

Git-Dch: Ignore
---
 methods/file.cc                          | 43 ++++++++++++++++++--------------
 test/integration/framework               |  2 +-
 test/integration/test-compressed-indexes |  2 +-
 3 files changed, 26 insertions(+), 21 deletions(-)

(limited to 'methods')

diff --git a/methods/file.cc b/methods/file.cc
index 12db62203..5d9d7b951 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -16,6 +16,7 @@
 #include 
 
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -33,7 +34,7 @@ class FileMethod : public pkgAcqMethod
    
    public:
    
-   FileMethod() : pkgAcqMethod("1.0",SingleInstance | LocalOnly) {};
+   FileMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig | LocalOnly) {};
 };
 
 // FileMethod::Fetch - Fetch a file					/*{{{*/
@@ -58,27 +59,31 @@ bool FileMethod::Fetch(FetchItem *Itm)
       if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
 	 Res.IMSHit = true;
    }
-   
-   // See if we can compute a file without a .gz exentsion
-   std::string::size_type Pos = File.rfind(".gz");
-   if (Pos + 3 == File.length())
+
+   // See if the uncompressed file exists and reuse it
+   std::vector extensions = APT::Configuration::getCompressorExtensions();
+   for (std::vector::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext)
    {
-      File = std::string(File,0,Pos);
-      if (stat(File.c_str(),&Buf) == 0)
+      if (APT::String::Endswith(File, *ext) == true)
       {
-	 FetchResult AltRes;
-	 AltRes.Size = Buf.st_size;
-	 AltRes.Filename = File;
-	 AltRes.LastModified = Buf.st_mtime;
-	 AltRes.IMSHit = false;
-	 if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
-	    AltRes.IMSHit = true;
-	 
-	 URIDone(Res,&AltRes);
-	 return true;
-      }      
+	 std::string const unfile = File.substr(0, File.length() - ext->length() - 1);
+	 if (stat(unfile.c_str(),&Buf) == 0)
+	 {
+	    FetchResult AltRes;
+	    AltRes.Size = Buf.st_size;
+	    AltRes.Filename = unfile;
+	    AltRes.LastModified = Buf.st_mtime;
+	    AltRes.IMSHit = false;
+	    if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+	       AltRes.IMSHit = true;
+
+	    URIDone(Res,&AltRes);
+	    return true;
+	 }
+	 // no break here as we could have situations similar to '.gz' vs '.tar.gz' here
+      }
    }
-   
+
    if (Res.Filename.empty() == true)
       return _error->Error(_("File not found"));
 
diff --git a/test/integration/framework b/test/integration/framework
index 50c027a2c..994956b74 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -3,7 +3,7 @@
 EXIT_CODE=0
 
 # we all like colorful messages
-if [ "$MSGCOLOR" != 'NO' ]; then
+if [ "$MSGCOLOR" != 'NO' ] && [ "$MSGCOLOR" != 'ALWAYS' ]; then
 	if [ ! -t 1 ]; then # but check that we output to a terminal
 		export MSGCOLOR='NO'
 	fi
diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
index 5b966754c..c6b292baa 100755
--- a/test/integration/test-compressed-indexes
+++ b/test/integration/test-compressed-indexes
@@ -61,7 +61,7 @@ testrun() {
 	cd downloaded
 	testsuccess --nomsg aptget download testpkg
 	msgtest '\tdeb file is present'; testsuccess --nomsg test -f testpkg_1.0_i386.deb
-	rm testpkg_1.0_i386.deb
+	rm -f testpkg_1.0_i386.deb
 	cd - >/dev/null
 	testsuccessequal 'Reading package lists...
 Building dependency tree...
-- 
cgit v1.2.3-70-g09d2


From 117038bac90261351518870b3f48136f134d4bfc Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 30 Mar 2015 19:52:32 +0200
Subject: handle servers closing encoded connections correctly

Servers who advertise that they close the connection get the 'Closes'
encoding flag, but this conflicts with servers who response with a
transfer-encoding (e.g. encoding) as it is saved in the same flag.

We have a better flag for the keep-alive (or not) of the connection
anyway, so we check this instead of the encoding.

This is in practice not much of a problem as real servers we talk to are
HTTP1.1 servers (with keep-alive) and there isn't much point in doing
chunked encoding if you are going to close anyway, but our simple
testserver stumbles over this if pressed and its a bit cleaner, too.

Git-Dch: Ignore
---
 methods/http.cc                         |  6 +++---
 test/interactive-helper/aptwebserver.cc | 28 +++++++++++++++++++++-------
 2 files changed, 24 insertions(+), 10 deletions(-)

(limited to 'methods')

diff --git a/methods/http.cc b/methods/http.cc
index 021b284d0..947002cc6 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -442,7 +442,7 @@ bool HttpServerState::RunData(FileFd * const File)
    {
       /* Closes encoding is used when the server did not specify a size, the
          loss of the connection means we are done */
-      if (Encoding == Closes)
+      if (Persistent == false)
 	 In.Limit(-1);
       else if (JunkSize != 0)
 	 In.Limit(JunkSize);
@@ -524,7 +524,7 @@ bool HttpServerState::Die(FileFd &File)
 
    // See if this is because the server finished the data stream
    if (In.IsLimit() == false && State != HttpServerState::Header &&
-       Encoding != HttpServerState::Closes)
+       Persistent == true)
    {
       Close();
       if (LErrno == 0)
@@ -571,7 +571,7 @@ bool HttpServerState::Flush(FileFd * const File)
 	    return true;
       }
 
-      if (In.IsLimit() == true || Encoding == ServerState::Closes)
+      if (In.IsLimit() == true || Persistent == false)
 	 return true;
    }
    return false;
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 644629a33..86d5c06f0 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -598,17 +598,24 @@ static void * handleClient(void * voidclient)				/*{{{*/
 {
    int client = *((int*)(voidclient));
    std::clog << "ACCEPT client " << client << std::endl;
-   std::vector messages;
    bool closeConnection = false;
-   std::list headers;
-   while (closeConnection == false && ReadMessages(client, messages))
+   while (closeConnection == false)
    {
-      // if we announced a closing, do the close
-      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+      std::vector messages;
+      if (ReadMessages(client, messages) == false)
 	 break;
-      headers.clear();
+
+      std::list headers;
       for (std::vector::const_iterator m = messages.begin();
 	    m != messages.end() && closeConnection == false; ++m) {
+	 // if we announced a closing in previous response, do the close now
+	 if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 {
+	    closeConnection = true;
+	    break;
+	 }
+	 headers.clear();
+
 	 std::clog << ">>> REQUEST from " << client << " >>>" << std::endl << *m
 	    << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
 	 std::string filename;
@@ -760,9 +767,16 @@ static void * handleClient(void * voidclient)				/*{{{*/
 	 else
 	    sendError(client, 404, *m, sendContent, "", headers);
       }
+
+      // if we announced a closing in the last response, do the close now
+      if (std::find(headers.begin(), headers.end(), std::string("Connection: close")) != headers.end())
+	 closeConnection = true;
+
+      if (_error->PendingError() == true)
+	 break;
       _error->DumpErrors(std::cerr);
-      messages.clear();
    }
+   _error->DumpErrors(std::cerr);
    close(client);
    std::clog << "CLOSE client " << client << std::endl;
    return NULL;
-- 
cgit v1.2.3-70-g09d2


From 9224ce3d4d1ea0428a70e75134998e08aa45b1e6 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Mon, 30 Mar 2015 20:47:13 +0200
Subject: calculate only expected hashes in methods

Methods get told which hashes are expected by the acquire system, which
means we can use this list to restrict what we calculate in the methods
as any extra we are calculating is wasted effort as we can't compare it
with anything anyway.

Adding support for a new hash algorithm is therefore 'free' now and if a
algorithm is no longer provided in a repository for a file, we
automatically stop calculating it.

In practice this results in a speed-up in Debian as we don't have SHA512
here (so far), so we practically stop calculating it.
---
 apt-pkg/contrib/hashes.cc    | 63 +++++++++++++++++++++++++++++++++-----------
 apt-pkg/contrib/hashes.h     | 21 ++++++++++++---
 ftparchive/cachedb.cc        |  4 +--
 ftparchive/writer.cc         |  4 +--
 methods/cdrom.cc             |  2 +-
 methods/copy.cc              | 10 +++----
 methods/file.cc              |  2 +-
 methods/ftp.cc               |  2 +-
 methods/gzip.cc              |  2 +-
 methods/http.cc              | 14 +++++-----
 methods/http.h               |  2 +-
 methods/https.cc             |  2 +-
 methods/https.h              |  2 +-
 methods/rred.cc              |  2 +-
 methods/rsh.cc               |  2 +-
 methods/server.cc            |  2 +-
 methods/server.h             |  2 +-
 test/libapt/hashsums_test.cc | 24 +++++++++++++++++
 18 files changed, 116 insertions(+), 46 deletions(-)

(limited to 'methods')

diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index 6e7080bc9..953465091 100644
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -250,28 +250,34 @@ bool HashStringList::operator!=(HashStringList const &other) const
 class PrivateHashes {
 public:
    unsigned long long FileSize;
+   unsigned int CalcHashes;
 
-   PrivateHashes() : FileSize(0) {}
+   PrivateHashes(unsigned int const CalcHashes) : FileSize(0), CalcHashes(CalcHashes) {}
 };
 									/*}}}*/
 // Hashes::Add* - Add the contents of data or FD			/*{{{*/
-bool Hashes::Add(const unsigned char * const Data,unsigned long long const Size, unsigned int const Hashes)
+bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size)
 {
    bool Res = true;
 APT_IGNORE_DEPRECATED_PUSH
-   if ((Hashes & MD5SUM) == MD5SUM)
+   if ((d->CalcHashes & MD5SUM) == MD5SUM)
       Res &= MD5.Add(Data, Size);
-   if ((Hashes & SHA1SUM) == SHA1SUM)
+   if ((d->CalcHashes & SHA1SUM) == SHA1SUM)
       Res &= SHA1.Add(Data, Size);
-   if ((Hashes & SHA256SUM) == SHA256SUM)
+   if ((d->CalcHashes & SHA256SUM) == SHA256SUM)
       Res &= SHA256.Add(Data, Size);
-   if ((Hashes & SHA512SUM) == SHA512SUM)
+   if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
       Res &= SHA512.Add(Data, Size);
 APT_IGNORE_DEPRECATED_POP
    d->FileSize += Size;
    return Res;
 }
-bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes)
+bool Hashes::Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes)
+{
+   d->CalcHashes = Hashes;
+   return Add(Data, Size);
+}
+bool Hashes::AddFD(int const Fd,unsigned long long Size)
 {
    unsigned char Buf[64*64];
    bool const ToEOF = (Size == UntilEOF);
@@ -285,12 +291,17 @@ bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hash
       if (ToEOF && Res == 0) // EOF
 	 break;
       Size -= Res;
-      if (Add(Buf, Res, Hashes) == false)
+      if (Add(Buf, Res) == false)
 	 return false;
    }
    return true;
 }
-bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes)
+bool Hashes::AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes)
+{
+   d->CalcHashes = Hashes;
+   return AddFD(Fd, Size);
+}
+bool Hashes::AddFD(FileFd &Fd,unsigned long long Size)
 {
    unsigned char Buf[64*64];
    bool const ToEOF = (Size == 0);
@@ -309,20 +320,29 @@ bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes
       else if (a == 0) // EOF
 	 break;
       Size -= a;
-      if (Add(Buf, a, Hashes) == false)
+      if (Add(Buf, a) == false)
 	 return false;
    }
    return true;
+}
+bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes)
+{
+   d->CalcHashes = Hashes;
+   return AddFD(Fd, Size);
 }
 									/*}}}*/
 HashStringList Hashes::GetHashStringList()
 {
    HashStringList hashes;
 APT_IGNORE_DEPRECATED_PUSH
-   hashes.push_back(HashString("MD5Sum", MD5.Result().Value()));
-   hashes.push_back(HashString("SHA1", SHA1.Result().Value()));
-   hashes.push_back(HashString("SHA256", SHA256.Result().Value()));
-   hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
+   if ((d->CalcHashes & MD5SUM) == MD5SUM)
+      hashes.push_back(HashString("MD5Sum", MD5.Result().Value()));
+   if ((d->CalcHashes & SHA1SUM) == SHA1SUM)
+      hashes.push_back(HashString("SHA1", SHA1.Result().Value()));
+   if ((d->CalcHashes & SHA256SUM) == SHA256SUM)
+      hashes.push_back(HashString("SHA256", SHA256.Result().Value()));
+   if ((d->CalcHashes & SHA512SUM) == SHA512SUM)
+      hashes.push_back(HashString("SHA512", SHA512.Result().Value()));
 APT_IGNORE_DEPRECATED_POP
    std::string SizeStr;
    strprintf(SizeStr, "%llu", d->FileSize);
@@ -330,6 +350,19 @@ APT_IGNORE_DEPRECATED_POP
    return hashes;
 }
 APT_IGNORE_DEPRECATED_PUSH
-Hashes::Hashes() { d = new PrivateHashes(); }
+Hashes::Hashes() { d = new PrivateHashes(~0); }
+Hashes::Hashes(unsigned int const Hashes) { d = new PrivateHashes(Hashes); }
+Hashes::Hashes(HashStringList const &Hashes) {
+   unsigned int calcHashes = Hashes.usable() ? 0 : ~0;
+   if (Hashes.find("MD5Sum") != NULL)
+      calcHashes |= MD5SUM;
+   if (Hashes.find("SHA1") != NULL)
+      calcHashes |= SHA1SUM;
+   if (Hashes.find("SHA256") != NULL)
+      calcHashes |= SHA256SUM;
+   if (Hashes.find("SHA512") != NULL)
+      calcHashes |= SHA512SUM;
+   d = new PrivateHashes(calcHashes);
+}
 Hashes::~Hashes() { delete d; }
 APT_IGNORE_DEPRECATED_POP
diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h
index 154862457..ac13c8ace 100644
--- a/apt-pkg/contrib/hashes.h
+++ b/apt-pkg/contrib/hashes.h
@@ -178,7 +178,8 @@ class Hashes
 
    static const int UntilEOF = 0;
 
-   bool Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes = ~0);
+   bool Add(const unsigned char * const Data, unsigned long long const Size);
+   APT_DEPRECATED bool Add(const unsigned char * const Data, unsigned long long const Size, unsigned int const Hashes);
    inline bool Add(const char * const Data)
    {return Add((unsigned char const * const)Data,strlen(Data));};
    inline bool Add(const unsigned char * const Beg,const unsigned char * const End)
@@ -186,13 +187,24 @@ class Hashes
 
    enum SupportedHashes { MD5SUM = (1 << 0), SHA1SUM = (1 << 1), SHA256SUM = (1 << 2),
       SHA512SUM = (1 << 3) };
-   bool AddFD(int const Fd,unsigned long long Size = 0, unsigned int const Hashes = ~0);
-   bool AddFD(FileFd &Fd,unsigned long long Size = 0, unsigned int const Hashes = ~0);
+   bool AddFD(int const Fd,unsigned long long Size = 0);
+   APT_DEPRECATED bool AddFD(int const Fd,unsigned long long Size, unsigned int const Hashes);
+   bool AddFD(FileFd &Fd,unsigned long long Size = 0);
+   APT_DEPRECATED bool AddFD(FileFd &Fd,unsigned long long Size, unsigned int const Hashes);
 
    HashStringList GetHashStringList();
 
 APT_IGNORE_DEPRECATED_PUSH
+   /** create a Hashes object to calculate all supported hashes
+    *
+    * If ALL is too much, you can limit which Hashes are calculated
+    * with the following other constructors which mention explicitly
+    * which hashes to generate. */
    Hashes();
+   /** @param Hashes bitflag composed of #SupportedHashes */
+   Hashes(unsigned int const Hashes);
+   /** @param Hashes is a list of hashes */
+   Hashes(HashStringList const &Hashes);
    virtual ~Hashes();
 APT_IGNORE_DEPRECATED_POP
 
@@ -208,15 +220,16 @@ APT_IGNORE_DEPRECATED_POP
    }
 
    public:
+APT_IGNORE_DEPRECATED_PUSH
    APT_DEPRECATED bool AddFD(int const Fd, unsigned long long Size, bool const addMD5,
 	 bool const addSHA1, bool const addSHA256, bool const addSHA512) {
       return AddFD(Fd, Size, boolsToFlag(addMD5, addSHA1, addSHA256, addSHA512));
    };
-
    APT_DEPRECATED bool AddFD(FileFd &Fd, unsigned long long Size, bool const addMD5,
 	 bool const addSHA1, bool const addSHA256, bool const addSHA512) {
       return AddFD(Fd, Size, boolsToFlag(addMD5, addSHA1, addSHA256, addSHA512));
    };
+APT_IGNORE_DEPRECATED_POP
 };
 
 #endif
diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc
index 1dc268594..cc3527ea4 100644
--- a/ftparchive/cachedb.cc
+++ b/ftparchive/cachedb.cc
@@ -441,8 +441,8 @@ bool CacheDB::GetHashes(bool const GenOnly, unsigned int const DoHashes)
       if (OpenFile() == false)
 	 return false;
 
-      Hashes hashes;
-      if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize, FlHashes) == false)
+      Hashes hashes(FlHashes);
+      if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize) == false)
 	 return false;
 
       HashStringList hl = hashes.GetHashStringList();
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index db68c21f0..593278590 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -1075,8 +1075,8 @@ bool ReleaseWriter::DoPackage(string FileName)
 
    CheckSums[NewFileName].size = fd.Size();
 
-   Hashes hs;
-   hs.AddFD(fd, 0, DoHashes);
+   Hashes hs(DoHashes);
+   hs.AddFD(fd);
    CheckSums[NewFileName].Hashes = hs.GetHashStringList();
    fd.Close();
 
diff --git a/methods/cdrom.cc b/methods/cdrom.cc
index 74e2ecc6b..10cb29f66 100644
--- a/methods/cdrom.cc
+++ b/methods/cdrom.cc
@@ -266,7 +266,7 @@ bool CDROMMethod::Fetch(FetchItem *Itm)
    Res.LastModified = Buf.st_mtime;
    Res.Size = Buf.st_size;
 
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    FileFd Fd(Res.Filename, FileFd::ReadOnly);
    Hash.AddFD(Fd);
    Res.TakeHashes(Hash);
diff --git a/methods/copy.cc b/methods/copy.cc
index f3cd84215..a8e289df5 100644
--- a/methods/copy.cc
+++ b/methods/copy.cc
@@ -28,16 +28,16 @@
 class CopyMethod : public pkgAcqMethod
 {
    virtual bool Fetch(FetchItem *Itm);
-   void CalculateHashes(FetchResult &Res);
+   void CalculateHashes(FetchItem const * const Itm, FetchResult &Res);
    
    public:
    
    CopyMethod() : pkgAcqMethod("1.0",SingleInstance | SendConfig) {};
 };
 
-void CopyMethod::CalculateHashes(FetchResult &Res)
+void CopyMethod::CalculateHashes(FetchItem const * const Itm, FetchResult &Res)
 {
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    FileFd::CompressMode CompressMode = FileFd::None;
    if (_config->FindB("Acquire::GzipIndexes", false) == true)
       CompressMode = FileFd::Extension;
@@ -71,7 +71,7 @@ bool CopyMethod::Fetch(FetchItem *Itm)
    // just calc the hashes if the source and destination are identical
    if (File == Itm->DestFile)
    {
-      CalculateHashes(Res);
+      CalculateHashes(Itm, Res);
       URIDone(Res);
       return true;
    }
@@ -104,7 +104,7 @@ bool CopyMethod::Fetch(FetchItem *Itm)
    if (utimes(Res.Filename.c_str(), times) != 0)
       return _error->Errno("utimes",_("Failed to set modification time"));
 
-   CalculateHashes(Res);
+   CalculateHashes(Itm, Res);
 
    URIDone(Res);
    return true;
diff --git a/methods/file.cc b/methods/file.cc
index 5d9d7b951..043ab04b8 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -87,7 +87,7 @@ bool FileMethod::Fetch(FetchItem *Itm)
    if (Res.Filename.empty() == true)
       return _error->Error(_("File not found"));
 
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    FileFd Fd(Res.Filename, FileFd::ReadOnly);
    Hash.AddFD(Fd);
    Res.TakeHashes(Hash);
diff --git a/methods/ftp.cc b/methods/ftp.cc
index 7764acf6a..92d8573f1 100644
--- a/methods/ftp.cc
+++ b/methods/ftp.cc
@@ -1064,7 +1064,7 @@ bool FtpMethod::Fetch(FetchItem *Itm)
    }
    
    // Open the file
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    {
       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
       if (_error->PendingError() == true)
diff --git a/methods/gzip.cc b/methods/gzip.cc
index 387c05f2e..65519633c 100644
--- a/methods/gzip.cc
+++ b/methods/gzip.cc
@@ -91,7 +91,7 @@ bool GzipMethod::Fetch(FetchItem *Itm)
       return false;
 
    // Read data from source, generate checksums and write
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    bool Failed = false;
    while (1) 
    {
diff --git a/methods/http.cc b/methods/http.cc
index 947002cc6..e4773b0e2 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -64,8 +64,8 @@ const unsigned int CircleBuf::BW_HZ=10;
 // CircleBuf::CircleBuf - Circular input buffer				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-CircleBuf::CircleBuf(unsigned long long Size) 
-   : Size(Size), Hash(0), TotalWriten(0)
+CircleBuf::CircleBuf(unsigned long long Size)
+   : Size(Size), Hash(NULL), TotalWriten(0)
 {
    Buf = new unsigned char[Size];
    Reset();
@@ -84,10 +84,10 @@ void CircleBuf::Reset()
    TotalWriten = 0;
    MaxGet = (unsigned long long)-1;
    OutQueue = string();
-   if (Hash != 0)
+   if (Hash != NULL)
    {
       delete Hash;
-      Hash = new Hashes;
+      Hash = NULL;
    }
 }
 									/*}}}*/
@@ -222,7 +222,7 @@ bool CircleBuf::Write(int Fd)
 
       TotalWriten += Res;
       
-      if (Hash != 0)
+      if (Hash != NULL)
 	 Hash->Add(Buf + (OutP%Size),Res);
       
       OutP += Res;
@@ -484,10 +484,10 @@ APT_PURE bool HttpServerState::IsOpen()					/*{{{*/
    return (ServerFd != -1);
 }
 									/*}}}*/
-bool HttpServerState::InitHashes(FileFd &File)				/*{{{*/
+bool HttpServerState::InitHashes(FileFd &File, HashStringList const &ExpectedHashes)/*{{{*/
 {
    delete In.Hash;
-   In.Hash = new Hashes;
+   In.Hash = new Hashes(ExpectedHashes);
 
    // Set the expected size and read file for the hashes
    File.Truncate(StartPos);
diff --git a/methods/http.h b/methods/http.h
index 40a88a7be..6dc872659 100644
--- a/methods/http.h
+++ b/methods/http.h
@@ -111,7 +111,7 @@ struct HttpServerState: public ServerState
    virtual bool Open();
    virtual bool IsOpen();
    virtual bool Close();
-   virtual bool InitHashes(FileFd &File);
+   virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes);
    virtual Hashes * GetHashes();
    virtual bool Die(FileFd &File);
    virtual bool Flush(FileFd * const File);
diff --git a/methods/https.cc b/methods/https.cc
index 70f6a1046..81903b239 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -443,7 +443,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       Res.LastModified = resultStat.st_mtime;
 
    // take hashes
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    FileFd Fd(Res.Filename, FileFd::ReadOnly);
    Hash.AddFD(Fd);
    Res.TakeHashes(Hash);
diff --git a/methods/https.h b/methods/https.h
index 4cc48fc34..dc0ff3322 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -42,7 +42,7 @@ class HttpsServerState : public ServerState
    virtual bool Open() { return false; }
    virtual bool IsOpen() { return false; }
    virtual bool Close() { return false; }
-   virtual bool InitHashes(FileFd &/*File*/) { return false; }
+   virtual bool InitHashes(FileFd &/*File*/, HashStringList const &/*ExpectedHashes*/) { return false; }
    virtual Hashes * GetHashes() { return NULL; }
    virtual bool Die(FileFd &/*File*/) { return false; }
    virtual bool Flush(FileFd * const /*File*/) { return false; }
diff --git a/methods/rred.cc b/methods/rred.cc
index 774b58a40..554ac99b4 100644
--- a/methods/rred.cc
+++ b/methods/rred.cc
@@ -581,7 +581,7 @@ class RredMethod : public pkgAcqMethod {
 	 FILE *inp = fopen(Path.c_str(), "r");
 	 FILE *out = fopen(Itm->DestFile.c_str(), "w");
 
-	 Hashes hash;
+	 Hashes hash(Itm->ExpectedHashes);
 
 	 patch.apply_against_file(out, inp, &hash);
 
diff --git a/methods/rsh.cc b/methods/rsh.cc
index 0e949160b..52349c61c 100644
--- a/methods/rsh.cc
+++ b/methods/rsh.cc
@@ -477,7 +477,7 @@ bool RSHMethod::Fetch(FetchItem *Itm)
    }
 
    // Open the file
-   Hashes Hash;
+   Hashes Hash(Itm->ExpectedHashes);
    {
       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
       if (_error->PendingError() == true)
diff --git a/methods/server.cc b/methods/server.cc
index 91ec824d1..e403f1071 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -357,7 +357,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
    FailFd = File->Fd();
    FailTime = Server->Date;
 
-   if (Server->InitHashes(*File) == false)
+   if (Server->InitHashes(*File, Queue->ExpectedHashes) == false)
    {
       _error->Errno("read",_("Problem hashing file"));
       return ERROR_NOT_FROM_SERVER;
diff --git a/methods/server.h b/methods/server.h
index b974ec89a..45622dd34 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -85,7 +85,7 @@ struct ServerState
    virtual bool Open() = 0;
    virtual bool IsOpen() = 0;
    virtual bool Close() = 0;
-   virtual bool InitHashes(FileFd &File) = 0;
+   virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes) = 0;
    virtual Hashes * GetHashes() = 0;
    virtual bool Die(FileFd &File) = 0;
    virtual bool Flush(FileFd * const File) = 0;
diff --git a/test/libapt/hashsums_test.cc b/test/libapt/hashsums_test.cc
index a19a0befd..edcd8a11a 100644
--- a/test/libapt/hashsums_test.cc
+++ b/test/libapt/hashsums_test.cc
@@ -193,6 +193,30 @@ TEST(HashSumsTest, FileBased)
       EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
    }
    fd.Seek(0);
+   {
+      Hashes hashes(Hashes::MD5SUM | Hashes::SHA512SUM);
+      hashes.AddFD(fd);
+      HashStringList list = hashes.GetHashStringList();
+      EXPECT_FALSE(list.empty());
+      EXPECT_EQ(3, list.size());
+      EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue());
+      EXPECT_EQ(NULL, list.find("SHA1"));
+      EXPECT_EQ(NULL, list.find("SHA256"));
+      EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue());
+      EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
+      fd.Seek(0);
+      Hashes hashes2(list);
+      hashes2.AddFD(fd);
+      list = hashes2.GetHashStringList();
+      EXPECT_FALSE(list.empty());
+      EXPECT_EQ(3, list.size());
+      EXPECT_EQ(md5.Value(), list.find("MD5Sum")->HashValue());
+      EXPECT_EQ(NULL, list.find("SHA1"));
+      EXPECT_EQ(NULL, list.find("SHA256"));
+      EXPECT_EQ(sha512.Value(), list.find("SHA512")->HashValue());
+      EXPECT_EQ(FileSize, list.find("Checksum-FileSize")->HashValue());
+   }
+   fd.Seek(0);
    {
       MD5Summation MD5;
       MD5.AddFD(fd.Fd());
-- 
cgit v1.2.3-70-g09d2


From 34faa8f7ae2526f46cd1f84bb6962ad06d841e5e Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Sat, 11 Apr 2015 10:23:52 +0200
Subject: calculate hashes while downloading in https
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We do this in HTTP already to give the CPU some exercise while the disk
is heavily spinning (or flashing?) to store the data avoiding the need
to reread the entire file again later on to calculate the hashes – which
happens outside of the eyes of progress reporting, so you might ended up
with a bunch of https workers 'stuck' at 100% while they were busy
calculating hashes.

This is a bummer for everyone using apt as a connection speedtest as the
https method works slower now (not really, it just isn't reporting done
too early anymore).
---
 methods/http.cc                             |  8 +++---
 methods/http.h                              |  2 +-
 methods/https.cc                            | 38 ++++++++++++++++++++---------
 methods/https.h                             |  6 +++--
 methods/server.cc                           |  8 +++++-
 methods/server.h                            |  3 ++-
 test/integration/test-apt-download-progress |  6 +++--
 7 files changed, 47 insertions(+), 24 deletions(-)

(limited to 'methods')

diff --git a/methods/http.cc b/methods/http.cc
index e4773b0e2..af3d5ccb6 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -484,16 +484,14 @@ APT_PURE bool HttpServerState::IsOpen()					/*{{{*/
    return (ServerFd != -1);
 }
 									/*}}}*/
-bool HttpServerState::InitHashes(FileFd &File, HashStringList const &ExpectedHashes)/*{{{*/
+bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes)	/*{{{*/
 {
    delete In.Hash;
    In.Hash = new Hashes(ExpectedHashes);
-
-   // Set the expected size and read file for the hashes
-   File.Truncate(StartPos);
-   return In.Hash->AddFD(File, StartPos);
+   return true;
 }
 									/*}}}*/
+
 APT_PURE Hashes * HttpServerState::GetHashes()				/*{{{*/
 {
    return In.Hash;
diff --git a/methods/http.h b/methods/http.h
index 6dc872659..e73871931 100644
--- a/methods/http.h
+++ b/methods/http.h
@@ -111,7 +111,7 @@ struct HttpServerState: public ServerState
    virtual bool Open();
    virtual bool IsOpen();
    virtual bool Close();
-   virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes);
+   virtual bool InitHashes(HashStringList const &ExpectedHashes);
    virtual Hashes * GetHashes();
    virtual bool Die(FileFd &File);
    virtual bool Flush(FileFd * const File);
diff --git a/methods/https.cc b/methods/https.cc
index 81903b239..c6b75d9ad 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -72,18 +72,18 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       else
 	 me->https->Server->StartPos = 0;
 
-      me->https->File->Truncate(me->https->Server->StartPos);
-      me->https->File->Seek(me->https->Server->StartPos);
-
       me->Res->LastModified = me->https->Server->Date;
       me->Res->Size = me->https->Server->Size;
       me->Res->ResumePoint = me->https->Server->StartPos;
 
       // we expect valid data, so tell our caller we get the file now
-      if (me->https->Server->Result >= 200 && me->https->Server->Result < 300 &&
-	    me->https->Server->JunkSize == 0 &&
-	    me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
-	 me->https->URIStart(*me->Res);
+      if (me->https->Server->Result >= 200 && me->https->Server->Result < 300)
+      {
+	 if (me->https->Server->JunkSize == 0 && me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
+	    me->https->URIStart(*me->Res);
+	 if (me->https->Server->AddPartialFileToHashes(*(me->https->File)) == false)
+	    return 0;
+      }
    }
    else if (me->https->Server->HeaderLine(line) == false)
       return 0;
@@ -116,16 +116,31 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
       }
    }
 
+   if (me->Server->GetHashes()->Add((unsigned char const * const)buffer, buffer_size) == false)
+      return 0;
+
    return buffer_size;
 }
 
 // HttpsServerState::HttpsServerState - Constructor			/*{{{*/
-HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner)
+HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner), Hash(NULL)
 {
    TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
    Reset();
 }
 									/*}}}*/
+bool HttpsServerState::InitHashes(HashStringList const &ExpectedHashes)	/*{{{*/
+{
+   delete Hash;
+   Hash = new Hashes(ExpectedHashes);
+   return true;
+}
+									/*}}}*/
+APT_PURE Hashes * HttpsServerState::GetHashes()				/*{{{*/
+{
+   return Hash;
+}
+									/*}}}*/
 
 void HttpsMethod::SetupProxy()						/*{{{*/
 {
@@ -365,6 +380,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    // go for it - if the file exists, append on it
    File = new FileFd(Itm->DestFile, FileFd::WriteAny);
    Server = CreateServerState(Itm->Uri);
+   if (Server->InitHashes(Itm->ExpectedHashes) == false)
+      return false;
 
    // keep apt updated
    Res.Filename = Itm->DestFile;
@@ -443,10 +460,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       Res.LastModified = resultStat.st_mtime;
 
    // take hashes
-   Hashes Hash(Itm->ExpectedHashes);
-   FileFd Fd(Res.Filename, FileFd::ReadOnly);
-   Hash.AddFD(Fd);
-   Res.TakeHashes(Hash);
+   Res.TakeHashes(*(Server->GetHashes()));
 
    // keep apt updated
    URIDone(Res);
diff --git a/methods/https.h b/methods/https.h
index dc0ff3322..6e32e8d3d 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -29,6 +29,8 @@ class FileFd;
 
 class HttpsServerState : public ServerState
 {
+   Hashes * Hash;
+
    protected:
    virtual bool ReadHeaderLines(std::string &/*Data*/) { return false; }
    virtual bool LoadNextResponse(bool const /*ToFile*/, FileFd * const /*File*/) { return false; }
@@ -42,8 +44,8 @@ class HttpsServerState : public ServerState
    virtual bool Open() { return false; }
    virtual bool IsOpen() { return false; }
    virtual bool Close() { return false; }
-   virtual bool InitHashes(FileFd &/*File*/, HashStringList const &/*ExpectedHashes*/) { return false; }
-   virtual Hashes * GetHashes() { return NULL; }
+   virtual bool InitHashes(HashStringList const &ExpectedHashes);
+   virtual Hashes * GetHashes();
    virtual bool Die(FileFd &/*File*/) { return false; }
    virtual bool Flush(FileFd * const /*File*/) { return false; }
    virtual bool Go(bool /*ToFile*/, FileFd * const /*File*/) { return false; }
diff --git a/methods/server.cc b/methods/server.cc
index e403f1071..2116926b0 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -235,6 +235,12 @@ ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOu
    Reset();
 }
 									/*}}}*/
+bool ServerState::AddPartialFileToHashes(FileFd &File)			/*{{{*/
+{
+   File.Truncate(StartPos);
+   return GetHashes()->AddFD(File, StartPos);
+}
+									/*}}}*/
 
 bool ServerMethod::Configuration(string Message)			/*{{{*/
 {
@@ -357,7 +363,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
    FailFd = File->Fd();
    FailTime = Server->Date;
 
-   if (Server->InitHashes(*File, Queue->ExpectedHashes) == false)
+   if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
    {
       _error->Errno("read",_("Problem hashing file"));
       return ERROR_NOT_FROM_SERVER;
diff --git a/methods/server.h b/methods/server.h
index 45622dd34..1b1f754a3 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -72,6 +72,7 @@ struct ServerState
    };
    /** \brief Get the headers before the data */
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
+   bool AddPartialFileToHashes(FileFd &File);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
    virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
@@ -85,7 +86,7 @@ struct ServerState
    virtual bool Open() = 0;
    virtual bool IsOpen() = 0;
    virtual bool Close() = 0;
-   virtual bool InitHashes(FileFd &File, HashStringList const &ExpectedHashes) = 0;
+   virtual bool InitHashes(HashStringList const &ExpectedHashes) = 0;
    virtual Hashes * GetHashes() = 0;
    virtual bool Die(FileFd &File) = 0;
    virtual bool Flush(FileFd * const File) = 0;
diff --git a/test/integration/test-apt-download-progress b/test/integration/test-apt-download-progress
index b2c9effe6..07c5e09c5 100755
--- a/test/integration/test-apt-download-progress
+++ b/test/integration/test-apt-download-progress
@@ -26,14 +26,16 @@ assertprogress() {
 TESTFILE=testfile.big
 testsuccess dd if=/dev/zero of=./aptarchive/$TESTFILE bs=800k count=1 
 
+OPT='-o APT::Status-Fd=3 -o Debug::pkgAcquire::Worker=1 -o Debug::Acquire::http=1 -o Debug::Acquire::https=1'
+
 msgtest 'download progress works via' 'http'
 exec 3> apt-progress.log
-testsuccess --nomsg apthelper download-file "http://localhost:8080/$TESTFILE" http-$TESTFILE -o APT::Status-Fd=3 -o Acquire::http::Dl-Limit=800
+testsuccess --nomsg apthelper download-file "http://localhost:8080/$TESTFILE" http-$TESTFILE $OPT -o Acquire::http::Dl-Limit=800
 assertprogress apt-progress.log
 
 msgtest 'download progress works via' 'https'
 exec 3> apt-progress.log
-testsuccess --nomsg apthelper download-file "https://localhost:4433/$TESTFILE" https-$TESTFILE -o APT::Status-Fd=3 -o Acquire::https::Dl-Limit=800
+testsuccess --nomsg apthelper download-file "https://localhost:4433/$TESTFILE" https-$TESTFILE $OPT -o Acquire::https::Dl-Limit=800
 assertprogress apt-progress.log
 
 # cleanup
-- 
cgit v1.2.3-70-g09d2


From dcbb364fc69e1108b3fea3adb12a7ba83d9af467 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Tue, 12 May 2015 00:30:16 +0200
Subject: detect 416 complete file in partial by expected hash

If we have the expected hashes we can check with them if the file we
have in partial we got a 416 for is the expected file. We detected this
with same-size before, but not every server sends a good Content-Range
header with a 416 response.
---
 methods/https.cc                              | 37 +++++++++++++++++++++------
 methods/https.h                               |  1 +
 methods/server.cc                             | 19 +++++++++++---
 test/integration/framework                    |  2 +-
 test/integration/test-apt-update-transactions |  1 +
 test/integration/test-partial-file-support    | 10 ++++++--
 test/interactive-helper/aptwebserver.cc       |  9 ++++---
 7 files changed, 62 insertions(+), 17 deletions(-)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index c6b75d9ad..712e9ee73 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -40,7 +40,9 @@ using namespace std;
 struct APT_HIDDEN CURLUserPointer {
    HttpsMethod * const https;
    HttpsMethod::FetchResult * const Res;
-   CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res) : https(https), Res(Res) {}
+   HttpsMethod::FetchItem const * const Itm;
+   CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res,
+	 HttpsMethod::FetchItem const * const Itm) : https(https), Res(Res), Itm(Itm) {}
 };
 
 size_t
@@ -61,13 +63,32 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
    {
       if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0)
 	 ;
-      else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize())
+      else if (me->https->Server->Result == 416)
       {
-         me->https->Server->Result = 200;
-	 me->https->Server->StartPos = me->https->Server->Size;
-	 // the actual size is not important for https as curl will deal with it
-	 // by itself and e.g. doesn't bother us with transport-encoding…
-	 me->https->Server->JunkSize = std::numeric_limits::max();
+	 bool partialHit = false;
+	 if (me->Itm->ExpectedHashes.usable() == true)
+	 {
+	    Hashes resultHashes(me->Itm->ExpectedHashes);
+	    FileFd file(me->Itm->DestFile, FileFd::ReadOnly);
+	    me->https->Server->Size = file.FileSize();
+	    me->https->Server->Date = file.ModificationTime();
+	    resultHashes.AddFD(file);
+	    HashStringList const hashList = resultHashes.GetHashStringList();
+	    partialHit = (me->Itm->ExpectedHashes == hashList);
+	 }
+	 else if (me->https->Server->Result == 416 && me->https->Server->Size == me->https->File->FileSize())
+	    partialHit = true;
+
+	 if (partialHit == true)
+	 {
+	    me->https->Server->Result = 200;
+	    me->https->Server->StartPos = me->https->Server->Size;
+	    // the actual size is not important for https as curl will deal with it
+	    // by itself and e.g. doesn't bother us with transport-encoding…
+	    me->https->Server->JunkSize = std::numeric_limits::max();
+	 }
+	 else
+	    me->https->Server->StartPos = 0;
       }
       else
 	 me->https->Server->StartPos = 0;
@@ -221,7 +242,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
 
    FetchResult Res;
-   CURLUserPointer userp(this, &Res);
+   CURLUserPointer userp(this, &Res, Itm);
    // callbacks
    curl_easy_setopt(curl, CURLOPT_URL, static_cast(Uri).c_str());
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
diff --git a/methods/https.h b/methods/https.h
index 6e32e8d3d..57fc292ee 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -79,6 +79,7 @@ class HttpsMethod : public ServerMethod
    virtual bool Configuration(std::string Message);
    virtual ServerState * CreateServerState(URI uri);
    using pkgAcqMethod::FetchResult;
+   using pkgAcqMethod::FetchItem;
 
    HttpsMethod() : ServerMethod("1.2",Pipeline | SendConfig), File(NULL)
    {
diff --git a/methods/server.cc b/methods/server.cc
index 2116926b0..bd01c3e98 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -269,7 +269,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
       Res.LastModified = Queue->LastModified;
       return IMS_HIT;
    }
-   
+
    /* Redirect
     *
     * Note that it is only OK for us to treat all redirection the same
@@ -314,7 +314,20 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
       struct stat SBuf;
       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
       {
-	 if ((unsigned long long)SBuf.st_size == Server->Size)
+	 bool partialHit = false;
+	 if (Queue->ExpectedHashes.usable() == true)
+	 {
+	    Hashes resultHashes(Queue->ExpectedHashes);
+	    FileFd file(Queue->DestFile, FileFd::ReadOnly);
+	    Server->Size = file.FileSize();
+	    Server->Date = file.ModificationTime();
+	    resultHashes.AddFD(file);
+	    HashStringList const hashList = resultHashes.GetHashStringList();
+	    partialHit = (Queue->ExpectedHashes == hashList);
+	 }
+	 else if ((unsigned long long)SBuf.st_size == Server->Size)
+	    partialHit = true;
+	 if (partialHit == true)
 	 {
 	    // the file is completely downloaded, but was not moved
 	    if (Server->HaveContent == true)
@@ -351,7 +364,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
    // This is some sort of 2xx 'data follows' reply
    Res.LastModified = Server->Date;
    Res.Size = Server->Size;
-   
+
    // Open the file
    delete File;
    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
diff --git a/test/integration/framework b/test/integration/framework
index 03c188189..2a53e8365 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1128,7 +1128,7 @@ acquire::cdrom::autodetect 0;" > rootdir/etc/apt/apt.conf.d/00cdrom
 downloadfile() {
 	local PROTO="${1%%:*}"
 	if ! apthelper -o Debug::Acquire::${PROTO}=1 -o Debug::pkgAcquire::Worker=1 \
-		download-file "$1" "$2" 2>&1 ; then
+		download-file "$1" "$2" "$3" 2>&1 ; then
 		return 1
 	fi
 	# only if the file exists the download was successful
diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions
index f028ac0c7..67dd633f9 100755
--- a/test/integration/test-apt-update-transactions
+++ b/test/integration/test-apt-update-transactions
@@ -63,6 +63,7 @@ testsetup 'file'
 changetowebserver
 webserverconfig 'aptwebserver::support::modified-since' 'false' "$1"
 webserverconfig 'aptwebserver::support::last-modified' 'false' "$1"  # curl is clever and sees hits here also
+webserverconfig 'aptwebserver::support::range' 'false' "$1"
 
 testsetup 'http'
 
diff --git a/test/integration/test-partial-file-support b/test/integration/test-partial-file-support
index 85046b3eb..c07af7bd0 100755
--- a/test/integration/test-partial-file-support
+++ b/test/integration/test-partial-file-support
@@ -17,8 +17,8 @@ DOWNLOADLOG='rootdir/tmp/testdownloadfile.log'
 
 testdownloadfile() {
 	rm -f "$DOWNLOADLOG"
-	msgtest "Testing download of file $2 with" "$1"
-	if ! downloadfile "$2" "$3" > "$DOWNLOADLOG"; then
+	msgtest "Testing download of file $2 with" "$1 $5"
+	if ! downloadfile "$2" "$3" "$5" > "$DOWNLOADLOG"; then
 		cat >&2 "$DOWNLOADLOG"
 		msgfail
 	else
@@ -78,6 +78,12 @@ followuprequest() {
 	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '='
 	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
 
+	webserverconfig 'aptwebserver::support::content-range' 'false'
+	copysource $TESTFILE 1M $DOWN
+	testdownloadfile 'completely downloaded file' "${1}/testfile" "$DOWN" '=' "SHA1:$(sha1sum "$TESTFILE" | cut -d' ' -f 1)"
+	testwebserverlaststatuscode '416' "$DOWNLOADLOG"
+	webserverconfig 'aptwebserver::support::content-range' 'true'
+
 	copysource $TESTFILE 1M $DOWN
 	copysource "${TESTFILE}2" 20 "${DOWN}2"
 	msgtest 'Testing download of files with' 'completely downloaded file + partial file'
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 6a411e24e..c933060e7 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -745,9 +745,12 @@ static void * handleClient(void * voidclient)				/*{{{*/
 		     }
 		     else
 		     {
-			std::ostringstream contentrange;
-			contentrange << "Content-Range: bytes */" << filesize;
-			headers.push_back(contentrange.str());
+			if (_config->FindB("aptwebserver::support::content-range", true) == true)
+			{
+			   std::ostringstream contentrange;
+			   contentrange << "Content-Range: bytes */" << filesize;
+			   headers.push_back(contentrange.str());
+			}
 			sendError(client, 416, *m, sendContent, "", headers);
 			break;
 		     }
-- 
cgit v1.2.3-70-g09d2


From 8eafc759544298211cd0bfaa3919afc0fadd47d1 Mon Sep 17 00:00:00 2001
From: David Kalnischkies 
Date: Wed, 13 May 2015 16:09:12 +0200
Subject: detect Releasefile IMS hits even if the server doesn't

Not all servers we are talking to support If-Modified-Since and some are
not even sending Last-Modified for us, so in an effort to detect such
hits we run a hashsum check on the 'old' compared to the 'new' file, we
got the hashes for the 'new' already for "free" from the methods anyway
and hence just need to calculated the old ones.

This allows us to detect hits even with unsupported servers, which in
turn means we benefit from all the new hit behavior also here.
---
 apt-pkg/acquire-item.cc                        | 31 ++++++++++----
 apt-pkg/acquire-item.h                         |  2 +-
 apt-pkg/acquire-worker.cc                      |  8 ++--
 methods/https.cc                               |  2 +-
 test/integration/framework                     |  9 ++++
 test/integration/test-apt-update-expected-size |  1 +
 test/integration/test-apt-update-not-modified  | 58 +++++++++++++++++++++++++-
 test/integration/test-apt-update-rollback      |  1 +
 test/integration/test-apt-update-transactions  |  2 +
 9 files changed, 99 insertions(+), 15 deletions(-)

(limited to 'methods')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 1090912f5..78dace12c 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -253,7 +253,10 @@ bool pkgAcquire::Item::Rename(string From,string To)
    strprintf(S, _("rename failed, %s (%s -> %s)."), strerror(errno),
 	 From.c_str(),To.c_str());
    Status = StatError;
-   ErrorText += S;
+   if (ErrorText.empty())
+      ErrorText = S;
+   else
+      ErrorText = ErrorText + ": " + S;
    return false;
 }
 									/*}}}*/
@@ -1794,7 +1797,7 @@ void pkgAcqMetaSig::Done(string Message,unsigned long long Size,
 
    if(AuthPass == false)
    {
-      if(CheckDownloadDone(Message) == true)
+      if(CheckDownloadDone(Message, Hashes) == true)
       {
          // destfile will be modified to point to MetaIndexFile for the
          // gpgv method, so we need to save it here
@@ -1837,7 +1840,8 @@ void pkgAcqMetaSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf)/*{{{*/
          Status = StatDone;
       } else {
          _error->Error("%s", downgrade_msg.c_str());
-         Rename(MetaIndexFile, MetaIndexFile+".FAILED");
+	 if (TransactionManager->IMSHit == false)
+	    Rename(MetaIndexFile, MetaIndexFile+".FAILED");
 	 Item::Failed("Message: " + downgrade_msg, Cnf);
          TransactionManager->AbortTransaction();
          return;
@@ -1922,12 +1926,12 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long long Size,	/*{{{*/
 {
    Item::Done(Message,Size,Hashes,Cfg);
 
-   if(CheckDownloadDone(Message))
+   if(CheckDownloadDone(Message, Hashes))
    {
       // we have a Release file, now download the Signature, all further
       // verify/queue for additional downloads will be done in the
       // pkgAcqMetaSig::Done() code
-      std::string MetaIndexFile = DestFile;
+      std::string const MetaIndexFile = DestFile;
       new pkgAcqMetaSig(Owner, TransactionManager, 
                         MetaIndexSigURI, MetaIndexSigURIDesc,
                         MetaIndexSigShortDesc, MetaIndexFile, IndexTargets, 
@@ -2008,7 +2012,7 @@ void pkgAcqMetaBase::QueueForSignatureVerify(const std::string &MetaIndexFile,
 }
 									/*}}}*/
 // pkgAcqMetaBase::CheckDownloadDone					/*{{{*/
-bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message)
+bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message, HashStringList const &Hashes)
 {
    // We have just finished downloading a Release file (it is not
    // verified yet)
@@ -2031,7 +2035,18 @@ bool pkgAcqMetaBase::CheckDownloadDone(const std::string &Message)
 
    // make sure to verify against the right file on I-M-S hit
    IMSHit = StringToBool(LookupTag(Message,"IMS-Hit"),false);
-   if(IMSHit)
+   if (IMSHit == false)
+   {
+      // detect IMS-Hits servers haven't detected by Hash comparison
+      std::string FinalFile = GetFinalFilename();
+      if (RealFileExists(FinalFile) && Hashes.VerifyFile(FinalFile) == true)
+      {
+	 IMSHit = true;
+	 unlink(DestFile.c_str());
+      }
+   }
+
+   if(IMSHit == true)
    {
       // for simplicity, the transaction manager is always InRelease
       // even if it doesn't exist.
@@ -2273,7 +2288,7 @@ void pkgAcqMetaClearSig::Done(std::string Message,unsigned long long Size,
 
    if(AuthPass == false)
    {
-      if(CheckDownloadDone(Message) == true)
+      if(CheckDownloadDone(Message, Hashes) == true)
          QueueForSignatureVerify(DestFile, DestFile);
       return;
    }
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 33a28671c..646de8416 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -429,7 +429,7 @@ class pkgAcqMetaBase  : public pkgAcquire::Item				/*{{{*/
     *  \param Message The message block received from the fetch
     *  subprocess.
     */
-   bool CheckDownloadDone(const std::string &Message);
+   bool CheckDownloadDone(const std::string &Message, HashStringList const &Hashes);
 
    /** \brief Queue the downloaded Signature for verification */
    void QueueForSignatureVerify(const std::string &MetaIndexFile,
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index bee01e620..9254e20a3 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -333,13 +333,12 @@ bool pkgAcquire::Worker::RunMessages()
 
 	    // see if there is a hash to verify
 	    HashStringList ReceivedHashes;
-	    HashStringList expectedHashes = Owner->HashSums();
-	    for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs)
+	    for (char const * const * type = HashString::SupportedHashes(); *type != NULL; ++type)
 	    {
-	       std::string const tagname = hs->HashType() + "-Hash";
+	       std::string const tagname = std::string(*type) + "-Hash";
 	       std::string const hashsum = LookupTag(Message, tagname.c_str());
 	       if (hashsum.empty() == false)
-		  ReceivedHashes.push_back(HashString(hs->HashType(), hashsum));
+		  ReceivedHashes.push_back(HashString(*type, hashsum));
 	    }
 
 	    if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
@@ -349,6 +348,7 @@ bool pkgAcquire::Worker::RunMessages()
 	       for (HashStringList::const_iterator hs = ReceivedHashes.begin(); hs != ReceivedHashes.end(); ++hs)
 		  std::clog <<  "\t- " << hs->toStr() << std::endl;
 	       std::clog << "ExpectedHash:" << endl;
+	       HashStringList expectedHashes = Owner->HashSums();
 	       for (HashStringList::const_iterator hs = expectedHashes.begin(); hs != expectedHashes.end(); ++hs)
 		  std::clog <<  "\t- " << hs->toStr() << std::endl;
 	       std::clog << endl;
diff --git a/methods/https.cc b/methods/https.cc
index 712e9ee73..fa143439a 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -444,7 +444,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       char err[255];
       snprintf(err, sizeof(err) - 1, "HttpError%i", Server->Result);
       SetFailReason(err);
-      _error->Error("%s", err);
+      _error->Error("%i %s", Server->Result, Server->Code);
       // unlink, no need keep 401/404 page content in partial/
       unlink(File->Name().c_str());
       return false;
diff --git a/test/integration/framework b/test/integration/framework
index 2a53e8365..8c8936ead 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -1005,6 +1005,15 @@ signreleasefiles() {
 	msgdone "info"
 }
 
+redatereleasefiles() {
+	local DATE="$(date -d "$1" '+%a, %d %b %Y %H:%M:%S %Z')"
+	for release in $(find aptarchive/ -name 'Release'); do
+		sed -i "s/^Date: .*$/Date: ${DATE}/" $release
+		touch -d "$DATE" $release
+	done
+	signreleasefiles "${2:-Joe Sixpack}"
+}
+
 webserverconfig() {
 	local WEBSERVER="${3:-http://localhost:8080}"
 	local NOCHECK=false
diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size
index 7efccaa57..55a5da848 100755
--- a/test/integration/test-apt-update-expected-size
+++ b/test/integration/test-apt-update-expected-size
@@ -26,6 +26,7 @@ test_inreleasetoobig() {
 }
 
 test_packagestoobig() {
+	redatereleasefiles '+1hour'
 	# append junk at the end of the Packages.gz/Packages
 	SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)"
 	find aptarchive/dists -name 'Packages*' | while read pkg; do
diff --git a/test/integration/test-apt-update-not-modified b/test/integration/test-apt-update-not-modified
index 2dc56e76c..a67ecb760 100755
--- a/test/integration/test-apt-update-not-modified
+++ b/test/integration/test-apt-update-not-modified
@@ -12,7 +12,7 @@ insertpackage 'unstable' 'apt' 'all' '1.0'
 setupaptarchive --no-update
 
 methodtest() {
-	msgmsg 'Test with' "$1"
+	msgmsg 'Test InRelease with' "$1"
 	rm -rf rootdir/var/lib/apt/lists
 	# get our cache populated
 	testsuccess aptget update
@@ -36,6 +36,62 @@ Reading package lists..." aptget update
 Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
 Reading package lists..." aptget update
 	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+	webserverconfig 'aptwebserver::support::modified-since' 'false'
+	webserverconfig 'aptwebserver::support::last-modified' 'false'
+	testsuccessequal "Get:1 $1 unstable InRelease [$(stat -c '%s' 'aptarchive/dists/unstable/InRelease') B]
+Reading package lists..." aptget update
+	webserverconfig 'aptwebserver::support::modified-since' 'true'
+	webserverconfig 'aptwebserver::support::last-modified' 'true'
+
+	msgmsg 'Test Release.gpg with' "$1"
+	rm -rf rootdir/var/lib/apt/lists
+	cp -a aptarchive/dists  aptarchive/dists.good
+	find aptarchive/dists -name 'InRelease' -delete
+	# get our cache populated
+	testsuccess aptget update
+	listcurrentlistsdirectory > listsdir.lst
+
+	# hit again with a good cache
+	testsuccessequal "Ign $1 unstable InRelease
+  404  Not Found
+Hit $1 unstable Release
+Hit $1 unstable Release.gpg
+Reading package lists..." aptget update
+	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+	# drop an architecture, which means the file should be gone now
+	configarchitecture 'i386'
+	sed '/_binary-amd64_Packages/ d' listsdir.lst > listsdir-without-amd64.lst
+	testsuccessequal "Ign $1 unstable InRelease
+  404  Not Found
+Hit $1 unstable Release
+Hit $1 unstable Release.gpg
+Reading package lists..." aptget update
+	testfileequal 'listsdir-without-amd64.lst' "$(listcurrentlistsdirectory)"
+
+	# readd arch so its downloaded again
+	configarchitecture 'amd64' 'i386'
+	testsuccessequal "Ign $1 unstable InRelease
+  404  Not Found
+Hit $1 unstable Release
+Hit $1 unstable Release.gpg
+Get:1 $1 unstable/main amd64 Packages [$(stat -c '%s' 'aptarchive/dists/unstable/main/binary-amd64/Packages.gz') B]
+Reading package lists..." aptget update
+	testfileequal 'listsdir.lst' "$(listcurrentlistsdirectory)"
+
+	webserverconfig 'aptwebserver::support::modified-since' 'false'
+	webserverconfig 'aptwebserver::support::last-modified' 'false'
+	testsuccessequal "Ign $1 unstable InRelease
+  404  Not Found
+Get:1 $1 unstable Release [$(stat -c '%s' 'aptarchive/dists/unstable/Release') B]
+Get:2 $1 unstable Release.gpg [$(stat -c '%s' 'aptarchive/dists/unstable/Release.gpg') B]
+Reading package lists..." aptget update
+	webserverconfig 'aptwebserver::support::modified-since' 'true'
+	webserverconfig 'aptwebserver::support::last-modified' 'true'
+
+	rm -rf aptarchive/dists
+	cp -a aptarchive/dists.good aptarchive/dists
 }
 
 changetowebserver
diff --git a/test/integration/test-apt-update-rollback b/test/integration/test-apt-update-rollback
index f4500b69d..29fe1ab56 100755
--- a/test/integration/test-apt-update-rollback
+++ b/test/integration/test-apt-update-rollback
@@ -158,6 +158,7 @@ test_inrelease_to_broken_gzip() {
     msgmsg "Test InRelease to broken gzip"
     start_with_good_inrelease
 
+    redatereleasefiles '+2hours'
     # append junk at the end of the compressed file
     echo "lala" >> $APTARCHIVE/dists/unstable/main/source/Sources.gz
     touch -d '+2min' $APTARCHIVE/dists/unstable/main/source/Sources.gz
diff --git a/test/integration/test-apt-update-transactions b/test/integration/test-apt-update-transactions
index 67dd633f9..63b318056 100755
--- a/test/integration/test-apt-update-transactions
+++ b/test/integration/test-apt-update-transactions
@@ -47,6 +47,7 @@ testrun() {
 
 testsetup() {
 	msgmsg 'Test with no initial data over' "$1"
+	redatereleasefiles 'now'
 	rm -rf rootdir/var/lib/apt/lists
 	mkdir -p rootdir/var/lib/apt/lists/partial
 	listcurrentlistsdirectory > listsdir.lst
@@ -55,6 +56,7 @@ testsetup() {
 	msgmsg 'Test with initial data over' "$1"
 	testsuccess aptget update
 	listcurrentlistsdirectory > listsdir.lst
+	redatereleasefiles '+1hour'
 	testrun 'listsdir.lst'
 }
 
-- 
cgit v1.2.3-70-g09d2


From ceafe8a6edc815df2923ba892894617829e9d3c2 Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Fri, 22 May 2015 15:28:53 +0200
Subject: Fix endless loop in apt-get update that can cause disk fillup

The apt http code parses Content-Length and Content-Range. For
both requests the variable "Size" is used and the semantic for
this Size is the total file size. However Content-Length is not
the entire file size for partital file requests. For servers that
send the Content-Range header first and then the Content-Length
header this can lead to globbing of Size so that its less than
the real file size. This may lead to a subsequent passing of a
negative number into the CircleBuf which leads to a endless
loop that writes data.

Thanks to Anton Blanchard for the analysis and initial patch.

LP: #1445239
---
 methods/http.cc                         |  2 +-
 methods/server.cc                       | 20 +++++++++++++++-----
 methods/server.h                        |  3 ++-
 test/interactive-helper/aptwebserver.cc |  6 +++---
 4 files changed, 21 insertions(+), 10 deletions(-)

(limited to 'methods')

diff --git a/methods/http.cc b/methods/http.cc
index 1b996db98..ad90c9891 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -443,7 +443,7 @@ bool HttpServerState::RunData(FileFd * const File)
       else if (JunkSize != 0)
 	 In.Limit(JunkSize);
       else
-	 In.Limit(Size - StartPos);
+	 In.Limit(DownloadSize);
       
       // Just transfer the whole block.
       do
diff --git a/methods/server.cc b/methods/server.cc
index e321e0230..ba0a8864b 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -164,15 +164,22 @@ bool ServerState::HeaderLine(string Line)
 	 Encoding = Stream;
       HaveContent = true;
 
-      unsigned long long * SizePtr = &Size;
+      unsigned long long * DownloadSizePtr = &DownloadSize;
       if (Result == 416)
-	 SizePtr = &JunkSize;
+	 DownloadSizePtr = &JunkSize;
 
-      *SizePtr = strtoull(Val.c_str(), NULL, 10);
-      if (*SizePtr >= std::numeric_limits::max())
+      *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
+      if (*DownloadSizePtr >= std::numeric_limits::max())
 	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
-      else if (*SizePtr == 0)
+      else if (*DownloadSizePtr == 0)
 	 HaveContent = false;
+
+      // On partial content (206) the Content-Length less than the real
+      // size, so do not set it here but leave that to the Content-Range
+      // header instead
+      if(Result != 206 && Size == 0)
+         Size = DownloadSize;
+
       return true;
    }
 
@@ -193,6 +200,9 @@ bool ServerState::HeaderLine(string Line)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
       if ((unsigned long long)StartPos > Size)
 	 return _error->Error(_("This HTTP server has broken range support"));
+
+      // figure out what we will download
+      DownloadSize = Size - StartPos;
       return true;
    }
 
diff --git a/methods/server.h b/methods/server.h
index 1b81e3549..ed3cb456a 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,7 +34,8 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
-   unsigned long long Size; // size of the usable content (aka: the file)
+   unsigned long long Size; // total size of the usable content (aka: the file)
+   unsigned long long DownloadSize; // size we actually download (can be smaller than Size if we have partial content)
    unsigned long long JunkSize; // size of junk content (aka: server error pages)
    unsigned long long StartPos;
    time_t Date;
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index cd52da692..9c67b67e4 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -654,13 +654,13 @@ static void * handleClient(void * voidclient)				/*{{{*/
 		     if (filesize > filestart)
 		     {
 			data.Skip(filestart);
-			std::ostringstream contentlength;
-			contentlength << "Content-Length: " << (filesize - filestart);
-			headers.push_back(contentlength.str());
 			std::ostringstream contentrange;
 			contentrange << "Content-Range: bytes " << filestart << "-"
 			   << filesize - 1 << "/" << filesize;
 			headers.push_back(contentrange.str());
+			std::ostringstream contentlength;
+			contentlength << "Content-Length: " << (filesize - filestart);
+			headers.push_back(contentlength.str());
 			sendHead(client, 206, headers);
 			if (sendContent == true)
 			   sendFile(client, headers, data);
-- 
cgit v1.2.3-70-g09d2


From 6291f60e86718697f261519a6818e1d5ee433216 Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Fri, 22 May 2015 15:40:18 +0200
Subject: Rename "Size" in ServerState to TotalFileSize

The variable "Size" was misleading and caused bug #1445239. To
avoid similar issues in the future, rename it to make the meaning
more obvious.

git-dch: ignore
---
 methods/https.cc  |  4 ++--
 methods/server.cc | 20 ++++++++++----------
 methods/server.h  | 14 ++++++++++----
 3 files changed, 22 insertions(+), 16 deletions(-)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index 3a5981b58..12fc6c70f 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -55,10 +55,10 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
    {
       if (me->Server->Result != 416 && me->Server->StartPos != 0)
 	 ;
-      else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize())
+      else if (me->Server->Result == 416 && me->Server->TotalFileSize == me->File->FileSize())
       {
          me->Server->Result = 200;
-	 me->Server->StartPos = me->Server->Size;
+	 me->Server->StartPos = me->Server->TotalFileSize;
 	 // the actual size is not important for https as curl will deal with it
 	 // by itself and e.g. doesn't bother us with transport-encoding…
 	 me->Server->JunkSize = std::numeric_limits::max();
diff --git a/methods/server.cc b/methods/server.cc
index ba0a8864b..6c05700a5 100644
--- a/methods/server.cc
+++ b/methods/server.cc
@@ -54,7 +54,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
    Major = 0; 
    Minor = 0; 
    Result = 0; 
-   Size = 0; 
+   TotalFileSize = 0;
    JunkSize = 0;
    StartPos = 0;
    Encoding = Closes;
@@ -177,8 +177,8 @@ bool ServerState::HeaderLine(string Line)
       // On partial content (206) the Content-Length less than the real
       // size, so do not set it here but leave that to the Content-Range
       // header instead
-      if(Result != 206 && Size == 0)
-         Size = DownloadSize;
+      if(Result != 206 && TotalFileSize == 0)
+         TotalFileSize = DownloadSize;
 
       return true;
    }
@@ -194,15 +194,15 @@ bool ServerState::HeaderLine(string Line)
       HaveContent = true;
 
       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
-      if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
+      if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
 	 ; // we got the expected filesize which is all we wanted
-      else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
+      else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
 	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
-      if ((unsigned long long)StartPos > Size)
+      if ((unsigned long long)StartPos > TotalFileSize)
 	 return _error->Error(_("This HTTP server has broken range support"));
 
       // figure out what we will download
-      DownloadSize = Size - StartPos;
+      DownloadSize = TotalFileSize - StartPos;
       return true;
    }
 
@@ -313,7 +313,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
       struct stat SBuf;
       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
       {
-	 if ((unsigned long long)SBuf.st_size == Server->Size)
+	 if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
 	 {
 	    // the file is completely downloaded, but was not moved
 	    if (Server->HaveContent == true)
@@ -323,7 +323,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
 	       Server->RunData(&DevNull);
 	    }
 	    Server->HaveContent = false;
-	    Server->StartPos = Server->Size;
+	    Server->StartPos = Server->TotalFileSize;
 	    Server->Result = 200;
 	 }
 	 else if (unlink(Queue->DestFile.c_str()) == 0)
@@ -349,7 +349,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res)
 
    // This is some sort of 2xx 'data follows' reply
    Res.LastModified = Server->Date;
-   Res.Size = Server->Size;
+   Res.Size = Server->TotalFileSize;
    
    // Open the file
    delete File;
diff --git a/methods/server.h b/methods/server.h
index ed3cb456a..8c14282b6 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -34,10 +34,16 @@ struct ServerState
    char Code[360];
 
    // These are some statistics from the last parsed header lines
-   unsigned long long Size; // total size of the usable content (aka: the file)
-   unsigned long long DownloadSize; // size we actually download (can be smaller than Size if we have partial content)
-   unsigned long long JunkSize; // size of junk content (aka: server error pages)
+
+   // total size of the usable content (aka: the file)
+   unsigned long long TotalFileSize;
+   // size we actually download (can be smaller than Size if we have partial content)
+   unsigned long long DownloadSize;
+   // size of junk content (aka: server error pages)
+   unsigned long long JunkSize;
+   // The start of the data (for partial content)
    unsigned long long StartPos;
+
    time_t Date;
    bool HaveContent;
    enum {Chunked,Stream,Closes} Encoding;
@@ -73,7 +79,7 @@ struct ServerState
    RunHeadersResult RunHeaders(FileFd * const File, const std::string &Uri);
 
    bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;};
-   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; JunkSize = 0;
+   virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; TotalFileSize = 0; JunkSize = 0;
 		 StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false;
 		 State = Header; Persistent = false; Pipeline = true;};
    virtual bool WriteResponse(std::string const &Data) = 0;
-- 
cgit v1.2.3-70-g09d2


From 65759e00eff0513c34f584b99420b72fe0e5073e Mon Sep 17 00:00:00 2001
From: Michael Vogt 
Date: Fri, 22 May 2015 16:27:08 +0200
Subject: Update methods/https.cc now that ServerState::Size is renamed

Git-Dch: ignore
---
 methods/https.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'methods')

diff --git a/methods/https.cc b/methods/https.cc
index 81060122c..c97367323 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -69,7 +69,7 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
       me->File->Truncate(me->Server->StartPos);
       me->File->Seek(me->Server->StartPos);
 
-      me->Res.Size = me->Server->Size;
+      me->Res.Size = me->Server->TotalFileSize;
    }
    else if (me->Server->HeaderLine(line) == false)
       return 0;
-- 
cgit v1.2.3-70-g09d2