From dcd5856b11c685ca6d4629212d2978ce196ea65c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 26 Aug 2014 19:08:37 -0700 Subject: Pass ExpectedSize to tthe backend method This ensures that we can stop downloading if the server send too much data by accident (or by a malicious attempt) --- methods/server.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'methods/server.h') diff --git a/methods/server.h b/methods/server.h index 5299b3954..0d7333140 100644 --- a/methods/server.h +++ b/methods/server.h @@ -49,6 +49,8 @@ struct ServerState URI Proxy; unsigned long TimeOut; + unsigned long long ExpectedSize; + protected: ServerMethod *Owner; @@ -73,7 +75,7 @@ struct ServerState bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; - State = Header; Persistent = false; Pipeline = true;}; + State = Header; Persistent = false; Pipeline = true; ExpectedSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; /** \brief Transfer the data from the socket */ -- cgit v1.2.3-70-g09d2 From c48eea97b93920062ea26001081d4fdf7eb967e3 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Oct 2014 17:47:30 +0200 Subject: make expected-size a maximum-size check as this is what we want at this point --- apt-pkg/acquire-method.cc | 2 +- apt-pkg/acquire-method.h | 5 ++++- apt-pkg/acquire-worker.cc | 10 ++++++---- methods/ftp.cc | 8 ++++---- methods/ftp.h | 2 +- methods/http.cc | 4 ++-- methods/https.cc | 4 ++-- methods/server.cc | 4 ++-- methods/server.h | 4 ++-- test/integration/test-apt-update-expected-size | 7 +++++++ 10 files changed, 31 insertions(+), 19 deletions(-) (limited to 'methods/server.h') diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 330854e75..9c0558223 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -373,7 +373,7 @@ int pkgAcqMethod::Run(bool Single) Tmp->ExpectedHashes.push_back(HashString(*t, hash)); } char *End; - Tmp->ExpectedSize = strtoll(LookupTag(Message, "Expected-Size", "0").c_str(), &End, 10); + Tmp->MaximumSize = strtoll(LookupTag(Message, "Maximum-Size", "0").c_str(), &End, 10); Tmp->Next = 0; // Append it to the list diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index 2e4e8281a..675c4f844 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -48,7 +48,10 @@ class pkgAcqMethod bool IndexFile; bool FailIgnore; HashStringList ExpectedHashes; - unsigned long long ExpectedSize; + // a maximum size we will download, this can be the exact filesize + // for when we know it or a arbitrary limit when we don't know the + // filesize (like a InRelease file) + unsigned long long MaximumSize; }; struct FetchResult diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 8bd1618f4..ffa84eb68 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -526,9 +526,6 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) if (OutFd == -1) return false; - string ExpectedSize; - strprintf(ExpectedSize, "%llu", Item->Owner->FileSize); - string Message = "600 URI Acquire\n"; Message.reserve(300); Message += "URI: " + Item->URI; @@ -536,7 +533,12 @@ bool pkgAcquire::Worker::QueueItem(pkgAcquire::Queue::QItem *Item) HashStringList const hsl = Item->Owner->HashSums(); for (HashStringList::const_iterator hs = hsl.begin(); hs != hsl.end(); ++hs) Message += "\nExpected-" + hs->HashType() + ": " + hs->HashValue(); - Message += "\nExpected-Size: " + ExpectedSize; + if(Item->Owner->FileSize > 0) + { + string MaximumSize; + strprintf(MaximumSize, "%llu", Item->Owner->FileSize); + Message += "\nMaximum-Size: " + MaximumSize; + } Message += Item->Owner->Custom600Headers(); Message += "\n\n"; diff --git a/methods/ftp.cc b/methods/ftp.cc index 75ace1c5a..bc84dda7d 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -849,7 +849,7 @@ bool FTPConn::Finalize() /* This opens a data connection, sends REST and RETR and then transfers the file over. */ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &Hash,bool &Missing, unsigned long long ExpectedSize) + Hashes &Hash,bool &Missing, unsigned long long MaximumSize) { Missing = false; if (CreateDataFd() == false) @@ -924,9 +924,9 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, return false; } - if (ExpectedSize > 0 && To.Tell() > ExpectedSize) + if (MaximumSize > 0 && To.Tell() > MaximumSize) return _error->Error("Writing more data than expected (%llu > %llu)", - To.Tell(), ExpectedSize); + To.Tell(), MaximumSize); } // All done @@ -1067,7 +1067,7 @@ bool FtpMethod::Fetch(FetchItem *Itm) FailFd = Fd.Fd(); bool Missing; - if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->ExpectedSize) == false) + if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize) == false) { Fd.Close(); diff --git a/methods/ftp.h b/methods/ftp.h index 416a91980..a31ebc999 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -62,7 +62,7 @@ class FTPConn bool Size(const char *Path,unsigned long long &Size); bool ModTime(const char *Path, time_t &Time); bool Get(const char *Path,FileFd &To,unsigned long long Resume, - Hashes &MD5,bool &Missing, unsigned long long ExpectedSize); + Hashes &MD5,bool &Missing, unsigned long long MaximumSize); FTPConn(URI Srv); ~FTPConn(); diff --git a/methods/http.cc b/methods/http.cc index b076e59cc..f8faa0cf8 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -655,10 +655,10 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) return _error->Errno("write",_("Error writing to output file")); } - if (ExpectedSize > 0 && File && File->Tell() > ExpectedSize) + if (MaximumSize > 0 && File && File->Tell() > MaximumSize) { return _error->Error("Writing more data than expected (%llu > %llu)", - File->Tell(), ExpectedSize); + File->Tell(), MaximumSize); } // Handle commands from APT diff --git a/methods/https.cc b/methods/https.cc index a4794e705..787e4a507 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -82,9 +82,9 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) if(me->File->Write(buffer, size*nmemb) != true) return false; - if(me->Queue->ExpectedSize > 0 && me->File->Tell() > me->Queue->ExpectedSize) + if(me->Queue->MaximumSize > 0 && me->File->Tell() > me->Queue->MaximumSize) return _error->Error("Writing more data than expected (%llu > %llu)", - me->TotalWritten, me->Queue->ExpectedSize); + me->TotalWritten, me->Queue->MaximumSize); return size*nmemb; } diff --git a/methods/server.cc b/methods/server.cc index 223737901..82f9b4750 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -534,8 +534,8 @@ int ServerMethod::Loop() bool Result = true; // ensure we don't fetch too much - if (Queue->ExpectedSize > 0) - Server->ExpectedSize = Queue->ExpectedSize; + if (Queue->MaximumSize > 0) + Server->MaximumSize = Queue->MaximumSize; if (Server->HaveContent) Result = Server->RunData(File); diff --git a/methods/server.h b/methods/server.h index 0134a9538..3093e00c9 100644 --- a/methods/server.h +++ b/methods/server.h @@ -49,7 +49,7 @@ struct ServerState URI Proxy; unsigned long TimeOut; - unsigned long long ExpectedSize; + unsigned long long MaximumSize; protected: ServerMethod *Owner; @@ -75,7 +75,7 @@ struct ServerState bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; - State = Header; Persistent = false; Pipeline = true; ExpectedSize = 0;}; + State = Header; Persistent = false; Pipeline = true; MaximumSize = 0;}; virtual bool WriteResponse(std::string const &Data) = 0; /** \brief Transfer the data from the socket */ diff --git a/test/integration/test-apt-update-expected-size b/test/integration/test-apt-update-expected-size index 72812336d..c1eecc08a 100755 --- a/test/integration/test-apt-update-expected-size +++ b/test/integration/test-apt-update-expected-size @@ -15,6 +15,13 @@ changetowebserver # normal update works fine testsuccess aptget update +# make InRelease really big +mv aptarchive/dists/unstable/InRelease aptarchive/dists/unstable/InRelease.good +dd if=/dev/zero of=aptarchive/dists/unstable/InRelease bs=1M count=2 +touch -d '+1hour' aptarchive/dists/unstable/InRelease +aptget update -o acquire::MaxReleaseFileSize=$((1*1000*1000)) + + # append junk at the end of the Packages.gz/Packages SIZE="$(stat --printf=%s aptarchive/dists/unstable/main/binary-i386/Packages)" echo "1234567890" >> aptarchive/dists/unstable/main/binary-i386/Packages.gz -- cgit v1.2.3-70-g09d2 From f2b47ba290f3a178c584da83f007cf0f720baabb Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 8 Oct 2014 08:32:42 +0200 Subject: Fix http pipeline messup detection The Maximum-Size protection breaks the http pipeline reorder code because it relies on that the object got fetched entirely so that it can compare the hash of the downloaded data. So instead of stopping when the Maximum-Size of the expected item is reached we only stop when the maximum size of the biggest item in the queue is reached. This way the pipeline reoder code keeps working. --- methods/server.cc | 16 ++++++++++++++-- methods/server.h | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'methods/server.h') diff --git a/methods/server.cc b/methods/server.cc index 82f9b4750..cef809738 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -534,8 +534,10 @@ int ServerMethod::Loop() bool Result = true; // ensure we don't fetch too much - if (Queue->MaximumSize > 0) - Server->MaximumSize = Queue->MaximumSize; + // we could do "Server->MaximumSize = Queue->MaximumSize" here + // but that would break the clever pipeline messup detection + // so instead we use the size of the biggest item in the queue + Server->MaximumSize = FindMaximumObjectSizeInQueue(); if (Server->HaveContent) Result = Server->RunData(File); @@ -706,5 +708,15 @@ int ServerMethod::Loop() } return 0; +} + /*}}}*/ + /*{{{*/ +unsigned long long +ServerMethod::FindMaximumObjectSizeInQueue() const +{ + unsigned long long MaxSizeInQueue = 0; + for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize); + return MaxSizeInQueue; } /*}}}*/ diff --git a/methods/server.h b/methods/server.h index 3093e00c9..7d5198478 100644 --- a/methods/server.h +++ b/methods/server.h @@ -106,6 +106,10 @@ class ServerMethod : public pkgAcqMethod unsigned long PipelineDepth; bool AllowRedirect; + // Find the biggest item in the fetch queue for the checking of the maximum + // size + unsigned long long FindMaximumObjectSizeInQueue() const APT_PURE; + public: bool Debug; -- cgit v1.2.3-70-g09d2