handle complete responses to https range requests

Servers might respond with a complete file either because they don't
support Ranges at all or the If-Range condition isn't statisfied, so we
have to parse the headers curl gets ourself to seek or truncate the file
we have so far.

This also finially adds the testcase testing a bunch of partial
situations for both, http and https - which is now all green.

Closes: 617643, 667699
LP: 1157943

6 files changed, 92 insertions, 25 deletions

diff --git a/methods/http.cc b/methods/http.cc
index d2f084b04..71a02e53a 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -863,3 +863,8 @@ ServerState * HttpMethod::CreateServerState(URI uri)			/*{{{*/
    return new HttpServerState(uri, this);
 }
 									/*}}}*/
+void HttpMethod::RotateDNS()						/*{{{*/
+{
+   ::RotateDNS();
+}
+									/*}}}*/
diff --git a/methods/http.h b/methods/http.h
index 112ce171d..02c04e8ae 100644
--- a/methods/http.h
+++ b/methods/http.h
@@ -126,6 +126,7 @@ class HttpMethod : public ServerMethod
    virtual bool Configuration(std::string Message);
 
    virtual ServerState * CreateServerState(URI uri);
+   virtual void RotateDNS();
 
    protected:
    std::string AutoDetectProxyCmd;
diff --git a/methods/https.cc b/methods/https.cc
index 4f00842ba..2a562434b 100644
--- a/methods/https.cc
+++ b/methods/https.cc
@@ -36,6 +36,41 @@
 									/*}}}*/
 using namespace std;
 
+size_t
+HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
+{
+   size_t len = size * nmemb;
+   HttpsMethod *me = (HttpsMethod *)userp;
+   std::string line((char*) buffer, len);
+   for (--len; len > 0; --len)
+      if (isspace(line[len]) == 0)
+      {
+	 ++len;
+	 break;
+      }
+   line.erase(len);
+
+   if (line.empty() == true)
+   {
+      if (me->Server->Result != 416 && me->Server->StartPos != 0)
+	 ;
+      else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize())
+      {
+         me->Server->Result = 200;
+	 me->Server->StartPos = me->Server->Size;
+      }
+      else
+	 me->Server->StartPos = 0;
+
+      me->File->Truncate(me->Server->StartPos);
+      me->File->Seek(me->Server->StartPos);
+   }
+   else if (me->Server->HeaderLine(line) == false)
+      return 0;
+
+   return size*nmemb;
+}
+
 size_t 
 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 {
@@ -59,6 +94,14 @@ HttpsMethod::progress_callback(void *clientp, double dltotal, double dlnow,
    return 0;
 }
 
+// HttpsServerState::HttpsServerState - Constructor			/*{{{*/
+HttpsServerState::HttpsServerState(URI Srv,HttpsMethod *Owner) : ServerState(Srv, NULL)
+{
+   TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut);
+   Reset();
+}
+									/*}}}*/
+
 void HttpsMethod::SetupProxy()  					/*{{{*/
 {
    URI ServerName = Queue->Uri;
@@ -136,6 +179,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
 
    // callbacks
    curl_easy_setopt(curl, CURLOPT_URL, static_cast<string>(Uri).c_str());
+   curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
+   curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
    curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback);
@@ -289,16 +334,13 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
 
    // go for it - if the file exists, append on it
    File = new FileFd(Itm->DestFile, FileFd::WriteAny);
-   if (File->Size() > 0)
-      File->Seek(File->Size());
+   Server = new HttpsServerState(Itm->Uri, this);
 
    // keep apt updated
    Res.Filename = Itm->DestFile;
 
    // get it!
    CURLcode success = curl_easy_perform(curl);
-   long curl_responsecode;
-   curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &curl_responsecode);
 
    // If the server returns 200 OK but the If-Modified-Since condition is not
    // met, CURLINFO_CONDITION_UNMET will be set to 1
@@ -317,7 +359,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    }
 
    // server says file not modified
-   if (curl_responsecode == 304 || curl_condition_unmet == 1)
+   if (Server->Result == 304 || curl_condition_unmet == 1)
    {
       unlink(File->Name().c_str());
       Res.IMSHit = true;
@@ -326,13 +368,14 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       URIDone(Res);
       return true;
    }
+   Res.IMSHit = false;
 
-   if (curl_responsecode != 200 && // OK
-	 curl_responsecode != 206 && // Partial
-	 curl_responsecode != 416) // invalid Range
+   if (Server->Result != 200 && // OK
+	 Server->Result != 206 && // Partial
+	 Server->Result != 416) // invalid Range
    {
       char err[255];
-      snprintf(err, sizeof(err) - 1, "HttpError%ld", curl_responsecode);
+      snprintf(err, sizeof(err) - 1, "HttpError%i", Server->Result);
       SetFailReason(err);
       _error->Error("%s", err);
       // unlink, no need keep 401/404 page content in partial/
@@ -349,7 +392,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
    Res.Size = resultStat.st_size;
 
    // invalid range-request
-   if (curl_responsecode == 416)
+   if (Server->Result == 416)
    {
       unlink(File->Name().c_str());
       Res.Size = 0;
@@ -358,18 +401,6 @@ bool HttpsMethod::Fetch(FetchItem *Itm)
       return true;
    }
 
-   // check the downloaded result
-   if (curl_responsecode == 304 || curl_condition_unmet)
-   {
-      unlink(File->Name().c_str());
-      Res.IMSHit = true;
-      Res.LastModified = Itm->LastModified;
-      Res.Size = 0;
-      URIDone(Res);
-      return true;
-   }
-   Res.IMSHit = false;
-
    // Timestamp
    curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified);
    if (Res.LastModified != -1)
@@ -408,4 +439,3 @@ int main()
    return Mth.Run();
 }
 
-
diff --git a/methods/https.h b/methods/https.h
index 293e288e0..8632d6d02 100644
--- a/methods/https.h
+++ b/methods/https.h
@@ -14,24 +14,53 @@
 #include <iostream>
 #include <curl/curl.h>
 
+#include "server.h"
+
 using std::cout;
 using std::endl;
 
 class HttpsMethod;
 class FileFd;
 
+class HttpsServerState : public ServerState
+{
+   protected:
+   virtual bool ReadHeaderLines(std::string &Data) { return false; }
+   virtual bool LoadNextResponse(bool const ToFile, FileFd * const File) { return false; }
+
+   public:
+   virtual bool WriteResponse(std::string const &Data) { return false; }
+
+   /** \brief Transfer the data from the socket */
+   virtual bool RunData(FileFd * const File) { return false; }
+
+   virtual bool Open() { return false; }
+   virtual bool IsOpen() { return false; }
+   virtual bool Close() { return false; }
+   virtual bool InitHashes(FileFd &File) { return false; }
+   virtual Hashes * GetHashes() { return NULL; }
+   virtual bool Die(FileFd &File) { return false; }
+   virtual bool Flush(FileFd * const File) { return false; }
+   virtual bool Go(bool ToFile, FileFd * const File) { return false; }
+
+   HttpsServerState(URI Srv, HttpsMethod *Owner);
+   virtual ~HttpsServerState() {Close();};
+};
+
 class HttpsMethod : public pkgAcqMethod
 {
    // minimum speed in bytes/se that triggers download timeout handling
    static const int DL_MIN_SPEED = 10;
 
    virtual bool Fetch(FetchItem *);
+   static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp);
    static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp);
    static int progress_callback(void *clientp, double dltotal, double dlnow, 
 				double ultotal, double ulnow);
    void SetupProxy();
    CURL *curl;
    FetchResult Res;
+   HttpsServerState *Server;
 
    public:
    FileFd *File;
diff --git a/methods/makefile b/methods/makefile
index f8098de74..6b7781294 100644
--- a/methods/makefile
+++ b/methods/makefile
@@ -55,7 +55,7 @@ include $(PROGRAM_H)
 PROGRAM=https
 SLIBS = -lapt-pkg -lcurl $(INTLLIBS)
 LIB_MAKES = apt-pkg/makefile
-SOURCE = https.cc
+SOURCE = https.cc server.cc
 include $(PROGRAM_H)
 
 # The ftp method
diff --git a/methods/server.h b/methods/server.h
index 2d43b332f..4dc6a1f2f 100644
--- a/methods/server.h
+++ b/methods/server.h
@@ -49,11 +49,12 @@ struct ServerState
    protected:
    ServerMethod *Owner;
 
-   bool HeaderLine(std::string Line);
    virtual bool ReadHeaderLines(std::string &Data) = 0;
    virtual bool LoadNextResponse(bool const ToFile, FileFd * const File) = 0;
 
    public:
+   bool HeaderLine(std::string Line);
+
    /** \brief Result of the header acquire */
    enum RunHeadersResult {
       /** \brief Header ok */
@@ -134,6 +135,7 @@ class ServerMethod : public pkgAcqMethod
 
    virtual void SendReq(FetchItem *Itm) = 0;
    virtual ServerState * CreateServerState(URI uri) = 0;
+   virtual void RotateDNS() = 0;
 
    ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), PipelineDepth(0), AllowRedirect(false), Debug(false) {};
    virtual ~ServerMethod() {};