refactor http client implementation

No effective behavior change, just shuffling big junks of code between
methods and classes to split them into those strongly related to our
client implementation and those implementing HTTP.

The idea is to get HTTPS to a point in which most of the implementation
can be shared even though the client implementations itself is
completely different. This isn't anywhere near yet though, but it should
beenough to reuse at least a few lines from http in https now.

Git-Dch: Ignore

1 files changed, 201 insertions, 790 deletions

diff --git a/methods/http.cc b/methods/http.cc
index 5f0dc03d9..d2f084b04 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -58,15 +58,6 @@
 									/*}}}*/
 using namespace std;
 
-string HttpMethod::FailFile;
-int HttpMethod::FailFd = -1;
-time_t HttpMethod::FailTime = 0;
-unsigned long PipelineDepth = 0;
-unsigned long TimeOut = 120;
-bool AllowRedirect = false;
-bool Debug = false;
-URI Proxy;
-
 unsigned long long CircleBuf::BwReadLimit=0;
 unsigned long long CircleBuf::BwTickReadData=0;
 struct timeval CircleBuf::BwReadTick={0,0};
@@ -296,20 +287,17 @@ CircleBuf::~CircleBuf()
    delete Hash;
 }
 
-// ServerState::ServerState - Constructor				/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
-                        In(64*1024), Out(4*1024),
-                        ServerName(Srv)
+// HttpServerState::HttpServerState - Constructor			/*{{{*/
+HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(64*1024), Out(4*1024)
 {
+   TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
    Reset();
 }
 									/*}}}*/
-// ServerState::Open - Open a connection to the server			/*{{{*/
+// HttpServerState::Open - Open a connection to the server		/*{{{*/
 // ---------------------------------------------------------------------
 /* This opens a connection to the server. */
-bool ServerState::Open()
+bool HttpServerState::Open()
 {
    // Use the already open connection if possible.
    if (ServerFd != -1)
@@ -373,72 +361,18 @@ bool ServerState::Open()
    return true;
 }
 									/*}}}*/
-// ServerState::Close - Close a connection to the server		/*{{{*/
+// HttpServerState::Close - Close a connection to the server		/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool ServerState::Close()
+bool HttpServerState::Close()
 {
    close(ServerFd);
    ServerFd = -1;
    return true;
 }
 									/*}}}*/
-// ServerState::RunHeaders - Get the headers before the data		/*{{{*/
-// ---------------------------------------------------------------------
-/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
-   parse error occurred */
-ServerState::RunHeadersResult ServerState::RunHeaders()
-{
-   State = Header;
-   
-   Owner->Status(_("Waiting for headers"));
-
-   Major = 0; 
-   Minor = 0; 
-   Result = 0; 
-   Size = 0; 
-   StartPos = 0;
-   Encoding = Closes;
-   HaveContent = false;
-   time(&Date);
-
-   do
-   {
-      string Data;
-      if (In.WriteTillEl(Data) == false)
-	 continue;
-
-      if (Debug == true)
-	 clog << Data;
-      
-      for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
-      {
-	 string::const_iterator J = I;
-	 for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
-	 if (HeaderLine(string(I,J)) == false)
-	    return RUN_HEADERS_PARSE_ERROR;
-	 I = J;
-      }
-
-      // 100 Continue is a Nop...
-      if (Result == 100)
-	 continue;
-      
-      // Tidy up the connection persistance state.
-      if (Encoding == Closes && HaveContent == true)
-	 Persistent = false;
-      
-      return RUN_HEADERS_OK;
-   }
-   while (Owner->Go(false,this) == true);
-   
-   return RUN_HEADERS_IO_ERROR;
-}
-									/*}}}*/
-// ServerState::RunData - Transfer the data from the socket		/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool ServerState::RunData()
+// HttpServerState::RunData - Transfer the data from the socket		/*{{{*/
+bool HttpServerState::RunData(FileFd * const File)
 {
    State = Data;
    
@@ -456,7 +390,7 @@ bool ServerState::RunData()
 	    if (In.WriteTillEl(Data,true) == true)
 	       break;
 	 }
-	 while ((Last = Owner->Go(false,this)) == true);
+	 while ((Last = Go(false, File)) == true);
 
 	 if (Last == false)
 	    return false;
@@ -474,7 +408,7 @@ bool ServerState::RunData()
 	       if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 		  break;
 	    }
-	    while ((Last = Owner->Go(false,this)) == true);
+	    while ((Last = Go(false, File)) == true);
 	    if (Last == false)
 	       return false;
 	    return !_error->PendingError();
@@ -482,7 +416,7 @@ bool ServerState::RunData()
 	 
 	 // Transfer the block
 	 In.Limit(Len);
-	 while (Owner->Go(true,this) == true)
+	 while (Go(true, File) == true)
 	    if (In.IsLimit() == true)
 	       break;
 	 
@@ -498,7 +432,7 @@ bool ServerState::RunData()
 	    if (In.WriteTillEl(Data,true) == true)
 	       break;
 	 }
-	 while ((Last = Owner->Go(false,this)) == true);
+	 while ((Last = Go(false, File)) == true);
 	 if (Last == false)
 	    return false;
       }
@@ -521,147 +455,217 @@ bool ServerState::RunData()
 	 In.Limit(-1);
 	 return !_error->PendingError();
       }
-      while (Owner->Go(true,this) == true);
+      while (Go(true, File) == true);
    }
 
-   return Owner->Flush(this) && !_error->PendingError();
+   return Owner->Flush() && !_error->PendingError();
 }
 									/*}}}*/
-// ServerState::HeaderLine - Process a header line			/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool ServerState::HeaderLine(string Line)
+bool HttpServerState::ReadHeaderLines(std::string &Data)		/*{{{*/
 {
-   if (Line.empty() == true)
-      return true;
+   return In.WriteTillEl(Data);
+}
+									/*}}}*/
+bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
+{
+   return Go(ToFile, File);
+}
+									/*}}}*/
+bool HttpServerState::WriteResponse(const std::string &Data)		/*{{{*/
+{
+   return Out.Read(Data);
+}
+									/*}}}*/
+bool HttpServerState::IsOpen()						/*{{{*/
+{
+   return (ServerFd != -1);
+}
+									/*}}}*/
+bool HttpServerState::InitHashes(FileFd &File)				/*{{{*/
+{
+   delete In.Hash;
+   In.Hash = new Hashes;
 
-   string::size_type Pos = Line.find(' ');
-   if (Pos == string::npos || Pos+1 > Line.length())
+   // Set the expected size and read file for the hashes
+   if (StartPos >= 0)
    {
-      // Blah, some servers use "connection:closes", evil.
-      Pos = Line.find(':');
-      if (Pos == string::npos || Pos + 2 > Line.length())
-	 return _error->Error(_("Bad header line"));
-      Pos++;
+      File.Truncate(StartPos);
+
+      return In.Hash->AddFD(File, StartPos);
    }
+   return true;
+}
+									/*}}}*/
+Hashes * HttpServerState::GetHashes()					/*{{{*/
+{
+   return In.Hash;
+}
+									/*}}}*/
+// HttpServerState::Die - The server has closed the connection.		/*{{{*/
+bool HttpServerState::Die(FileFd &File)
+{
+   unsigned int LErrno = errno;
 
-   // Parse off any trailing spaces between the : and the next word.
-   string::size_type Pos2 = Pos;
-   while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
-      Pos2++;
-      
-   string Tag = string(Line,0,Pos);
-   string Val = string(Line,Pos2);
-   
-   if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
+   // Dump the buffer to the file
+   if (State == ServerState::Data)
    {
-      // Evil servers return no version
-      if (Line[4] == '/')
-      {
-	 int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
-	 if (elements == 3)
-	 {
-	    Code[0] = '\0';
-	    if (Debug == true)
-	       clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
-	 }
-	 else if (elements != 4)
-	    return _error->Error(_("The HTTP server sent an invalid reply header"));
-      }
-      else
+      // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
+      // can't be set
+      if (File.Name() != "/dev/null")
+	 SetNonBlock(File.Fd(),false);
+      while (In.WriteSpace() == true)
       {
-	 Major = 0;
-	 Minor = 9;
-	 if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
-	    return _error->Error(_("The HTTP server sent an invalid reply header"));
-      }
+	 if (In.Write(File.Fd()) == false)
+	    return _error->Errno("write",_("Error writing to the file"));
 
-      /* Check the HTTP response header to get the default persistance
-         state. */
-      if (Major < 1)
-	 Persistent = false;
-      else
-      {
-	 if (Major == 1 && Minor == 0)
-	    Persistent = false;
-	 else
-	    Persistent = true;
+	 // Done
+	 if (In.IsLimit() == true)
+	    return true;
       }
+   }
 
-      return true;
-   }      
-      
-   if (stringcasecmp(Tag,"Content-Length:") == 0)
+   // See if this is because the server finished the data stream
+   if (In.IsLimit() == false && State != HttpServerState::Header &&
+       Encoding != HttpServerState::Closes)
    {
-      if (Encoding == Closes)
-	 Encoding = Stream;
-      HaveContent = true;
-      
-      // The length is already set from the Content-Range header
-      if (StartPos != 0)
-	 return true;
-
-      Size = strtoull(Val.c_str(), NULL, 10);
-      if (Size >= std::numeric_limits<unsigned long long>::max())
-	 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
-      else if (Size == 0)
-	 HaveContent = false;
-      return true;
+      Close();
+      if (LErrno == 0)
+	 return _error->Error(_("Error reading from server. Remote end closed connection"));
+      errno = LErrno;
+      return _error->Errno("read",_("Error reading from server"));
    }
-
-   if (stringcasecmp(Tag,"Content-Type:") == 0)
+   else
    {
-      HaveContent = true;
+      In.Limit(-1);
+
+      // Nothing left in the buffer
+      if (In.WriteSpace() == false)
+	 return false;
+
+      // We may have got multiple responses back in one packet..
+      Close();
       return true;
    }
-   
-   if (stringcasecmp(Tag,"Content-Range:") == 0)
-   {
-      HaveContent = true;
 
-      // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
-      if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
+   return false;
+}
+									/*}}}*/
+// HttpServerState::Flush - Dump the buffer into the file		/*{{{*/
+// ---------------------------------------------------------------------
+/* This takes the current input buffer from the Server FD and writes it
+   into the file */
+bool HttpServerState::Flush(FileFd * const File)
+{
+   if (File != NULL)
+   {
+      // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
+      // can't be set
+      if (File->Name() != "/dev/null")
+	 SetNonBlock(File->Fd(),false);
+      if (In.WriteSpace() == false)
+	 return true;
+      
+      while (In.WriteSpace() == true)
       {
-	 StartPos = 1; // ignore Content-Length, it would override Size
-	 HaveContent = false;
+	 if (In.Write(File->Fd()) == false)
+	    return _error->Errno("write",_("Error writing to file"));
+	 if (In.IsLimit() == true)
+	    return true;
       }
-      else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
-	 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
-      if ((unsigned long long)StartPos > Size)
-	 return _error->Error(_("This HTTP server has broken range support"));
-      return true;
+
+      if (In.IsLimit() == true || Encoding == ServerState::Closes)
+	 return true;
    }
+   return false;
+}
+									/*}}}*/
+// HttpServerState::Go - Run a single loop				/*{{{*/
+// ---------------------------------------------------------------------
+/* This runs the select loop over the server FDs, Output file FDs and
+   stdin. */
+bool HttpServerState::Go(bool ToFile, FileFd * const File)
+{
+   // Server has closed the connection
+   if (ServerFd == -1 && (In.WriteSpace() == false || 
+			       ToFile == false))
+      return false;
+   
+   fd_set rfds,wfds;
+   FD_ZERO(&rfds);
+   FD_ZERO(&wfds);
+   
+   /* Add the server. We only send more requests if the connection will 
+      be persisting */
+   if (Out.WriteSpace() == true && ServerFd != -1 
+       && Persistent == true)
+      FD_SET(ServerFd,&wfds);
+   if (In.ReadSpace() == true && ServerFd != -1)
+      FD_SET(ServerFd,&rfds);
    
-   if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
+   // Add the file
+   int FileFD = -1;
+   if (File != NULL)
+      FileFD = File->Fd();
+   
+   if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
+      FD_SET(FileFD,&wfds);
+
+   // Add stdin
+   if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
+      FD_SET(STDIN_FILENO,&rfds);
+	  
+   // Figure out the max fd
+   int MaxFd = FileFD;
+   if (MaxFd < ServerFd)
+      MaxFd = ServerFd;
+
+   // Select
+   struct timeval tv;
+   tv.tv_sec = TimeOut;
+   tv.tv_usec = 0;
+   int Res = 0;
+   if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
    {
-      HaveContent = true;
-      if (stringcasecmp(Val,"chunked") == 0)
-	 Encoding = Chunked;      
-      return true;
+      if (errno == EINTR)
+	 return true;
+      return _error->Errno("select",_("Select failed"));
    }
-
-   if (stringcasecmp(Tag,"Connection:") == 0)
+   
+   if (Res == 0)
    {
-      if (stringcasecmp(Val,"close") == 0)
-	 Persistent = false;
-      if (stringcasecmp(Val,"keep-alive") == 0)
-	 Persistent = true;
-      return true;
+      _error->Error(_("Connection timed out"));
+      return Die(*File);
    }
    
-   if (stringcasecmp(Tag,"Last-Modified:") == 0)
+   // Handle server IO
+   if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
    {
-      if (RFC1123StrToTime(Val.c_str(), Date) == false)
-	 return _error->Error(_("Unknown date format"));
-      return true;
+      errno = 0;
+      if (In.Read(ServerFd) == false)
+	 return Die(*File);
+   }
+	 
+   if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
+   {
+      errno = 0;
+      if (Out.Write(ServerFd) == false)
+	 return Die(*File);
    }
 
-   if (stringcasecmp(Tag,"Location:") == 0)
+   // Send data to the file
+   if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
    {
-      Location = Val;
-      return true;
+      if (In.Write(FileFD) == false)
+	 return _error->Errno("write",_("Error writing to output file"));
    }
 
+   // Handle commands from APT
+   if (FD_ISSET(STDIN_FILENO,&rfds))
+   {
+      if (Owner->Run(true) != -1)
+	 exit(100);
+   }   
+       
    return true;
 }
 									/*}}}*/
@@ -669,7 +673,7 @@ bool ServerState::HeaderLine(string Line)
 // HttpMethod::SendReq - Send the HTTP request				/*{{{*/
 // ---------------------------------------------------------------------
 /* This places the http request in the outbound buffer */
-void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
+void HttpMethod::SendReq(FetchItem *Itm)
 {
    URI Uri = Itm->Uri;
 
@@ -695,7 +699,7 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       but while its a must for all servers to accept absolute URIs,
       it is assumed clients will sent an absolute path for non-proxies */
    std::string requesturi;
-   if (Proxy.empty() == true || Proxy.Host.empty())
+   if (Server->Proxy.empty() == true || Server->Proxy.Host.empty())
       requesturi = Uri.Path;
    else
       requesturi = Itm->Uri;
@@ -763,9 +767,9 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
       }
    }
 
-   if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
+   if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)
       Req += string("Proxy-Authorization: Basic ") + 
-          Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
+          Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) + "\r\n";
 
    maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
    if (Uri.User.empty() == false || Uri.Password.empty() == false)
@@ -779,360 +783,18 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
    if (Debug == true)
       cerr << Req << endl;
 
-   Out.Read(Req);
-}
-									/*}}}*/
-// HttpMethod::Go - Run a single loop					/*{{{*/
-// ---------------------------------------------------------------------
-/* This runs the select loop over the server FDs, Output file FDs and
-   stdin. */
-bool HttpMethod::Go(bool ToFile,ServerState *Srv)
-{
-   // Server has closed the connection
-   if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false || 
-			       ToFile == false))
-      return false;
-   
-   fd_set rfds,wfds;
-   FD_ZERO(&rfds);
-   FD_ZERO(&wfds);
-   
-   /* Add the server. We only send more requests if the connection will 
-      be persisting */
-   if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1 
-       && Srv->Persistent == true)
-      FD_SET(Srv->ServerFd,&wfds);
-   if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
-      FD_SET(Srv->ServerFd,&rfds);
-   
-   // Add the file
-   int FileFD = -1;
-   if (File != 0)
-      FileFD = File->Fd();
-   
-   if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
-      FD_SET(FileFD,&wfds);
-
-   // Add stdin
-   if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
-      FD_SET(STDIN_FILENO,&rfds);
-	  
-   // Figure out the max fd
-   int MaxFd = FileFD;
-   if (MaxFd < Srv->ServerFd)
-      MaxFd = Srv->ServerFd;
-
-   // Select
-   struct timeval tv;
-   tv.tv_sec = TimeOut;
-   tv.tv_usec = 0;
-   int Res = 0;
-   if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
-   {
-      if (errno == EINTR)
-	 return true;
-      return _error->Errno("select",_("Select failed"));
-   }
-   
-   if (Res == 0)
-   {
-      _error->Error(_("Connection timed out"));
-      return ServerDie(Srv);
-   }
-   
-   // Handle server IO
-   if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
-   {
-      errno = 0;
-      if (Srv->In.Read(Srv->ServerFd) == false)
-	 return ServerDie(Srv);
-   }
-	 
-   if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
-   {
-      errno = 0;
-      if (Srv->Out.Write(Srv->ServerFd) == false)
-	 return ServerDie(Srv);
-   }
-
-   // Send data to the file
-   if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
-   {
-      if (Srv->In.Write(FileFD) == false)
-	 return _error->Errno("write",_("Error writing to output file"));
-   }
-
-   // Handle commands from APT
-   if (FD_ISSET(STDIN_FILENO,&rfds))
-   {
-      if (Run(true) != -1)
-	 exit(100);
-   }   
-       
-   return true;
-}
-									/*}}}*/
-// HttpMethod::Flush - Dump the buffer into the file			/*{{{*/
-// ---------------------------------------------------------------------
-/* This takes the current input buffer from the Server FD and writes it
-   into the file */
-bool HttpMethod::Flush(ServerState *Srv)
-{
-   if (File != 0)
-   {
-      // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
-      // can't be set
-      if (File->Name() != "/dev/null")
-	 SetNonBlock(File->Fd(),false);
-      if (Srv->In.WriteSpace() == false)
-	 return true;
-      
-      while (Srv->In.WriteSpace() == true)
-      {
-	 if (Srv->In.Write(File->Fd()) == false)
-	    return _error->Errno("write",_("Error writing to file"));
-	 if (Srv->In.IsLimit() == true)
-	    return true;
-      }
-
-      if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
-	 return true;
-   }
-   return false;
+   Server->WriteResponse(Req);
 }
 									/*}}}*/
-// HttpMethod::ServerDie - The server has closed the connection.	/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool HttpMethod::ServerDie(ServerState *Srv)
-{
-   unsigned int LErrno = errno;
-   
-   // Dump the buffer to the file
-   if (Srv->State == ServerState::Data)
-   {
-      // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
-      // can't be set
-      if (File->Name() != "/dev/null")
-	 SetNonBlock(File->Fd(),false);
-      while (Srv->In.WriteSpace() == true)
-      {
-	 if (Srv->In.Write(File->Fd()) == false)
-	    return _error->Errno("write",_("Error writing to the file"));
-
-	 // Done
-	 if (Srv->In.IsLimit() == true)
-	    return true;
-      }
-   }
-   
-   // See if this is because the server finished the data stream
-   if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header && 
-       Srv->Encoding != ServerState::Closes)
-   {
-      Srv->Close();
-      if (LErrno == 0)
-	 return _error->Error(_("Error reading from server. Remote end closed connection"));
-      errno = LErrno;
-      return _error->Errno("read",_("Error reading from server"));
-   }
-   else
-   {
-      Srv->In.Limit(-1);
-
-      // Nothing left in the buffer
-      if (Srv->In.WriteSpace() == false)
-	 return false;
-      
-      // We may have got multiple responses back in one packet..
-      Srv->Close();
-      return true;
-   }
-   
-   return false;
-}
-									/*}}}*/
-// HttpMethod::DealWithHeaders - Handle the retrieved header data	/*{{{*/
-// ---------------------------------------------------------------------
-/* We look at the header data we got back from the server and decide what
-   to do. Returns DealWithHeadersResult (see http.h for details).
- */
-HttpMethod::DealWithHeadersResult
-HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
-{
-   // Not Modified
-   if (Srv->Result == 304)
-   {
-      unlink(Queue->DestFile.c_str());
-      Res.IMSHit = true;
-      Res.LastModified = Queue->LastModified;
-      return IMS_HIT;
-   }
-   
-   /* Redirect
-    *
-    * Note that it is only OK for us to treat all redirection the same
-    * because we *always* use GET, not other HTTP methods.  There are
-    * three redirection codes for which it is not appropriate that we
-    * redirect.  Pass on those codes so the error handling kicks in.
-    */
-   if (AllowRedirect
-       && (Srv->Result > 300 && Srv->Result < 400)
-       && (Srv->Result != 300       // Multiple Choices
-           && Srv->Result != 304    // Not Modified
-           && Srv->Result != 306))  // (Not part of HTTP/1.1, reserved)
-   {
-      if (Srv->Location.empty() == true);
-      else if (Srv->Location[0] == '/' && Queue->Uri.empty() == false)
-      {
-	 URI Uri = Queue->Uri;
-	 if (Uri.Host.empty() == false)
-            NextURI = URI::SiteOnly(Uri);
-	 else
-	    NextURI.clear();
-	 NextURI.append(DeQuoteString(Srv->Location));
-	 return TRY_AGAIN_OR_REDIRECT;
-      }
-      else
-      {
-         NextURI = DeQuoteString(Srv->Location);
-         URI tmpURI = NextURI;
-         // Do not allow a redirection to switch protocol
-         if (tmpURI.Access == "http")
-            return TRY_AGAIN_OR_REDIRECT;
-      }
-      /* else pass through for error message */
-   }
-   // retry after an invalid range response without partial data
-   else if (Srv->Result == 416)
-   {
-      struct stat SBuf;
-      if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
-      {
-	 if ((unsigned long long)SBuf.st_size == Srv->Size)
-	 {
-	    // the file is completely downloaded, but was not moved
-	    Srv->StartPos = Srv->Size;
-	    Srv->Result = 200;
-	    Srv->HaveContent = false;
-	 }
-	 else if (unlink(Queue->DestFile.c_str()) == 0)
-	 {
-	    NextURI = Queue->Uri;
-	    return TRY_AGAIN_OR_REDIRECT;
-	 }
-      }
-   }
- 
-   /* We have a reply we dont handle. This should indicate a perm server
-      failure */
-   if (Srv->Result < 200 || Srv->Result >= 300)
-   {
-      char err[255];
-      snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result);
-      SetFailReason(err);
-      _error->Error("%u %s",Srv->Result,Srv->Code);
-      if (Srv->HaveContent == true)
-	 return ERROR_WITH_CONTENT_PAGE;
-      return ERROR_UNRECOVERABLE;
-   }
-
-   // This is some sort of 2xx 'data follows' reply
-   Res.LastModified = Srv->Date;
-   Res.Size = Srv->Size;
-   
-   // Open the file
-   delete File;
-   File = new FileFd(Queue->DestFile,FileFd::WriteAny);
-   if (_error->PendingError() == true)
-      return ERROR_NOT_FROM_SERVER;
-
-   FailFile = Queue->DestFile;
-   FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
-   FailFd = File->Fd();
-   FailTime = Srv->Date;
-
-   delete Srv->In.Hash;
-   Srv->In.Hash = new Hashes;
-
-   // Set the expected size and read file for the hashes
-   if (Srv->StartPos >= 0)
-   {
-      Res.ResumePoint = Srv->StartPos;
-      File->Truncate(Srv->StartPos);
-
-      if (Srv->In.Hash->AddFD(*File,Srv->StartPos) == false)
-      {
-	 _error->Errno("read",_("Problem hashing file"));
-	 return ERROR_NOT_FROM_SERVER;
-      }
-   }
-   
-   SetNonBlock(File->Fd(),true);
-   return FILE_IS_OPEN;
-}
-									/*}}}*/
-// HttpMethod::SigTerm - Handle a fatal signal				/*{{{*/
-// ---------------------------------------------------------------------
-/* This closes and timestamps the open file. This is neccessary to get 
-   resume behavoir on user abort */
-void HttpMethod::SigTerm(int)
-{
-   if (FailFd == -1)
-      _exit(100);
-   close(FailFd);
-   
-   // Timestamp
-   struct utimbuf UBuf;
-   UBuf.actime = FailTime;
-   UBuf.modtime = FailTime;
-   utime(FailFile.c_str(),&UBuf);
-   
-   _exit(100);
-}
-									/*}}}*/
-// HttpMethod::Fetch - Fetch an item					/*{{{*/
-// ---------------------------------------------------------------------
-/* This adds an item to the pipeline. We keep the pipeline at a fixed
-   depth. */
-bool HttpMethod::Fetch(FetchItem *)
-{
-   if (Server == 0)
-      return true;
-
-   // Queue the requests
-   int Depth = -1;
-   for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; 
-	I = I->Next, Depth++)
-   {
-      // If pipelining is disabled, we only queue 1 request
-      if (Server->Pipeline == false && Depth >= 0)
-	 break;
-      
-      // Make sure we stick with the same server
-      if (Server->Comp(I->Uri) == false)
-	 break;
-      if (QueueBack == I)
-      {
-	 QueueBack = I->Next;
-	 SendReq(I,Server->Out);
-	 continue;
-      }
-   }
-   
-   return true;
-};
-									/*}}}*/
 // HttpMethod::Configuration - Handle a configuration message		/*{{{*/
 // ---------------------------------------------------------------------
 /* We stash the desired pipeline depth */
 bool HttpMethod::Configuration(string Message)
 {
-   if (pkgAcqMethod::Configuration(Message) == false)
+   if (ServerMethod::Configuration(Message) == false)
       return false;
-   
+
    AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
-   TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 				  PipelineDepth);
    Debug = _config->FindB("Debug::Acquire::http",false);
@@ -1144,260 +806,6 @@ bool HttpMethod::Configuration(string Message)
    return true;
 }
 									/*}}}*/
-// HttpMethod::Loop - Main loop						/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-int HttpMethod::Loop()
-{
-   typedef vector<string> StringVector;
-   typedef vector<string>::iterator StringVectorIterator;
-   map<string, StringVector> Redirected;
-
-   signal(SIGTERM,SigTerm);
-   signal(SIGINT,SigTerm);
-   
-   Server = 0;
-   
-   int FailCounter = 0;
-   while (1)
-   {      
-      // We have no commands, wait for some to arrive
-      if (Queue == 0)
-      {
-	 if (WaitFd(STDIN_FILENO) == false)
-	    return 0;
-      }
-      
-      /* Run messages, we can accept 0 (no message) if we didn't
-         do a WaitFd above.. Otherwise the FD is closed. */
-      int Result = Run(true);
-      if (Result != -1 && (Result != 0 || Queue == 0))
-      {
-	 if(FailReason.empty() == false ||
-	    _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
-	    return 100;
-	 else
-	    return 0;
-      }
-
-      if (Queue == 0)
-	 continue;
-      
-      // Connect to the server
-      if (Server == 0 || Server->Comp(Queue->Uri) == false)
-      {
-	 delete Server;
-	 Server = new ServerState(Queue->Uri,this);
-      }
-      /* If the server has explicitly said this is the last connection
-         then we pre-emptively shut down the pipeline and tear down 
-	 the connection. This will speed up HTTP/1.0 servers a tad
-	 since we don't have to wait for the close sequence to
-         complete */
-      if (Server->Persistent == false)
-	 Server->Close();
-      
-      // Reset the pipeline
-      if (Server->ServerFd == -1)
-	 QueueBack = Queue;	 
-	 
-      // Connnect to the host
-      if (Server->Open() == false)
-      {
-	 Fail(true);
-	 delete Server;
-	 Server = 0;
-	 continue;
-      }
-
-      // Fill the pipeline.
-      Fetch(0);
-      
-      // Fetch the next URL header data from the server.
-      switch (Server->RunHeaders())
-      {
-	 case ServerState::RUN_HEADERS_OK:
-	 break;
-	 
-	 // The header data is bad
-	 case ServerState::RUN_HEADERS_PARSE_ERROR:
-	 {
-	    _error->Error(_("Bad header data"));
-	    Fail(true);
-	    RotateDNS();
-	    continue;
-	 }
-	 
-	 // The server closed a connection during the header get..
-	 default:
-	 case ServerState::RUN_HEADERS_IO_ERROR:
-	 {
-	    FailCounter++;
-	    _error->Discard();
-	    Server->Close();
-	    Server->Pipeline = false;
-	    
-	    if (FailCounter >= 2)
-	    {
-	       Fail(_("Connection failed"),true);
-	       FailCounter = 0;
-	    }
-	    
-	    RotateDNS();
-	    continue;
-	 }
-      };
-
-      // Decide what to do.
-      FetchResult Res;
-      Res.Filename = Queue->DestFile;
-      switch (DealWithHeaders(Res,Server))
-      {
-	 // Ok, the file is Open
-	 case FILE_IS_OPEN:
-	 {
-	    URIStart(Res);
-
-	    // Run the data
-	    bool Result = true;
-            if (Server->HaveContent)
-	       Result = Server->RunData();
-
-	    /* If the server is sending back sizeless responses then fill in
-	       the size now */
-	    if (Res.Size == 0)
-	       Res.Size = File->Size();
-	    
-	    // Close the file, destroy the FD object and timestamp it
-	    FailFd = -1;
-	    delete File;
-	    File = 0;
-	    
-	    // Timestamp
-	    struct utimbuf UBuf;
-	    time(&UBuf.actime);
-	    UBuf.actime = Server->Date;
-	    UBuf.modtime = Server->Date;
-	    utime(Queue->DestFile.c_str(),&UBuf);
-
-	    // Send status to APT
-	    if (Result == true)
-	    {
-	       Res.TakeHashes(*Server->In.Hash);
-	       URIDone(Res);
-	    }
-	    else
-	    {
-	       if (Server->ServerFd == -1)
-	       {
-		  FailCounter++;
-		  _error->Discard();
-		  Server->Close();
-		  
-		  if (FailCounter >= 2)
-		  {
-		     Fail(_("Connection failed"),true);
-		     FailCounter = 0;
-		  }
-		  
-		  QueueBack = Queue;
-	       }
-	       else
-		  Fail(true);
-	    }
-	    break;
-	 }
-	 
-	 // IMS hit
-	 case IMS_HIT:
-	 {
-	    URIDone(Res);
-	    break;
-	 }
-	 
-	 // Hard server error, not found or something
-	 case ERROR_UNRECOVERABLE:
-	 {
-	    Fail();
-	    break;
-	 }
-	  
-	 // Hard internal error, kill the connection and fail
-	 case ERROR_NOT_FROM_SERVER:
-	 {
-	    delete File;
-	    File = 0;
-
-	    Fail();
-	    RotateDNS();
-	    Server->Close();
-	    break;
-	 }
-
-	 // We need to flush the data, the header is like a 404 w/ error text
-	 case ERROR_WITH_CONTENT_PAGE:
-	 {
-	    Fail();
-	    
-	    // Send to content to dev/null
-	    File = new FileFd("/dev/null",FileFd::WriteExists);
-	    Server->RunData();
-	    delete File;
-	    File = 0;
-	    break;
-	 }
-	 
-         // Try again with a new URL
-         case TRY_AGAIN_OR_REDIRECT:
-         {
-            // Clear rest of response if there is content
-            if (Server->HaveContent)
-            {
-               File = new FileFd("/dev/null",FileFd::WriteExists);
-               Server->RunData();
-               delete File;
-               File = 0;
-            }
-
-            /* Detect redirect loops.  No more redirects are allowed
-               after the same URI is seen twice in a queue item. */
-            StringVector &R = Redirected[Queue->DestFile];
-            bool StopRedirects = false;
-            if (R.empty() == true)
-               R.push_back(Queue->Uri);
-            else if (R[0] == "STOP" || R.size() > 10)
-               StopRedirects = true;
-            else
-            {
-               for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
-                  if (Queue->Uri == *I)
-                  {
-                     R[0] = "STOP";
-                     break;
-                  }
- 
-               R.push_back(Queue->Uri);
-            }
- 
-            if (StopRedirects == false)
-               Redirect(NextURI);
-            else
-               Fail();
- 
-            break;
-         }
-
-	 default:
-	 Fail(_("Internal error"));
-	 break;
-      }
-      
-      FailCounter = 0;
-   }
-   
-   return 0;
-}
-									/*}}}*/
 // HttpMethod::AutoDetectProxy - auto detect proxy			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
@@ -1450,5 +858,8 @@ bool HttpMethod::AutoDetectProxy()
    return true;
 }
 									/*}}}*/
-
-
+ServerState * HttpMethod::CreateServerState(URI uri)			/*{{{*/
+{
+   return new HttpServerState(uri, this);
+}
+									/*}}}*/