From 3b501823ab07123588a365bddc330ce36b94fec8 Mon Sep 17 00:00:00 2001 From: Tim Retout Date: Sat, 19 Apr 2014 23:03:48 +0100 Subject: Copy https apt transport, and use SOCKS proxy --- apti18n.h | 31 +++ config.h | 1 + server.cc | 663 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ server.h | 147 ++++++++++++++ tor.cc | 429 ++++++++++++++++++++++++++++++++++++++++ tor.h | 88 +++++++++ 6 files changed, 1359 insertions(+) create mode 100644 apti18n.h create mode 100644 config.h create mode 100644 server.cc create mode 100644 server.h create mode 100644 tor.cc create mode 100644 tor.h diff --git a/apti18n.h b/apti18n.h new file mode 100644 index 0000000..ae7515e --- /dev/null +++ b/apti18n.h @@ -0,0 +1,31 @@ +/* include/apti18n.h. Generated from apti18n.h.in by configure. */ +// -*- mode: cpp; mode: fold -*- +// $Id: apti18n.h.in,v 1.6 2003/01/11 07:18:18 jgg Exp $ +/* Internationalization macros for apt. This header should be included last + in each C file. */ + +// Set by autoconf +#define USE_NLS 1 + +#ifdef USE_NLS +// apt will use the gettext implementation of the C library +#include +#include +# ifdef APT_DOMAIN +# define _(x) dgettext(APT_DOMAIN,x) +# define P_(msg,plural,n) dngettext(APT_DOMAIN,msg,plural,n) +# else +# define _(x) gettext(x) +# define P_(msg,plural,n) ngettext(msg,plural,n) +# endif +# define N_(x) x +#else +// apt will not use any gettext +# define setlocale(a, b) +# define textdomain(a) +# define bindtextdomain(a, b) +# define _(x) x +# define P_(msg,plural,n) (n == 1 ? msg : plural) +# define N_(x) x +# define dgettext(d, m) m +#endif diff --git a/config.h b/config.h new file mode 100644 index 0000000..aac2e04 --- /dev/null +++ b/config.h @@ -0,0 +1 @@ +#define PACKAGE_VERSION "1" diff --git a/server.cc b/server.cc new file mode 100644 index 0000000..5a13f18 --- /dev/null +++ b/server.cc @@ -0,0 +1,663 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + HTTP and HTTPS share a lot of common code and these classes are + exactly the dumping ground for this common code + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "server.h" + +#include + /*}}}*/ +using namespace std; + +string ServerMethod::FailFile; +int ServerMethod::FailFd = -1; +time_t ServerMethod::FailTime = 0; + +// ServerState::RunHeaders - Get the headers before the data /*{{{*/ +// --------------------------------------------------------------------- +/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header + parse error occurred */ +ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) +{ + State = Header; + + Owner->Status(_("Waiting for headers")); + + Major = 0; + Minor = 0; + Result = 0; + Size = 0; + StartPos = 0; + Encoding = Closes; + HaveContent = false; + time(&Date); + + do + { + string Data; + if (ReadHeaderLines(Data) == false) + continue; + + if (Owner->Debug == true) + clog << Data; + + for (string::const_iterator I = Data.begin(); I < Data.end(); ++I) + { + string::const_iterator J = I; + for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J); + if (HeaderLine(string(I,J)) == false) + return RUN_HEADERS_PARSE_ERROR; + I = J; + } + + // 100 Continue is a Nop... + if (Result == 100) + continue; + + // Tidy up the connection persistence state. + if (Encoding == Closes && HaveContent == true) + Persistent = false; + + return RUN_HEADERS_OK; + } + while (LoadNextResponse(false, File) == true); + + return RUN_HEADERS_IO_ERROR; +} + /*}}}*/ +// ServerState::HeaderLine - Process a header line /*{{{*/ +// --------------------------------------------------------------------- +/* */ +bool ServerState::HeaderLine(string Line) +{ + if (Line.empty() == true) + return true; + + string::size_type Pos = Line.find(' '); + if (Pos == string::npos || Pos+1 > Line.length()) + { + // Blah, some servers use "connection:closes", evil. + Pos = Line.find(':'); + if (Pos == string::npos || Pos + 2 > Line.length()) + return _error->Error(_("Bad header line")); + Pos++; + } + + // Parse off any trailing spaces between the : and the next word. + string::size_type Pos2 = Pos; + while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0) + Pos2++; + + string Tag = string(Line,0,Pos); + string Val = string(Line,Pos2); + + if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0) + { + // Evil servers return no version + if (Line[4] == '/') + { + int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code); + if (elements == 3) + { + Code[0] = '\0'; + if (Owner->Debug == true) + clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl; + } + else if (elements != 4) + return _error->Error(_("The HTTP server sent an invalid reply header")); + } + else + { + Major = 0; + Minor = 9; + if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2) + return _error->Error(_("The HTTP server sent an invalid reply header")); + } + + /* Check the HTTP response header to get the default persistence + state. */ + if (Major < 1) + Persistent = false; + else + { + if (Major == 1 && Minor == 0) + Persistent = false; + else + Persistent = true; + } + + return true; + } + + if (stringcasecmp(Tag,"Content-Length:") == 0) + { + if (Encoding == Closes) + Encoding = Stream; + HaveContent = true; + + // The length is already set from the Content-Range header + if (StartPos != 0) + return true; + + Size = strtoull(Val.c_str(), NULL, 10); + if (Size >= std::numeric_limits::max()) + return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); + else if (Size == 0) + HaveContent = false; + return true; + } + + if (stringcasecmp(Tag,"Content-Type:") == 0) + { + HaveContent = true; + return true; + } + + if (stringcasecmp(Tag,"Content-Range:") == 0) + { + HaveContent = true; + + // ยง14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 + if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) + { + StartPos = 1; // ignore Content-Length, it would override Size + HaveContent = false; + } + else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) + return _error->Error(_("The HTTP server sent an invalid Content-Range header")); + if ((unsigned long long)StartPos > Size) + return _error->Error(_("This HTTP server has broken range support")); + return true; + } + + if (stringcasecmp(Tag,"Transfer-Encoding:") == 0) + { + HaveContent = true; + if (stringcasecmp(Val,"chunked") == 0) + Encoding = Chunked; + return true; + } + + if (stringcasecmp(Tag,"Connection:") == 0) + { + if (stringcasecmp(Val,"close") == 0) + Persistent = false; + if (stringcasecmp(Val,"keep-alive") == 0) + Persistent = true; + return true; + } + + if (stringcasecmp(Tag,"Last-Modified:") == 0) + { + if (RFC1123StrToTime(Val.c_str(), Date) == false) + return _error->Error(_("Unknown date format")); + return true; + } + + if (stringcasecmp(Tag,"Location:") == 0) + { + Location = Val; + return true; + } + + return true; +} + /*}}}*/ +// ServerState::ServerState - Constructor /*{{{*/ +ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner) +{ + Reset(); +} + /*}}}*/ + +bool ServerMethod::Configuration(string Message) /*{{{*/ +{ + return pkgAcqMethod::Configuration(Message); +} + /*}}}*/ + +// ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/ +// --------------------------------------------------------------------- +/* We look at the header data we got back from the server and decide what + to do. Returns DealWithHeadersResult (see http.h for details). + */ +ServerMethod::DealWithHeadersResult +ServerMethod::DealWithHeaders(FetchResult &Res) +{ + // Not Modified + if (Server->Result == 304) + { + unlink(Queue->DestFile.c_str()); + Res.IMSHit = true; + Res.LastModified = Queue->LastModified; + return IMS_HIT; + } + + /* Redirect + * + * Note that it is only OK for us to treat all redirection the same + * because we *always* use GET, not other HTTP methods. There are + * three redirection codes for which it is not appropriate that we + * redirect. Pass on those codes so the error handling kicks in. + */ + if (AllowRedirect + && (Server->Result > 300 && Server->Result < 400) + && (Server->Result != 300 // Multiple Choices + && Server->Result != 304 // Not Modified + && Server->Result != 306)) // (Not part of HTTP/1.1, reserved) + { + if (Server->Location.empty() == true); + else if (Server->Location[0] == '/' && Queue->Uri.empty() == false) + { + URI Uri = Queue->Uri; + if (Uri.Host.empty() == false) + NextURI = URI::SiteOnly(Uri); + else + NextURI.clear(); + NextURI.append(DeQuoteString(Server->Location)); + return TRY_AGAIN_OR_REDIRECT; + } + else + { + NextURI = DeQuoteString(Server->Location); + URI tmpURI = NextURI; + URI Uri = Queue->Uri; + // same protocol redirects are okay + if (tmpURI.Access == Uri.Access) + return TRY_AGAIN_OR_REDIRECT; + // as well as http to https + else if (Uri.Access == "http" && tmpURI.Access == "https") + return TRY_AGAIN_OR_REDIRECT; + } + /* else pass through for error message */ + } + // retry after an invalid range response without partial data + else if (Server->Result == 416) + { + struct stat SBuf; + if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) + { + if ((unsigned long long)SBuf.st_size == Server->Size) + { + // the file is completely downloaded, but was not moved + Server->StartPos = Server->Size; + Server->Result = 200; + Server->HaveContent = false; + } + else if (unlink(Queue->DestFile.c_str()) == 0) + { + NextURI = Queue->Uri; + return TRY_AGAIN_OR_REDIRECT; + } + } + } + + /* We have a reply we dont handle. This should indicate a perm server + failure */ + if (Server->Result < 200 || Server->Result >= 300) + { + char err[255]; + snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result); + SetFailReason(err); + _error->Error("%u %s",Server->Result,Server->Code); + if (Server->HaveContent == true) + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; + } + + // This is some sort of 2xx 'data follows' reply + Res.LastModified = Server->Date; + Res.Size = Server->Size; + + // Open the file + delete File; + File = new FileFd(Queue->DestFile,FileFd::WriteAny); + if (_error->PendingError() == true) + return ERROR_NOT_FROM_SERVER; + + FailFile = Queue->DestFile; + FailFile.c_str(); // Make sure we dont do a malloc in the signal handler + FailFd = File->Fd(); + FailTime = Server->Date; + + if (Server->InitHashes(*File) == false) + { + _error->Errno("read",_("Problem hashing file")); + return ERROR_NOT_FROM_SERVER; + } + if (Server->StartPos > 0) + Res.ResumePoint = Server->StartPos; + + SetNonBlock(File->Fd(),true); + return FILE_IS_OPEN; +} + /*}}}*/ +// ServerMethod::SigTerm - Handle a fatal signal /*{{{*/ +// --------------------------------------------------------------------- +/* This closes and timestamps the open file. This is necessary to get + resume behavoir on user abort */ +void ServerMethod::SigTerm(int) +{ + if (FailFd == -1) + _exit(100); + + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(FailFile.c_str(), times); + close(FailFd); + + _exit(100); +} + /*}}}*/ +// ServerMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool ServerMethod::Fetch(FetchItem *) +{ + if (Server == 0) + return true; + + // Queue the requests + int Depth = -1; + for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; + I = I->Next, Depth++) + { + // If pipelining is disabled, we only queue 1 request + if (Server->Pipeline == false && Depth >= 0) + break; + + // Make sure we stick with the same server + if (Server->Comp(I->Uri) == false) + break; + if (QueueBack == I) + { + QueueBack = I->Next; + SendReq(I); + continue; + } + } + + return true; +} + /*}}}*/ +// ServerMethod::Loop - Main loop /*{{{*/ +int ServerMethod::Loop() +{ + typedef vector StringVector; + typedef vector::iterator StringVectorIterator; + map Redirected; + + signal(SIGTERM,SigTerm); + signal(SIGINT,SigTerm); + + Server = 0; + + int FailCounter = 0; + while (1) + { + // We have no commands, wait for some to arrive + if (Queue == 0) + { + if (WaitFd(STDIN_FILENO) == false) + return 0; + } + + /* Run messages, we can accept 0 (no message) if we didn't + do a WaitFd above.. Otherwise the FD is closed. */ + int Result = Run(true); + if (Result != -1 && (Result != 0 || Queue == 0)) + { + if(FailReason.empty() == false || + _config->FindB("Acquire::http::DependOnSTDIN", true) == true) + return 100; + else + return 0; + } + + if (Queue == 0) + continue; + + // Connect to the server + if (Server == 0 || Server->Comp(Queue->Uri) == false) + { + delete Server; + Server = CreateServerState(Queue->Uri); + } + /* If the server has explicitly said this is the last connection + then we pre-emptively shut down the pipeline and tear down + the connection. This will speed up HTTP/1.0 servers a tad + since we don't have to wait for the close sequence to + complete */ + if (Server->Persistent == false) + Server->Close(); + + // Reset the pipeline + if (Server->IsOpen() == false) + QueueBack = Queue; + + // Connnect to the host + if (Server->Open() == false) + { + Fail(true); + delete Server; + Server = 0; + continue; + } + + // Fill the pipeline. + Fetch(0); + + // Fetch the next URL header data from the server. + switch (Server->RunHeaders(File)) + { + case ServerState::RUN_HEADERS_OK: + break; + + // The header data is bad + case ServerState::RUN_HEADERS_PARSE_ERROR: + { + _error->Error(_("Bad header data")); + Fail(true); + RotateDNS(); + continue; + } + + // The server closed a connection during the header get.. + default: + case ServerState::RUN_HEADERS_IO_ERROR: + { + FailCounter++; + _error->Discard(); + Server->Close(); + Server->Pipeline = false; + + if (FailCounter >= 2) + { + Fail(_("Connection failed"),true); + FailCounter = 0; + } + + RotateDNS(); + continue; + } + }; + + // Decide what to do. + FetchResult Res; + Res.Filename = Queue->DestFile; + switch (DealWithHeaders(Res)) + { + // Ok, the file is Open + case FILE_IS_OPEN: + { + URIStart(Res); + + // Run the data + bool Result = true; + if (Server->HaveContent) + Result = Server->RunData(File); + + /* If the server is sending back sizeless responses then fill in + the size now */ + if (Res.Size == 0) + Res.Size = File->Size(); + + // Close the file, destroy the FD object and timestamp it + FailFd = -1; + delete File; + File = 0; + + // Timestamp + struct timeval times[2]; + times[0].tv_sec = times[1].tv_sec = Server->Date; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(Queue->DestFile.c_str(), times); + + // Send status to APT + if (Result == true) + { + Res.TakeHashes(*Server->GetHashes()); + URIDone(Res); + } + else + { + if (Server->IsOpen() == false) + { + FailCounter++; + _error->Discard(); + Server->Close(); + + if (FailCounter >= 2) + { + Fail(_("Connection failed"),true); + FailCounter = 0; + } + + QueueBack = Queue; + } + else + Fail(true); + } + break; + } + + // IMS hit + case IMS_HIT: + { + URIDone(Res); + break; + } + + // Hard server error, not found or something + case ERROR_UNRECOVERABLE: + { + Fail(); + break; + } + + // Hard internal error, kill the connection and fail + case ERROR_NOT_FROM_SERVER: + { + delete File; + File = 0; + + Fail(); + RotateDNS(); + Server->Close(); + break; + } + + // We need to flush the data, the header is like a 404 w/ error text + case ERROR_WITH_CONTENT_PAGE: + { + Fail(); + + // Send to content to dev/null + File = new FileFd("/dev/null",FileFd::WriteExists); + Server->RunData(File); + delete File; + File = 0; + break; + } + + // Try again with a new URL + case TRY_AGAIN_OR_REDIRECT: + { + // Clear rest of response if there is content + if (Server->HaveContent) + { + File = new FileFd("/dev/null",FileFd::WriteExists); + Server->RunData(File); + delete File; + File = 0; + } + + /* Detect redirect loops. No more redirects are allowed + after the same URI is seen twice in a queue item. */ + StringVector &R = Redirected[Queue->DestFile]; + bool StopRedirects = false; + if (R.empty() == true) + R.push_back(Queue->Uri); + else if (R[0] == "STOP" || R.size() > 10) + StopRedirects = true; + else + { + for (StringVectorIterator I = R.begin(); I != R.end(); ++I) + if (Queue->Uri == *I) + { + R[0] = "STOP"; + break; + } + + R.push_back(Queue->Uri); + } + + if (StopRedirects == false) + Redirect(NextURI); + else + Fail(); + + break; + } + + default: + Fail(_("Internal error")); + break; + } + + FailCounter = 0; + } + + return 0; +} + /*}}}*/ diff --git a/server.h b/server.h new file mode 100644 index 0000000..0f45ab9 --- /dev/null +++ b/server.h @@ -0,0 +1,147 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +/* ###################################################################### + + Classes dealing with the abstraction of talking to a end via a text + protocol like HTTP (which is used by the http and https methods) + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_SERVER_H +#define APT_SERVER_H + +#include +#include + +#include +#include +#include + +using std::cout; +using std::endl; + +class Hashes; +class ServerMethod; +class FileFd; + +struct ServerState +{ + // This is the last parsed Header Line + unsigned int Major; + unsigned int Minor; + unsigned int Result; + char Code[360]; + + // These are some statistics from the last parsed header lines + unsigned long long Size; + unsigned long long StartPos; + time_t Date; + bool HaveContent; + enum {Chunked,Stream,Closes} Encoding; + enum {Header, Data} State; + bool Persistent; + std::string Location; + + // This is a Persistent attribute of the server itself. + bool Pipeline; + URI ServerName; + URI Proxy; + unsigned long TimeOut; + + protected: + ServerMethod *Owner; + + virtual bool ReadHeaderLines(std::string &Data) = 0; + virtual bool LoadNextResponse(bool const ToFile, FileFd * const File) = 0; + + public: + bool HeaderLine(std::string Line); + + /** \brief Result of the header acquire */ + enum RunHeadersResult { + /** \brief Header ok */ + RUN_HEADERS_OK, + /** \brief IO error while retrieving */ + RUN_HEADERS_IO_ERROR, + /** \brief Parse error after retrieving */ + RUN_HEADERS_PARSE_ERROR + }; + /** \brief Get the headers before the data */ + RunHeadersResult RunHeaders(FileFd * const File); + + bool Comp(URI Other) const {return Other.Host == ServerName.Host && Other.Port == ServerName.Port;}; + virtual void Reset() {Major = 0; Minor = 0; Result = 0; Code[0] = '\0'; Size = 0; + StartPos = 0; Encoding = Closes; time(&Date); HaveContent = false; + State = Header; Persistent = false; Pipeline = true;}; + virtual bool WriteResponse(std::string const &Data) = 0; + + /** \brief Transfer the data from the socket */ + virtual bool RunData(FileFd * const File) = 0; + + virtual bool Open() = 0; + virtual bool IsOpen() = 0; + virtual bool Close() = 0; + virtual bool InitHashes(FileFd &File) = 0; + virtual Hashes * GetHashes() = 0; + virtual bool Die(FileFd &File) = 0; + virtual bool Flush(FileFd * const File) = 0; + virtual bool Go(bool ToFile, FileFd * const File) = 0; + + ServerState(URI Srv, ServerMethod *Owner); + virtual ~ServerState() {}; +}; + +class ServerMethod : public pkgAcqMethod +{ + protected: + virtual bool Fetch(FetchItem *); + + ServerState *Server; + std::string NextURI; + FileFd *File; + + unsigned long PipelineDepth; + bool AllowRedirect; + + public: + bool Debug; + + /** \brief Result of the header parsing */ + enum DealWithHeadersResult { + /** \brief The file is open and ready */ + FILE_IS_OPEN, + /** \brief We got a IMS hit, the file has not changed */ + IMS_HIT, + /** \brief The server reported a unrecoverable error */ + ERROR_UNRECOVERABLE, + /** \brief The server reported a error with a error content page */ + ERROR_WITH_CONTENT_PAGE, + /** \brief An error on the client side */ + ERROR_NOT_FROM_SERVER, + /** \brief A redirect or retry request */ + TRY_AGAIN_OR_REDIRECT + }; + /** \brief Handle the retrieved header data */ + DealWithHeadersResult DealWithHeaders(FetchResult &Res); + + // In the event of a fatal signal this file will be closed and timestamped. + static std::string FailFile; + static int FailFd; + static time_t FailTime; + static APT_NORETURN void SigTerm(int); + + virtual bool Configuration(std::string Message); + virtual bool Flush() { return Server->Flush(File); }; + + int Loop(); + + virtual void SendReq(FetchItem *Itm) = 0; + virtual ServerState * CreateServerState(URI uri) = 0; + virtual void RotateDNS() = 0; + + ServerMethod(const char *Ver,unsigned long Flags = 0) : pkgAcqMethod(Ver, Flags), Server(NULL), File(NULL), PipelineDepth(0), AllowRedirect(false), Debug(false) {}; + virtual ~ServerMethod() {}; +}; + +#endif diff --git a/tor.cc b/tor.cc new file mode 100644 index 0000000..c764840 --- /dev/null +++ b/tor.cc @@ -0,0 +1,429 @@ +//-*- mode: cpp; mode: fold -*- +// Description /*{{{*/ +// $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $ +/* ###################################################################### + + HTTPS Acquire Method - This is the HTTPS acquire method for APT. + + It uses libcurl + + ##################################################################### */ + /*}}}*/ +// Include Files /*{{{*/ +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tor.h" + +#include + /*}}}*/ +using namespace std; + +size_t +HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) +{ + size_t len = size * nmemb; + HttpsMethod *me = (HttpsMethod *)userp; + std::string line((char*) buffer, len); + for (--len; len > 0; --len) + if (isspace(line[len]) == 0) + { + ++len; + break; + } + line.erase(len); + + if (line.empty() == true) + { + if (me->Server->Result != 416 && me->Server->StartPos != 0) + ; + else if (me->Server->Result == 416 && me->Server->Size == me->File->FileSize()) + { + me->Server->Result = 200; + me->Server->StartPos = me->Server->Size; + } + else + me->Server->StartPos = 0; + + me->File->Truncate(me->Server->StartPos); + me->File->Seek(me->Server->StartPos); + } + else if (me->Server->HeaderLine(line) == false) + return 0; + + return size*nmemb; +} + +size_t +HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) +{ + HttpsMethod *me = (HttpsMethod *)userp; + + if (me->Res.Size == 0) + me->URIStart(me->Res); + if(me->File->Write(buffer, size*nmemb) != true) + return false; + + return size*nmemb; +} + +int +HttpsMethod::progress_callback(void *clientp, double dltotal, double /*dlnow*/, + double /*ultotal*/, double /*ulnow*/) +{ + HttpsMethod *me = (HttpsMethod *)clientp; + if(dltotal > 0 && me->Res.Size == 0) { + me->Res.Size = (unsigned long long)dltotal; + } + return 0; +} + +// HttpsServerState::HttpsServerState - Constructor /*{{{*/ +HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * /*Owner*/) : ServerState(Srv, NULL) +{ + TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut); + Reset(); +} + /*}}}*/ + +void HttpsMethod::SetupProxy() /*{{{*/ +{ + URI ServerName = Queue->Uri; + + // Curl should never read proxy settings from the environment, as + // we determine which proxy to use. Do this for consistency among + // methods and prevent an environment variable overriding a + // no-proxy ("DIRECT") setting in apt.conf. + curl_easy_setopt(curl, CURLOPT_PROXY, ""); + + // Determine the proxy setting - try https first, fallback to http and use env at last + string UseProxy = _config->Find("Acquire::tor::Proxy", _config->Find("Acquire::tor::Proxy").c_str()); + + if (UseProxy.empty() == true) + { + UseProxy = "socks5h://localhost:9050"; + } + + // Determine what host and port to use based on the proxy settings + Proxy = UseProxy; + if (Proxy.Port != 1) + curl_easy_setopt(curl, CURLOPT_PROXYPORT, Proxy.Port); + curl_easy_setopt(curl, CURLOPT_PROXY, Proxy.Host.c_str()); + if (Proxy.User.empty() == false || Proxy.Password.empty() == false) + { + curl_easy_setopt(curl, CURLOPT_PROXYUSERNAME, Proxy.User.c_str()); + curl_easy_setopt(curl, CURLOPT_PROXYPASSWORD, Proxy.Password.c_str()); + } + + // Set proxy type to SOCKS5, and let proxy do DNS resolution + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME); +} /*}}}*/ +// HttpsMethod::Fetch - Fetch an item /*{{{*/ +// --------------------------------------------------------------------- +/* This adds an item to the pipeline. We keep the pipeline at a fixed + depth. */ +bool HttpsMethod::Fetch(FetchItem *Itm) +{ + struct stat SBuf; + struct curl_slist *headers=NULL; + char curl_errorstr[CURL_ERROR_SIZE]; + URI Uri = Itm->Uri; + string remotehost = Uri.Host; + + // Undo the "tor" at the start + Uri.Access = "http"; + + // TODO: + // - http::Pipeline-Depth + // - error checking/reporting + // - more debug options? (CURLOPT_DEBUGFUNCTION?) + + curl_easy_reset(curl); + SetupProxy(); + + maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc")); + + // callbacks + curl_easy_setopt(curl, CURLOPT_URL, static_cast(Uri).c_str()); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header); + curl_easy_setopt(curl, CURLOPT_WRITEHEADER, this); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this); + // options + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, false); + curl_easy_setopt(curl, CURLOPT_FILETIME, true); + // only allow curl to handle https, not the other stuff it supports + //curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS); + //curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS); + + // SSL parameters are set by default to the common (non mirror-specific) value + // if available (or a default one) and gets overload by mirror-specific ones. + + // File containing the list of trusted CA. + string cainfo = _config->Find("Acquire::https::CaInfo",""); + string knob = "Acquire::https::"+remotehost+"::CaInfo"; + cainfo = _config->Find(knob.c_str(),cainfo.c_str()); + if(cainfo.empty() == false) + curl_easy_setopt(curl, CURLOPT_CAINFO,cainfo.c_str()); + + // Check server certificate against previous CA list ... + bool peer_verify = _config->FindB("Acquire::https::Verify-Peer",true); + knob = "Acquire::https::" + remotehost + "::Verify-Peer"; + peer_verify = _config->FindB(knob.c_str(), peer_verify); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, peer_verify); + + // ... and hostname against cert CN or subjectAltName + bool verify = _config->FindB("Acquire::https::Verify-Host",true); + knob = "Acquire::https::"+remotehost+"::Verify-Host"; + verify = _config->FindB(knob.c_str(),verify); + int const default_verify = (verify == true) ? 2 : 0; + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, default_verify); + + // Also enforce issuer of server certificate using its cert + string issuercert = _config->Find("Acquire::https::IssuerCert",""); + knob = "Acquire::https::"+remotehost+"::IssuerCert"; + issuercert = _config->Find(knob.c_str(),issuercert.c_str()); + if(issuercert.empty() == false) + curl_easy_setopt(curl, CURLOPT_ISSUERCERT,issuercert.c_str()); + + // For client authentication, certificate file ... + string pem = _config->Find("Acquire::https::SslCert",""); + knob = "Acquire::https::"+remotehost+"::SslCert"; + pem = _config->Find(knob.c_str(),pem.c_str()); + if(pem.empty() == false) + curl_easy_setopt(curl, CURLOPT_SSLCERT, pem.c_str()); + + // ... and associated key. + string key = _config->Find("Acquire::https::SslKey",""); + knob = "Acquire::https::"+remotehost+"::SslKey"; + key = _config->Find(knob.c_str(),key.c_str()); + if(key.empty() == false) + curl_easy_setopt(curl, CURLOPT_SSLKEY, key.c_str()); + + // Allow forcing SSL version to SSLv3 or TLSv1 (SSLv2 is not + // supported by GnuTLS). + long final_version = CURL_SSLVERSION_DEFAULT; + string sslversion = _config->Find("Acquire::https::SslForceVersion",""); + knob = "Acquire::https::"+remotehost+"::SslForceVersion"; + sslversion = _config->Find(knob.c_str(),sslversion.c_str()); + if(sslversion == "TLSv1") + final_version = CURL_SSLVERSION_TLSv1; + else if(sslversion == "SSLv3") + final_version = CURL_SSLVERSION_SSLv3; + curl_easy_setopt(curl, CURLOPT_SSLVERSION, final_version); + + // CRL file + string crlfile = _config->Find("Acquire::https::CrlFile",""); + knob = "Acquire::https::"+remotehost+"::CrlFile"; + crlfile = _config->Find(knob.c_str(),crlfile.c_str()); + if(crlfile.empty() == false) + curl_easy_setopt(curl, CURLOPT_CRLFILE, crlfile.c_str()); + + // cache-control + if(_config->FindB("Acquire::https::No-Cache", + _config->FindB("Acquire::http::No-Cache",false)) == false) + { + // cache enabled + if (_config->FindB("Acquire::https::No-Store", + _config->FindB("Acquire::http::No-Store",false)) == true) + headers = curl_slist_append(headers,"Cache-Control: no-store"); + stringstream ss; + ioprintf(ss, "Cache-Control: max-age=%u", _config->FindI("Acquire::https::Max-Age", + _config->FindI("Acquire::http::Max-Age",0))); + headers = curl_slist_append(headers, ss.str().c_str()); + } else { + // cache disabled by user + headers = curl_slist_append(headers, "Cache-Control: no-cache"); + headers = curl_slist_append(headers, "Pragma: no-cache"); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + // speed limit + int const dlLimit = _config->FindI("Acquire::https::Dl-Limit", + _config->FindI("Acquire::http::Dl-Limit",0))*1024; + if (dlLimit > 0) + curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, dlLimit); + + // set header + curl_easy_setopt(curl, CURLOPT_USERAGENT, + _config->Find("Acquire::https::User-Agent", + _config->Find("Acquire::http::User-Agent", + "Debian APT-CURL/1.0 (" PACKAGE_VERSION ")").c_str()).c_str()); + + // set timeout + int const timeout = _config->FindI("Acquire::https::Timeout", + _config->FindI("Acquire::http::Timeout",120)); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout); + //set really low lowspeed timeout (see #497983) + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, DL_MIN_SPEED); + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout); + + // set redirect options and default to 10 redirects + bool const AllowRedirect = _config->FindB("Acquire::https::AllowRedirect", + _config->FindB("Acquire::http::AllowRedirect",true)); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10); + + // debug + if(_config->FindB("Debug::Acquire::https", false)) + curl_easy_setopt(curl, CURLOPT_VERBOSE, true); + + // error handling + curl_errorstr[0] = '\0'; + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr); + + // If we ask for uncompressed files servers might respond with content- + // negotiation which lets us end up with compressed files we do not support, + // see 657029, 657560 and co, so if we have no extension on the request + // ask for text only. As a sidenote: If there is nothing to negotate servers + // seem to be nice and ignore it. + if (_config->FindB("Acquire::https::SendAccept", _config->FindB("Acquire::http::SendAccept", true)) == true) + { + size_t const filepos = Itm->Uri.find_last_of('/'); + string const file = Itm->Uri.substr(filepos + 1); + if (flExtension(file) == file) + headers = curl_slist_append(headers, "Accept: text/*"); + } + + // if we have the file send an if-range query with a range header + if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) + { + char Buf[1000]; + sprintf(Buf, "Range: bytes=%li-", (long) SBuf.st_size); + headers = curl_slist_append(headers, Buf); + sprintf(Buf, "If-Range: %s", TimeRFC1123(SBuf.st_mtime).c_str()); + headers = curl_slist_append(headers, Buf); + } + else if(Itm->LastModified > 0) + { + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified); + } + + // go for it - if the file exists, append on it + File = new FileFd(Itm->DestFile, FileFd::WriteAny); + Server = new HttpsServerState(Itm->Uri, this); + + // keep apt updated + Res.Filename = Itm->DestFile; + + // get it! + CURLcode success = curl_easy_perform(curl); + + // If the server returns 200 OK but the If-Modified-Since condition is not + // met, CURLINFO_CONDITION_UNMET will be set to 1 + long curl_condition_unmet = 0; + curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &curl_condition_unmet); + + File->Close(); + curl_slist_free_all(headers); + + // cleanup + if (success != 0) + { + _error->Error("%s", curl_errorstr); + unlink(File->Name().c_str()); + return false; + } + + // server says file not modified + if (Server->Result == 304 || curl_condition_unmet == 1) + { + unlink(File->Name().c_str()); + Res.IMSHit = true; + Res.LastModified = Itm->LastModified; + Res.Size = 0; + URIDone(Res); + return true; + } + Res.IMSHit = false; + + if (Server->Result != 200 && // OK + Server->Result != 206 && // Partial + Server->Result != 416) // invalid Range + { + char err[255]; + snprintf(err, sizeof(err) - 1, "HttpError%i", Server->Result); + SetFailReason(err); + _error->Error("%s", err); + // unlink, no need keep 401/404 page content in partial/ + unlink(File->Name().c_str()); + return false; + } + + struct stat resultStat; + if (unlikely(stat(File->Name().c_str(), &resultStat) != 0)) + { + _error->Errno("stat", "Unable to access file %s", File->Name().c_str()); + return false; + } + Res.Size = resultStat.st_size; + + // invalid range-request + if (Server->Result == 416) + { + unlink(File->Name().c_str()); + Res.Size = 0; + delete File; + Redirect(Itm->Uri); + return true; + } + + // Timestamp + curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified); + if (Res.LastModified != -1) + { + struct timeval times[2]; + times[0].tv_sec = Res.LastModified; + times[1].tv_sec = Res.LastModified; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(File->Name().c_str(), times); + } + else + Res.LastModified = resultStat.st_mtime; + + // take hashes + Hashes Hash; + FileFd Fd(Res.Filename, FileFd::ReadOnly); + Hash.AddFD(Fd); + Res.TakeHashes(Hash); + + // keep apt updated + URIDone(Res); + + // cleanup + Res.Size = 0; + delete File; + + return true; +} + +int main() +{ + setlocale(LC_ALL, ""); + + HttpsMethod Mth; + curl_global_init(CURL_GLOBAL_SSL) ; + + return Mth.Run(); +} + diff --git a/tor.h b/tor.h new file mode 100644 index 0000000..faac8a3 --- /dev/null +++ b/tor.h @@ -0,0 +1,88 @@ +// -*- mode: cpp; mode: fold -*- +// Description /*{{{*/// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +// $Id: http.h,v 1.12 2002/04/18 05:09:38 jgg Exp $ +/* ###################################################################### + + HTTP Acquire Method - This is the HTTP acquire method for APT. + + ##################################################################### */ + /*}}}*/ + +#ifndef APT_HTTPS_H +#define APT_HTTPS_H + +#include + +#include +#include +#include +#include + +#include "server.h" + +using std::cout; +using std::endl; + +class Hashes; +class HttpsMethod; +class FileFd; + +class HttpsServerState : public ServerState +{ + protected: + virtual bool ReadHeaderLines(std::string &/*Data*/) { return false; } + virtual bool LoadNextResponse(bool const /*ToFile*/, FileFd * const /*File*/) { return false; } + + public: + virtual bool WriteResponse(std::string const &/*Data*/) { return false; } + + /** \brief Transfer the data from the socket */ + virtual bool RunData(FileFd * const /*File*/) { return false; } + + virtual bool Open() { return false; } + virtual bool IsOpen() { return false; } + virtual bool Close() { return false; } + virtual bool InitHashes(FileFd &/*File*/) { return false; } + virtual Hashes * GetHashes() { return NULL; } + virtual bool Die(FileFd &/*File*/) { return false; } + virtual bool Flush(FileFd * const /*File*/) { return false; } + virtual bool Go(bool /*ToFile*/, FileFd * const /*File*/) { return false; } + + HttpsServerState(URI Srv, HttpsMethod *Owner); + virtual ~HttpsServerState() {Close();}; +}; + +class HttpsMethod : public pkgAcqMethod +{ + // minimum speed in bytes/se that triggers download timeout handling + static const int DL_MIN_SPEED = 10; + + virtual bool Fetch(FetchItem *); + static size_t parse_header(void *buffer, size_t size, size_t nmemb, void *userp); + static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp); + static int progress_callback(void *clientp, double dltotal, double dlnow, + double ultotal, double ulnow); + void SetupProxy(); + CURL *curl; + FetchResult Res; + HttpsServerState *Server; + + public: + FileFd *File; + + HttpsMethod() : pkgAcqMethod("1.2",Pipeline | SendConfig), File(NULL) + { + File = 0; + curl = curl_easy_init(); + }; + + ~HttpsMethod() + { + curl_easy_cleanup(curl); + }; +}; + +#include +URI Proxy; + +#endif -- cgit v1.2.3-18-g5258