diff options
Diffstat (limited to 'methods')
| -rw-r--r-- | methods/aptmethod.h | 73 | ||||
| -rw-r--r-- | methods/cdrom.cc | 4 | ||||
| -rw-r--r-- | methods/connect.cc | 20 | ||||
| -rw-r--r-- | methods/gpgv.cc | 6 | ||||
| -rw-r--r-- | methods/http.cc | 286 | ||||
| -rw-r--r-- | methods/http.h | 14 | ||||
| -rw-r--r-- | methods/http_main.cc | 8 | ||||
| -rw-r--r-- | methods/https.cc | 359 | ||||
| -rw-r--r-- | methods/https.h | 16 | ||||
| -rw-r--r-- | methods/mirror.cc | 4 | ||||
| -rw-r--r-- | methods/rred.cc | 2 | ||||
| -rw-r--r-- | methods/rsh.cc | 12 | ||||
| -rw-r--r-- | methods/rsh.h | 3 | ||||
| -rw-r--r-- | methods/server.cc | 204 | ||||
| -rw-r--r-- | methods/server.h | 8 | ||||
| -rw-r--r-- | methods/store.cc | 11 |
16 files changed, 662 insertions, 368 deletions
diff --git a/methods/aptmethod.h b/methods/aptmethod.h index d3c948636..38f451708 100644 --- a/methods/aptmethod.h +++ b/methods/aptmethod.h @@ -5,8 +5,10 @@ #include <apt-pkg/configuration.h> #include <apt-pkg/error.h> +#include <algorithm> #include <locale> #include <string> +#include <vector> #include <sys/time.h> #include <sys/types.h> @@ -15,9 +17,15 @@ #include <apti18n.h> +static bool hasDoubleColon(std::string const &n) +{ + return n.find("::") != std::string::npos; +} + class aptMethod : public pkgAcqMethod { - char const * const Binary; +protected: + std::string const Binary; public: virtual bool Configuration(std::string Message) APT_OVERRIDE @@ -33,7 +41,7 @@ public: return true; } - bool CalculateHashes(FetchItem const * const Itm, FetchResult &Res) const + bool CalculateHashes(FetchItem const * const Itm, FetchResult &Res) const APT_NONNULL(2) { Hashes Hash(Itm->ExpectedHashes); FileFd Fd; @@ -51,7 +59,62 @@ public: va_end(args); } - bool TransferModificationTimes(char const * const From, char const * const To, time_t &LastModified) + std::vector<std::string> methodNames; + void setPostfixForMethodNames(char const * const postfix) APT_NONNULL(2) + { + methodNames.erase(std::remove_if(methodNames.begin(), methodNames.end(), hasDoubleColon), methodNames.end()); + decltype(methodNames) toAdd; + for (auto && name: methodNames) + toAdd.emplace_back(name + "::" + postfix); + std::move(toAdd.begin(), toAdd.end(), std::back_inserter(methodNames)); + } + bool DebugEnabled() const + { + if (methodNames.empty()) + return false; + auto const sni = std::find_if_not(methodNames.crbegin(), methodNames.crend(), hasDoubleColon); + if (unlikely(sni == methodNames.crend())) + return false; + auto const ln = methodNames[methodNames.size() - 1]; + // worst case: all three are the same + std::string confln, confsn, confpn; + strprintf(confln, "Debug::Acquire::%s", ln.c_str()); + strprintf(confsn, "Debug::Acquire::%s", sni->c_str()); + auto const pni = sni->substr(0, sni->find('+')); + strprintf(confpn, "Debug::Acquire::%s", pni.c_str()); + return _config->FindB(confln,_config->FindB(confsn, _config->FindB(confpn, false))); + } + std::string ConfigFind(char const * const postfix, std::string const &defValue) const APT_NONNULL(2) + { + for (auto && name: methodNames) + { + std::string conf; + strprintf(conf, "Acquire::%s::%s", name.c_str(), postfix); + auto const value = _config->Find(conf); + if (value.empty() == false) + return value; + } + return defValue; + } + std::string ConfigFind(std::string const &postfix, std::string const &defValue) const + { + return ConfigFind(postfix.c_str(), defValue); + } + bool ConfigFindB(char const * const postfix, bool const defValue) const APT_NONNULL(2) + { + return StringToBool(ConfigFind(postfix, defValue ? "yes" : "no"), defValue); + } + int ConfigFindI(char const * const postfix, int const defValue) const APT_NONNULL(2) + { + char *End; + std::string const value = ConfigFind(postfix, ""); + auto const Res = strtol(value.c_str(), &End, 0); + if (value.c_str() == End) + return defValue; + return Res; + } + + bool TransferModificationTimes(char const * const From, char const * const To, time_t &LastModified) APT_NONNULL(2, 3) { if (strcmp(To, "/dev/null") == 0) return true; @@ -74,8 +137,8 @@ public: return true; } - aptMethod(char const * const Binary, char const * const Ver, unsigned long const Flags) : - pkgAcqMethod(Ver, Flags), Binary(Binary) + aptMethod(std::string &&Binary, char const * const Ver, unsigned long const Flags) APT_NONNULL(3) : + pkgAcqMethod(Ver, Flags), Binary(Binary), methodNames({Binary}) { try { std::locale::global(std::locale("")); diff --git a/methods/cdrom.cc b/methods/cdrom.cc index 161822ac6..87a58e948 100644 --- a/methods/cdrom.cc +++ b/methods/cdrom.cc @@ -177,7 +177,7 @@ bool CDROMMethod::Fetch(FetchItem *Itm) URI Get = Itm->Uri; string File = Get.Path; - Debug = _config->FindB("Debug::Acquire::cdrom", false); + Debug = DebugEnabled(); if (Debug) clog << "CDROMMethod::Fetch " << Itm->Uri << endl; @@ -224,7 +224,7 @@ bool CDROMMethod::Fetch(FetchItem *Itm) return true; } - bool AutoDetect = _config->FindB("Acquire::cdrom::AutoDetect", true); + bool const AutoDetect = ConfigFindB("AutoDetect", true); CDROM = _config->FindDir("Acquire::cdrom::mount"); if (Debug) clog << "Looking for CDROM at " << CDROM << endl; diff --git a/methods/connect.cc b/methods/connect.cc index f768169d1..c819c1dfb 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -61,10 +61,23 @@ void RotateDNS() LastUsed = LastHostAddr; } /*}}}*/ +static bool ConnectionAllowed(char const * const Service, std::string const &Host)/*{{{*/ +{ + if (APT::String::Endswith(Host, ".onion") && _config->FindB("Acquire::BlockDotOnion", true)) + { + // TRANSLATOR: %s is e.g. Tor's ".onion" which would likely fail or leak info (RFC7686) + _error->Error(_("Direct connection to %s domains is blocked by default."), ".onion"); + if (strcmp(Service, "http") == 0) + _error->Error(_("If you meant to use Tor remember to use %s instead of %s."), "tor+http", "http"); + return false; + } + return true; +} + /*}}}*/ // DoConnect - Attempt a connect operation /*{{{*/ // --------------------------------------------------------------------- /* This helper function attempts a connection to a single address. */ -static bool DoConnect(struct addrinfo *Addr,std::string Host, +static bool DoConnect(struct addrinfo *Addr,std::string const &Host, unsigned long TimeOut,int &Fd,pkgAcqMethod *Owner) { // Show a status indicator @@ -138,6 +151,8 @@ static bool ConnectToHostname(std::string const &Host, int const Port, const char * const Service, int DefPort, int &Fd, unsigned long const TimeOut, pkgAcqMethod * const Owner) { + if (ConnectionAllowed(Service, Host) == false) + return false; // Convert the port name/number char ServStr[300]; if (Port != 0) @@ -274,6 +289,9 @@ bool Connect(std::string Host,int Port,const char *Service, if (_error->PendingError() == true) return false; + if (ConnectionAllowed(Service, Host) == false) + return false; + if(LastHost != Host || LastPort != Port) { SrvRecords.clear(); diff --git a/methods/gpgv.cc b/methods/gpgv.cc index fc6eb9159..2fed53a39 100644 --- a/methods/gpgv.cc +++ b/methods/gpgv.cc @@ -147,7 +147,7 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile, vector<Signer> &SoonWorthlessSigners, vector<string> &NoPubKeySigners) { - bool const Debug = _config->FindB("Debug::Acquire::gpgv", false); + bool const Debug = DebugEnabled(); if (Debug == true) std::clog << "inside VerifyGetSigners" << std::endl; @@ -415,10 +415,8 @@ bool GPGVMethod::URIAcquire(std::string const &Message, FetchItem *Itm) std::move(NoPubKeySigners.begin(), NoPubKeySigners.end(), std::back_inserter(Res.GPGVOutput)); URIDone(Res); - if (_config->FindB("Debug::Acquire::gpgv", false)) - { + if (DebugEnabled()) std::clog << "apt-key succeeded\n"; - } return true; } diff --git a/methods/http.cc b/methods/http.cc index 9fcc80103..0358b50cd 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -44,6 +44,7 @@ #include <unistd.h> #include <stdio.h> #include <errno.h> +#include <arpa/inet.h> #include <iostream> #include <sstream> @@ -63,13 +64,13 @@ const unsigned int CircleBuf::BW_HZ=10; // CircleBuf::CircleBuf - Circular input buffer /*{{{*/ // --------------------------------------------------------------------- /* */ -CircleBuf::CircleBuf(unsigned long long Size) +CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size) : Size(Size), Hash(NULL), TotalWriten(0) { Buf = new unsigned char[Size]; Reset(); - CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024; + CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024; } /*}}}*/ // CircleBuf::Reset - Reset to the default state /*{{{*/ @@ -151,9 +152,9 @@ bool CircleBuf::Read(int Fd) // CircleBuf::Read - Put the string into the buffer /*{{{*/ // --------------------------------------------------------------------- /* This will hold the string in and fill the buffer with it as it empties */ -bool CircleBuf::Read(string Data) +bool CircleBuf::Read(string const &Data) { - OutQueue += Data; + OutQueue.append(Data); FillOut(); return true; } @@ -287,15 +288,33 @@ CircleBuf::~CircleBuf() } // HttpServerState::HttpServerState - Constructor /*{{{*/ -HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(64*1024), Out(4*1024) +HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024) { - TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut); + TimeOut = Owner->ConfigFindI("Timeout", TimeOut); Reset(); } /*}}}*/ // HttpServerState::Open - Open a connection to the server /*{{{*/ // --------------------------------------------------------------------- /* This opens a connection to the server. */ +static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, + char const * const type, bool const ReadWrite, uint8_t * const ToFrom, + unsigned int const Size, unsigned int const Timeout) +{ + if (WaitFd(ServerFd, ReadWrite, Timeout) == false) + return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type); + if (ReadWrite == false) + { + if (FileFd::Read(ServerFd, ToFrom, Size) == false) + return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); + } + else + { + if (FileFd::Write(ServerFd, ToFrom, Size) == false) + return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); + } + return true; +} bool HttpServerState::Open() { // Use the already open connection if possible. @@ -309,7 +328,7 @@ bool HttpServerState::Open() // Determine the proxy setting AutoDetectProxy(ServerName); - string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host); + string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, ""); if (!SpecificProxy.empty()) { if (SpecificProxy == "DIRECT") @@ -319,7 +338,7 @@ bool HttpServerState::Open() } else { - string DefProxy = _config->Find("Acquire::http::Proxy"); + string DefProxy = Owner->ConfigFind("Proxy", ""); if (!DefProxy.empty()) { Proxy = DefProxy; @@ -337,27 +356,159 @@ bool HttpServerState::Open() if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true) Proxy = ""; } - - // Determine what host and port to use based on the proxy settings - int Port = 0; - string Host; - if (Proxy.empty() == true || Proxy.Host.empty() == true) + + if (Proxy.empty() == false) + Owner->AddProxyAuth(Proxy, ServerName); + + if (Proxy.Access == "socks5h") { - if (ServerName.Port != 0) - Port = ServerName.Port; - Host = ServerName.Host; + if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false) + return false; + + /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect + * for not offering GSSAPI auth which is a must (we only do no or user/pass auth). + * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */ + std::string const ProxyInfo = URI::SiteOnly(Proxy); + Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str()); + auto const Timeout = Owner->ConfigFindI("TimeOut", 120); + #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false + #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false + if (ServerName.Host.length() > 255) + return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str()); + if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) + return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length()); + if (Proxy.User.empty()) + { + uint8_t greeting[] = { 0x05, 0x01, 0x00 }; + APT_WriteOrFail("greet-1", greeting, sizeof(greeting)); + } + else + { + uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 }; + APT_WriteOrFail("greet-2", greeting, sizeof(greeting)); + } + uint8_t greeting[2]; + APT_ReadOrFail("greet back", greeting, sizeof(greeting)); + if (greeting[0] != 0x05) + return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]); + if (greeting[1] == 0x00) + ; // no auth has no method-dependent sub-negotiations + else if (greeting[1] == 0x02) + { + if (Proxy.User.empty()) + return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str()); + // user&pass auth sub-negotiations are defined by RFC1929 + std::vector<uint8_t> auth = {{ 0x01, static_cast<uint8_t>(Proxy.User.length()) }}; + std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth)); + auth.push_back(static_cast<uint8_t>(Proxy.Password.length())); + std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth)); + APT_WriteOrFail("user&pass auth", auth.data(), auth.size()); + uint8_t authstatus[2]; + APT_ReadOrFail("auth report", authstatus, sizeof(authstatus)); + if (authstatus[0] != 0x01) + return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]); + if (authstatus[1] != 0x00) + return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]); + } + else + return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]); + union { uint16_t * i; uint8_t * b; } portu; + uint16_t port = htons(static_cast<uint16_t>(ServerName.Port == 0 ? 80 : ServerName.Port)); + portu.i = &port; + std::vector<uint8_t> request = {{ 0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(ServerName.Host.length()) }}; + std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request)); + request.push_back(portu.b[0]); + request.push_back(portu.b[1]); + APT_WriteOrFail("request", request.data(), request.size()); + uint8_t response[4]; + APT_ReadOrFail("first part of response", response, sizeof(response)); + if (response[0] != 0x05) + return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]); + if (response[2] != 0x00) + return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]); + std::string bindaddr; + if (response[3] == 0x01) // IPv4 address + { + uint8_t ip4port[6]; + APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port)); + portu.b[0] = ip4port[4]; + portu.b[1] = ip4port[5]; + port = ntohs(*portu.i); + strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port); + } + else if (response[3] == 0x03) // hostname + { + uint8_t namelength; + APT_ReadOrFail("hostname length of response", &namelength, 1); + uint8_t hostname[namelength + 2]; + APT_ReadOrFail("hostname of response", hostname, sizeof(hostname)); + portu.b[0] = hostname[namelength]; + portu.b[1] = hostname[namelength + 1]; + port = ntohs(*portu.i); + hostname[namelength] = '\0'; + strprintf(bindaddr, "%s:%d", hostname, port); + } + else if (response[3] == 0x04) // IPv6 address + { + uint8_t ip6port[18]; + APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port)); + portu.b[0] = ip6port[16]; + portu.b[1] = ip6port[17]; + port = ntohs(*portu.i); + strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d", + ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7], + ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15], + port); + } + else + return _error->Error("SOCKS proxy %s destination address is of unknown type: %d", + ProxyInfo.c_str(), response[3]); + if (response[1] != 0x00) + { + char const * errstr; + switch (response[1]) + { + case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break; + case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break; + case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break; + case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break; + case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break; + case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break; + case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break; + case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break; + default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break; + } + return _error->Error("SOCKS proxy %s didn't grant the connect to %s due to: %s (%d)", ProxyInfo.c_str(), bindaddr.c_str(), errstr, response[1]); + } + else if (Owner->DebugEnabled()) + ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s\n", ProxyInfo.c_str(), bindaddr.c_str()); + + if (WaitFd(ServerFd, true, Timeout) == false) + return _error->Error("SOCKS proxy %s reported connection, but timed out", ProxyInfo.c_str()); + #undef APT_ReadOrFail + #undef APT_WriteOrFail } else { - if (Proxy.Port != 0) - Port = Proxy.Port; - Host = Proxy.Host; + // Determine what host and port to use based on the proxy settings + int Port = 0; + string Host; + if (Proxy.empty() == true || Proxy.Host.empty() == true) + { + if (ServerName.Port != 0) + Port = ServerName.Port; + Host = ServerName.Host; + } + else if (Proxy.Access != "http") + return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str()); + else + { + if (Proxy.Port != 0) + Port = Proxy.Port; + Host = Proxy.Host; + } + return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner); } - - // Connect to the remote server - if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false) - return false; - return true; } /*}}}*/ @@ -463,6 +614,12 @@ bool HttpServerState::RunData(FileFd * const File) return Owner->Flush() && !_error->PendingError(); } /*}}}*/ +bool HttpServerState::RunDataToDevNull() /*{{{*/ +{ + FileFd DevNull("/dev/null", FileFd::WriteOnly); + return RunData(&DevNull); +} + /*}}}*/ bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/ { return In.WriteTillEl(Data); @@ -608,7 +765,7 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) FD_SET(FileFD,&wfds); // Add stdin - if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true) + if (Owner->ConfigFindB("DependOnSTDIN", true) == true) FD_SET(STDIN_FILENO,&rfds); // Figure out the max fd @@ -680,6 +837,11 @@ bool HttpServerState::Go(bool ToFile, FileFd * const File) void HttpMethod::SendReq(FetchItem *Itm) { URI Uri = Itm->Uri; + { + auto const plus = Binary.find('+'); + if (plus != std::string::npos) + Uri.Access = Binary.substr(plus + 1); + } // The HTTP server expects a hostname with a trailing :port std::stringstream Req; @@ -694,10 +856,10 @@ void HttpMethod::SendReq(FetchItem *Itm) but while its a must for all servers to accept absolute URIs, it is assumed clients will sent an absolute path for non-proxies */ std::string requesturi; - if (Server->Proxy.empty() == true || Server->Proxy.Host.empty()) + if (Server->Proxy.Access != "http" || Server->Proxy.empty() == true || Server->Proxy.Host.empty()) requesturi = Uri.Path; else - requesturi = Itm->Uri; + requesturi = Uri; // The "+" is encoded as a workaround for a amazon S3 bug // see LP bugs #1003633 and #1086997. @@ -714,12 +876,12 @@ void HttpMethod::SendReq(FetchItem *Itm) Req << "Host: " << ProperHost << "\r\n"; // generate a cache control header (if needed) - if (_config->FindB("Acquire::http::No-Cache",false) == true) + if (ConfigFindB("No-Cache",false) == true) Req << "Cache-Control: no-cache\r\n" << "Pragma: no-cache\r\n"; else if (Itm->IndexFile == true) - Req << "Cache-Control: max-age=" << std::to_string(_config->FindI("Acquire::http::Max-Age",0)) << "\r\n"; - else if (_config->FindB("Acquire::http::No-Store",false) == true) + Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n"; + else if (ConfigFindB("No-Store", false) == true) Req << "Cache-Control: no-store\r\n"; // If we ask for uncompressed files servers might respond with content- @@ -727,7 +889,7 @@ void HttpMethod::SendReq(FetchItem *Itm) // see 657029, 657560 and co, so if we have no extension on the request // ask for text only. As a sidenote: If there is nothing to negotate servers // seem to be nice and ignore it. - if (_config->FindB("Acquire::http::SendAccept", true) == true) + if (ConfigFindB("SendAccept", true) == true) { size_t const filepos = Itm->Uri.find_last_of('/'); string const file = Itm->Uri.substr(filepos + 1); @@ -743,7 +905,8 @@ void HttpMethod::SendReq(FetchItem *Itm) else if (Itm->LastModified != 0) Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n"; - if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false) + if (Server->Proxy.Access == "http" && + (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)) Req << "Proxy-Authorization: Basic " << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n"; @@ -752,7 +915,7 @@ void HttpMethod::SendReq(FetchItem *Itm) Req << "Authorization: Basic " << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n"; - Req << "User-Agent: " << _config->Find("Acquire::http::User-Agent", + Req << "User-Agent: " << ConfigFind("User-Agent", "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n"; Req << "\r\n"; @@ -763,22 +926,6 @@ void HttpMethod::SendReq(FetchItem *Itm) Server->WriteResponse(Req.str()); } /*}}}*/ -// HttpMethod::Configuration - Handle a configuration message /*{{{*/ -// --------------------------------------------------------------------- -/* We stash the desired pipeline depth */ -bool HttpMethod::Configuration(string Message) -{ - if (ServerMethod::Configuration(Message) == false) - return false; - - AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true); - PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth", - PipelineDepth); - Debug = _config->FindB("Debug::Acquire::http",false); - - return true; -} - /*}}}*/ std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/ { return std::unique_ptr<ServerState>(new HttpServerState(uri, this)); @@ -789,3 +936,44 @@ void HttpMethod::RotateDNS() /*{{{*/ ::RotateDNS(); } /*}}}*/ +ServerMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res)/*{{{*/ +{ + auto ret = ServerMethod::DealWithHeaders(Res); + if (ret != ServerMethod::FILE_IS_OPEN) + return ret; + + // Open the file + delete File; + File = new FileFd(Queue->DestFile,FileFd::WriteAny); + if (_error->PendingError() == true) + return ERROR_NOT_FROM_SERVER; + + FailFile = Queue->DestFile; + FailFile.c_str(); // Make sure we don't do a malloc in the signal handler + FailFd = File->Fd(); + FailTime = Server->Date; + + if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false) + { + _error->Errno("read",_("Problem hashing file")); + return ERROR_NOT_FROM_SERVER; + } + if (Server->StartPos > 0) + Res.ResumePoint = Server->StartPos; + + SetNonBlock(File->Fd(),true); + return FILE_IS_OPEN; +} + /*}}}*/ +HttpMethod::HttpMethod(std::string &&pProg) : ServerMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/ +{ + auto addName = std::inserter(methodNames, methodNames.begin()); + if (Binary != "http") + addName = "http"; + auto const plus = Binary.find('+'); + if (plus != std::string::npos) + addName = Binary.substr(0, plus); + File = 0; + Server = 0; +} + /*}}}*/ diff --git a/methods/http.h b/methods/http.h index ac5f52314..644e576f0 100644 --- a/methods/http.h +++ b/methods/http.h @@ -67,7 +67,7 @@ class CircleBuf // Read data in bool Read(int Fd); - bool Read(std::string Data); + bool Read(std::string const &Data); // Write data out bool Write(int Fd); @@ -86,7 +86,7 @@ class CircleBuf // Dump everything void Stats(); - explicit CircleBuf(unsigned long long Size); + CircleBuf(HttpMethod const * const Owner, unsigned long long Size); ~CircleBuf(); }; @@ -106,6 +106,7 @@ struct HttpServerState: public ServerState virtual void Reset() APT_OVERRIDE { ServerState::Reset(); ServerFd = -1; }; virtual bool RunData(FileFd * const File) APT_OVERRIDE; + virtual bool RunDataToDevNull() APT_OVERRIDE; virtual bool Open() APT_OVERRIDE; virtual bool IsOpen() APT_OVERRIDE; @@ -125,10 +126,9 @@ class HttpMethod : public ServerMethod public: virtual void SendReq(FetchItem *Itm) APT_OVERRIDE; - virtual bool Configuration(std::string Message) APT_OVERRIDE; - virtual std::unique_ptr<ServerState> CreateServerState(URI const &uri) APT_OVERRIDE; virtual void RotateDNS() APT_OVERRIDE; + virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res) APT_OVERRIDE; protected: std::string AutoDetectProxyCmd; @@ -136,11 +136,7 @@ class HttpMethod : public ServerMethod public: friend struct HttpServerState; - HttpMethod() : ServerMethod("http", "1.2",Pipeline | SendConfig) - { - File = 0; - Server = 0; - }; + explicit HttpMethod(std::string &&pProg); }; #endif diff --git a/methods/http_main.cc b/methods/http_main.cc index fa183ddb3..1e56044b7 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -5,11 +5,13 @@ #include "http.h" -int main() +int main(int, const char *argv[]) { // ignore SIGPIPE, this can happen on write() if the socket // closes the connection (this is dealt with via ServerDie()) signal(SIGPIPE, SIG_IGN); - - return HttpMethod().Loop(); + std::string Binary = flNotDir(argv[0]); + if (Binary.find('+') == std::string::npos && Binary != "http") + Binary.append("+http"); + return HttpMethod(std::move(Binary)).Loop(); } diff --git a/methods/https.cc b/methods/https.cc index bbdbd8fa9..283126f6b 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -25,11 +25,14 @@ #include <sys/time.h> #include <unistd.h> #include <stdio.h> -#include <iostream> -#include <sstream> #include <ctype.h> #include <stdlib.h> +#include <array> +#include <iostream> +#include <sstream> + + #include "https.h" #include <apti18n.h> @@ -60,6 +63,7 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) if (line.empty() == true) { + me->https->Server->JunkSize = 0; if (me->https->Server->Result != 416 && me->https->Server->StartPos != 0) ; else if (me->https->Server->Result == 416) @@ -99,11 +103,13 @@ HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp) // we expect valid data, so tell our caller we get the file now if (me->https->Server->Result >= 200 && me->https->Server->Result < 300) { - if (me->https->Server->JunkSize == 0 && me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint) + if (me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint) me->https->URIStart(*me->Res); if (me->https->Server->AddPartialFileToHashes(*(me->https->File)) == false) return 0; } + else + me->https->Server->JunkSize = std::numeric_limits<decltype(me->https->Server->JunkSize)>::max(); } else if (me->https->Server->HeaderLine(line) == false) return 0; @@ -145,7 +151,7 @@ HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp) // HttpsServerState::HttpsServerState - Constructor /*{{{*/ HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner), Hash(NULL) { - TimeOut = _config->FindI("Acquire::https::Timeout",TimeOut); + TimeOut = Owner->ConfigFindI("Timeout", TimeOut); Reset(); } /*}}}*/ @@ -162,7 +168,7 @@ APT_PURE Hashes * HttpsServerState::GetHashes() /*{{{*/ } /*}}}*/ -void HttpsMethod::SetupProxy() /*{{{*/ +bool HttpsMethod::SetupProxy() /*{{{*/ { URI ServerName = Queue->Uri; @@ -176,38 +182,52 @@ void HttpsMethod::SetupProxy() /*{{{*/ curl_easy_setopt(curl, CURLOPT_PROXY, ""); // Determine the proxy setting - try https first, fallback to http and use env at last - string UseProxy = _config->Find("Acquire::https::Proxy::" + ServerName.Host, - _config->Find("Acquire::http::Proxy::" + ServerName.Host).c_str()); - + string UseProxy = ConfigFind("Proxy::" + ServerName.Host, ""); if (UseProxy.empty() == true) - UseProxy = _config->Find("Acquire::https::Proxy", _config->Find("Acquire::http::Proxy").c_str()); - - // User want to use NO proxy, so nothing to setup + UseProxy = ConfigFind("Proxy", ""); + // User wants to use NO proxy, so nothing to setup if (UseProxy == "DIRECT") - return; + return true; - // Parse no_proxy, a comma (,) separated list of domains we don't want to use + // Parse no_proxy, a comma (,) separated list of domains we don't want to use // a proxy for so we stop right here if it is in the list if (getenv("no_proxy") != 0 && CheckDomainList(ServerName.Host,getenv("no_proxy")) == true) - return; + return true; if (UseProxy.empty() == true) { - const char* result = getenv("https_proxy"); + const char* result = nullptr; + if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end()) + result = getenv("https_proxy"); // FIXME: Fall back to http_proxy is to remain compatible with // existing setups and behaviour of apt.conf. This should be // deprecated in the future (including apt.conf). Most other // programs do not fall back to http proxy settings and neither // should Apt. - if (result == NULL) - result = getenv("http_proxy"); - UseProxy = result == NULL ? "" : result; + if (result == nullptr && std::find(methodNames.begin(), methodNames.end(), "http") != methodNames.end()) + result = getenv("http_proxy"); + UseProxy = result == nullptr ? "" : result; } // Determine what host and port to use based on the proxy settings - if (UseProxy.empty() == false) + if (UseProxy.empty() == false) { Proxy = UseProxy; + AddProxyAuth(Proxy, ServerName); + + if (Proxy.Access == "socks5h") + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME); + else if (Proxy.Access == "socks5") + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); + else if (Proxy.Access == "socks4a") + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A); + else if (Proxy.Access == "socks") + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4); + else if (Proxy.Access == "http" || Proxy.Access == "https") + curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); + else + return false; + if (Proxy.Port != 1) curl_easy_setopt(curl, CURLOPT_PROXYPORT, Proxy.Port); curl_easy_setopt(curl, CURLOPT_PROXY, Proxy.Host.c_str()); @@ -217,6 +237,7 @@ void HttpsMethod::SetupProxy() /*{{{*/ curl_easy_setopt(curl, CURLOPT_PROXYPASSWORD, Proxy.Password.c_str()); } } + return true; } /*}}}*/ // HttpsMethod::Fetch - Fetch an item /*{{{*/ // --------------------------------------------------------------------- @@ -228,15 +249,23 @@ bool HttpsMethod::Fetch(FetchItem *Itm) struct curl_slist *headers=NULL; char curl_errorstr[CURL_ERROR_SIZE]; URI Uri = Itm->Uri; - string remotehost = Uri.Host; + setPostfixForMethodNames(Uri.Host.c_str()); + AllowRedirect = ConfigFindB("AllowRedirect", true); + Debug = DebugEnabled(); // TODO: // - http::Pipeline-Depth // - error checking/reporting // - more debug options? (CURLOPT_DEBUGFUNCTION?) + { + auto const plus = Binary.find('+'); + if (plus != std::string::npos) + Uri.Access = Binary.substr(plus + 1); + } curl_easy_reset(curl); - SetupProxy(); + if (SetupProxy() == false) + return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str()); maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc")); @@ -251,116 +280,87 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // options curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true); curl_easy_setopt(curl, CURLOPT_FILETIME, true); - // only allow curl to handle https, not the other stuff it supports - curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS); - curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS); - - // SSL parameters are set by default to the common (non mirror-specific) value - // if available (or a default one) and gets overload by mirror-specific ones. - - // File containing the list of trusted CA. - string cainfo = _config->Find("Acquire::https::CaInfo",""); - string knob = "Acquire::https::"+remotehost+"::CaInfo"; - cainfo = _config->Find(knob.c_str(),cainfo.c_str()); - if(cainfo.empty() == false) - curl_easy_setopt(curl, CURLOPT_CAINFO,cainfo.c_str()); - - // Check server certificate against previous CA list ... - bool peer_verify = _config->FindB("Acquire::https::Verify-Peer",true); - knob = "Acquire::https::" + remotehost + "::Verify-Peer"; - peer_verify = _config->FindB(knob.c_str(), peer_verify); - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, peer_verify); - - // ... and hostname against cert CN or subjectAltName - bool verify = _config->FindB("Acquire::https::Verify-Host",true); - knob = "Acquire::https::"+remotehost+"::Verify-Host"; - verify = _config->FindB(knob.c_str(),verify); - int const default_verify = (verify == true) ? 2 : 0; - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, default_verify); - - // Also enforce issuer of server certificate using its cert - string issuercert = _config->Find("Acquire::https::IssuerCert",""); - knob = "Acquire::https::"+remotehost+"::IssuerCert"; - issuercert = _config->Find(knob.c_str(),issuercert.c_str()); - if(issuercert.empty() == false) - curl_easy_setopt(curl, CURLOPT_ISSUERCERT,issuercert.c_str()); - - // For client authentication, certificate file ... - string pem = _config->Find("Acquire::https::SslCert",""); - knob = "Acquire::https::"+remotehost+"::SslCert"; - pem = _config->Find(knob.c_str(),pem.c_str()); - if(pem.empty() == false) - curl_easy_setopt(curl, CURLOPT_SSLCERT, pem.c_str()); - - // ... and associated key. - string key = _config->Find("Acquire::https::SslKey",""); - knob = "Acquire::https::"+remotehost+"::SslKey"; - key = _config->Find(knob.c_str(),key.c_str()); - if(key.empty() == false) - curl_easy_setopt(curl, CURLOPT_SSLKEY, key.c_str()); - - // Allow forcing SSL version to SSLv3 or TLSv1 (SSLv2 is not - // supported by GnuTLS). - long final_version = CURL_SSLVERSION_DEFAULT; - string sslversion = _config->Find("Acquire::https::SslForceVersion",""); - knob = "Acquire::https::"+remotehost+"::SslForceVersion"; - sslversion = _config->Find(knob.c_str(),sslversion.c_str()); - if(sslversion == "TLSv1") - final_version = CURL_SSLVERSION_TLSv1; - else if(sslversion == "SSLv3") - final_version = CURL_SSLVERSION_SSLv3; - curl_easy_setopt(curl, CURLOPT_SSLVERSION, final_version); - - // CRL file - string crlfile = _config->Find("Acquire::https::CrlFile",""); - knob = "Acquire::https::"+remotehost+"::CrlFile"; - crlfile = _config->Find(knob.c_str(),crlfile.c_str()); - if(crlfile.empty() == false) - curl_easy_setopt(curl, CURLOPT_CRLFILE, crlfile.c_str()); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0); + if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end()) + { + curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS); + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS); + + // File containing the list of trusted CA. + std::string const cainfo = ConfigFind("CaInfo", ""); + if(cainfo.empty() == false) + curl_easy_setopt(curl, CURLOPT_CAINFO, cainfo.c_str()); + // Check server certificate against previous CA list ... + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, ConfigFindB("Verify-Peer", true) ? 1 : 0); + // ... and hostname against cert CN or subjectAltName + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, ConfigFindB("Verify-Host", true) ? 2 : 0); + // Also enforce issuer of server certificate using its cert + std::string const issuercert = ConfigFind("IssuerCert", ""); + if(issuercert.empty() == false) + curl_easy_setopt(curl, CURLOPT_ISSUERCERT, issuercert.c_str()); + // For client authentication, certificate file ... + std::string const pem = ConfigFind("SslCert", ""); + if(pem.empty() == false) + curl_easy_setopt(curl, CURLOPT_SSLCERT, pem.c_str()); + // ... and associated key. + std::string const key = ConfigFind("SslKey", ""); + if(key.empty() == false) + curl_easy_setopt(curl, CURLOPT_SSLKEY, key.c_str()); + // Allow forcing SSL version to SSLv3 or TLSv1 + long final_version = CURL_SSLVERSION_DEFAULT; + std::string const sslversion = ConfigFind("SslForceVersion", ""); + if(sslversion == "TLSv1") + final_version = CURL_SSLVERSION_TLSv1; + else if(sslversion == "TLSv1.0") + final_version = CURL_SSLVERSION_TLSv1_0; + else if(sslversion == "TLSv1.1") + final_version = CURL_SSLVERSION_TLSv1_1; + else if(sslversion == "TLSv1.2") + final_version = CURL_SSLVERSION_TLSv1_2; + else if(sslversion == "SSLv3") + final_version = CURL_SSLVERSION_SSLv3; + curl_easy_setopt(curl, CURLOPT_SSLVERSION, final_version); + // CRL file + std::string const crlfile = ConfigFind("CrlFile", ""); + if(crlfile.empty() == false) + curl_easy_setopt(curl, CURLOPT_CRLFILE, crlfile.c_str()); + } + else + { + curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP); + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP); + } // cache-control - if(_config->FindB("Acquire::https::No-Cache", - _config->FindB("Acquire::http::No-Cache",false)) == false) + if(ConfigFindB("No-Cache", false) == false) { // cache enabled - if (_config->FindB("Acquire::https::No-Store", - _config->FindB("Acquire::http::No-Store",false)) == true) + if (ConfigFindB("No-Store", false) == true) headers = curl_slist_append(headers,"Cache-Control: no-store"); - stringstream ss; - ioprintf(ss, "Cache-Control: max-age=%u", _config->FindI("Acquire::https::Max-Age", - _config->FindI("Acquire::http::Max-Age",0))); - headers = curl_slist_append(headers, ss.str().c_str()); + std::string ss; + strprintf(ss, "Cache-Control: max-age=%u", ConfigFindI("Max-Age", 0)); + headers = curl_slist_append(headers, ss.c_str()); } else { // cache disabled by user headers = curl_slist_append(headers, "Cache-Control: no-cache"); headers = curl_slist_append(headers, "Pragma: no-cache"); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - // speed limit - int const dlLimit = _config->FindI("Acquire::https::Dl-Limit", - _config->FindI("Acquire::http::Dl-Limit",0))*1024; + int const dlLimit = ConfigFindI("Dl-Limit", 0) * 1024; if (dlLimit > 0) curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, dlLimit); // set header - curl_easy_setopt(curl, CURLOPT_USERAGENT, - _config->Find("Acquire::https::User-Agent", - _config->Find("Acquire::http::User-Agent", - "Debian APT-CURL/1.0 (" PACKAGE_VERSION ")").c_str()).c_str()); + curl_easy_setopt(curl, CURLOPT_USERAGENT, ConfigFind("User-Agent", "Debian APT-CURL/1.0 (" PACKAGE_VERSION ")").c_str()); // set timeout - int const timeout = _config->FindI("Acquire::https::Timeout", - _config->FindI("Acquire::http::Timeout",120)); + int const timeout = ConfigFindI("Timeout", 120); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout); //set really low lowspeed timeout (see #497983) curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, DL_MIN_SPEED); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout); - // set redirect options and default to 10 redirects - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, AllowRedirect); - curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 10); - // debug if (Debug == true) curl_easy_setopt(curl, CURLOPT_VERBOSE, true); @@ -374,7 +374,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // see 657029, 657560 and co, so if we have no extension on the request // ask for text only. As a sidenote: If there is nothing to negotate servers // seem to be nice and ignore it. - if (_config->FindB("Acquire::https::SendAccept", _config->FindB("Acquire::http::SendAccept", true)) == true) + if (ConfigFindB("SendAccept", true)) { size_t const filepos = Itm->Uri.find_last_of('/'); string const file = Itm->Uri.substr(filepos + 1); @@ -413,6 +413,8 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // met, CURLINFO_CONDITION_UNMET will be set to 1 long curl_condition_unmet = 0; curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &curl_condition_unmet); + if (curl_condition_unmet == 1) + Server->Result = 304; File->Close(); curl_slist_free_all(headers); @@ -445,94 +447,91 @@ bool HttpsMethod::Fetch(FetchItem *Itm) return false; } - // server says file not modified - if (Server->Result == 304 || curl_condition_unmet == 1) + switch (DealWithHeaders(Res)) { - RemoveFile("https", File->Name()); - Res.IMSHit = true; - Res.LastModified = Itm->LastModified; - Res.Size = 0; - URIDone(Res); - return true; - } - Res.IMSHit = false; + case ServerMethod::IMS_HIT: + URIDone(Res); + break; - if (Server->Result != 200 && // OK - Server->Result != 206 && // Partial - Server->Result != 416) // invalid Range - { - char err[255]; - snprintf(err, sizeof(err) - 1, "HttpError%i", Server->Result); - SetFailReason(err); - _error->Error("%i %s", Server->Result, Server->Code); - // unlink, no need keep 401/404 page content in partial/ - RemoveFile("https", File->Name()); - return false; - } + case ServerMethod::ERROR_WITH_CONTENT_PAGE: + // unlink, no need keep 401/404 page content in partial/ + RemoveFile(Binary.c_str(), File->Name()); + case ServerMethod::ERROR_UNRECOVERABLE: + case ServerMethod::ERROR_NOT_FROM_SERVER: + return false; - // invalid range-request - if (Server->Result == 416) - { - RemoveFile("https", File->Name()); - delete File; - Redirect(Itm->Uri); - return true; - } + case ServerMethod::TRY_AGAIN_OR_REDIRECT: + Redirect(NextURI); + break; - struct stat resultStat; - if (unlikely(stat(File->Name().c_str(), &resultStat) != 0)) - { - _error->Errno("stat", "Unable to access file %s", File->Name().c_str()); - return false; - } - Res.Size = resultStat.st_size; + case ServerMethod::FILE_IS_OPEN: + struct stat resultStat; + if (unlikely(stat(File->Name().c_str(), &resultStat) != 0)) + { + _error->Errno("stat", "Unable to access file %s", File->Name().c_str()); + return false; + } + Res.Size = resultStat.st_size; - // Timestamp - curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified); - if (Res.LastModified != -1) - { - struct timeval times[2]; - times[0].tv_sec = Res.LastModified; - times[1].tv_sec = Res.LastModified; - times[0].tv_usec = times[1].tv_usec = 0; - utimes(File->Name().c_str(), times); - } - else - Res.LastModified = resultStat.st_mtime; + // Timestamp + curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified); + if (Res.LastModified != -1) + { + struct timeval times[2]; + times[0].tv_sec = Res.LastModified; + times[1].tv_sec = Res.LastModified; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(File->Name().c_str(), times); + } + else + Res.LastModified = resultStat.st_mtime; - // take hashes - Res.TakeHashes(*(Server->GetHashes())); + // take hashes + Res.TakeHashes(*(Server->GetHashes())); - // keep apt updated - URIDone(Res); + // keep apt updated + URIDone(Res); + break; + } - // cleanup delete File; - return true; } /*}}}*/ -// HttpsMethod::Configuration - Handle a configuration message /*{{{*/ -bool HttpsMethod::Configuration(string Message) +std::unique_ptr<ServerState> HttpsMethod::CreateServerState(URI const &uri)/*{{{*/ { - if (ServerMethod::Configuration(Message) == false) - return false; - - AllowRedirect = _config->FindB("Acquire::https::AllowRedirect", - _config->FindB("Acquire::http::AllowRedirect", true)); - Debug = _config->FindB("Debug::Acquire::https",false); - - return true; + return std::unique_ptr<ServerState>(new HttpsServerState(uri, this)); } /*}}}*/ -std::unique_ptr<ServerState> HttpsMethod::CreateServerState(URI const &uri)/*{{{*/ +HttpsMethod::HttpsMethod(std::string &&pProg) : ServerMethod(std::move(pProg),"1.2",Pipeline | SendConfig)/*{{{*/ { - return std::unique_ptr<ServerState>(new HttpsServerState(uri, this)); + auto addName = std::inserter(methodNames, methodNames.begin()); + addName = "http"; + auto const plus = Binary.find('+'); + if (plus != std::string::npos) + { + addName = Binary.substr(plus + 1); + auto base = Binary.substr(0, plus); + if (base != "https") + addName = base; + } + if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end()) + curl_global_init(CURL_GLOBAL_SSL); + else + curl_global_init(CURL_GLOBAL_NOTHING); + curl = curl_easy_init(); } /*}}}*/ - -int main() +HttpsMethod::~HttpsMethod() /*{{{*/ { - return HttpsMethod().Run(); + curl_easy_cleanup(curl); } - + /*}}}*/ +int main(int, const char *argv[]) /*{{{*/ +{ + std::string Binary = flNotDir(argv[0]); + if (Binary.find('+') == std::string::npos && Binary != "https") + Binary.append("+https"); + return HttpsMethod(std::move(Binary)).Run(); +} + /*}}}*/ diff --git a/methods/https.h b/methods/https.h index 2fa714c7b..04e72e815 100644 --- a/methods/https.h +++ b/methods/https.h @@ -39,6 +39,7 @@ class HttpsServerState : public ServerState /** \brief Transfer the data from the socket */ virtual bool RunData(FileFd * const /*File*/) APT_OVERRIDE { return false; } + virtual bool RunDataToDevNull() APT_OVERRIDE { return false; } virtual bool Open() APT_OVERRIDE { return false; } virtual bool IsOpen() APT_OVERRIDE { return false; } @@ -64,7 +65,7 @@ class HttpsMethod : public ServerMethod static size_t write_data(void *buffer, size_t size, size_t nmemb, void *userp); static int progress_callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow); - void SetupProxy(); + bool SetupProxy(); CURL *curl; // Used by ServerMethods unused by https @@ -73,21 +74,12 @@ class HttpsMethod : public ServerMethod public: - virtual bool Configuration(std::string Message) APT_OVERRIDE; virtual std::unique_ptr<ServerState> CreateServerState(URI const &uri) APT_OVERRIDE; using pkgAcqMethod::FetchResult; using pkgAcqMethod::FetchItem; - HttpsMethod() : ServerMethod("https","1.2",Pipeline | SendConfig) - { - curl_global_init(CURL_GLOBAL_SSL); - curl = curl_easy_init(); - }; - - ~HttpsMethod() - { - curl_easy_cleanup(curl); - }; + explicit HttpsMethod(std::string &&pProg); + virtual ~HttpsMethod(); }; #include <apt-pkg/strutl.h> diff --git a/methods/mirror.cc b/methods/mirror.cc index 9d900771b..71faaf591 100644 --- a/methods/mirror.cc +++ b/methods/mirror.cc @@ -57,7 +57,7 @@ using namespace std; */ MirrorMethod::MirrorMethod() - : HttpMethod(), DownloadedMirrorFile(false), Debug(false) + : HttpMethod("mirror"), DownloadedMirrorFile(false), Debug(false) { } @@ -68,7 +68,7 @@ bool MirrorMethod::Configuration(string Message) { if (pkgAcqMethod::Configuration(Message) == false) return false; - Debug = _config->FindB("Debug::Acquire::mirror",false); + Debug = DebugEnabled(); return true; } diff --git a/methods/rred.cc b/methods/rred.cc index 2d2333f91..0c641ad82 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -572,7 +572,7 @@ class RredMethod : public aptMethod { protected: virtual bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE { - Debug = _config->FindB("Debug::pkgAcquire::RRed", false); + Debug = DebugEnabled(); URI Get = Itm->Uri; std::string Path = Get.Host + Get.Path; // rred:/path - no host diff --git a/methods/rsh.cc b/methods/rsh.cc index 74a908ef7..7b8af6f9b 100644 --- a/methods/rsh.cc +++ b/methods/rsh.cc @@ -383,9 +383,7 @@ bool RSHConn::Get(const char *Path,FileFd &To,unsigned long long Resume, /*}}}*/ // RSHMethod::RSHMethod - Constructor /*{{{*/ -// --------------------------------------------------------------------- -/* */ -RSHMethod::RSHMethod(std::string const &pProg) : aptMethod(pProg.c_str(),"1.0",SendConfig), Prog(pProg) +RSHMethod::RSHMethod(std::string &&pProg) : aptMethod(std::move(pProg),"1.0",SendConfig) { signal(SIGTERM,SigTerm); signal(SIGINT,SigTerm); @@ -399,14 +397,14 @@ bool RSHMethod::Configuration(std::string Message) { // enabling privilege dropping for this method requires configuration… // … which is otherwise lifted straight from root, so use it by default. - _config->Set(std::string("Binary::") + Prog + "::APT::Sandbox::User", ""); + _config->Set(std::string("Binary::") + Binary + "::APT::Sandbox::User", ""); if (aptMethod::Configuration(Message) == false) return false; - std::string const timeconf = std::string("Acquire::") + Prog + "::Timeout"; + std::string const timeconf = std::string("Acquire::") + Binary + "::Timeout"; TimeOut = _config->FindI(timeconf, TimeOut); - std::string const optsconf = std::string("Acquire::") + Prog + "::Options"; + std::string const optsconf = std::string("Acquire::") + Binary + "::Options"; RshOptions = _config->Tree(optsconf.c_str()); return true; @@ -445,7 +443,7 @@ bool RSHMethod::Fetch(FetchItem *Itm) // Connect to the server if (Server == 0 || Server->Comp(Get) == false) { delete Server; - Server = new RSHConn(Prog, Get); + Server = new RSHConn(Binary, Get); } // Could not connect is a transient error.. diff --git a/methods/rsh.h b/methods/rsh.h index 571e38ba6..dee4ad647 100644 --- a/methods/rsh.h +++ b/methods/rsh.h @@ -58,7 +58,6 @@ class RSHConn class RSHMethod : public aptMethod { - std::string const Prog; virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; virtual bool Configuration(std::string Message) APT_OVERRIDE; @@ -71,7 +70,7 @@ class RSHMethod : public aptMethod public: - explicit RSHMethod(std::string const &Prog); + explicit RSHMethod(std::string &&Prog); }; #endif diff --git a/methods/server.cc b/methods/server.cc index 6d147fe12..0888617b1 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -170,7 +170,7 @@ bool ServerState::HeaderLine(string Line) HaveContent = true; unsigned long long * DownloadSizePtr = &DownloadSize; - if (Result == 416) + if (Result == 416 || (Result >= 300 && Result < 400)) DownloadSizePtr = &JunkSize; *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10); @@ -272,6 +272,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) RemoveFile("server", Queue->DestFile); Res.IMSHit = true; Res.LastModified = Queue->LastModified; + Res.Size = 0; return IMS_HIT; } @@ -288,7 +289,8 @@ ServerMethod::DealWithHeaders(FetchResult &Res) && Server->Result != 304 // Not Modified && Server->Result != 306)) // (Not part of HTTP/1.1, reserved) { - if (Server->Location.empty() == true); + if (Server->Location.empty() == true) + ; else if (Server->Location[0] == '/' && Queue->Uri.empty() == false) { URI Uri = Queue->Uri; @@ -297,19 +299,77 @@ ServerMethod::DealWithHeaders(FetchResult &Res) else NextURI.clear(); NextURI.append(DeQuoteString(Server->Location)); + if (Queue->Uri == NextURI) + { + SetFailReason("RedirectionLoop"); + _error->Error("Redirection loop encountered"); + if (Server->HaveContent == true) + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; + } return TRY_AGAIN_OR_REDIRECT; } else { NextURI = DeQuoteString(Server->Location); URI tmpURI = NextURI; + if (tmpURI.Access.find('+') != std::string::npos) + { + _error->Error("Server tried to trick us into using a specific implementation: %s", tmpURI.Access.c_str()); + if (Server->HaveContent == true) + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; + } URI Uri = Queue->Uri; + if (Binary.find('+') != std::string::npos) + { + auto base = Binary.substr(0, Binary.find('+')); + if (base != tmpURI.Access) + { + tmpURI.Access = base + '+' + tmpURI.Access; + if (tmpURI.Access == Binary) + { + std::string tmpAccess = Uri.Access; + std::swap(tmpURI.Access, Uri.Access); + NextURI = tmpURI; + std::swap(tmpURI.Access, Uri.Access); + } + else + NextURI = tmpURI; + } + } + if (Queue->Uri == NextURI) + { + SetFailReason("RedirectionLoop"); + _error->Error("Redirection loop encountered"); + if (Server->HaveContent == true) + return ERROR_WITH_CONTENT_PAGE; + return ERROR_UNRECOVERABLE; + } + Uri.Access = Binary; // same protocol redirects are okay if (tmpURI.Access == Uri.Access) return TRY_AGAIN_OR_REDIRECT; // as well as http to https - else if (Uri.Access == "http" && tmpURI.Access == "https") + else if ((Uri.Access == "http" || Uri.Access == "https+http") && tmpURI.Access == "https") return TRY_AGAIN_OR_REDIRECT; + else + { + auto const tmpplus = tmpURI.Access.find('+'); + if (tmpplus != std::string::npos && tmpURI.Access.substr(tmpplus + 1) == "https") + { + auto const uriplus = Uri.Access.find('+'); + if (uriplus == std::string::npos) + { + if (Uri.Access == tmpURI.Access.substr(0, tmpplus)) // foo -> foo+https + return TRY_AGAIN_OR_REDIRECT; + } + else if (Uri.Access.substr(uriplus + 1) == "http" && + Uri.Access.substr(0, uriplus) == tmpURI.Access.substr(0, tmpplus)) // foo+http -> foo+https + return TRY_AGAIN_OR_REDIRECT; + } + } + _error->Error("Redirection from %s to '%s' is forbidden", Uri.Access.c_str(), NextURI.c_str()); } /* else pass through for error message */ } @@ -337,11 +397,10 @@ ServerMethod::DealWithHeaders(FetchResult &Res) // the file is completely downloaded, but was not moved if (Server->HaveContent == true) { - // Send to error page to dev/null - FileFd DevNull("/dev/null",FileFd::WriteExists); - Server->RunData(&DevNull); + // nuke the sent error page + Server->RunDataToDevNull(); + Server->HaveContent = false; } - Server->HaveContent = false; Server->StartPos = Server->TotalFileSize; Server->Result = 200; } @@ -357,10 +416,13 @@ ServerMethod::DealWithHeaders(FetchResult &Res) failure */ if (Server->Result < 200 || Server->Result >= 300) { - std::string err; - strprintf(err, "HttpError%u", Server->Result); - SetFailReason(err); - _error->Error("%u %s", Server->Result, Server->Code); + if (_error->PendingError() == false) + { + std::string err; + strprintf(err, "HttpError%u", Server->Result); + SetFailReason(err); + _error->Error("%u %s", Server->Result, Server->Code); + } if (Server->HaveContent == true) return ERROR_WITH_CONTENT_PAGE; return ERROR_UNRECOVERABLE; @@ -369,27 +431,6 @@ ServerMethod::DealWithHeaders(FetchResult &Res) // This is some sort of 2xx 'data follows' reply Res.LastModified = Server->Date; Res.Size = Server->TotalFileSize; - - // Open the file - delete File; - File = new FileFd(Queue->DestFile,FileFd::WriteAny); - if (_error->PendingError() == true) - return ERROR_NOT_FROM_SERVER; - - FailFile = Queue->DestFile; - FailFile.c_str(); // Make sure we don't do a malloc in the signal handler - FailFd = File->Fd(); - FailTime = Server->Date; - - if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false) - { - _error->Errno("read",_("Problem hashing file")); - return ERROR_NOT_FROM_SERVER; - } - if (Server->StartPos > 0) - Res.ResumePoint = Server->StartPos; - - SetNonBlock(File->Fd(),true); return FILE_IS_OPEN; } /*}}}*/ @@ -486,10 +527,6 @@ bool ServerMethod::Fetch(FetchItem *) // ServerMethod::Loop - Main loop /*{{{*/ int ServerMethod::Loop() { - typedef vector<string> StringVector; - typedef vector<string>::iterator StringVectorIterator; - map<string, StringVector> Redirected; - signal(SIGTERM,SigTerm); signal(SIGINT,SigTerm); @@ -511,7 +548,7 @@ int ServerMethod::Loop() if (Result != -1 && (Result != 0 || Queue == 0)) { if(FailReason.empty() == false || - _config->FindB("Acquire::http::DependOnSTDIN", true) == true) + ConfigFindB("DependOnSTDIN", true) == true) return 100; else return 0; @@ -522,7 +559,13 @@ int ServerMethod::Loop() // Connect to the server if (Server == 0 || Server->Comp(Queue->Uri) == false) + { Server = CreateServerState(Queue->Uri); + setPostfixForMethodNames(::URI(Queue->Uri).Host.c_str()); + AllowRedirect = ConfigFindB("AllowRedirect", true); + PipelineDepth = ConfigFindI("Pipeline-Depth", 10); + Debug = DebugEnabled(); + } /* If the server has explicitly said this is the last connection then we pre-emptively shut down the pipeline and tear down @@ -712,54 +755,19 @@ int ServerMethod::Loop() case ERROR_WITH_CONTENT_PAGE: { Fail(); - - // Send to content to dev/null - File = new FileFd("/dev/null",FileFd::WriteExists); - Server->RunData(File); - delete File; - File = 0; + Server->RunDataToDevNull(); + break; + } + + // Try again with a new URL + case TRY_AGAIN_OR_REDIRECT: + { + // Clear rest of response if there is content + if (Server->HaveContent) + Server->RunDataToDevNull(); + Redirect(NextURI); break; } - - // Try again with a new URL - case TRY_AGAIN_OR_REDIRECT: - { - // Clear rest of response if there is content - if (Server->HaveContent) - { - File = new FileFd("/dev/null",FileFd::WriteExists); - Server->RunData(File); - delete File; - File = 0; - } - - /* Detect redirect loops. No more redirects are allowed - after the same URI is seen twice in a queue item. */ - StringVector &R = Redirected[Queue->DestFile]; - bool StopRedirects = false; - if (R.empty() == true) - R.push_back(Queue->Uri); - else if (R[0] == "STOP" || R.size() > 10) - StopRedirects = true; - else - { - for (StringVectorIterator I = R.begin(); I != R.end(); ++I) - if (Queue->Uri == *I) - { - R[0] = "STOP"; - break; - } - - R.push_back(Queue->Uri); - } - - if (StopRedirects == false) - Redirect(NextURI); - else - Fail(); - - break; - } default: Fail(_("Internal error")); @@ -780,9 +788,35 @@ unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/ return MaxSizeInQueue; } /*}}}*/ -ServerMethod::ServerMethod(char const * const Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/ - aptMethod(Binary, Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10), +ServerMethod::ServerMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/ + aptMethod(std::move(Binary), Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10), AllowRedirect(false), Debug(false) { } /*}}}*/ +bool ServerMethod::Configuration(std::string Message) /*{{{*/ +{ + if (aptMethod::Configuration(Message) == false) + return false; + + _config->CndSet("Acquire::tor::Proxy", + "socks5h://apt-transport-tor@localhost:9050"); + return true; +} + /*}}}*/ +bool ServerMethod::AddProxyAuth(URI &Proxy, URI const &Server) const /*{{{*/ +{ + if (std::find(methodNames.begin(), methodNames.end(), "tor") != methodNames.end() && + Proxy.User == "apt-transport-tor" && Proxy.Password.empty()) + { + std::string pass = Server.Host; + pass.erase(std::remove_if(pass.begin(), pass.end(), [](char const c) { return std::isalnum(c) == 0; }), pass.end()); + if (pass.length() > 255) + Proxy.Password = pass.substr(0, 255); + else + Proxy.Password = std::move(pass); + } + // FIXME: should we support auth.conf for proxies? + return true; +} + /*}}}*/ diff --git a/methods/server.h b/methods/server.h index af0914b9c..1d114354f 100644 --- a/methods/server.h +++ b/methods/server.h @@ -91,6 +91,7 @@ struct ServerState /** \brief Transfer the data from the socket */ virtual bool RunData(FileFd * const File) = 0; + virtual bool RunDataToDevNull() = 0; virtual bool Open() = 0; virtual bool IsOpen() = 0; @@ -140,7 +141,7 @@ class ServerMethod : public aptMethod TRY_AGAIN_OR_REDIRECT }; /** \brief Handle the retrieved header data */ - DealWithHeadersResult DealWithHeaders(FetchResult &Res); + virtual DealWithHeadersResult DealWithHeaders(FetchResult &Res); // In the event of a fatal signal this file will be closed and timestamped. static std::string FailFile; @@ -155,8 +156,11 @@ class ServerMethod : public aptMethod virtual void SendReq(FetchItem *Itm) = 0; virtual std::unique_ptr<ServerState> CreateServerState(URI const &uri) = 0; virtual void RotateDNS() = 0; + virtual bool Configuration(std::string Message) APT_OVERRIDE; - ServerMethod(char const * const Binary, char const * const Ver,unsigned long const Flags); + bool AddProxyAuth(URI &Proxy, URI const &Server) const; + + ServerMethod(std::string &&Binary, char const * const Ver,unsigned long const Flags); virtual ~ServerMethod() {}; }; diff --git a/methods/store.cc b/methods/store.cc index fa02d4597..1faaa4fb4 100644 --- a/methods/store.cc +++ b/methods/store.cc @@ -32,12 +32,15 @@ class StoreMethod : public aptMethod { - std::string const Prog; virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE; public: - explicit StoreMethod(std::string const &pProg) : aptMethod(pProg.c_str(),"1.2",SingleInstance | SendConfig), Prog(pProg) {}; + explicit StoreMethod(std::string &&pProg) : aptMethod(std::move(pProg),"1.2",SingleInstance | SendConfig) + { + if (Binary != "store") + methodNames.insert(methodNames.begin(), "store"); + } }; static bool OpenFileWithCompressorByName(FileFd &fileFd, std::string const &Filename, unsigned int const Mode, std::string const &Name) @@ -70,7 +73,7 @@ bool StoreMethod::Fetch(FetchItem *Itm) /*{{{*/ FileFd From; if (_config->FindB("Method::Compress", false) == false) { - if (OpenFileWithCompressorByName(From, Path, FileFd::ReadOnly, Prog) == false) + if (OpenFileWithCompressorByName(From, Path, FileFd::ReadOnly, Binary) == false) return false; if(From.IsCompressed() && From.FileSize() == 0) return _error->Error(_("Empty files can't be valid archives")); @@ -85,7 +88,7 @@ bool StoreMethod::Fetch(FetchItem *Itm) /*{{{*/ { if (_config->FindB("Method::Compress", false) == false) To.Open(Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Atomic, FileFd::Extension); - else if (OpenFileWithCompressorByName(To, Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Empty, Prog) == false) + else if (OpenFileWithCompressorByName(To, Itm->DestFile, FileFd::WriteOnly | FileFd::Create | FileFd::Empty, Binary) == false) return false; if (To.IsOpen() == false || To.Failed() == true) |