From 5666084ecfe140aaa3f89388de557c2f875b4244 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 10:55:08 +0200 Subject: methods: connect: Switch from int fds to new MethodFd Use std::unique_ptr everywhere we used an integer-based file descriptor before. This allows us to implement stuff like TLS support easily. Gbp-Dch: ignore --- methods/http.cc | 77 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 40 insertions(+), 37 deletions(-) (limited to 'methods/http.cc') diff --git a/methods/http.cc b/methods/http.cc index 9f5959548..d62f97be3 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -95,7 +95,7 @@ void CircleBuf::Reset() // --------------------------------------------------------------------- /* This fills up the buffer with as much data as is in the FD, assuming it is non-blocking.. */ -bool CircleBuf::Read(int Fd) +bool CircleBuf::Read(std::unique_ptr const &Fd) { while (1) { @@ -126,11 +126,11 @@ bool CircleBuf::Read(int Fd) // Write the buffer segment ssize_t Res; if(CircleBuf::BwReadLimit) { - Res = read(Fd,Buf + (InP%Size), - BwReadMax > LeftRead() ? LeftRead() : BwReadMax); + Res = Fd->Read(Buf + (InP % Size), + BwReadMax > LeftRead() ? LeftRead() : BwReadMax); } else - Res = read(Fd,Buf + (InP%Size),LeftRead()); - + Res = Fd->Read(Buf + (InP % Size), LeftRead()); + if(Res > 0 && BwReadLimit > 0) CircleBuf::BwTickReadData += Res; @@ -193,7 +193,7 @@ void CircleBuf::FillOut() // CircleBuf::Write - Write from the buffer into a FD /*{{{*/ // --------------------------------------------------------------------- /* This empties the buffer into the FD. */ -bool CircleBuf::Write(int Fd) +bool CircleBuf::Write(std::unique_ptr const &Fd) { while (1) { @@ -208,7 +208,7 @@ bool CircleBuf::Write(int Fd) // Write the buffer segment ssize_t Res; - Res = write(Fd,Buf + (OutP%Size),LeftWrite()); + Res = Fd->Write(Buf + (OutP % Size), LeftWrite()); if (Res == 0) return false; @@ -291,6 +291,7 @@ CircleBuf::~CircleBuf() HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024) { TimeOut = Owner->ConfigFindI("Timeout", TimeOut); + ServerFd = MethodFd::FromFd(-1); Reset(); } /*}}}*/ @@ -318,7 +319,7 @@ static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, bool HttpServerState::Open() { // Use the already open connection if possible. - if (ServerFd != -1) + if (ServerFd->Fd() != -1) return true; Close(); @@ -371,8 +372,12 @@ bool HttpServerState::Open() std::string const ProxyInfo = URI::SiteOnly(Proxy); Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str()); auto const Timeout = Owner->ConfigFindI("TimeOut", 120); - #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false - #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false +#define APT_WriteOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(ServerFd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \ + return false +#define APT_ReadOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(ServerFd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \ + return false if (ServerName.Host.length() > 255) return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str()); if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) @@ -514,7 +519,7 @@ bool HttpServerState::Open() ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n", ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); - if (WaitFd(ServerFd, true, Timeout) == false) + if (WaitFd(ServerFd->Fd(), true, Timeout) == false) return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out", ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); #undef APT_ReadOrFail @@ -549,8 +554,7 @@ bool HttpServerState::Open() /* */ bool HttpServerState::Close() { - close(ServerFd); - ServerFd = -1; + ServerFd->Close(); return true; } /*}}}*/ @@ -672,7 +676,7 @@ bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/ /*}}}*/ APT_PURE bool HttpServerState::IsOpen() /*{{{*/ { - return (ServerFd != -1); + return (ServerFd->Fd() != -1); } /*}}}*/ bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/ @@ -685,7 +689,7 @@ bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/ void HttpServerState::Reset() /*{{{*/ { ServerState::Reset(); - ServerFd = -1; + ServerFd->Close(); } /*}}}*/ @@ -710,7 +714,7 @@ bool HttpServerState::Die(RequestState &Req) SetNonBlock(Req.File.Fd(),false); while (In.WriteSpace() == true) { - if (In.Write(Req.File.Fd()) == false) + if (In.Write(MethodFd::FromFd(Req.File.Fd())) == false) return _error->Errno("write",_("Error writing to the file")); // Done @@ -762,7 +766,7 @@ bool HttpServerState::Flush(FileFd * const File) while (In.WriteSpace() == true) { - if (In.Write(File->Fd()) == false) + if (In.Write(MethodFd::FromFd(File->Fd())) == false) return _error->Errno("write",_("Error writing to file")); if (In.IsLimit() == true) return true; @@ -781,8 +785,8 @@ bool HttpServerState::Flush(FileFd * const File) bool HttpServerState::Go(bool ToFile, RequestState &Req) { // Server has closed the connection - if (ServerFd == -1 && (In.WriteSpace() == false || - ToFile == false)) + if (ServerFd->Fd() == -1 && (In.WriteSpace() == false || + ToFile == false)) return false; fd_set rfds,wfds; @@ -791,28 +795,27 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) /* Add the server. We only send more requests if the connection will be persisting */ - if (Out.WriteSpace() == true && ServerFd != -1 - && Persistent == true) - FD_SET(ServerFd,&wfds); - if (In.ReadSpace() == true && ServerFd != -1) - FD_SET(ServerFd,&rfds); - + if (Out.WriteSpace() == true && ServerFd->Fd() != -1 && Persistent == true) + FD_SET(ServerFd->Fd(), &wfds); + if (In.ReadSpace() == true && ServerFd->Fd() != -1) + FD_SET(ServerFd->Fd(), &rfds); + // Add the file - int FileFD = -1; + auto FileFD = MethodFd::FromFd(-1); if (Req.File.IsOpen()) - FileFD = Req.File.Fd(); - - if (In.WriteSpace() == true && ToFile == true && FileFD != -1) - FD_SET(FileFD,&wfds); + FileFD = MethodFd::FromFd(Req.File.Fd()); + + if (In.WriteSpace() == true && ToFile == true && FileFD->Fd() != -1) + FD_SET(FileFD->Fd(), &wfds); // Add stdin if (Owner->ConfigFindB("DependOnSTDIN", true) == true) FD_SET(STDIN_FILENO,&rfds); // Figure out the max fd - int MaxFd = FileFD; - if (MaxFd < ServerFd) - MaxFd = ServerFd; + int MaxFd = FileFD->Fd(); + if (MaxFd < ServerFd->Fd()) + MaxFd = ServerFd->Fd(); // Select struct timeval tv; @@ -833,14 +836,14 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) } // Handle server IO - if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds)) + if (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &rfds)) { errno = 0; if (In.Read(ServerFd) == false) return Die(Req); } - - if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds)) + + if (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &wfds)) { errno = 0; if (Out.Write(ServerFd) == false) @@ -848,7 +851,7 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) } // Send data to the file - if (FileFD != -1 && FD_ISSET(FileFD,&wfds)) + if (FileFD->Fd() != -1 && FD_ISSET(FileFD->Fd(), &wfds)) { if (In.Write(FileFD) == false) return _error->Errno("write",_("Error writing to output file")); -- cgit v1.2.3-70-g09d2 From 2851ec6cf037d552118b885be0dd7796d74730c6 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 10:55:57 +0200 Subject: methods: Add HTTPS support to http method, using GnuTLS The http method will eventually replace the curl-based https method, but for now, this is an opt-in experiment that can be enabled by setting Dir::Bin::Methods::https to "http". Known issues: - We do not support HTTPS proxies yet - We do not support proxying HTTPS connections yet (CONNECT) - IssuerCert and SslForceVersion are unsupported Gbp-Dch: Full --- .travis.yml | 2 +- CMakeLists.txt | 4 + debian/control | 1 + methods/CMakeLists.txt | 9 +- methods/connect.cc | 361 +++++++++++++++++++++++++++++++++++++++++++++++++ methods/connect.h | 3 + methods/http.cc | 187 ++----------------------- 7 files changed, 385 insertions(+), 182 deletions(-) (limited to 'methods/http.cc') diff --git a/.travis.yml b/.travis.yml index 023107a68..74dff44b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - sudo apt-get update -qq install: - sudo apt-get -qq -y -t wily install gettext liblz4-dev python3-apt - - sudo apt-get -qq -y -t xenial install cmake ninja-build + - sudo apt-get -qq -y -t xenial install cmake ninja-build libgnutls28-dev libcurl4-gnutls-dev - sudo ./prepare-release travis-ci before_script: - ( mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Coverage -G Ninja $CMAKE_FLAGS .. ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 83af9bd5a..8f8ffb898 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,6 +81,10 @@ find_package(CURL REQUIRED) if (CURL_FOUND) set(HAVE_CURL 1) endif() +find_package(GnuTLS REQUIRED) +if (GNUTLS_FOUND) + set(HAVE_GNUTLS 1) +endif() # (De)Compressor libraries find_package(ZLIB REQUIRED) diff --git a/debian/control b/debian/control index 96bbef348..04c44eb93 100644 --- a/debian/control +++ b/debian/control @@ -14,6 +14,7 @@ Build-Depends: cmake (>= 3.4), dpkg-dev (>= 1.17.14), gettext (>= 0.12), libbz2-dev, + libgnutls28-dev (>= 3.4.6), libcurl4-gnutls-dev (>= 7.19.4~), libdb-dev, googletest | libgtest-dev , diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index a74c2ce07..64807e310 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -11,7 +11,8 @@ add_executable(ftp ftp.cc rfc2553emu.cc connect.cc) add_executable(rred rred.cc) add_executable(rsh rsh.cc) -# Add target-specific header directories +target_compile_definitions(http PRIVATE ${GNUTLS_DEFINITIONS}) +target_include_directories(http PRIVATE ${GNUTLS_INCLUDE_DIR}) target_include_directories(https PRIVATE ${CURL_INCLUDE_DIRS}) # Link the executables against the libraries @@ -20,10 +21,10 @@ target_link_libraries(copy apt-pkg) target_link_libraries(store apt-pkg) target_link_libraries(gpgv apt-pkg) target_link_libraries(cdrom apt-pkg) -target_link_libraries(http apt-pkg) -target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES}) +target_link_libraries(http apt-pkg ${GNUTLS_LIBRARIES}) +target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES} ${GNUTLS_LIBRARIES}) target_link_libraries(https apt-pkg ${CURL_LIBRARIES}) -target_link_libraries(ftp apt-pkg) +target_link_libraries(ftp apt-pkg ${GNUTLS_LIBRARIES}) target_link_libraries(rred apt-pkg) target_link_libraries(rsh apt-pkg) diff --git a/methods/connect.cc b/methods/connect.cc index 9e9e09ac4..c40f34f2d 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -20,6 +20,9 @@ #include #include +#include +#include + #include #include #include @@ -367,3 +370,361 @@ bool Connect(std::string Host, int Port, const char *Service, _error->MergeWithStack(); return ret; } + /*}}}*/ +// UnwrapSocks - Handle SOCKS setup /*{{{*/ +// --------------------------------------------------------------------- +/* This does socks magic */ +static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, + char const *const type, bool const ReadWrite, uint8_t *const ToFrom, + unsigned int const Size, unsigned int const Timeout) +{ + if (WaitFd(ServerFd, ReadWrite, Timeout) == false) + return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type); + if (ReadWrite == false) + { + if (FileFd::Read(ServerFd, ToFrom, Size) == false) + return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); + } + else + { + if (FileFd::Write(ServerFd, ToFrom, Size) == false) + return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); + } + return true; +} + +bool UnwrapSocks(std::string Host, int Port, URI Proxy, std::unique_ptr &Fd, + unsigned long Timeout, aptMethod *Owner) +{ + /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect + * for not offering GSSAPI auth which is a must (we only do no or user/pass auth). + * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */ + std::string const ProxyInfo = URI::SiteOnly(Proxy); + Owner->Status(_("Connecting to %s (%s)"), "SOCKS5h proxy", ProxyInfo.c_str()); +#define APT_WriteOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \ + return false +#define APT_ReadOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \ + return false + if (Host.length() > 255) + return _error->Error("Can't use SOCKS5h as hostname %s is too long!", Host.c_str()); + if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) + return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length()); + if (Proxy.User.empty()) + { + uint8_t greeting[] = {0x05, 0x01, 0x00}; + APT_WriteOrFail("greet-1", greeting, sizeof(greeting)); + } + else + { + uint8_t greeting[] = {0x05, 0x02, 0x00, 0x02}; + APT_WriteOrFail("greet-2", greeting, sizeof(greeting)); + } + uint8_t greeting[2]; + APT_ReadOrFail("greet back", greeting, sizeof(greeting)); + if (greeting[0] != 0x05) + return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]); + if (greeting[1] == 0x00) + ; // no auth has no method-dependent sub-negotiations + else if (greeting[1] == 0x02) + { + if (Proxy.User.empty()) + return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str()); + // user&pass auth sub-negotiations are defined by RFC1929 + std::vector auth = {{0x01, static_cast(Proxy.User.length())}}; + std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth)); + auth.push_back(static_cast(Proxy.Password.length())); + std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth)); + APT_WriteOrFail("user&pass auth", auth.data(), auth.size()); + uint8_t authstatus[2]; + APT_ReadOrFail("auth report", authstatus, sizeof(authstatus)); + if (authstatus[0] != 0x01) + return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]); + if (authstatus[1] != 0x00) + return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]); + } + else + return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]); + union { + uint16_t *i; + uint8_t *b; + } portu; + uint16_t port = htons(static_cast(Port)); + portu.i = &port; + std::vector request = {{0x05, 0x01, 0x00, 0x03, static_cast(Host.length())}}; + std::copy(Host.begin(), Host.end(), std::back_inserter(request)); + request.push_back(portu.b[0]); + request.push_back(portu.b[1]); + APT_WriteOrFail("request", request.data(), request.size()); + uint8_t response[4]; + APT_ReadOrFail("first part of response", response, sizeof(response)); + if (response[0] != 0x05) + return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]); + if (response[2] != 0x00) + return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]); + std::string bindaddr; + if (response[3] == 0x01) // IPv4 address + { + uint8_t ip4port[6]; + APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port)); + portu.b[0] = ip4port[4]; + portu.b[1] = ip4port[5]; + port = ntohs(*portu.i); + strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port); + } + else if (response[3] == 0x03) // hostname + { + uint8_t namelength; + APT_ReadOrFail("hostname length of response", &namelength, 1); + uint8_t hostname[namelength + 2]; + APT_ReadOrFail("hostname of response", hostname, sizeof(hostname)); + portu.b[0] = hostname[namelength]; + portu.b[1] = hostname[namelength + 1]; + port = ntohs(*portu.i); + hostname[namelength] = '\0'; + strprintf(bindaddr, "%s:%d", hostname, port); + } + else if (response[3] == 0x04) // IPv6 address + { + uint8_t ip6port[18]; + APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port)); + portu.b[0] = ip6port[16]; + portu.b[1] = ip6port[17]; + port = ntohs(*portu.i); + strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d", + ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7], + ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15], + port); + } + else + return _error->Error("SOCKS proxy %s destination address is of unknown type: %d", + ProxyInfo.c_str(), response[3]); + if (response[1] != 0x00) + { + char const *errstr = nullptr; + auto errcode = response[1]; + // Tor error reporting can be a bit arcane, lets try to detect & fix it up + if (bindaddr == "0.0.0.0:0") + { + auto const lastdot = Host.rfind('.'); + if (lastdot == std::string::npos || Host.substr(lastdot) != ".onion") + ; + else if (errcode == 0x01) + { + auto const prevdot = Host.rfind('.', lastdot - 1); + if (lastdot == 16 && prevdot == std::string::npos) + ; // valid .onion address + else if (prevdot != std::string::npos && (lastdot - prevdot) == 17) + ; // valid .onion address with subdomain(s) + else + { + errstr = "Invalid hostname: onion service name must be 16 characters long"; + Owner->SetFailReason("SOCKS"); + } + } + // in all likelihood the service is either down or the address has + // a typo and so "Host unreachable" is the better understood error + // compared to the technically correct "TLL expired". + else if (errcode == 0x06) + errcode = 0x04; + } + if (errstr == nullptr) + { + switch (errcode) + { + case 0x01: + errstr = "general SOCKS server failure"; + Owner->SetFailReason("SOCKS"); + break; + case 0x02: + errstr = "connection not allowed by ruleset"; + Owner->SetFailReason("SOCKS"); + break; + case 0x03: + errstr = "Network unreachable"; + Owner->SetFailReason("ConnectionTimedOut"); + break; + case 0x04: + errstr = "Host unreachable"; + Owner->SetFailReason("ConnectionTimedOut"); + break; + case 0x05: + errstr = "Connection refused"; + Owner->SetFailReason("ConnectionRefused"); + break; + case 0x06: + errstr = "TTL expired"; + Owner->SetFailReason("Timeout"); + break; + case 0x07: + errstr = "Command not supported"; + Owner->SetFailReason("SOCKS"); + break; + case 0x08: + errstr = "Address type not supported"; + Owner->SetFailReason("SOCKS"); + break; + default: + errstr = "Unknown error"; + Owner->SetFailReason("SOCKS"); + break; + } + } + return _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str(), errstr, response[1]); + } + else if (Owner->DebugEnabled()) + ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str()); + + if (WaitFd(Fd->Fd(), true, Timeout) == false) + return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str()); +#undef APT_ReadOrFail +#undef APT_WriteOrFail + + return true; +} + /*}}}*/ +// UnwrapTLS - Handle TLS connections /*{{{*/ +// --------------------------------------------------------------------- +/* Performs a TLS handshake on the socket */ +struct TlsFd : public MethodFd +{ + std::unique_ptr UnderlyingFd; + gnutls_session_t session; + gnutls_certificate_credentials_t credentials; + std::string hostname; + + int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); } + + ssize_t Read(void *buf, size_t count) APT_OVERRIDE + { + return HandleError(gnutls_record_recv(session, buf, count)); + } + ssize_t Write(void *buf, size_t count) APT_OVERRIDE + { + return HandleError(gnutls_record_send(session, buf, count)); + } + + template + T HandleError(T err) + { + if (err < 0 && gnutls_error_is_fatal(err)) + errno = EIO; + else if (err < 0) + errno = EAGAIN; + else + errno = 0; + return err; + } + + int Close() APT_OVERRIDE + { + if (HandleError(gnutls_bye(session, GNUTLS_SHUT_RDWR)) < 0) + return -1; + return UnderlyingFd->Close(); + } +}; + +bool UnwrapTLS(std::string Host, std::unique_ptr &Fd, + unsigned long Timeout, aptMethod *Owner) +{ + int err; + TlsFd *tlsFd = new TlsFd(); + + tlsFd->hostname = Host; + tlsFd->UnderlyingFd = MethodFd::FromFd(-1); // For now + + gnutls_init(&tlsFd->session, GNUTLS_CLIENT | GNUTLS_NONBLOCK); + gnutls_transport_set_int(tlsFd->session, dynamic_cast(Fd.get())->fd); + gnutls_certificate_allocate_credentials(&tlsFd->credentials); + + // Credential setup + if ((err = gnutls_certificate_set_x509_system_trust(tlsFd->credentials)) <= 0) + return _error->Error("Could not load TLS certificates: %s", gnutls_strerror(err)); + + std::string fileinfo = Owner->ConfigFind("CaInfo", ""); + if (!fileinfo.empty()) + { + gnutls_certificate_set_verify_flags(tlsFd->credentials, GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT); + err = gnutls_certificate_set_x509_trust_file(tlsFd->credentials, fileinfo.c_str(), GNUTLS_X509_FMT_PEM); + if (err < 0) + return _error->Error("Could not load CaInfo certificate: %s", gnutls_strerror(err)); + } + + // TODO: IssuerCert AKA CURLOPT_ISSUERCERT + // TODO: Emulate SslForceVersion AKA CURLOPT_SSLVERSION? + + // For client authentication, certificate file ... + std::string const cert = Owner->ConfigFind("SslCert", ""); + std::string const key = Owner->ConfigFind("SslKey", ""); + if (cert.empty() == false) + { + if ((err = gnutls_certificate_set_x509_key_file( + tlsFd->credentials, + cert.c_str(), + key.empty() ? cert.c_str() : key.c_str(), + GNUTLS_X509_FMT_PEM)) < 0) + { + return _error->Error("Could not load client certificate or key: %s", gnutls_strerror(err)); + } + } + + // CRL file + std::string const crlfile = Owner->ConfigFind("CrlFile", ""); + if (crlfile.empty() == false) + { + if ((err = gnutls_certificate_set_x509_crl_file(tlsFd->credentials, + crlfile.c_str(), + GNUTLS_X509_FMT_PEM)) < 0) + return _error->Error("Could not load CrlFile: %s", gnutls_strerror(err)); + } + + if ((err = gnutls_credentials_set(tlsFd->session, GNUTLS_CRD_CERTIFICATE, tlsFd->credentials)) < 0) + return _error->Error("Could not set CaInfo certificate: %s", gnutls_strerror(err)); + + if ((err = gnutls_set_default_priority(tlsFd->session)) < 0) + return _error->Error("Could not set algorithm preferences: %s", gnutls_strerror(err)); + + if (Owner->ConfigFindB("Verify-Peer", true) || Owner->ConfigFindB("Verify-Host", true)) + { + gnutls_session_set_verify_cert(tlsFd->session, tlsFd->hostname.c_str(), 0); + } + if ((err = gnutls_server_name_set(tlsFd->session, GNUTLS_NAME_DNS, tlsFd->hostname.c_str(), tlsFd->hostname.length())) < 0) + return _error->Error("Could not set SNI name: %s", gnutls_strerror(err)); + + // Do the handshake. Our socket is non-blocking, so we need to call WaitFd() + // accordingly. + do + { + err = gnutls_handshake(tlsFd->session); + if ((err == GNUTLS_E_INTERRUPTED || err == GNUTLS_E_AGAIN) && + WaitFd(Fd->Fd(), gnutls_record_get_direction(tlsFd->session) == 1, Timeout) == false) + return _error->Errno("select", "Could not wait for server fd"); + } while (err < 0 && gnutls_error_is_fatal(err) == 0); + + if (err < 0) + { + // Print reason why validation failed. + if (err == GNUTLS_E_CERTIFICATE_VERIFICATION_ERROR) + { + gnutls_datum_t txt; + auto type = gnutls_certificate_type_get(tlsFd->session); + auto status = gnutls_session_get_verify_cert_status(tlsFd->session); + if (gnutls_certificate_verification_status_print(status, + type, &txt, 0) == 0) + { + _error->Error("Certificate verification failed: %s", txt.data); + } + gnutls_free(txt.data); + } + return _error->Error("Could not handshake: %s", gnutls_strerror(err)); + } + + tlsFd->UnderlyingFd = std::move(Fd); + Fd.reset(tlsFd); + return true; +} + /*}}}*/ diff --git a/methods/connect.h b/methods/connect.h index 5eae77d09..39f3c1ce2 100644 --- a/methods/connect.h +++ b/methods/connect.h @@ -40,6 +40,9 @@ struct MethodFd bool Connect(std::string To, int Port, const char *Service, int DefPort, std::unique_ptr &Fd, unsigned long TimeOut, aptMethod *Owner); +bool UnwrapSocks(std::string To, int Port, URI Proxy, std::unique_ptr &Fd, unsigned long Timeout, aptMethod *Owner); +bool UnwrapTLS(std::string To, std::unique_ptr &Fd, unsigned long Timeout, aptMethod *Owner); + void RotateDNS(); #endif diff --git a/methods/http.cc b/methods/http.cc index d62f97be3..4d8c0808b 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -298,24 +298,6 @@ HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, O // HttpServerState::Open - Open a connection to the server /*{{{*/ // --------------------------------------------------------------------- /* This opens a connection to the server. */ -static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, - char const * const type, bool const ReadWrite, uint8_t * const ToFrom, - unsigned int const Size, unsigned int const Timeout) -{ - if (WaitFd(ServerFd, ReadWrite, Timeout) == false) - return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type); - if (ReadWrite == false) - { - if (FileFd::Read(ServerFd, ToFrom, Size) == false) - return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); - } - else - { - if (FileFd::Write(ServerFd, ToFrom, Size) == false) - return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); - } - return true; -} bool HttpServerState::Open() { // Use the already open connection if possible. @@ -361,169 +343,15 @@ bool HttpServerState::Open() if (Proxy.empty() == false) Owner->AddProxyAuth(Proxy, ServerName); + bool tls = ServerName.Access == "https"; if (Proxy.Access == "socks5h") { if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false) return false; - /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect - * for not offering GSSAPI auth which is a must (we only do no or user/pass auth). - * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */ - std::string const ProxyInfo = URI::SiteOnly(Proxy); - Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str()); - auto const Timeout = Owner->ConfigFindI("TimeOut", 120); -#define APT_WriteOrFail(TYPE, DATA, LENGTH) \ - if (TalkToSocksProxy(ServerFd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \ - return false -#define APT_ReadOrFail(TYPE, DATA, LENGTH) \ - if (TalkToSocksProxy(ServerFd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \ - return false - if (ServerName.Host.length() > 255) - return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str()); - if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) - return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length()); - if (Proxy.User.empty()) - { - uint8_t greeting[] = { 0x05, 0x01, 0x00 }; - APT_WriteOrFail("greet-1", greeting, sizeof(greeting)); - } - else - { - uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 }; - APT_WriteOrFail("greet-2", greeting, sizeof(greeting)); - } - uint8_t greeting[2]; - APT_ReadOrFail("greet back", greeting, sizeof(greeting)); - if (greeting[0] != 0x05) - return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]); - if (greeting[1] == 0x00) - ; // no auth has no method-dependent sub-negotiations - else if (greeting[1] == 0x02) - { - if (Proxy.User.empty()) - return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str()); - // user&pass auth sub-negotiations are defined by RFC1929 - std::vector auth = {{ 0x01, static_cast(Proxy.User.length()) }}; - std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth)); - auth.push_back(static_cast(Proxy.Password.length())); - std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth)); - APT_WriteOrFail("user&pass auth", auth.data(), auth.size()); - uint8_t authstatus[2]; - APT_ReadOrFail("auth report", authstatus, sizeof(authstatus)); - if (authstatus[0] != 0x01) - return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]); - if (authstatus[1] != 0x00) - return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]); - } - else - return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]); - union { uint16_t * i; uint8_t * b; } portu; - uint16_t port = htons(static_cast(ServerName.Port == 0 ? 80 : ServerName.Port)); - portu.i = &port; - std::vector request = {{ 0x05, 0x01, 0x00, 0x03, static_cast(ServerName.Host.length()) }}; - std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request)); - request.push_back(portu.b[0]); - request.push_back(portu.b[1]); - APT_WriteOrFail("request", request.data(), request.size()); - uint8_t response[4]; - APT_ReadOrFail("first part of response", response, sizeof(response)); - if (response[0] != 0x05) - return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]); - if (response[2] != 0x00) - return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]); - std::string bindaddr; - if (response[3] == 0x01) // IPv4 address - { - uint8_t ip4port[6]; - APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port)); - portu.b[0] = ip4port[4]; - portu.b[1] = ip4port[5]; - port = ntohs(*portu.i); - strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port); - } - else if (response[3] == 0x03) // hostname - { - uint8_t namelength; - APT_ReadOrFail("hostname length of response", &namelength, 1); - uint8_t hostname[namelength + 2]; - APT_ReadOrFail("hostname of response", hostname, sizeof(hostname)); - portu.b[0] = hostname[namelength]; - portu.b[1] = hostname[namelength + 1]; - port = ntohs(*portu.i); - hostname[namelength] = '\0'; - strprintf(bindaddr, "%s:%d", hostname, port); - } - else if (response[3] == 0x04) // IPv6 address - { - uint8_t ip6port[18]; - APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port)); - portu.b[0] = ip6port[16]; - portu.b[1] = ip6port[17]; - port = ntohs(*portu.i); - strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d", - ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7], - ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15], - port); - } - else - return _error->Error("SOCKS proxy %s destination address is of unknown type: %d", - ProxyInfo.c_str(), response[3]); - if (response[1] != 0x00) - { - char const * errstr = nullptr; - auto errcode = response[1]; - // Tor error reporting can be a bit arcane, lets try to detect & fix it up - if (bindaddr == "0.0.0.0:0") - { - auto const lastdot = ServerName.Host.rfind('.'); - if (lastdot == std::string::npos || ServerName.Host.substr(lastdot) != ".onion") - ; - else if (errcode == 0x01) - { - auto const prevdot = ServerName.Host.rfind('.', lastdot - 1); - if (lastdot == 16 && prevdot == std::string::npos) - ; // valid .onion address - else if (prevdot != std::string::npos && (lastdot - prevdot) == 17) - ; // valid .onion address with subdomain(s) - else - { - errstr = "Invalid hostname: onion service name must be 16 characters long"; - Owner->SetFailReason("SOCKS"); - } - } - // in all likelihood the service is either down or the address has - // a typo and so "Host unreachable" is the better understood error - // compared to the technically correct "TLL expired". - else if (errcode == 0x06) - errcode = 0x04; - } - if (errstr == nullptr) - { - switch (errcode) - { - case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break; - case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break; - case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break; - case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break; - case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break; - case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break; - case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break; - case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break; - default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break; - } - } - return _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str(), errstr, response[1]); - } - else if (Owner->DebugEnabled()) - ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); - - if (WaitFd(ServerFd->Fd(), true, Timeout) == false) - return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); - #undef APT_ReadOrFail - #undef APT_WriteOrFail + if (UnwrapSocks(ServerName.Host, ServerName.Port == 0 ? 80 : ServerName.Port, + Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 120), Owner) == false) + return false; } else { @@ -544,8 +372,13 @@ bool HttpServerState::Open() Port = Proxy.Port; Host = Proxy.Host; } - return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner); + if (!Connect(Host, Port, tls ? "https" : "http", tls ? 443 : 80, ServerFd, TimeOut, Owner)) + return false; } + + if (tls && UnwrapTLS(ServerName.Host, ServerFd, TimeOut, Owner) == false) + return false; + return true; } /*}}}*/ -- cgit v1.2.3-70-g09d2 From f806530b9ea858ca6bda8fb8f43d988aba02dab3 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 13:20:54 +0200 Subject: methods: http: Drain pending data before selecting GnuTLS can already have data pending in its buffers, we need to to drain that first otherwise select() might block indefinitely. Gbp-Dch: ignore --- methods/connect.cc | 9 +++++++++ methods/connect.h | 2 ++ methods/http.cc | 10 +++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) (limited to 'methods/http.cc') diff --git a/methods/connect.cc b/methods/connect.cc index c40f34f2d..e48008214 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -98,6 +98,10 @@ struct FdFd : public MethodFd } }; +bool MethodFd::HasPending() +{ + return false; +} std::unique_ptr MethodFd::FromFd(int iFd) { FdFd *fd = new FdFd(); @@ -626,6 +630,11 @@ struct TlsFd : public MethodFd return -1; return UnderlyingFd->Close(); } + + bool HasPending() APT_OVERRIDE + { + return gnutls_record_check_pending(session) > 0; + } }; bool UnwrapTLS(std::string Host, std::unique_ptr &Fd, diff --git a/methods/connect.h b/methods/connect.h index 39f3c1ce2..4456d660d 100644 --- a/methods/connect.h +++ b/methods/connect.h @@ -35,6 +35,8 @@ struct MethodFd virtual ~MethodFd(){}; /// \brief Construct a MethodFd from a UNIX file descriptor static std::unique_ptr FromFd(int iFd); + /// \brief If there is pending data. + virtual bool HasPending(); }; bool Connect(std::string To, int Port, const char *Service, int DefPort, diff --git a/methods/http.cc b/methods/http.cc index 4d8c0808b..b302c896d 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -621,7 +621,15 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) if (ServerFd->Fd() == -1 && (In.WriteSpace() == false || ToFile == false)) return false; - + + // Handle server IO + if (ServerFd->HasPending() && In.ReadSpace() == true) + { + errno = 0; + if (In.Read(ServerFd) == false) + return Die(Req); + } + fd_set rfds,wfds; FD_ZERO(&rfds); FD_ZERO(&wfds); -- cgit v1.2.3-70-g09d2