From 3cb78eef6fd8dcfa964c4ebb83db71efc8cabd64 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 10:43:06 +0200 Subject: Add clang-format definitions Also add git-clang-format helper to help with }}} Gbp-Dch: ignore --- .clang-format | 9 +++++++++ git-clang-format.sh | 2 ++ 2 files changed, 11 insertions(+) create mode 100644 .clang-format create mode 100755 git-clang-format.sh diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..00edbfc92 --- /dev/null +++ b/.clang-format @@ -0,0 +1,9 @@ +Language: Cpp +# BasedOnStyle + +TabWidth: 8 +UseTab: Always +IndentWidth: 3 +ColumnLimit: 0 +BreakBeforeBraces: Allman +AccessModifierOffset: 0 diff --git a/git-clang-format.sh b/git-clang-format.sh new file mode 100755 index 000000000..7cbb5ccd7 --- /dev/null +++ b/git-clang-format.sh @@ -0,0 +1,2 @@ +#!/bin/sh +git clang-format-3.8 --diff "$@" | sed "s#+/\*\}\}\}\*/#+ /*}}}*/#" | patch -p1 -- cgit v1.2.3-70-g09d2 From b4afd84a59578965120f175b410ea325d60bcb58 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 27 Jun 2017 20:47:47 +0200 Subject: methods: connect: Change PkgAcqMethod to aptMethod This will allow us to access ConfigFind() and stuff which makes it possible for us to implement TLS support. Gbp-Dch: ignore --- methods/connect.cc | 15 ++++++++------- methods/connect.h | 7 ++++--- methods/ftp.cc | 2 +- methods/ftp.h | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/methods/connect.cc b/methods/connect.cc index f6fb14769..d5e40fbab 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -34,6 +34,7 @@ #include #include +#include "aptmethod.h" #include "connect.h" #include "rfc2553emu.h" #include @@ -79,8 +80,8 @@ static bool ConnectionAllowed(char const * const Service, std::string const &Hos // DoConnect - Attempt a connect operation /*{{{*/ // --------------------------------------------------------------------- /* This helper function attempts a connection to a single address. */ -static bool DoConnect(struct addrinfo *Addr,std::string const &Host, - unsigned long TimeOut,int &Fd,pkgAcqMethod *Owner) +static bool DoConnect(struct addrinfo *Addr, std::string const &Host, + unsigned long TimeOut, int &Fd, aptMethod *Owner) { // Show a status indicator char Name[NI_MAXHOST]; @@ -150,8 +151,8 @@ static bool DoConnect(struct addrinfo *Addr,std::string const &Host, /*}}}*/ // Connect to a given Hostname /*{{{*/ static bool ConnectToHostname(std::string const &Host, int const Port, - const char * const Service, int DefPort, int &Fd, - unsigned long const TimeOut, pkgAcqMethod * const Owner) + const char *const Service, int DefPort, int &Fd, + unsigned long const TimeOut, aptMethod *const Owner) { if (ConnectionAllowed(Service, Host) == false) return false; @@ -286,9 +287,9 @@ static bool ConnectToHostname(std::string const &Host, int const Port, // Connect - Connect to a server /*{{{*/ // --------------------------------------------------------------------- /* Performs a connection to the server (including SRV record lookup) */ -bool Connect(std::string Host,int Port,const char *Service, - int DefPort,int &Fd, - unsigned long TimeOut,pkgAcqMethod *Owner) +bool Connect(std::string Host, int Port, const char *Service, + int DefPort, int &Fd, + unsigned long TimeOut, aptMethod *Owner) { if (_error->PendingError() == true) return false; diff --git a/methods/connect.h b/methods/connect.h index bbe1bb35d..44473a07c 100644 --- a/methods/connect.h +++ b/methods/connect.h @@ -12,10 +12,11 @@ #include -class pkgAcqMethod; +class aptMethod; + +bool Connect(std::string To, int Port, const char *Service, int DefPort, + int &Fd, unsigned long TimeOut, aptMethod *Owner); -bool Connect(std::string To,int Port,const char *Service,int DefPort, - int &Fd,unsigned long TimeOut,pkgAcqMethod *Owner); void RotateDNS(); #endif diff --git a/methods/ftp.cc b/methods/ftp.cc index d789637a8..5b30b5486 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -112,7 +112,7 @@ void FTPConn::Close() // --------------------------------------------------------------------- /* Connect to the server using a non-blocking connection and perform a login. */ -bool FTPConn::Open(pkgAcqMethod *Owner) +bool FTPConn::Open(aptMethod *Owner) { // Use the already open connection if possible. if (ServerFd != -1) diff --git a/methods/ftp.h b/methods/ftp.h index 6a12475a0..3a482fa42 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -55,7 +55,7 @@ class FTPConn bool WriteMsg(unsigned int &Ret,std::string &Text,const char *Fmt,...); // Connection control - bool Open(pkgAcqMethod *Owner); + bool Open(aptMethod *Owner); void Close(); bool GoPasv(); bool ExtGoPasv(); -- cgit v1.2.3-70-g09d2 From 5666084ecfe140aaa3f89388de557c2f875b4244 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 10:55:08 +0200 Subject: methods: connect: Switch from int fds to new MethodFd Use std::unique_ptr everywhere we used an integer-based file descriptor before. This allows us to implement stuff like TLS support easily. Gbp-Dch: ignore --- methods/connect.cc | 56 ++++++++++++++++++++++++++++----------- methods/connect.h | 25 +++++++++++++++++- methods/ftp.cc | 25 +++++++++--------- methods/ftp.h | 5 ++-- methods/http.cc | 77 ++++++++++++++++++++++++++++-------------------------- methods/http.h | 10 ++++--- 6 files changed, 126 insertions(+), 72 deletions(-) diff --git a/methods/connect.cc b/methods/connect.cc index d5e40fbab..9e9e09ac4 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -77,16 +77,42 @@ static bool ConnectionAllowed(char const * const Service, std::string const &Hos return true; } /*}}}*/ + +// File Descriptor based Fd /*{{{*/ +struct FdFd : public MethodFd +{ + int fd = -1; + int Fd() APT_OVERRIDE { return fd; } + ssize_t Read(void *buf, size_t count) APT_OVERRIDE { return ::read(fd, buf, count); } + ssize_t Write(void *buf, size_t count) APT_OVERRIDE { return ::write(fd, buf, count); } + int Close() APT_OVERRIDE + { + int result = 0; + if (fd != -1) + result = ::close(fd); + fd = -1; + return result; + } +}; + +std::unique_ptr MethodFd::FromFd(int iFd) +{ + FdFd *fd = new FdFd(); + fd->fd = iFd; + return std::unique_ptr(fd); +} + /*}}}*/ // DoConnect - Attempt a connect operation /*{{{*/ // --------------------------------------------------------------------- /* This helper function attempts a connection to a single address. */ static bool DoConnect(struct addrinfo *Addr, std::string const &Host, - unsigned long TimeOut, int &Fd, aptMethod *Owner) + unsigned long TimeOut, std::unique_ptr &Fd, aptMethod *Owner) { // Show a status indicator char Name[NI_MAXHOST]; char Service[NI_MAXSERV]; - + Fd.reset(new FdFd()); + Name[0] = 0; Service[0] = 0; getnameinfo(Addr->ai_addr,Addr->ai_addrlen, @@ -108,20 +134,21 @@ static bool DoConnect(struct addrinfo *Addr, std::string const &Host, } // Get a socket - if ((Fd = socket(Addr->ai_family,Addr->ai_socktype, - Addr->ai_protocol)) < 0) + if ((static_cast(Fd.get())->fd = socket(Addr->ai_family, Addr->ai_socktype, + Addr->ai_protocol)) < 0) return _error->Errno("socket",_("Could not create a socket for %s (f=%u t=%u p=%u)"), Name,Addr->ai_family,Addr->ai_socktype,Addr->ai_protocol); - - SetNonBlock(Fd,true); - if (connect(Fd,Addr->ai_addr,Addr->ai_addrlen) < 0 && + + SetNonBlock(Fd->Fd(), true); + if (connect(Fd->Fd(), Addr->ai_addr, Addr->ai_addrlen) < 0 && errno != EINPROGRESS) return _error->Errno("connect",_("Cannot initiate the connection " "to %s:%s (%s)."),Host.c_str(),Service,Name); /* This implements a timeout for connect by opening the connection nonblocking */ - if (WaitFd(Fd,true,TimeOut) == false) { + if (WaitFd(Fd->Fd(), true, TimeOut) == false) + { bad_addr.insert(bad_addr.begin(), std::string(Name)); Owner->SetFailReason("Timeout"); return _error->Error(_("Could not connect to %s:%s (%s), " @@ -131,7 +158,7 @@ static bool DoConnect(struct addrinfo *Addr, std::string const &Host, // Check the socket for an error condition unsigned int Err; unsigned int Len = sizeof(Err); - if (getsockopt(Fd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0) + if (getsockopt(Fd->Fd(), SOL_SOCKET, SO_ERROR, &Err, &Len) != 0) return _error->Errno("getsockopt",_("Failed")); if (Err != 0) @@ -151,7 +178,7 @@ static bool DoConnect(struct addrinfo *Addr, std::string const &Host, /*}}}*/ // Connect to a given Hostname /*{{{*/ static bool ConnectToHostname(std::string const &Host, int const Port, - const char *const Service, int DefPort, int &Fd, + const char *const Service, int DefPort, std::unique_ptr &Fd, unsigned long const TimeOut, aptMethod *const Owner) { if (ConnectionAllowed(Service, Host) == false) @@ -255,10 +282,9 @@ static bool ConnectToHostname(std::string const &Host, int const Port, { LastUsed = CurHost; return true; - } - close(Fd); - Fd = -1; - + } + Fd->Close(); + // Ignore UNIX domain sockets do { @@ -288,7 +314,7 @@ static bool ConnectToHostname(std::string const &Host, int const Port, // --------------------------------------------------------------------- /* Performs a connection to the server (including SRV record lookup) */ bool Connect(std::string Host, int Port, const char *Service, - int DefPort, int &Fd, + int DefPort, std::unique_ptr &Fd, unsigned long TimeOut, aptMethod *Owner) { if (_error->PendingError() == true) diff --git a/methods/connect.h b/methods/connect.h index 44473a07c..5eae77d09 100644 --- a/methods/connect.h +++ b/methods/connect.h @@ -10,12 +10,35 @@ #ifndef CONNECT_H #define CONNECT_H +#include +#include #include class aptMethod; +/** + * \brief Small representation of a file descriptor for network traffic. + * + * This provides support for TLS, SOCKS, and HTTP CONNECT proxies. + */ +struct MethodFd +{ + /// \brief Returns -1 for unusable, or an fd to select() on otherwise + virtual int Fd() = 0; + /// \brief Should behave like read(2) + virtual ssize_t Read(void *buf, size_t count) = 0; + /// \brief Should behave like write(2) + virtual ssize_t Write(void *buf, size_t count) = 0; + /// \brief Closes the file descriptor. Can be called multiple times. + virtual int Close() = 0; + /// \brief Destructor + virtual ~MethodFd(){}; + /// \brief Construct a MethodFd from a UNIX file descriptor + static std::unique_ptr FromFd(int iFd); +}; + bool Connect(std::string To, int Port, const char *Service, int DefPort, - int &Fd, unsigned long TimeOut, aptMethod *Owner); + std::unique_ptr &Fd, unsigned long TimeOut, aptMethod *Owner); void RotateDNS(); diff --git a/methods/ftp.cc b/methods/ftp.cc index 5b30b5486..f4986f648 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -73,8 +73,8 @@ time_t FtpMethod::FailTime = 0; // FTPConn::FTPConn - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1), - DataListenFd(-1), ServerName(Srv), +FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(MethodFd::FromFd(-1)), DataFd(-1), + DataListenFd(-1), ServerName(Srv), ForceExtended(false), TryPassive(true), PeerAddrLen(0), ServerAddrLen(0) { @@ -96,8 +96,7 @@ FTPConn::~FTPConn() /* Just tear down the socket and data socket */ void FTPConn::Close() { - close(ServerFd); - ServerFd = -1; + ServerFd->Close(); close(DataFd); DataFd = -1; close(DataListenFd); @@ -115,7 +114,7 @@ void FTPConn::Close() bool FTPConn::Open(aptMethod *Owner) { // Use the already open connection if possible. - if (ServerFd != -1) + if (ServerFd->Fd() != -1) return true; Close(); @@ -178,12 +177,12 @@ bool FTPConn::Open(aptMethod *Owner) // Get the remote server's address PeerAddrLen = sizeof(PeerAddr); - if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0) + if (getpeername(ServerFd->Fd(), (sockaddr *)&PeerAddr, &PeerAddrLen) != 0) return _error->Errno("getpeername",_("Unable to determine the peer name")); // Get the local machine's address ServerAddrLen = sizeof(ServerAddr); - if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0) + if (getsockname(ServerFd->Fd(), (sockaddr *)&ServerAddr, &ServerAddrLen) != 0) return _error->Errno("getsockname",_("Unable to determine the local name")); return Res; @@ -314,7 +313,7 @@ bool FTPConn::Login() /* This performs a very simple buffered read. */ bool FTPConn::ReadLine(string &Text) { - if (ServerFd == -1) + if (ServerFd->Fd() == -1) return false; // Suck in a line @@ -339,14 +338,14 @@ bool FTPConn::ReadLine(string &Text) } // Wait for some data.. - if (WaitFd(ServerFd,false,TimeOut) == false) + if (WaitFd(ServerFd->Fd(), false, TimeOut) == false) { Close(); return _error->Error(_("Connection timeout")); } // Suck it back - int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len); + int Res = ServerFd->Read(Buffer + Len, sizeof(Buffer) - Len); if (Res == 0) _error->Error(_("Server closed the connection")); if (Res <= 0) @@ -451,13 +450,13 @@ bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...) unsigned long Start = 0; while (Len != 0) { - if (WaitFd(ServerFd,true,TimeOut) == false) + if (WaitFd(ServerFd->Fd(), true, TimeOut) == false) { Close(); return _error->Error(_("Connection timeout")); } - - int Res = write(ServerFd,S + Start,Len); + + int Res = ServerFd->Write(S + Start, Len); if (Res <= 0) { _error->Errno("write",_("Write error")); diff --git a/methods/ftp.h b/methods/ftp.h index 3a482fa42..5b23dba41 100644 --- a/methods/ftp.h +++ b/methods/ftp.h @@ -10,8 +10,9 @@ #ifndef APT_FTP_H #define APT_FTP_H -#include #include "aptmethod.h" +#include "connect.h" +#include #include #include @@ -22,7 +23,7 @@ class FTPConn { char Buffer[1024*10]; unsigned long Len; - int ServerFd; + std::unique_ptr ServerFd; int DataFd; int DataListenFd; URI ServerName; diff --git a/methods/http.cc b/methods/http.cc index 9f5959548..d62f97be3 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -95,7 +95,7 @@ void CircleBuf::Reset() // --------------------------------------------------------------------- /* This fills up the buffer with as much data as is in the FD, assuming it is non-blocking.. */ -bool CircleBuf::Read(int Fd) +bool CircleBuf::Read(std::unique_ptr const &Fd) { while (1) { @@ -126,11 +126,11 @@ bool CircleBuf::Read(int Fd) // Write the buffer segment ssize_t Res; if(CircleBuf::BwReadLimit) { - Res = read(Fd,Buf + (InP%Size), - BwReadMax > LeftRead() ? LeftRead() : BwReadMax); + Res = Fd->Read(Buf + (InP % Size), + BwReadMax > LeftRead() ? LeftRead() : BwReadMax); } else - Res = read(Fd,Buf + (InP%Size),LeftRead()); - + Res = Fd->Read(Buf + (InP % Size), LeftRead()); + if(Res > 0 && BwReadLimit > 0) CircleBuf::BwTickReadData += Res; @@ -193,7 +193,7 @@ void CircleBuf::FillOut() // CircleBuf::Write - Write from the buffer into a FD /*{{{*/ // --------------------------------------------------------------------- /* This empties the buffer into the FD. */ -bool CircleBuf::Write(int Fd) +bool CircleBuf::Write(std::unique_ptr const &Fd) { while (1) { @@ -208,7 +208,7 @@ bool CircleBuf::Write(int Fd) // Write the buffer segment ssize_t Res; - Res = write(Fd,Buf + (OutP%Size),LeftWrite()); + Res = Fd->Write(Buf + (OutP % Size), LeftWrite()); if (Res == 0) return false; @@ -291,6 +291,7 @@ CircleBuf::~CircleBuf() HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024) { TimeOut = Owner->ConfigFindI("Timeout", TimeOut); + ServerFd = MethodFd::FromFd(-1); Reset(); } /*}}}*/ @@ -318,7 +319,7 @@ static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, bool HttpServerState::Open() { // Use the already open connection if possible. - if (ServerFd != -1) + if (ServerFd->Fd() != -1) return true; Close(); @@ -371,8 +372,12 @@ bool HttpServerState::Open() std::string const ProxyInfo = URI::SiteOnly(Proxy); Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str()); auto const Timeout = Owner->ConfigFindI("TimeOut", 120); - #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false - #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false +#define APT_WriteOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(ServerFd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \ + return false +#define APT_ReadOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(ServerFd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \ + return false if (ServerName.Host.length() > 255) return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str()); if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) @@ -514,7 +519,7 @@ bool HttpServerState::Open() ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n", ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); - if (WaitFd(ServerFd, true, Timeout) == false) + if (WaitFd(ServerFd->Fd(), true, Timeout) == false) return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out", ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); #undef APT_ReadOrFail @@ -549,8 +554,7 @@ bool HttpServerState::Open() /* */ bool HttpServerState::Close() { - close(ServerFd); - ServerFd = -1; + ServerFd->Close(); return true; } /*}}}*/ @@ -672,7 +676,7 @@ bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/ /*}}}*/ APT_PURE bool HttpServerState::IsOpen() /*{{{*/ { - return (ServerFd != -1); + return (ServerFd->Fd() != -1); } /*}}}*/ bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/ @@ -685,7 +689,7 @@ bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/ void HttpServerState::Reset() /*{{{*/ { ServerState::Reset(); - ServerFd = -1; + ServerFd->Close(); } /*}}}*/ @@ -710,7 +714,7 @@ bool HttpServerState::Die(RequestState &Req) SetNonBlock(Req.File.Fd(),false); while (In.WriteSpace() == true) { - if (In.Write(Req.File.Fd()) == false) + if (In.Write(MethodFd::FromFd(Req.File.Fd())) == false) return _error->Errno("write",_("Error writing to the file")); // Done @@ -762,7 +766,7 @@ bool HttpServerState::Flush(FileFd * const File) while (In.WriteSpace() == true) { - if (In.Write(File->Fd()) == false) + if (In.Write(MethodFd::FromFd(File->Fd())) == false) return _error->Errno("write",_("Error writing to file")); if (In.IsLimit() == true) return true; @@ -781,8 +785,8 @@ bool HttpServerState::Flush(FileFd * const File) bool HttpServerState::Go(bool ToFile, RequestState &Req) { // Server has closed the connection - if (ServerFd == -1 && (In.WriteSpace() == false || - ToFile == false)) + if (ServerFd->Fd() == -1 && (In.WriteSpace() == false || + ToFile == false)) return false; fd_set rfds,wfds; @@ -791,28 +795,27 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) /* Add the server. We only send more requests if the connection will be persisting */ - if (Out.WriteSpace() == true && ServerFd != -1 - && Persistent == true) - FD_SET(ServerFd,&wfds); - if (In.ReadSpace() == true && ServerFd != -1) - FD_SET(ServerFd,&rfds); - + if (Out.WriteSpace() == true && ServerFd->Fd() != -1 && Persistent == true) + FD_SET(ServerFd->Fd(), &wfds); + if (In.ReadSpace() == true && ServerFd->Fd() != -1) + FD_SET(ServerFd->Fd(), &rfds); + // Add the file - int FileFD = -1; + auto FileFD = MethodFd::FromFd(-1); if (Req.File.IsOpen()) - FileFD = Req.File.Fd(); - - if (In.WriteSpace() == true && ToFile == true && FileFD != -1) - FD_SET(FileFD,&wfds); + FileFD = MethodFd::FromFd(Req.File.Fd()); + + if (In.WriteSpace() == true && ToFile == true && FileFD->Fd() != -1) + FD_SET(FileFD->Fd(), &wfds); // Add stdin if (Owner->ConfigFindB("DependOnSTDIN", true) == true) FD_SET(STDIN_FILENO,&rfds); // Figure out the max fd - int MaxFd = FileFD; - if (MaxFd < ServerFd) - MaxFd = ServerFd; + int MaxFd = FileFD->Fd(); + if (MaxFd < ServerFd->Fd()) + MaxFd = ServerFd->Fd(); // Select struct timeval tv; @@ -833,14 +836,14 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) } // Handle server IO - if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds)) + if (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &rfds)) { errno = 0; if (In.Read(ServerFd) == false) return Die(Req); } - - if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds)) + + if (ServerFd->Fd() != -1 && FD_ISSET(ServerFd->Fd(), &wfds)) { errno = 0; if (Out.Write(ServerFd) == false) @@ -848,7 +851,7 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) } // Send data to the file - if (FileFD != -1 && FD_ISSET(FileFD,&wfds)) + if (FileFD->Fd() != -1 && FD_ISSET(FileFD->Fd(), &wfds)) { if (In.Write(FileFD) == false) return _error->Errno("write",_("Error writing to output file")); diff --git a/methods/http.h b/methods/http.h index c79a6454e..3336fb780 100644 --- a/methods/http.h +++ b/methods/http.h @@ -13,11 +13,13 @@ #include +#include +#include #include #include -#include #include "basehttp.h" +#include "connect.h" using std::cout; using std::endl; @@ -66,11 +68,11 @@ class CircleBuf unsigned long long TotalWriten; // Read data in - bool Read(int Fd); + bool Read(std::unique_ptr const &Fd); bool Read(std::string const &Data); // Write data out - bool Write(int Fd); + bool Write(std::unique_ptr const &Fd); bool WriteTillEl(std::string &Data,bool Single = false); // Control the write limit @@ -95,7 +97,7 @@ struct HttpServerState: public ServerState // This is the connection itself. Output is data FROM the server CircleBuf In; CircleBuf Out; - int ServerFd; + std::unique_ptr ServerFd; protected: virtual bool ReadHeaderLines(std::string &Data) APT_OVERRIDE; -- cgit v1.2.3-70-g09d2 From 2851ec6cf037d552118b885be0dd7796d74730c6 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 10:55:57 +0200 Subject: methods: Add HTTPS support to http method, using GnuTLS The http method will eventually replace the curl-based https method, but for now, this is an opt-in experiment that can be enabled by setting Dir::Bin::Methods::https to "http". Known issues: - We do not support HTTPS proxies yet - We do not support proxying HTTPS connections yet (CONNECT) - IssuerCert and SslForceVersion are unsupported Gbp-Dch: Full --- .travis.yml | 2 +- CMakeLists.txt | 4 + debian/control | 1 + methods/CMakeLists.txt | 9 +- methods/connect.cc | 361 +++++++++++++++++++++++++++++++++++++++++++++++++ methods/connect.h | 3 + methods/http.cc | 187 ++----------------------- 7 files changed, 385 insertions(+), 182 deletions(-) diff --git a/.travis.yml b/.travis.yml index 023107a68..74dff44b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ before_install: - sudo apt-get update -qq install: - sudo apt-get -qq -y -t wily install gettext liblz4-dev python3-apt - - sudo apt-get -qq -y -t xenial install cmake ninja-build + - sudo apt-get -qq -y -t xenial install cmake ninja-build libgnutls28-dev libcurl4-gnutls-dev - sudo ./prepare-release travis-ci before_script: - ( mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Coverage -G Ninja $CMAKE_FLAGS .. ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 83af9bd5a..8f8ffb898 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,6 +81,10 @@ find_package(CURL REQUIRED) if (CURL_FOUND) set(HAVE_CURL 1) endif() +find_package(GnuTLS REQUIRED) +if (GNUTLS_FOUND) + set(HAVE_GNUTLS 1) +endif() # (De)Compressor libraries find_package(ZLIB REQUIRED) diff --git a/debian/control b/debian/control index 96bbef348..04c44eb93 100644 --- a/debian/control +++ b/debian/control @@ -14,6 +14,7 @@ Build-Depends: cmake (>= 3.4), dpkg-dev (>= 1.17.14), gettext (>= 0.12), libbz2-dev, + libgnutls28-dev (>= 3.4.6), libcurl4-gnutls-dev (>= 7.19.4~), libdb-dev, googletest | libgtest-dev , diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index a74c2ce07..64807e310 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -11,7 +11,8 @@ add_executable(ftp ftp.cc rfc2553emu.cc connect.cc) add_executable(rred rred.cc) add_executable(rsh rsh.cc) -# Add target-specific header directories +target_compile_definitions(http PRIVATE ${GNUTLS_DEFINITIONS}) +target_include_directories(http PRIVATE ${GNUTLS_INCLUDE_DIR}) target_include_directories(https PRIVATE ${CURL_INCLUDE_DIRS}) # Link the executables against the libraries @@ -20,10 +21,10 @@ target_link_libraries(copy apt-pkg) target_link_libraries(store apt-pkg) target_link_libraries(gpgv apt-pkg) target_link_libraries(cdrom apt-pkg) -target_link_libraries(http apt-pkg) -target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES}) +target_link_libraries(http apt-pkg ${GNUTLS_LIBRARIES}) +target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES} ${GNUTLS_LIBRARIES}) target_link_libraries(https apt-pkg ${CURL_LIBRARIES}) -target_link_libraries(ftp apt-pkg) +target_link_libraries(ftp apt-pkg ${GNUTLS_LIBRARIES}) target_link_libraries(rred apt-pkg) target_link_libraries(rsh apt-pkg) diff --git a/methods/connect.cc b/methods/connect.cc index 9e9e09ac4..c40f34f2d 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -20,6 +20,9 @@ #include #include +#include +#include + #include #include #include @@ -367,3 +370,361 @@ bool Connect(std::string Host, int Port, const char *Service, _error->MergeWithStack(); return ret; } + /*}}}*/ +// UnwrapSocks - Handle SOCKS setup /*{{{*/ +// --------------------------------------------------------------------- +/* This does socks magic */ +static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, + char const *const type, bool const ReadWrite, uint8_t *const ToFrom, + unsigned int const Size, unsigned int const Timeout) +{ + if (WaitFd(ServerFd, ReadWrite, Timeout) == false) + return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type); + if (ReadWrite == false) + { + if (FileFd::Read(ServerFd, ToFrom, Size) == false) + return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); + } + else + { + if (FileFd::Write(ServerFd, ToFrom, Size) == false) + return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); + } + return true; +} + +bool UnwrapSocks(std::string Host, int Port, URI Proxy, std::unique_ptr &Fd, + unsigned long Timeout, aptMethod *Owner) +{ + /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect + * for not offering GSSAPI auth which is a must (we only do no or user/pass auth). + * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */ + std::string const ProxyInfo = URI::SiteOnly(Proxy); + Owner->Status(_("Connecting to %s (%s)"), "SOCKS5h proxy", ProxyInfo.c_str()); +#define APT_WriteOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \ + return false +#define APT_ReadOrFail(TYPE, DATA, LENGTH) \ + if (TalkToSocksProxy(Fd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \ + return false + if (Host.length() > 255) + return _error->Error("Can't use SOCKS5h as hostname %s is too long!", Host.c_str()); + if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) + return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length()); + if (Proxy.User.empty()) + { + uint8_t greeting[] = {0x05, 0x01, 0x00}; + APT_WriteOrFail("greet-1", greeting, sizeof(greeting)); + } + else + { + uint8_t greeting[] = {0x05, 0x02, 0x00, 0x02}; + APT_WriteOrFail("greet-2", greeting, sizeof(greeting)); + } + uint8_t greeting[2]; + APT_ReadOrFail("greet back", greeting, sizeof(greeting)); + if (greeting[0] != 0x05) + return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]); + if (greeting[1] == 0x00) + ; // no auth has no method-dependent sub-negotiations + else if (greeting[1] == 0x02) + { + if (Proxy.User.empty()) + return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str()); + // user&pass auth sub-negotiations are defined by RFC1929 + std::vector auth = {{0x01, static_cast(Proxy.User.length())}}; + std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth)); + auth.push_back(static_cast(Proxy.Password.length())); + std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth)); + APT_WriteOrFail("user&pass auth", auth.data(), auth.size()); + uint8_t authstatus[2]; + APT_ReadOrFail("auth report", authstatus, sizeof(authstatus)); + if (authstatus[0] != 0x01) + return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]); + if (authstatus[1] != 0x00) + return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]); + } + else + return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]); + union { + uint16_t *i; + uint8_t *b; + } portu; + uint16_t port = htons(static_cast(Port)); + portu.i = &port; + std::vector request = {{0x05, 0x01, 0x00, 0x03, static_cast(Host.length())}}; + std::copy(Host.begin(), Host.end(), std::back_inserter(request)); + request.push_back(portu.b[0]); + request.push_back(portu.b[1]); + APT_WriteOrFail("request", request.data(), request.size()); + uint8_t response[4]; + APT_ReadOrFail("first part of response", response, sizeof(response)); + if (response[0] != 0x05) + return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]); + if (response[2] != 0x00) + return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]); + std::string bindaddr; + if (response[3] == 0x01) // IPv4 address + { + uint8_t ip4port[6]; + APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port)); + portu.b[0] = ip4port[4]; + portu.b[1] = ip4port[5]; + port = ntohs(*portu.i); + strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port); + } + else if (response[3] == 0x03) // hostname + { + uint8_t namelength; + APT_ReadOrFail("hostname length of response", &namelength, 1); + uint8_t hostname[namelength + 2]; + APT_ReadOrFail("hostname of response", hostname, sizeof(hostname)); + portu.b[0] = hostname[namelength]; + portu.b[1] = hostname[namelength + 1]; + port = ntohs(*portu.i); + hostname[namelength] = '\0'; + strprintf(bindaddr, "%s:%d", hostname, port); + } + else if (response[3] == 0x04) // IPv6 address + { + uint8_t ip6port[18]; + APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port)); + portu.b[0] = ip6port[16]; + portu.b[1] = ip6port[17]; + port = ntohs(*portu.i); + strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d", + ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7], + ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15], + port); + } + else + return _error->Error("SOCKS proxy %s destination address is of unknown type: %d", + ProxyInfo.c_str(), response[3]); + if (response[1] != 0x00) + { + char const *errstr = nullptr; + auto errcode = response[1]; + // Tor error reporting can be a bit arcane, lets try to detect & fix it up + if (bindaddr == "0.0.0.0:0") + { + auto const lastdot = Host.rfind('.'); + if (lastdot == std::string::npos || Host.substr(lastdot) != ".onion") + ; + else if (errcode == 0x01) + { + auto const prevdot = Host.rfind('.', lastdot - 1); + if (lastdot == 16 && prevdot == std::string::npos) + ; // valid .onion address + else if (prevdot != std::string::npos && (lastdot - prevdot) == 17) + ; // valid .onion address with subdomain(s) + else + { + errstr = "Invalid hostname: onion service name must be 16 characters long"; + Owner->SetFailReason("SOCKS"); + } + } + // in all likelihood the service is either down or the address has + // a typo and so "Host unreachable" is the better understood error + // compared to the technically correct "TLL expired". + else if (errcode == 0x06) + errcode = 0x04; + } + if (errstr == nullptr) + { + switch (errcode) + { + case 0x01: + errstr = "general SOCKS server failure"; + Owner->SetFailReason("SOCKS"); + break; + case 0x02: + errstr = "connection not allowed by ruleset"; + Owner->SetFailReason("SOCKS"); + break; + case 0x03: + errstr = "Network unreachable"; + Owner->SetFailReason("ConnectionTimedOut"); + break; + case 0x04: + errstr = "Host unreachable"; + Owner->SetFailReason("ConnectionTimedOut"); + break; + case 0x05: + errstr = "Connection refused"; + Owner->SetFailReason("ConnectionRefused"); + break; + case 0x06: + errstr = "TTL expired"; + Owner->SetFailReason("Timeout"); + break; + case 0x07: + errstr = "Command not supported"; + Owner->SetFailReason("SOCKS"); + break; + case 0x08: + errstr = "Address type not supported"; + Owner->SetFailReason("SOCKS"); + break; + default: + errstr = "Unknown error"; + Owner->SetFailReason("SOCKS"); + break; + } + } + return _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str(), errstr, response[1]); + } + else if (Owner->DebugEnabled()) + ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str()); + + if (WaitFd(Fd->Fd(), true, Timeout) == false) + return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out", + ProxyInfo.c_str(), Host.c_str(), bindaddr.c_str()); +#undef APT_ReadOrFail +#undef APT_WriteOrFail + + return true; +} + /*}}}*/ +// UnwrapTLS - Handle TLS connections /*{{{*/ +// --------------------------------------------------------------------- +/* Performs a TLS handshake on the socket */ +struct TlsFd : public MethodFd +{ + std::unique_ptr UnderlyingFd; + gnutls_session_t session; + gnutls_certificate_credentials_t credentials; + std::string hostname; + + int Fd() APT_OVERRIDE { return UnderlyingFd->Fd(); } + + ssize_t Read(void *buf, size_t count) APT_OVERRIDE + { + return HandleError(gnutls_record_recv(session, buf, count)); + } + ssize_t Write(void *buf, size_t count) APT_OVERRIDE + { + return HandleError(gnutls_record_send(session, buf, count)); + } + + template + T HandleError(T err) + { + if (err < 0 && gnutls_error_is_fatal(err)) + errno = EIO; + else if (err < 0) + errno = EAGAIN; + else + errno = 0; + return err; + } + + int Close() APT_OVERRIDE + { + if (HandleError(gnutls_bye(session, GNUTLS_SHUT_RDWR)) < 0) + return -1; + return UnderlyingFd->Close(); + } +}; + +bool UnwrapTLS(std::string Host, std::unique_ptr &Fd, + unsigned long Timeout, aptMethod *Owner) +{ + int err; + TlsFd *tlsFd = new TlsFd(); + + tlsFd->hostname = Host; + tlsFd->UnderlyingFd = MethodFd::FromFd(-1); // For now + + gnutls_init(&tlsFd->session, GNUTLS_CLIENT | GNUTLS_NONBLOCK); + gnutls_transport_set_int(tlsFd->session, dynamic_cast(Fd.get())->fd); + gnutls_certificate_allocate_credentials(&tlsFd->credentials); + + // Credential setup + if ((err = gnutls_certificate_set_x509_system_trust(tlsFd->credentials)) <= 0) + return _error->Error("Could not load TLS certificates: %s", gnutls_strerror(err)); + + std::string fileinfo = Owner->ConfigFind("CaInfo", ""); + if (!fileinfo.empty()) + { + gnutls_certificate_set_verify_flags(tlsFd->credentials, GNUTLS_VERIFY_ALLOW_X509_V1_CA_CRT); + err = gnutls_certificate_set_x509_trust_file(tlsFd->credentials, fileinfo.c_str(), GNUTLS_X509_FMT_PEM); + if (err < 0) + return _error->Error("Could not load CaInfo certificate: %s", gnutls_strerror(err)); + } + + // TODO: IssuerCert AKA CURLOPT_ISSUERCERT + // TODO: Emulate SslForceVersion AKA CURLOPT_SSLVERSION? + + // For client authentication, certificate file ... + std::string const cert = Owner->ConfigFind("SslCert", ""); + std::string const key = Owner->ConfigFind("SslKey", ""); + if (cert.empty() == false) + { + if ((err = gnutls_certificate_set_x509_key_file( + tlsFd->credentials, + cert.c_str(), + key.empty() ? cert.c_str() : key.c_str(), + GNUTLS_X509_FMT_PEM)) < 0) + { + return _error->Error("Could not load client certificate or key: %s", gnutls_strerror(err)); + } + } + + // CRL file + std::string const crlfile = Owner->ConfigFind("CrlFile", ""); + if (crlfile.empty() == false) + { + if ((err = gnutls_certificate_set_x509_crl_file(tlsFd->credentials, + crlfile.c_str(), + GNUTLS_X509_FMT_PEM)) < 0) + return _error->Error("Could not load CrlFile: %s", gnutls_strerror(err)); + } + + if ((err = gnutls_credentials_set(tlsFd->session, GNUTLS_CRD_CERTIFICATE, tlsFd->credentials)) < 0) + return _error->Error("Could not set CaInfo certificate: %s", gnutls_strerror(err)); + + if ((err = gnutls_set_default_priority(tlsFd->session)) < 0) + return _error->Error("Could not set algorithm preferences: %s", gnutls_strerror(err)); + + if (Owner->ConfigFindB("Verify-Peer", true) || Owner->ConfigFindB("Verify-Host", true)) + { + gnutls_session_set_verify_cert(tlsFd->session, tlsFd->hostname.c_str(), 0); + } + if ((err = gnutls_server_name_set(tlsFd->session, GNUTLS_NAME_DNS, tlsFd->hostname.c_str(), tlsFd->hostname.length())) < 0) + return _error->Error("Could not set SNI name: %s", gnutls_strerror(err)); + + // Do the handshake. Our socket is non-blocking, so we need to call WaitFd() + // accordingly. + do + { + err = gnutls_handshake(tlsFd->session); + if ((err == GNUTLS_E_INTERRUPTED || err == GNUTLS_E_AGAIN) && + WaitFd(Fd->Fd(), gnutls_record_get_direction(tlsFd->session) == 1, Timeout) == false) + return _error->Errno("select", "Could not wait for server fd"); + } while (err < 0 && gnutls_error_is_fatal(err) == 0); + + if (err < 0) + { + // Print reason why validation failed. + if (err == GNUTLS_E_CERTIFICATE_VERIFICATION_ERROR) + { + gnutls_datum_t txt; + auto type = gnutls_certificate_type_get(tlsFd->session); + auto status = gnutls_session_get_verify_cert_status(tlsFd->session); + if (gnutls_certificate_verification_status_print(status, + type, &txt, 0) == 0) + { + _error->Error("Certificate verification failed: %s", txt.data); + } + gnutls_free(txt.data); + } + return _error->Error("Could not handshake: %s", gnutls_strerror(err)); + } + + tlsFd->UnderlyingFd = std::move(Fd); + Fd.reset(tlsFd); + return true; +} + /*}}}*/ diff --git a/methods/connect.h b/methods/connect.h index 5eae77d09..39f3c1ce2 100644 --- a/methods/connect.h +++ b/methods/connect.h @@ -40,6 +40,9 @@ struct MethodFd bool Connect(std::string To, int Port, const char *Service, int DefPort, std::unique_ptr &Fd, unsigned long TimeOut, aptMethod *Owner); +bool UnwrapSocks(std::string To, int Port, URI Proxy, std::unique_ptr &Fd, unsigned long Timeout, aptMethod *Owner); +bool UnwrapTLS(std::string To, std::unique_ptr &Fd, unsigned long Timeout, aptMethod *Owner); + void RotateDNS(); #endif diff --git a/methods/http.cc b/methods/http.cc index d62f97be3..4d8c0808b 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -298,24 +298,6 @@ HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, O // HttpServerState::Open - Open a connection to the server /*{{{*/ // --------------------------------------------------------------------- /* This opens a connection to the server. */ -static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy, - char const * const type, bool const ReadWrite, uint8_t * const ToFrom, - unsigned int const Size, unsigned int const Timeout) -{ - if (WaitFd(ServerFd, ReadWrite, Timeout) == false) - return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type); - if (ReadWrite == false) - { - if (FileFd::Read(ServerFd, ToFrom, Size) == false) - return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); - } - else - { - if (FileFd::Write(ServerFd, ToFrom, Size) == false) - return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str()); - } - return true; -} bool HttpServerState::Open() { // Use the already open connection if possible. @@ -361,169 +343,15 @@ bool HttpServerState::Open() if (Proxy.empty() == false) Owner->AddProxyAuth(Proxy, ServerName); + bool tls = ServerName.Access == "https"; if (Proxy.Access == "socks5h") { if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false) return false; - /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect - * for not offering GSSAPI auth which is a must (we only do no or user/pass auth). - * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */ - std::string const ProxyInfo = URI::SiteOnly(Proxy); - Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str()); - auto const Timeout = Owner->ConfigFindI("TimeOut", 120); -#define APT_WriteOrFail(TYPE, DATA, LENGTH) \ - if (TalkToSocksProxy(ServerFd->Fd(), ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) \ - return false -#define APT_ReadOrFail(TYPE, DATA, LENGTH) \ - if (TalkToSocksProxy(ServerFd->Fd(), ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) \ - return false - if (ServerName.Host.length() > 255) - return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str()); - if (Proxy.User.length() > 255 || Proxy.Password.length() > 255) - return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length()); - if (Proxy.User.empty()) - { - uint8_t greeting[] = { 0x05, 0x01, 0x00 }; - APT_WriteOrFail("greet-1", greeting, sizeof(greeting)); - } - else - { - uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 }; - APT_WriteOrFail("greet-2", greeting, sizeof(greeting)); - } - uint8_t greeting[2]; - APT_ReadOrFail("greet back", greeting, sizeof(greeting)); - if (greeting[0] != 0x05) - return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]); - if (greeting[1] == 0x00) - ; // no auth has no method-dependent sub-negotiations - else if (greeting[1] == 0x02) - { - if (Proxy.User.empty()) - return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str()); - // user&pass auth sub-negotiations are defined by RFC1929 - std::vector auth = {{ 0x01, static_cast(Proxy.User.length()) }}; - std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth)); - auth.push_back(static_cast(Proxy.Password.length())); - std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth)); - APT_WriteOrFail("user&pass auth", auth.data(), auth.size()); - uint8_t authstatus[2]; - APT_ReadOrFail("auth report", authstatus, sizeof(authstatus)); - if (authstatus[0] != 0x01) - return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]); - if (authstatus[1] != 0x00) - return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]); - } - else - return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]); - union { uint16_t * i; uint8_t * b; } portu; - uint16_t port = htons(static_cast(ServerName.Port == 0 ? 80 : ServerName.Port)); - portu.i = &port; - std::vector request = {{ 0x05, 0x01, 0x00, 0x03, static_cast(ServerName.Host.length()) }}; - std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request)); - request.push_back(portu.b[0]); - request.push_back(portu.b[1]); - APT_WriteOrFail("request", request.data(), request.size()); - uint8_t response[4]; - APT_ReadOrFail("first part of response", response, sizeof(response)); - if (response[0] != 0x05) - return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]); - if (response[2] != 0x00) - return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]); - std::string bindaddr; - if (response[3] == 0x01) // IPv4 address - { - uint8_t ip4port[6]; - APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port)); - portu.b[0] = ip4port[4]; - portu.b[1] = ip4port[5]; - port = ntohs(*portu.i); - strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port); - } - else if (response[3] == 0x03) // hostname - { - uint8_t namelength; - APT_ReadOrFail("hostname length of response", &namelength, 1); - uint8_t hostname[namelength + 2]; - APT_ReadOrFail("hostname of response", hostname, sizeof(hostname)); - portu.b[0] = hostname[namelength]; - portu.b[1] = hostname[namelength + 1]; - port = ntohs(*portu.i); - hostname[namelength] = '\0'; - strprintf(bindaddr, "%s:%d", hostname, port); - } - else if (response[3] == 0x04) // IPv6 address - { - uint8_t ip6port[18]; - APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port)); - portu.b[0] = ip6port[16]; - portu.b[1] = ip6port[17]; - port = ntohs(*portu.i); - strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d", - ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7], - ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15], - port); - } - else - return _error->Error("SOCKS proxy %s destination address is of unknown type: %d", - ProxyInfo.c_str(), response[3]); - if (response[1] != 0x00) - { - char const * errstr = nullptr; - auto errcode = response[1]; - // Tor error reporting can be a bit arcane, lets try to detect & fix it up - if (bindaddr == "0.0.0.0:0") - { - auto const lastdot = ServerName.Host.rfind('.'); - if (lastdot == std::string::npos || ServerName.Host.substr(lastdot) != ".onion") - ; - else if (errcode == 0x01) - { - auto const prevdot = ServerName.Host.rfind('.', lastdot - 1); - if (lastdot == 16 && prevdot == std::string::npos) - ; // valid .onion address - else if (prevdot != std::string::npos && (lastdot - prevdot) == 17) - ; // valid .onion address with subdomain(s) - else - { - errstr = "Invalid hostname: onion service name must be 16 characters long"; - Owner->SetFailReason("SOCKS"); - } - } - // in all likelihood the service is either down or the address has - // a typo and so "Host unreachable" is the better understood error - // compared to the technically correct "TLL expired". - else if (errcode == 0x06) - errcode = 0x04; - } - if (errstr == nullptr) - { - switch (errcode) - { - case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break; - case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break; - case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break; - case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break; - case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break; - case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break; - case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break; - case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break; - default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break; - } - } - return _error->Error("SOCKS proxy %s could not connect to %s (%s) due to: %s (%d)", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str(), errstr, response[1]); - } - else if (Owner->DebugEnabled()) - ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s (%s)\n", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); - - if (WaitFd(ServerFd->Fd(), true, Timeout) == false) - return _error->Error("SOCKS proxy %s reported connection to %s (%s), but timed out", - ProxyInfo.c_str(), ServerName.Host.c_str(), bindaddr.c_str()); - #undef APT_ReadOrFail - #undef APT_WriteOrFail + if (UnwrapSocks(ServerName.Host, ServerName.Port == 0 ? 80 : ServerName.Port, + Proxy, ServerFd, Owner->ConfigFindI("TimeOut", 120), Owner) == false) + return false; } else { @@ -544,8 +372,13 @@ bool HttpServerState::Open() Port = Proxy.Port; Host = Proxy.Host; } - return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner); + if (!Connect(Host, Port, tls ? "https" : "http", tls ? 443 : 80, ServerFd, TimeOut, Owner)) + return false; } + + if (tls && UnwrapTLS(ServerName.Host, ServerFd, TimeOut, Owner) == false) + return false; + return true; } /*}}}*/ -- cgit v1.2.3-70-g09d2 From 0fe2161020d6e331639ed11872a947dd20035890 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 00:12:11 +0200 Subject: Allow building without curl This makes testing easier and prepares us for the transition. --- CMakeLists.txt | 12 +++++++++--- methods/CMakeLists.txt | 17 ++++++++++++++--- po/CMakeLists.txt | 8 +++++++- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f8ffb898..a0690923c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,10 +77,16 @@ endif() # apt-transport-https dependencies -find_package(CURL REQUIRED) -if (CURL_FOUND) - set(HAVE_CURL 1) +option(WITH_CURL "Build curl-based methods" ON) +if (WITH_CURL) + find_package(CURL REQUIRED) + if (CURL_FOUND) + set(HAVE_CURL 1) + endif() +else() + set(HAVE_CURL 0) endif() + find_package(GnuTLS REQUIRED) if (GNUTLS_FOUND) set(HAVE_GNUTLS 1) diff --git a/methods/CMakeLists.txt b/methods/CMakeLists.txt index 64807e310..9f01ec506 100644 --- a/methods/CMakeLists.txt +++ b/methods/CMakeLists.txt @@ -6,14 +6,18 @@ add_executable(gpgv gpgv.cc) add_executable(cdrom cdrom.cc) add_executable(http http.cc http_main.cc rfc2553emu.cc connect.cc basehttp.cc) add_executable(mirror mirror.cc http.cc rfc2553emu.cc connect.cc basehttp.cc) -add_executable(https https.cc basehttp.cc) +if (HAVE_CURL) + add_executable(https https.cc basehttp.cc) +endif() add_executable(ftp ftp.cc rfc2553emu.cc connect.cc) add_executable(rred rred.cc) add_executable(rsh rsh.cc) target_compile_definitions(http PRIVATE ${GNUTLS_DEFINITIONS}) target_include_directories(http PRIVATE ${GNUTLS_INCLUDE_DIR}) +if (HAVE_CURL) target_include_directories(https PRIVATE ${CURL_INCLUDE_DIRS}) +endif() # Link the executables against the libraries target_link_libraries(file apt-pkg) @@ -23,14 +27,21 @@ target_link_libraries(gpgv apt-pkg) target_link_libraries(cdrom apt-pkg) target_link_libraries(http apt-pkg ${GNUTLS_LIBRARIES}) target_link_libraries(mirror apt-pkg ${RESOLV_LIBRARIES} ${GNUTLS_LIBRARIES}) -target_link_libraries(https apt-pkg ${CURL_LIBRARIES}) +if (HAVE_CURL) + target_link_libraries(https apt-pkg ${CURL_LIBRARIES}) +endif() target_link_libraries(ftp apt-pkg ${GNUTLS_LIBRARIES}) target_link_libraries(rred apt-pkg) target_link_libraries(rsh apt-pkg) # Install the library -install(TARGETS file copy store gpgv cdrom http https ftp rred rsh mirror +install(TARGETS file copy store gpgv cdrom http ftp rred rsh mirror RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods) add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods store gzip lzma bzip2 xz) add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods rsh ssh) +if (HAVE_CURL) + install(TARGETS https RUNTIME DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}/apt/methods) +else() + add_slaves(${CMAKE_INSTALL_LIBEXECDIR}/apt/methods http https) +endif() diff --git a/po/CMakeLists.txt b/po/CMakeLists.txt index 258c9b050..2630a2f89 100644 --- a/po/CMakeLists.txt +++ b/po/CMakeLists.txt @@ -10,12 +10,18 @@ apt_add_translation_domain( EXCLUDE_LANGUAGES ${languages_excluded} ) +if (HAVE_CURL) + set(curl_methods https) +else() + set(curl_methods) +endif() + apt_add_translation_domain( DOMAIN apt TARGETS apt apt-cache apt-get apt-config apt-cdrom apt-helper apt-mark apt-private # Methods - file copy store gpgv cdrom http https ftp rred rsh mirror + file copy store gpgv cdrom http ${curl_methods} ftp rred rsh mirror SCRIPTS ../dselect/install ../dselect/update EXCLUDE_LANGUAGES ${languages_excluded} ) -- cgit v1.2.3-70-g09d2 From f806530b9ea858ca6bda8fb8f43d988aba02dab3 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 13:20:54 +0200 Subject: methods: http: Drain pending data before selecting GnuTLS can already have data pending in its buffers, we need to to drain that first otherwise select() might block indefinitely. Gbp-Dch: ignore --- methods/connect.cc | 9 +++++++++ methods/connect.h | 2 ++ methods/http.cc | 10 +++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/methods/connect.cc b/methods/connect.cc index c40f34f2d..e48008214 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -98,6 +98,10 @@ struct FdFd : public MethodFd } }; +bool MethodFd::HasPending() +{ + return false; +} std::unique_ptr MethodFd::FromFd(int iFd) { FdFd *fd = new FdFd(); @@ -626,6 +630,11 @@ struct TlsFd : public MethodFd return -1; return UnderlyingFd->Close(); } + + bool HasPending() APT_OVERRIDE + { + return gnutls_record_check_pending(session) > 0; + } }; bool UnwrapTLS(std::string Host, std::unique_ptr &Fd, diff --git a/methods/connect.h b/methods/connect.h index 39f3c1ce2..4456d660d 100644 --- a/methods/connect.h +++ b/methods/connect.h @@ -35,6 +35,8 @@ struct MethodFd virtual ~MethodFd(){}; /// \brief Construct a MethodFd from a UNIX file descriptor static std::unique_ptr FromFd(int iFd); + /// \brief If there is pending data. + virtual bool HasPending(); }; bool Connect(std::string To, int Port, const char *Service, int DefPort, diff --git a/methods/http.cc b/methods/http.cc index 4d8c0808b..b302c896d 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -621,7 +621,15 @@ bool HttpServerState::Go(bool ToFile, RequestState &Req) if (ServerFd->Fd() == -1 && (In.WriteSpace() == false || ToFile == false)) return false; - + + // Handle server IO + if (ServerFd->HasPending() && In.ReadSpace() == true) + { + errno = 0; + if (In.Read(ServerFd) == false) + return Die(Req); + } + fd_set rfds,wfds; FD_ZERO(&rfds); FD_ZERO(&wfds); -- cgit v1.2.3-70-g09d2 From 29a08d8ab0c4d82f26c2712c456508784040cdbb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 28 Jun 2017 15:52:00 +0200 Subject: Fix https->http redirect issues Gbp-Dch: ignore --- methods/http_main.cc | 2 +- test/integration/test-apt-https-no-redirect | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/methods/http_main.cc b/methods/http_main.cc index 1e56044b7..90a0450e2 100644 --- a/methods/http_main.cc +++ b/methods/http_main.cc @@ -11,7 +11,7 @@ int main(int, const char *argv[]) // closes the connection (this is dealt with via ServerDie()) signal(SIGPIPE, SIG_IGN); std::string Binary = flNotDir(argv[0]); - if (Binary.find('+') == std::string::npos && Binary != "http") + if (Binary.find('+') == std::string::npos && Binary != "https" && Binary != "http") Binary.append("+http"); return HttpMethod(std::move(Binary)).Loop(); } diff --git a/test/integration/test-apt-https-no-redirect b/test/integration/test-apt-https-no-redirect index d6c630d5f..05e97159c 100755 --- a/test/integration/test-apt-https-no-redirect +++ b/test/integration/test-apt-https-no-redirect @@ -14,7 +14,7 @@ echo 'alright' > aptarchive/working changetohttpswebserver webserverconfig 'aptwebserver::redirect::replace::/redirectme/' "http://localhost:${APTHTTPPORT}/" webserverconfig 'aptwebserver::redirect::replace::/redirectme2/' "https://localhost:${APTHTTPSPORT}/" -echo 'Dir::Bin::Methods::https+http "https";' > rootdir/etc/apt/apt.conf.d/99add-https-http-method +echo 'Dir::Bin::Methods::https+http "http";' > rootdir/etc/apt/apt.conf.d/99add-https-http-method msgtest 'download of a file works via' 'http' testsuccess --nomsg downloadfile "http://localhost:${APTHTTPPORT}/working" httpfile @@ -26,13 +26,9 @@ testfileequal httpsfile 'alright' rm -f httpfile httpsfile msgtest 'download of http file works via' 'https+http' -testsuccess --nomsg downloadfile "http://localhost:${APTHTTPPORT}/working" httpfile +testsuccess --nomsg downloadfile "https+http://localhost:${APTHTTPPORT}/working" httpfile testfileequal httpfile 'alright' - -msgtest 'download of https file works via' 'https+http' -testsuccess --nomsg downloadfile "https://localhost:${APTHTTPSPORT}/working" httpsfile -testfileequal httpsfile 'alright' -rm -f httpfile httpsfile +rm -f httpfile msgtest 'download of a file does not work if' 'https redirected to http' testfailure --nomsg downloadfile "https://localhost:${APTHTTPSPORT}/redirectme/working" redirectfile -- cgit v1.2.3-70-g09d2 From c81b83864b7da79250a210ea7c49b5b03a4b2b16 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 15:29:13 +0200 Subject: Fix test suite and enable non-curl testing on travis, shippable Gbp-Dch: ignore --- .travis.yml | 1 + shippable.yml | 2 +- test/integration/test-apt-update-failure-propagation | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 74dff44b3..8aa8dceb6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ sudo: required dist: trusty env: - TEST_SUITE=user CMAKE_FLAGS= + - TEST_SUITE=user CMAKE_FLAGS="-DWITH_DOC=OFF -DWITH_CURL=OFF" - TEST_SUITE=root CMAKE_FLAGS=-DWITH_DOC=OFF before_install: - sudo add-apt-repository 'deb http://archive.ubuntu.com/ubuntu/ wily main universe' -y diff --git a/shippable.yml b/shippable.yml index ad72afc80..e144d94d5 100644 --- a/shippable.yml +++ b/shippable.yml @@ -9,7 +9,7 @@ build: - apt-get install -qq build-essential - ./prepare-release travis-ci - mkdir build - - ( cd build && cmake .. ) + - ( cd build && cmake -DWITH_CURL=OFF .. ) - make -C build -j 4 - CTEST_OUTPUT_ON_FAILURE=1 make -C build test - ./test/integration/run-tests -q diff --git a/test/integration/test-apt-update-failure-propagation b/test/integration/test-apt-update-failure-propagation index 1361b1b93..9ca6e481f 100755 --- a/test/integration/test-apt-update-failure-propagation +++ b/test/integration/test-apt-update-failure-propagation @@ -100,7 +100,13 @@ for FILE in rootdir/etc/apt/sources.list.d/*-stable-* ; do sed -i -e "s#:${APTHTTPSPORT}/#:666/#" "$FILE" done testwarning aptget update -o Dir::Bin::Methods::https="${OLDMETHODS}/https" +if grep -q WITH_CURL:BOOL=OFF $PROJECT_BINARY_DIR/CMakeCache.txt; then +testequalor2 "W: Failed to fetch https://localhost:666/dists/stable/InRelease Failed to connect to localhost port 666: Connection refused +W: Some index files failed to download. They have been ignored, or old ones used instead." "W: Failed to fetch https://localhost:666/dists/stable/InRelease Could not connect to localhost:666 (127.0.0.1). - connect (111: Connection refused) +W: Some index files failed to download. They have been ignored, or old ones used instead." tail -n 2 rootdir/tmp/testwarning.output +else testequalor2 "W: Failed to fetch https://localhost:666/dists/stable/InRelease Failed to connect to localhost port 666: Connection refused W: Some index files failed to download. They have been ignored, or old ones used instead." "W: Failed to fetch https://localhost:666/dists/stable/InRelease couldn't connect to host W: Some index files failed to download. They have been ignored, or old ones used instead." tail -n 2 rootdir/tmp/testwarning.output +fi posttest -- cgit v1.2.3-70-g09d2 From 147ac0fc90d972a11f5e91521ba3d385015b5945 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Wed, 28 Jun 2017 17:17:37 +0200 Subject: Introduce Acquire::AllowTLS to turn off TLS support As requested by Henrique de Moraes Holschuh, here comes an option to disable TLS support. If the option is set to false, the internal TLS layer is disabled. --- doc/apt.conf.5.xml | 9 +++++++++ doc/examples/configure-index | 2 ++ methods/connect.cc | 3 +++ 3 files changed, 14 insertions(+) diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index 260c66c46..54ed78c95 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -357,6 +357,15 @@ APT::Compressor::rev { + + + Allow use of the internal TLS support in the http method. If set to false, + this completely disables support for TLS in apt's own methods (excluding + the curl-based https method). No TLS-related functions will be called + anymore. + + + Try to download deltas called PDiffs for indexes (like Packages files) instead of diff --git a/doc/examples/configure-index b/doc/examples/configure-index index aada67bf5..a48d4cb99 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -206,6 +206,8 @@ Acquire Source-Symlinks ""; ForceHash ""; // hashmethod used for expected hash: sha256, sha1 or md5sum + AllowTLS ""; // whether support for tls is enabled + PDiffs ""; // try to get the IndexFile diffs PDiffs::FileLimit ""; // don't use diffs if we would need more than 4 diffs PDiffs::SizeLimit ""; // don't use diffs if size of all patches excess X% of the size of the original file diff --git a/methods/connect.cc b/methods/connect.cc index e48008214..0103b5873 100644 --- a/methods/connect.cc +++ b/methods/connect.cc @@ -640,6 +640,9 @@ struct TlsFd : public MethodFd bool UnwrapTLS(std::string Host, std::unique_ptr &Fd, unsigned long Timeout, aptMethod *Owner) { + if (_config->FindB("Acquire::AllowTLS", true) == false) + return _error->Error("TLS support has been disabled: Acquire::AllowTLS is false."); + int err; TlsFd *tlsFd = new TlsFd(); -- cgit v1.2.3-70-g09d2