From 57da1b4bd21aceced30f658993fb811a5232cff7 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Fri, 31 May 2013 18:19:09 +0200
Subject: stop building l10n if strings are unchanged

The buildsystem tried to build l10n for test applications which never
produced the output it expected causing it to try building it all the
time.
---
 test/interactive-helper/makefile | 1 +
 1 file changed, 1 insertion(+)

(limited to 'test/interactive-helper')

diff --git a/test/interactive-helper/makefile b/test/interactive-helper/makefile
index 10d1e44ec..fa4f5fecb 100644
--- a/test/interactive-helper/makefile
+++ b/test/interactive-helper/makefile
@@ -1,6 +1,7 @@
 # -*- make -*-
 BASE=../..
 SUBDIR=test/interactive-helper
+APT_DOMAIN=none
 
 # Bring in the default rules
 include ../../buildlib/defaults.mak
-- 
cgit v1.2.3-70-g09d2


From fbd29dd60a195a5c252ba1d28366d52d60447560 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Sat, 15 Jun 2013 15:45:22 +0200
Subject: add a simple webserver for our testcases

APT needs to acquire data in a secure fashion over an inherently
unsecure way, known as the internet, while communicating with
unreliable partners, known as webservers and proxies.

For your integration tests we so far relied on 'normal' webservers,
but all of them have certain quirks and none is able to provide us
with all quirks which can be observed in the wild and we therefore
have to test with, so this webserver isn't trying to be fast, secure
or feature complete, but to provide all the quirks we need in a
consistent way.

This webserver also makes the APT project self-contained, as it is now
able to generate, serve as well as acquire package indexes. ;)

Git-Dch: Ignore
---
 test/integration/framework              |  19 +-
 test/interactive-helper/aptwebserver.cc | 343 ++++++++++++++++++++++++++++++++
 test/interactive-helper/makefile        |   7 +
 3 files changed, 363 insertions(+), 6 deletions(-)
 create mode 100644 test/interactive-helper/aptwebserver.cc

(limited to 'test/interactive-helper')

diff --git a/test/integration/framework b/test/integration/framework
index 5c50498a2..9db4a1017 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -718,12 +718,20 @@ signreleasefiles() {
 }
 
 changetowebserver() {
-	if which weborf > /dev/null; then
-		weborf -xb aptarchive/ 2>&1 > /dev/null &
+	local LOG='/dev/null'
+	if test -x ${BUILDDIRECTORY}/aptwebserver; then
+		cd aptarchive
+		LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/aptwebserver "$@" >$LOG 2>&1 &
+		addtrap "kill $!;"
+		cd - > /dev/null
+	elif [ $# -gt 0 ]; then
+		msgdie 'Need the aptwebserver when passing arguments for the webserver'
+	elif which weborf > /dev/null; then
+		weborf -xb aptarchive/ >$LOG 2>&1 &
 		addtrap "kill $!;"
 	elif which gatling > /dev/null; then
 		cd aptarchive
-		gatling -p 8080 -F -S 2>&1 > /dev/null &
+		gatling -p 8080 -F -S >$LOG 2>&1 &
 		addtrap "kill $!;"
 		cd - > /dev/null
 	elif which lighttpd > /dev/null; then
@@ -731,11 +739,10 @@ changetowebserver() {
 server.port = 8080
 server.stat-cache-engine = \"disable\"" > lighttpd.conf
 		lighttpd -t -f lighttpd.conf >/dev/null || msgdie 'Can not change to webserver: our lighttpd config is invalid'
-		lighttpd -D -f lighttpd.conf 2>/dev/null >/dev/null &
+		lighttpd -D -f lighttpd.conf >$LOG 2>&1 &
 		addtrap "kill $!;"
 	else
-		msgdie 'You have to install weborf or lighttpd first'
-		return 1
+		msgdie 'You have to build aptwerbserver or install a webserver'
 	fi
 	local APTARCHIVE="file://$(readlink -f ./aptarchive)"
 	for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
new file mode 100644
index 000000000..98bbde21b
--- /dev/null
+++ b/test/interactive-helper/aptwebserver.cc
@@ -0,0 +1,343 @@
+#include <config.h>
+
+#include <apt-pkg/strutl.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/error.h>
+#include <apt-pkg/cmndline.h>
+#include <apt-pkg/configuration.h>
+#include <apt-pkg/init.h>
+
+#include <vector>
+#include <string>
+#include <list>
+#include <sstream>
+
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <netinet/in.h>
+#include <unistd.h>
+#include <errno.h>
+#include <time.h>
+#include <stdlib.h>
+#include <signal.h>
+
+char const * const httpcodeToStr(int const httpcode)			/*{{{*/
+{
+   switch (httpcode)
+   {
+      // Informational 1xx
+      case 100: return "100 Continue";
+      case 101: return "101 Switching Protocols";
+      // Successful 2xx
+      case 200: return "200 OK";
+      case 201: return "201 Created";
+      case 202: return "202 Accepted";
+      case 203: return "203 Non-Authoritative Information";
+      case 204: return "204 No Content";
+      case 205: return "205 Reset Content";
+      case 206: return "206 Partial Content";
+      // Redirections 3xx
+      case 300: return "300 Multiple Choices";
+      case 301: return "301 Moved Permanently";
+      case 302: return "302 Found";
+      case 303: return "303 See Other";
+      case 304: return "304 Not Modified";
+      case 305: return "304 Use Proxy";
+      case 307: return "307 Temporary Redirect";
+      // Client errors 4xx
+      case 400: return "400 Bad Request";
+      case 401: return "401 Unauthorized";
+      case 402: return "402 Payment Required";
+      case 403: return "403 Forbidden";
+      case 404: return "404 Not Found";
+      case 405: return "405 Method Not Allowed";
+      case 406: return "406 Not Acceptable";
+      case 407: return "407 Proxy Authentication Required";
+      case 408: return "408 Request Time-out";
+      case 409: return "409 Conflict";
+      case 410: return "410 Gone";
+      case 411: return "411 Length Required";
+      case 412: return "412 Precondition Failed";
+      case 413: return "413 Request Entity Too Large";
+      case 414: return "414 Request-URI Too Large";
+      case 415: return "415 Unsupported Media Type";
+      case 416: return "416 Requested range not satisfiable";
+      case 417: return "417 Expectation Failed";
+      case 418: return "418 I'm a teapot";
+      // Server error 5xx
+      case 500: return "500 Internal Server Error";
+      case 501: return "501 Not Implemented";
+      case 502: return "502 Bad Gateway";
+      case 503: return "503 Service Unavailable";
+      case 504: return "504 Gateway Time-out";
+      case 505: return "505 HTTP Version not supported";
+   }
+   return NULL;
+}
+									/*}}}*/
+void addFileHeaders(std::list<std::string> &headers, FileFd &data)	/*{{{*/
+{
+   std::ostringstream contentlength;
+   contentlength << "Content-Length: " << data.FileSize();
+   headers.push_back(contentlength.str());
+
+   std::string lastmodified("Last-Modified: ");
+   lastmodified.append(TimeRFC1123(data.ModificationTime()));
+   headers.push_back(lastmodified);
+}
+									/*}}}*/
+void addDataHeaders(std::list<std::string> &headers, std::string &data)	/*{{{*/
+{
+   std::ostringstream contentlength;
+   contentlength << "Content-Length: " << data.size();
+   headers.push_back(contentlength.str());
+}
+									/*}}}*/
+bool sendHead(int const client, int const httpcode, std::list<std::string> &headers)/*{{{*/
+{
+   std::string response("HTTP/1.1 ");
+   response.append(httpcodeToStr(httpcode));
+   headers.push_front(response);
+
+   headers.push_back("Server: APT webserver");
+
+   std::string date("Date: ");
+   date.append(TimeRFC1123(time(NULL)));
+   headers.push_back(date);
+
+   std::clog << ">>> RESPONSE >>>" << std::endl;
+   bool Success = true;
+   for (std::list<std::string>::const_iterator h = headers.begin();
+	Success == true && h != headers.end(); ++h)
+   {
+      Success &= FileFd::Write(client, h->c_str(), h->size());
+      if (Success == true)
+	 Success &= FileFd::Write(client, "\r\n", 2);
+      std::clog << *h << std::endl;
+   }
+   if (Success == true)
+      Success &= FileFd::Write(client, "\r\n", 2);
+   std::clog << "<<<<<<<<<<<<<<<<" << std::endl;
+   return Success;
+}
+									/*}}}*/
+bool sendFile(int const client, FileFd &data)				/*{{{*/
+{
+   bool Success = true;
+   char buffer[500];
+   unsigned long long actual = 0;
+   while ((Success &= data.Read(buffer, sizeof(buffer), &actual)) == true)
+   {
+      if (actual == 0)
+	 break;
+      if (Success == true)
+	 Success &= FileFd::Write(client, buffer, actual);
+   }
+   if (Success == true)
+      Success &= FileFd::Write(client, "\r\n", 2);
+   return Success;
+}
+									/*}}}*/
+bool sendData(int const client, std::string const &data)		/*{{{*/
+{
+   bool Success = true;
+   Success &= FileFd::Write(client, data.c_str(), data.size());
+   if (Success == true)
+      Success &= FileFd::Write(client, "\r\n", 2);
+   return Success;
+}
+									/*}}}*/
+void sendError(int const client, int const httpcode, std::string const &request,/*{{{*/
+	       bool content, std::string const &error = "")
+{
+   std::list<std::string> headers;
+   std::string response("<html><head><title>");
+   response.append(httpcodeToStr(httpcode)).append("</title></head>");
+   response.append("<body><h1>").append(httpcodeToStr(httpcode)).append("</h1>");
+   if (error.empty() == false)
+      response.append("<p><em>Error</em>: ").append(error).append("</p>");
+   response.append("This error is a result of the request: <pre>");
+   response.append(request).append("</pre></body></html>");
+   addDataHeaders(headers, response);
+   sendHead(client, httpcode, headers);
+   if (content == true)
+      sendData(client, response);
+}
+									/*}}}*/
+bool parseFirstLine(int const client, std::string const &request,	/*{{{*/
+		    std::string &filename, bool &sendContent,
+		    bool &closeConnection)
+{
+   if (strncmp(request.c_str(), "HEAD ", 5) == 0)
+      sendContent = false;
+   if (strncmp(request.c_str(), "GET ", 4) != 0)
+   {
+      sendError(client, 501, request, true);
+      return false;
+   }
+
+   size_t const lineend = request.find('\n');
+   size_t filestart = request.find(' ');
+   for (; request[filestart] == ' '; ++filestart);
+   size_t fileend = request.rfind(' ', lineend);
+   if (lineend == std::string::npos || filestart == std::string::npos ||
+	 fileend == std::string::npos || filestart == fileend)
+   {
+      sendError(client, 500, request, sendContent, "Filename can't be extracted");
+      return false;
+   }
+
+   size_t httpstart = fileend;
+   for (; request[httpstart] == ' '; ++httpstart);
+   if (strncmp(request.c_str() + httpstart, "HTTP/1.1\r", 9) == 0)
+      closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "Keep-Alive") != 0;
+   else if (strncmp(request.c_str() + httpstart, "HTTP/1.0\r", 9) == 0)
+      closeConnection = strcasecmp(LookupTag(request, "Connection", "Keep-Alive").c_str(), "close") == 0;
+   else
+   {
+      sendError(client, 500, request, sendContent, "Not a HTTP/1.{0,1} request");
+      return false;
+   }
+
+   filename = request.substr(filestart, fileend - filestart);
+   if (filename.find(' ') != std::string::npos)
+   {
+      sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
+      return false;
+   }
+   filename = DeQuoteString(filename);
+
+   // this is not a secure server, but at least prevent the obvious …
+   if (filename.empty() == true || filename[0] != '/' ||
+       strncmp(filename.c_str(), "//", 2) == 0 ||
+       filename.find_first_of("\r\n\t\f\v") != std::string::npos ||
+       filename.find("/../") != std::string::npos)
+   {
+      sendError(client, 400, request, sendContent, "Filename contains illegal character (sequence)");
+      return false;
+   }
+
+   // nuke the first character which is a / as we assured above
+   filename.erase(0, 1);
+   if (filename.empty() == true)
+      filename = ".";
+   return true;
+}
+									/*}}}*/
+int main(int const argc, const char * argv[])
+{
+   CommandLine::Args Args[] = {
+      {0, "port", "aptwebserver::port", CommandLine::HasArg},
+      {'c',"config-file",0,CommandLine::ConfigFile},
+      {'o',"option",0,CommandLine::ArbItem},
+      {0,0,0,0}
+   };
+
+   CommandLine CmdL(Args, _config);
+   if(CmdL.Parse(argc,argv) == false)
+   {
+      _error->DumpErrors();
+      exit(1);
+   }
+
+   // create socket, bind and listen to it {{{
+   // ignore SIGPIPE, this can happen on write() if the socket closes connection
+   signal(SIGPIPE, SIG_IGN);
+   int sock = socket(AF_INET6, SOCK_STREAM, 0);
+   if(sock < 0)
+   {
+      _error->Errno("aptwerbserver", "Couldn't create socket");
+      _error->DumpErrors(std::cerr);
+      return 1;
+   }
+
+   int const port = _config->FindI("aptwebserver::port", 8080);
+
+   // ensure that we accept all connections: v4 or v6
+   int const iponly = 0;
+   setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &iponly, sizeof(iponly));
+   // to not linger on an address
+   int const enable = 1;
+   setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(enable));
+
+   struct sockaddr_in6 locAddr;
+   memset(&locAddr, 0, sizeof(locAddr));
+   locAddr.sin6_family = AF_INET6;
+   locAddr.sin6_port = htons(port);
+   locAddr.sin6_addr = in6addr_any;
+
+   if (bind(sock, (struct sockaddr*) &locAddr, sizeof(locAddr)) < 0)
+   {
+      _error->Errno("aptwerbserver", "Couldn't bind");
+      _error->DumpErrors(std::cerr);
+      return 2;
+   }
+
+   std::clog << "Serving ANY file on port: " << port << std::endl;
+
+   listen(sock, 1);
+   /*}}}*/
+
+   std::vector<std::string> messages;
+   int client;
+   while ((client = accept(sock, NULL, NULL)) != -1)
+   {
+      std::clog << "ACCEPT client " << client
+		<< " on socket " << sock << std::endl;
+
+      while (ReadMessages(client, messages))
+      {
+	 bool closeConnection = false;
+	 for (std::vector<std::string>::const_iterator m = messages.begin();
+	      m != messages.end() && closeConnection == false; ++m) {
+	    std::clog << ">>> REQUEST >>>>" << std::endl << *m
+		      << std::endl << "<<<<<<<<<<<<<<<<" << std::endl;
+	    std::list<std::string> headers;
+	    std::string filename;
+	    bool sendContent = true;
+	    if (parseFirstLine(client, *m, filename, sendContent, closeConnection) == false)
+	       continue;
+
+	    std::string host = LookupTag(*m, "Host", "");
+	    if (host.empty() == true)
+	    {
+	       // RFC 2616 §14.23 requires Host
+	       sendError(client, 400, *m, sendContent, "Host header is required");
+	       continue;
+	    }
+
+	    if (RealFileExists(filename) == true)
+	    {
+	       FileFd data(filename, FileFd::ReadOnly);
+	       std::string condition = LookupTag(*m, "If-Modified-Since", "");
+	       if (condition.empty() == false)
+	       {
+		  time_t cache;
+		  if (RFC1123StrToTime(condition.c_str(), cache) == true &&
+			cache >= data.ModificationTime())
+		  {
+		     sendHead(client, 304, headers);
+		     continue;
+		  }
+	       }
+
+	       addFileHeaders(headers, data);
+	       sendHead(client, 200, headers);
+	       if (sendContent == true)
+		  sendFile(client, data);
+	    }
+	    else
+	       sendError(client, 404, *m, sendContent);
+	 }
+	 _error->DumpErrors(std::cerr);
+	 messages.clear();
+	 if (closeConnection == true)
+	    break;
+      }
+
+      std::clog << "CLOSE client " << client
+		<< " on socket " << sock << std::endl;
+      close(client);
+   }
+   return 0;
+}
diff --git a/test/interactive-helper/makefile b/test/interactive-helper/makefile
index fa4f5fecb..f43df97e3 100644
--- a/test/interactive-helper/makefile
+++ b/test/interactive-helper/makefile
@@ -38,3 +38,10 @@ include $(PROGRAM_H)
 #SLIBS = -lapt-pkg -lrpm
 #SOURCE = rpmver.cc
 #include $(PROGRAM_H)
+
+# Program for testing udevcdrom
+PROGRAM=aptwebserver
+SLIBS = -lapt-pkg
+LIB_MAKES = apt-pkg/makefile
+SOURCE = aptwebserver.cc
+include $(PROGRAM_H)
-- 
cgit v1.2.3-70-g09d2


From bf3daa15e9e744d9481d9e6d1e250b77d1f7b256 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Sat, 15 Jun 2013 16:45:12 +0200
Subject: add directory listing to the webserver

Git-Dch: Ignore
---
 test/interactive-helper/aptwebserver.cc | 121 ++++++++++++++++++++++++++++++++
 1 file changed, 121 insertions(+)

(limited to 'test/interactive-helper')

diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 98bbde21b..d25f50624 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -13,12 +13,14 @@
 #include <sstream>
 
 #include <sys/socket.h>
+#include <sys/types.h>
 #include <sys/stat.h>
 #include <netinet/in.h>
 #include <unistd.h>
 #include <errno.h>
 #include <time.h>
 #include <stdlib.h>
+#include <dirent.h>
 #include <signal.h>
 
 char const * const httpcodeToStr(int const httpcode)			/*{{{*/
@@ -164,6 +166,118 @@ void sendError(int const client, int const httpcode, std::string const &request,
       sendData(client, response);
 }
 									/*}}}*/
+void sendRedirect(int const client, int const httpcode, std::string const &uri,/*{{{*/
+		  std::string const &request, bool content)
+{
+   std::list<std::string> headers;
+   std::string response("<html><head><title>");
+   response.append(httpcodeToStr(httpcode)).append("</title></head>");
+   response.append("<body><h1>").append(httpcodeToStr(httpcode)).append("</h1");
+   response.append("<p>You should be redirected to <em>").append(uri).append("</em></p>");
+   response.append("This page is a result of the request: <pre>");
+   response.append(request).append("</pre></body></html>");
+   addDataHeaders(headers, response);
+   std::string location("Location: ");
+   if (strncmp(uri.c_str(), "http://", 7) != 0)
+      location.append("http://").append(LookupTag(request, "Host")).append("/").append(uri);
+   else
+      location.append(uri);
+   headers.push_back(location);
+   sendHead(client, httpcode, headers);
+   if (content == true)
+      sendData(client, response);
+}
+									/*}}}*/
+int filter_hidden_files(const struct dirent *a)				/*{{{*/
+{
+   if (a->d_name[0] == '.')
+      return 0;
+#ifdef _DIRENT_HAVE_D_TYPE
+   // if we have the d_type check that only files and dirs will be included
+   if (a->d_type != DT_UNKNOWN &&
+       a->d_type != DT_REG &&
+       a->d_type != DT_LNK && // this includes links to regular files
+       a->d_type != DT_DIR)
+      return 0;
+#endif
+   return 1;
+}
+int grouped_alpha_case_sort(const struct dirent **a, const struct dirent **b) {
+#ifdef _DIRENT_HAVE_D_TYPE
+   if ((*a)->d_type == DT_DIR && (*b)->d_type == DT_DIR);
+   else if ((*a)->d_type == DT_DIR && (*b)->d_type == DT_REG)
+      return -1;
+   else if ((*b)->d_type == DT_DIR && (*a)->d_type == DT_REG)
+      return 1;
+   else
+#endif
+   {
+      struct stat f_prop; //File's property
+      stat((*a)->d_name, &f_prop);
+      int const amode = f_prop.st_mode;
+      stat((*b)->d_name, &f_prop);
+      int const bmode = f_prop.st_mode;
+      if (S_ISDIR(amode) && S_ISDIR(bmode));
+      else if (S_ISDIR(amode))
+	 return -1;
+      else if (S_ISDIR(bmode))
+	 return 1;
+   }
+   return strcasecmp((*a)->d_name, (*b)->d_name);
+}
+									/*}}}*/
+void sendDirectoryListing(int const client, std::string const &dir,	/*{{{*/
+			  std::string const &request, bool content)
+{
+   std::list<std::string> headers;
+   std::ostringstream listing;
+
+   struct dirent **namelist;
+   int const counter = scandir(dir.c_str(), &namelist, filter_hidden_files, grouped_alpha_case_sort);
+   if (counter == -1)
+   {
+      sendError(client, 500, request, content);
+      return;
+   }
+
+   listing << "<html><head><title>Index of " << dir << "</title>"
+	   << "<style type=\"text/css\"><!-- td {padding: 0.02em 0.5em 0.02em 0.5em;}"
+	   << "tr:nth-child(even){background-color:#dfdfdf;}"
+	   << "h1, td:nth-child(3){text-align:center;}"
+	   << "table {margin-left:auto;margin-right:auto;} --></style>"
+	   << "</head>" << std::endl
+	   << "<body><h1>Index of " << dir << "</h1>" << std::endl
+	   << "<table><tr><th>#</th><th>Name</th><th>Size</th><th>Last-Modified</th></tr>" << std::endl;
+   if (dir != ".")
+      listing << "<tr><td>d</td><td><a href=\"..\">Parent Directory</a></td><td>-</td><td>-</td></tr>";
+   for (int i = 0; i < counter; ++i) {
+      struct stat fs;
+      std::string filename(dir);
+      filename.append("/").append(namelist[i]->d_name);
+      stat(filename.c_str(), &fs);
+      if (S_ISDIR(fs.st_mode))
+      {
+	 listing << "<tr><td>d</td>"
+		 << "<td><a href=\"" << namelist[i]->d_name << "/\">" << namelist[i]->d_name << "</a></td>"
+		 << "<td>-</td>";
+      }
+      else
+      {
+	 listing << "<tr><td>f</td>"
+		 << "<td><a href=\"" << namelist[i]->d_name << "\">" << namelist[i]->d_name << "</a></td>"
+		 << "<td>" << SizeToStr(fs.st_size) << "B</td>";
+      }
+      listing << "<td>" << TimeRFC1123(fs.st_mtime) << "</td></tr>" << std::endl;
+   }
+   listing << "</table></body></html>" << std::endl;
+
+   std::string response(listing.str());
+   addDataHeaders(headers, response);
+   sendHead(client, 200, headers);
+   if (content == true)
+      sendData(client, response);
+}
+									/*}}}*/
 bool parseFirstLine(int const client, std::string const &request,	/*{{{*/
 		    std::string &filename, bool &sendContent,
 		    bool &closeConnection)
@@ -326,6 +440,13 @@ int main(int const argc, const char * argv[])
 	       if (sendContent == true)
 		  sendFile(client, data);
 	    }
+	    else if (DirectoryExists(filename) == true)
+	    {
+	       if (filename == "." || filename[filename.length()-1] == '/')
+		  sendDirectoryListing(client, filename, *m, sendContent);
+	       else
+		  sendRedirect(client, 301, filename.append("/"), *m, sendContent);
+	    }
 	    else
 	       sendError(client, 404, *m, sendContent);
 	 }
-- 
cgit v1.2.3-70-g09d2


From 7ea27d2a6d10ee41f8fca5d8ad7373c8b3d90ea9 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Sat, 15 Jun 2013 23:58:12 +0200
Subject: simple URI rewrite rules config for webserver

we have a test which required traditionally lighttpd to be executed
as it requires a webserver supporting some kind of URI rewriting.
Now with some lines of code our own webserver can do this and the
testcase can be enabled by default. This test hinted at the bug fixed
in the previous commit, so having more tests which can easily be run
is a good thing.

Git-Dch: Ignore
---
 test/integration/skip-bug-602412-dequote-redirect | 38 -----------------------
 test/integration/test-bug-602412-dequote-redirect | 29 +++++++++++++++++
 test/interactive-helper/aptwebserver.cc           | 16 ++++++++++
 3 files changed, 45 insertions(+), 38 deletions(-)
 delete mode 100755 test/integration/skip-bug-602412-dequote-redirect
 create mode 100755 test/integration/test-bug-602412-dequote-redirect

(limited to 'test/interactive-helper')

diff --git a/test/integration/skip-bug-602412-dequote-redirect b/test/integration/skip-bug-602412-dequote-redirect
deleted file mode 100755
index 689b671ce..000000000
--- a/test/integration/skip-bug-602412-dequote-redirect
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/sh
-set -e
-
-TESTDIR=$(readlink -f $(dirname $0))
-. $TESTDIR/framework
-setupenvironment
-configarchitecture 'i386'
-
-if ! which lighttpd > /dev/null; then
-	msgdie 'You need lighttpd for this testcase, sorry…'
-	exit 1
-fi
-
-buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
-
-setupaptarchive
-
-echo "server.modules = ( \"mod_redirect\" )
-server.document-root = \"$(readlink -f ./aptarchive)\"
-server.port = 8080
-server.stat-cache-engine = \"disable\"
-url.redirect = ( \"^/pool/(.*)$\" => \"/newpool/\$1\",
- \"^/dists/(.*)$\" => \"/newdists/\$1\" )" > lighttpd.conf
-
-mv aptarchive/pool aptarchive/newpool
-mv aptarchive/dists aptarchive/newdists
-
-lighttpd -t -f lighttpd.conf >/dev/null || msgdie 'Can not change to webserver: our lighttpd config is invalid'
-lighttpd -D -f lighttpd.conf 2>/dev/null >/dev/null &
-addtrap "kill $!;"
-
-APTARCHIVE="file://$(readlink -f ./aptarchive)"
-for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
-	sed -i $LIST -e "s#$APTARCHIVE#http://localhost:8080/#"
-done
-
-aptget update || msgdie 'apt-get update failed'
-aptget install unrelated --download-only || msgdie 'downloading package failed'
diff --git a/test/integration/test-bug-602412-dequote-redirect b/test/integration/test-bug-602412-dequote-redirect
new file mode 100755
index 000000000..c20443559
--- /dev/null
+++ b/test/integration/test-bug-602412-dequote-redirect
@@ -0,0 +1,29 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
+
+setupaptarchive
+changetowebserver -o aptwebserver::redirect::replace::/pool/=/newpool/ \
+		  -o aptwebserver::redirect::replace::/dists/=/newdists/
+
+mv aptarchive/pool aptarchive/newpool
+mv aptarchive/dists aptarchive/newdists
+
+msgtest 'Test redirection works in' 'apt-get update'
+aptget update -qq && msgpass || msgfail
+
+# check that I-M-S header is kept in redirections
+testequal 'Hit http://localhost:8080 unstable InRelease
+Hit http://localhost:8080 unstable/main Sources
+Hit http://localhost:8080 unstable/main amd64 Packages
+Hit http://localhost:8080 unstable/main Translation-en
+Reading package lists...' aptget update #-o debug::pkgacquire=1 -o debug::pkgacquire::worker=1
+
+msgtest 'Test redirection works in' 'package download'
+aptget install unrelated --download-only -qq && msgpass || msgfail
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index d25f50624..de235fa05 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -420,6 +420,22 @@ int main(int const argc, const char * argv[])
 	       continue;
 	    }
 
+	    // string replacements in the requested filename
+	    ::Configuration::Item const *Replaces = _config->Tree("aptwebserver::redirect::replace");
+	    if (Replaces != NULL)
+	    {
+	       std::string redirect = "/" + filename;
+	       for (::Configuration::Item *I = Replaces->Child; I != NULL; I = I->Next)
+		  redirect = SubstVar(redirect, I->Tag, I->Value);
+	       redirect.erase(0,1);
+	       if (redirect != filename)
+	       {
+		  sendRedirect(client, 301, redirect, *m, sendContent);
+		  continue;
+	       }
+	    }
+
+	    // deal with the request
 	    if (RealFileExists(filename) == true)
 	    {
 	       FileFd data(filename, FileFd::ReadOnly);
-- 
cgit v1.2.3-70-g09d2


From ae99ce2e3cadb07c80b89ab2afc804875b1026c5 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Mon, 17 Jun 2013 11:23:13 +0200
Subject: trigger NODATA error for invalid InRelease files

With the selfgrown splitting we got the problem of not recovering
from networks which just reply with invalid data like those sending
us login pages to authenticate with the network (e.g. hotels) back.

The good thing about the InRelease file is that we know that it must
be clearsigned (a Release file might or might not have a detached sig)
so if we get a file but are unable to split it something is seriously
wrong, so there is not much point in trying further.

The Acquire system already looks out for a NODATA error from gpgv,
so this adds a new error message sent to the acquire system in case
the splitting we do now ourselves failed including this magic word.

Closes: #712486
---
 apt-pkg/contrib/gpgv.cc                            |  2 +-
 apt-pkg/contrib/gpgv.h                             | 15 ++++-
 debian/changelog                                   |  1 +
 methods/gpgv.cc                                    | 16 +++---
 test/integration/framework                         |  3 +
 .../test-ubuntu-bug-346386-apt-get-update-paywall  | 64 ++++++++++++++++++++++
 test/interactive-helper/aptwebserver.cc            | 26 +++++++++
 7 files changed, 114 insertions(+), 13 deletions(-)
 create mode 100755 test/integration/test-ubuntu-bug-346386-apt-get-update-paywall

(limited to 'test/interactive-helper')

diff --git a/apt-pkg/contrib/gpgv.cc b/apt-pkg/contrib/gpgv.cc
index 31db7d5fe..f47e7ea48 100644
--- a/apt-pkg/contrib/gpgv.cc
+++ b/apt-pkg/contrib/gpgv.cc
@@ -154,7 +154,7 @@ void ExecGPGV(std::string const &File, std::string const &FileGPG,
 	 if (sigFd != -1)
 	    unlink(data);
 	 ioprintf(std::cerr, "Splitting up %s into data and signature failed", File.c_str());
-	 exit(EINTERNAL);
+	 exit(112);
       }
       Args.push_back(sig);
       Args.push_back(data);
diff --git a/apt-pkg/contrib/gpgv.h b/apt-pkg/contrib/gpgv.h
index 08b10a97a..45f069058 100644
--- a/apt-pkg/contrib/gpgv.h
+++ b/apt-pkg/contrib/gpgv.h
@@ -23,9 +23,18 @@
 /** \brief generates and run the command to verify a file with gpgv
  *
  * If File and FileSig specify the same file it is assumed that we
- * deal with a clear-signed message. In that case the file will be
- * rewritten to be in a good-known format without uneeded whitespaces
- * and additional messages (unsigned or signed).
+ * deal with a clear-signed message. Note that the method will accept
+ * and validate files which include additional (unsigned) messages
+ * without complaining. Do NOT open files accepted by this method
+ * for reading. Use #OpenMaybeClearSignedFile to access the message
+ * instead to ensure you are only reading signed data.
+ *
+ * The method does not return, but has some noteable exit-codes:
+ * 111 signals an internal error like the inability to execute gpgv,
+ * 112 indicates a clear-signed file which doesn't include a message,
+ *  which can happen if APT is run while on a network requiring
+ *  authentication before usage (e.g. in hotels)
+ * All other exit-codes are passed-through from gpgv.
  *
  * @param File is the message (unsigned or clear-signed)
  * @param FileSig is the signature (detached or clear-signed)
diff --git a/debian/changelog b/debian/changelog
index 91d4ae536..bd25df1e2 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -20,6 +20,7 @@ apt (0.9.8.3) UNRELEASED; urgency=low
   * try defaults if auto-detection failed in apt-cdrom (Closes: #712433)
   * support \n and \r\n line endings in ReadMessages
   * do not redownload unchanged InRelease files
+  * trigger NODATA error for invalid InRelease files (Closes: #712486)
 
  -- David Kalnischkies <kalnischkies@gmail.com>  Sun, 09 Jun 2013 15:06:24 +0200
 
diff --git a/methods/gpgv.cc b/methods/gpgv.cc
index 3f814b9f0..fe8bac6c9 100644
--- a/methods/gpgv.cc
+++ b/methods/gpgv.cc
@@ -55,9 +55,6 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile,
 					 vector<string> &NoPubKeySigners)
 {
    bool const Debug = _config->FindB("Debug::Acquire::gpgv", false);
-   // setup a (empty) stringstream for formating the return value
-   std::stringstream ret;
-   ret.str("");
 
    if (Debug == true)
       std::clog << "inside VerifyGetSigners" << std::endl;
@@ -170,18 +167,19 @@ string GPGVMethod::VerifyGetSigners(const char *file, const char *outfile,
       return "";
    }
    else if (WEXITSTATUS(status) == 1)
-   {
       return _("At least one invalid signature was encountered.");
-   }
    else if (WEXITSTATUS(status) == 111)
+      return _("Could not execute 'gpgv' to verify signature (is gpgv installed?)");
+   else if (WEXITSTATUS(status) == 112)
    {
-      ioprintf(ret, _("Could not execute 'gpgv' to verify signature (is gpgv installed?)"));
-      return ret.str();
+      // acquire system checks for "NODATA" to generate GPG errors (the others are only warnings)
+      std::string errmsg;
+      //TRANSLATORS: %s is a single techy word like 'NODATA'
+      strprintf(errmsg, _("Clearsigned file isn't valid, got '%s' (does the network require authentication?)"), "NODATA");
+      return errmsg;
    }
    else
-   {
       return _("Unknown error executing gpgv");
-   }
 }
 
 bool GPGVMethod::Fetch(FetchItem *Itm)
diff --git a/test/integration/framework b/test/integration/framework
index 3f11ac23b..3a02cfb76 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -118,6 +118,9 @@ gdb() {
 	echo "gdb: run »$*«"
 	APT_CONFIG=aptconfig.conf LD_LIBRARY_PATH=${BUILDDIRECTORY} $(which gdb) ${BUILDDIRECTORY}/$1
 }
+http() {
+	LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/methods/http
+}
 
 exitwithstatus() {
         # error if we about to overflow, but ...
diff --git a/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall
new file mode 100755
index 000000000..1576c396c
--- /dev/null
+++ b/test/integration/test-ubuntu-bug-346386-apt-get-update-paywall
@@ -0,0 +1,64 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture 'native'
+
+insertpackage 'unstable' 'unrelated' 'all' '1.0' 'stable'
+insertsource 'unstable' 'unrelated' 'all' '1.0' 'stable'
+
+echo 'ni ni ni' > aptarchive/knights
+
+setupaptarchive
+changetowebserver -o 'aptwebserver::overwrite::.*::filename=/knights'
+
+msgtest 'Acquire test file from the webserver to check' 'overwrite'
+echo '601 Configuration
+Config-Item: Acquire::http::DependOnSTDIN=0
+
+600 Acquire URI
+URI: http://localhost:8080/holygrail
+Filename: knights-talking
+' | http >/dev/null 2>&1 && msgpass || msgfail
+testfileequal knights-talking 'ni ni ni'
+
+ensure_n_canary_strings_in_dir() {
+	local DIR="$1"
+	local CANARY_STRING="$2"
+	local EXPECTED_N="$3"
+
+	msgtest "Testing in $DIR for $EXPECTED_N canary" "$CANARY_STRING"
+	local N=$(grep "$CANARY_STRING" $DIR/* 2>/dev/null |wc -l )
+	test "$N" = "$EXPECTED_N" && msgpass || msgfail "Expected $EXPECTED_N canaries, got $N"
+}
+
+LISTS='rootdir/var/lib/apt/lists'
+rm -rf rootdir/var/lib/apt/lists
+msgtest 'Got expected NODATA failure in' 'apt-get update'
+aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail
+
+ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0
+testequal 'partial' ls $LISTS
+
+# and again with pre-existing files with "valid data" which should remain
+for f in Release Release.gpg main_binary-amd64_Packages main_source_Sources; do
+    echo 'peng neee-wom' > $LISTS/localhost:8080_dists_stable_${f}
+done
+
+msgtest 'Got expected NODATA failure in' 'apt-get update'
+aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail
+
+ensure_n_canary_strings_in_dir $LISTS 'peng neee-wom' 4
+ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0
+
+# and now with a pre-existing InRelease file
+echo 'peng neee-wom' > $LISTS/localhost:8080_dists_stable_InRelease
+rm -f $LISTS/localhost:8080_dists_stable_Release $LISTS/localhost:8080_dists_stable_Release.gpg
+msgtest 'excpected failure of' 'apt-get update'
+aptget update -qq 2>&1 | grep -q 'E: GPG error.*NODATA' && msgpass || msgfail
+
+ensure_n_canary_strings_in_dir $LISTS 'peng neee-wom' 3
+ensure_n_canary_strings_in_dir $LISTS 'ni ni ni' 0
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index de235fa05..05b875673 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -435,6 +435,32 @@ int main(int const argc, const char * argv[])
 	       }
 	    }
 
+	    ::Configuration::Item const *Overwrite = _config->Tree("aptwebserver::overwrite");
+	    if (Overwrite != NULL)
+	    {
+	       for (::Configuration::Item *I = Overwrite->Child; I != NULL; I = I->Next)
+	       {
+		  regex_t *pattern = new regex_t;
+		  int const res = regcomp(pattern, I->Tag.c_str(), REG_EXTENDED | REG_ICASE | REG_NOSUB);
+		  if (res != 0)
+		  {
+		     char error[300];
+		     regerror(res, pattern, error, sizeof(error));
+		     sendError(client, 500, *m, sendContent, error);
+		     continue;
+		  }
+		  if (regexec(pattern, filename.c_str(), 0, 0, 0) == 0)
+		  {
+		      filename = _config->Find("aptwebserver::overwrite::" + I->Tag + "::filename", filename);
+		      if (filename[0] == '/')
+			 filename.erase(0,1);
+		      regfree(pattern);
+		      break;
+		  }
+		  regfree(pattern);
+	       }
+	    }
+
 	    // deal with the request
 	    if (RealFileExists(filename) == true)
 	    {
-- 
cgit v1.2.3-70-g09d2


From e3c62328abbd548bb0da42fdbad954b3ce4f7102 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Mon, 24 Jun 2013 16:34:38 +0200
Subject: simple fork and pidfile aptwebserver
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Forking only after being ready to accept clients avoids running races
with the tests which sometimes failed on the first 'apt-get update'
(or similar) with the previous background-start and hope for the best…

The commit fixes also some oversight output-order changes in regards to
Description-md5 and (I-M-S) race conditions in various tests.

Git-Dch: Ignore
---
 test/integration/Packages-pdiff-usage              |  2 ++
 test/integration/Packages-pdiff-usage-new          |  2 ++
 test/integration/Packages-releasefile-verification |  1 +
 .../Packages-releasefile-verification-new          |  1 +
 test/integration/framework                         | 18 ++++++++---
 .../test-bug-590041-prefer-non-virtual-packages    |  2 ++
 .../test-bug-601016-description-translation        | 31 +++++++++++-------
 .../test-cve-2013-1051-InRelease-parsing           |  2 +-
 test/integration/test-pdiff-usage                  |  2 +-
 test/integration/test-releasefile-verification     |  4 +--
 test/interactive-helper/aptwebserver.cc            | 37 ++++++++++++++++++++++
 11 files changed, 83 insertions(+), 19 deletions(-)

(limited to 'test/interactive-helper')

diff --git a/test/integration/Packages-pdiff-usage b/test/integration/Packages-pdiff-usage
index d1530a95c..ac962f29a 100644
--- a/test/integration/Packages-pdiff-usage
+++ b/test/integration/Packages-pdiff-usage
@@ -19,6 +19,7 @@ Description: Advanced front-end for dpkg
  .
  APT features complete installation ordering, multiple source capability
  and several other unique features, see the Users Guide in apt-doc.
+Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c
 
 Package: oldstuff
 Version: 1.0
@@ -32,3 +33,4 @@ SHA1: 3c695e028f74d5c544deeddaaa1242desa81088c
 SHA256: b46fd1546151c545fe4bfa56a5cc0e7deaef23e2da3e4f129727fd660f28f050
 Description: some cool but old stuff
  This package will disappear in the next mirror update
+Description-md5: 1948af60eda0a41dfa9fe83f60eb8389
diff --git a/test/integration/Packages-pdiff-usage-new b/test/integration/Packages-pdiff-usage-new
index 4f374b37f..f8d7b1958 100644
--- a/test/integration/Packages-pdiff-usage-new
+++ b/test/integration/Packages-pdiff-usage-new
@@ -22,6 +22,7 @@ Description: Advanced front-end for dpkg
  .
  APT features complete installation ordering, multiple source capability
  and several other unique features, see the Users Guide in apt-doc.
+Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c
 
 Package: newstuff
 Version: 1.0
@@ -35,3 +36,4 @@ SHA1: 3c695e028f7a1ae324deeddaaa1242desa81088c
 SHA256: b46fd154615edefab321cc56a5cc0e7deaef23e2da3e4f129727fd660f28f050
 Description: some cool and shiny new stuff
  This package will appear in the next mirror update
+Description-md5: d5f89fbbc2ac69c43d7e4c9b67d82b6b
diff --git a/test/integration/Packages-releasefile-verification b/test/integration/Packages-releasefile-verification
index 29a385f4f..eb7327279 100644
--- a/test/integration/Packages-releasefile-verification
+++ b/test/integration/Packages-releasefile-verification
@@ -16,3 +16,4 @@ Description: Advanced front-end for dpkg
  .
  APT features complete installation ordering, multiple source capability
  and several other unique features, see the Users Guide in apt-doc.
+Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c
diff --git a/test/integration/Packages-releasefile-verification-new b/test/integration/Packages-releasefile-verification-new
index e3b2edf1f..61509d157 100644
--- a/test/integration/Packages-releasefile-verification-new
+++ b/test/integration/Packages-releasefile-verification-new
@@ -19,3 +19,4 @@ Description: Advanced front-end for dpkg
  .
  APT features complete installation ordering, multiple source capability
  and several other unique features, see the Users Guide in apt-doc.
+Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c
diff --git a/test/integration/framework b/test/integration/framework
index 3a02cfb76..7dd7c20a7 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -605,9 +605,12 @@ buildaptarchivefromfiles() {
 		cat ${line} | bzip2 > ${line}.bz2
 		cat ${line} | xz --format=lzma > ${line}.lzma
 		cat ${line} | xz > ${line}.xz
+		if [ -n "$1" ]; then
+			touch -d "$1" ${line}.gz ${line}.bz2 ${line}.lzma ${line}.xz
+		fi
 		msgdone "info"
 	done
-	generatereleasefiles
+	generatereleasefiles "$@"
 }
 
 # can be overridden by testcases for their pleasure
@@ -719,7 +722,10 @@ signreleasefiles() {
 	done
 	for RELEASE in $(find aptarchive/ -name Release); do
 		gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" -abs -o ${RELEASE}.gpg ${RELEASE}
-		gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" --clearsign -o "$(echo "${RELEASE}" | sed 's#/Release$#/InRelease#')" $RELEASE
+		local INRELEASE="$(echo "${RELEASE}" | sed 's#/Release$#/InRelease#')"
+		gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" --clearsign -o $INRELEASE $RELEASE
+		# we might have set a specific date for the Release file, so copy it
+		touch -d "$(stat --format "%y" ${RELEASE})" ${RELEASE}.gpg ${INRELEASE}
 	done
 	msgdone "info"
 }
@@ -728,8 +734,12 @@ changetowebserver() {
 	local LOG='/dev/null'
 	if test -x ${BUILDDIRECTORY}/aptwebserver; then
 		cd aptarchive
-		LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/aptwebserver "$@" >$LOG 2>&1 &
-		addtrap "kill $!;"
+		LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/aptwebserver -o aptwebserver::fork=1 "$@" >$LOG 2>&1
+		local PID="$(cat aptwebserver.pid)"
+		if [ -z "$PID" ]; then
+			msgdie 'Could not fork aptwebserver successfully'
+		fi
+		addtrap "kill $PID;"
 		cd - > /dev/null
 	elif [ $# -gt 0 ]; then
 		msgdie 'Need the aptwebserver when passing arguments for the webserver'
diff --git a/test/integration/test-bug-590041-prefer-non-virtual-packages b/test/integration/test-bug-590041-prefer-non-virtual-packages
index e0dd7737f..0ce4c1413 100755
--- a/test/integration/test-bug-590041-prefer-non-virtual-packages
+++ b/test/integration/test-bug-590041-prefer-non-virtual-packages
@@ -9,6 +9,7 @@ pkglibc6="Package: libc6
 Architecture: armel
 Version: 2.11.2-2~0.3
 Description: Embedded GNU C Library: Shared libraries
+Description-md5: b8c1e0561b75e2dc6b6482a99079c3e4
 Filename: pool/main/e/eglibc/libc6_2.11.2-2_armel.deb
 Installed-Size: 9740
 MD5sum: f5b878ce5fb8aa01a7927fa1460df537
@@ -25,6 +26,7 @@ Architecture: i386
 Version: 2.1.3-13~0.3
 Replaces: libc6 (<< 2.2.5-13~0.3)
 Description: The Berkeley database routines [glibc 2.0/2.1 compatibility]
+Description-md5: de1876f7fe7f7709a110875e145e38a8
 Filename: pool/main/d/db1-compat/libdb1-compat_2.1.3-13_armel.deb
 Installed-Size: 136
 MD5sum: 4043f176ab2b40b0c01bc1211b8c103c
diff --git a/test/integration/test-bug-601016-description-translation b/test/integration/test-bug-601016-description-translation
index 03fddbfda..33c209e9d 100755
--- a/test/integration/test-bug-601016-description-translation
+++ b/test/integration/test-bug-601016-description-translation
@@ -9,8 +9,9 @@ configarchitecture 'i386' 'amd64'
 # we need a valid locale here, otherwise the language configuration
 # will be overridden by LC_ALL=C
 LOCALE="$(echo "$LANG" | cut -d'_' -f 1)"
+MD5Sum='Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c'
 
-PACKAGESTANZA="Package: apt
+PACKAGESTANZA='Package: apt
 Priority: important
 Section: admin
 Installed-Size: 5984
@@ -19,8 +20,7 @@ Architecture: i386
 Version: 0.8.7
 Filename: pool/main/a/apt/apt_0.8.7_i386.deb
 Size: 2140230
-MD5sum: 74769bfbcef9ebc4fa74f7a5271b9c08
-Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c"
+MD5sum: 74769bfbcef9ebc4fa74f7a5271b9c08'
 
 PACKAGESTANZA2='Package: apt
 Priority: important
@@ -31,22 +31,23 @@ Architecture: amd64
 Version: 0.8.7
 Filename: pool/main/a/apt/apt_0.8.7_amd64.deb
 Size: 2210342
-MD5sum: 4a869bfbdef9ebc9fa74f7a5271e8d1a
-Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c'
+MD5sum: 4a869bfbdef9ebc9fa74f7a5271e8d1a'
 
 echo "$PACKAGESTANZA
 Description: Advanced front-end for dpkg
+$MD5Sum
 
 $PACKAGESTANZA2
-Description: Advanced front-end for dpkg" > aptarchive/Packages
+Description: Advanced front-end for dpkg
+$MD5Sum" > aptarchive/Packages
 
 echo "Package: apt
-Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c
 Description-${LOCALE}: Mächtige Oberfläche für dpkg
  Das Paket bietet dem Nutzer technisch führende Methoden für den Zugriff
  auf den dpkg-Paketmanager. Es beinhaltet das apt-get-Werkzeug und die
  APT-Dselect-Methode. Beides sind einfache und sicherere Wege,
- um Pakete zu installieren und Upgrades durchzuführen." | bzip2 > aptarchive/${LOCALE}.bz2
+ um Pakete zu installieren und Upgrades durchzuführen.
+$MD5Sum" | bzip2 > aptarchive/${LOCALE}.bz2
 
 # the $LOCALE translation file will not be included as it is a flat archive it came from and therefore
 # its name can not be guessed correctly… (in non-flat archives the files are called Translation-*)
@@ -54,10 +55,12 @@ echo 'APT::Cache::Generate "false";' > rootdir/etc/apt/apt.conf.d/00nogenerate
 
 NOLONGSTANZA="$PACKAGESTANZA
 Description: Advanced front-end for dpkg
+$MD5Sum
 "
 
 ENGLISHSTANZA="$PACKAGESTANZA
 Description: Advanced front-end for dpkg
+$MD5Sum
 "
 
 LOCALESTANZA="$PACKAGESTANZA
@@ -66,6 +69,7 @@ Description-${LOCALE}: Mächtige Oberfläche für dpkg
  auf den dpkg-Paketmanager. Es beinhaltet das apt-get-Werkzeug und die
  APT-Dselect-Methode. Beides sind einfache und sicherere Wege,
  um Pakete zu installieren und Upgrades durchzuführen.
+$MD5Sum
 "
 LOCALESTANZA2="$PACKAGESTANZA2
 Description-${LOCALE}: Mächtige Oberfläche für dpkg
@@ -73,6 +77,7 @@ Description-${LOCALE}: Mächtige Oberfläche für dpkg
  auf den dpkg-Paketmanager. Es beinhaltet das apt-get-Werkzeug und die
  APT-Dselect-Methode. Beides sind einfache und sicherere Wege,
  um Pakete zu installieren und Upgrades durchzuführen.
+$MD5Sum
 "
 
 testrun() {
@@ -97,28 +102,32 @@ testrun
 
 echo "$PACKAGESTANZA
 Description: Advanced front-end for dpkg
+$MD5Sum
 
 $PACKAGESTANZA2
-Description: Advanced front-end for dpkg" > aptarchive/Packages
+Description: Advanced front-end for dpkg
+$MD5Sum" > aptarchive/Packages
 
 echo "Package: apt
-Description-md5: d41ee493aa9fcc6cbc9ce4eb7069959c
 Description-en: Advanced front-end for dpkg
  This is Debian's next generation front-end for the dpkg package manager.
  It provides the apt-get utility and APT dselect method that provides a
- simpler, safer way to install and upgrade packages." | bzip2 > aptarchive/en.bz2
+ simpler, safer way to install and upgrade packages.
+$MD5Sum" | bzip2 > aptarchive/en.bz2
 
 ENGLISHSTANZA="$PACKAGESTANZA
 Description-en: Advanced front-end for dpkg
  This is Debian's next generation front-end for the dpkg package manager.
  It provides the apt-get utility and APT dselect method that provides a
  simpler, safer way to install and upgrade packages.
+$MD5Sum
 "
 ENGLISHSTANZA2="$PACKAGESTANZA2
 Description-en: Advanced front-end for dpkg
  This is Debian's next generation front-end for the dpkg package manager.
  It provides the apt-get utility and APT dselect method that provides a
  simpler, safer way to install and upgrade packages.
+$MD5Sum
 "
 
 testrun
diff --git a/test/integration/test-cve-2013-1051-InRelease-parsing b/test/integration/test-cve-2013-1051-InRelease-parsing
index 853da5ff6..6764fefff 100755
--- a/test/integration/test-cve-2013-1051-InRelease-parsing
+++ b/test/integration/test-cve-2013-1051-InRelease-parsing
@@ -37,7 +37,7 @@ sed -i '/^-----BEGIN PGP SIGNATURE-----/,/^-----END PGP SIGNATURE-----/ s/^$/  /
 
 # we append the (evil unsigned) Release file to the (good signed) InRelease
 cat aptarchive/dists/stable/Release >> aptarchive/dists/stable/InRelease
-
+touch -d '+1hour' aptarchive/dists/stable/InRelease
 
 # ensure the update fails
 # useful for debugging to add "-o Debug::pkgAcquire::auth=true"
diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
index 29301d07d..e45326970 100755
--- a/test/integration/test-pdiff-usage
+++ b/test/integration/test-pdiff-usage
@@ -35,7 +35,7 @@ SHA1-History:
 SHA1-Patches:
  7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28
  $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
-generatereleasefiles
+generatereleasefiles '+1hour'
 signreleasefiles
 find aptarchive -name 'Packages*' -type f -delete
 aptget update -qq
diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification
index fba7ab290..e56f458d3 100755
--- a/test/integration/test-releasefile-verification
+++ b/test/integration/test-releasefile-verification
@@ -184,5 +184,5 @@ runtest2
 
 DELETEFILE="InRelease"
 runtest
-#DELETEFILE="Release.gpg"
-#runtest
+DELETEFILE="Release.gpg"
+runtest
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index 05b875673..a8d191d0e 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -387,6 +387,41 @@ int main(int const argc, const char * argv[])
       return 2;
    }
 
+   FileFd pidfile;
+   if (_config->FindB("aptwebserver::fork", false) == true)
+   {
+      std::string const pidfilename = _config->Find("aptwebserver::pidfile", "aptwebserver.pid");
+      int const pidfilefd = GetLock(pidfilename);
+      if (pidfilefd < 0 || pidfile.OpenDescriptor(pidfilefd, FileFd::WriteOnly) == false)
+      {
+	 _error->Errno("aptwebserver", "Couldn't acquire lock on pidfile '%s'", pidfilename.c_str());
+	 _error->DumpErrors(std::cerr);
+	 return 3;
+      }
+
+      pid_t child = fork();
+      if (child < 0)
+      {
+	 _error->Errno("aptwebserver", "Forking failed");
+	 _error->DumpErrors(std::cerr);
+	 return 4;
+      }
+      else if (child != 0)
+      {
+	 // successfully forked: ready to serve!
+	 std::string pidcontent;
+	 strprintf(pidcontent, "%d", child);
+	 pidfile.Write(pidcontent.c_str(), pidcontent.size());
+	 if (_error->PendingError() == true)
+	 {
+	    _error->DumpErrors(std::cerr);
+	    return 5;
+	 }
+	 std::cout << "Successfully forked as " << child << std::endl;
+	 return 0;
+      }
+   }
+
    std::clog << "Serving ANY file on port: " << port << std::endl;
 
    listen(sock, 1);
@@ -502,5 +537,7 @@ int main(int const argc, const char * argv[])
 		<< " on socket " << sock << std::endl;
       close(client);
    }
+   pidfile.Close();
+
    return 0;
 }
-- 
cgit v1.2.3-70-g09d2


From f2380a78aa90ff8a3b76607c0c140810aff84086 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Fri, 26 Jul 2013 09:22:52 +0200
Subject: request absolute URIs from proxies again (0.9.9.3 regession)

Commit 2b9c9b7f28b18f6ae3e422020e8934872b06c9f3 not only removes
keep-alive, but also changes the request URI send to proxies which are
required to be absolute URIs rather than the usual absolute paths.

Closes: 717891
---
 methods/http.cc                                    | 20 +++++++++---
 .../test-bug-717891-abolute-uris-for-proxies       | 28 +++++++++++++++++
 test/interactive-helper/aptwebserver.cc            | 36 +++++++++++++++++-----
 3 files changed, 72 insertions(+), 12 deletions(-)
 create mode 100755 test/integration/test-bug-717891-abolute-uris-for-proxies

(limited to 'test/interactive-helper')

diff --git a/methods/http.cc b/methods/http.cc
index 6e03e9d63..82456d78b 100644
--- a/methods/http.cc
+++ b/methods/http.cc
@@ -682,15 +682,27 @@ void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
    // Just in case.
    if (Itm->Uri.length() >= sizeof(Buf))
        abort();
-       
+
+   /* RFC 2616 §5.1.2 requires absolute URIs for requests to proxies,
+      but while its a must for all servers to accept absolute URIs,
+      it is assumed clients will sent an absolute path for non-proxies */
+   std::string requesturi;
+   if (Proxy.empty() == true || Proxy.Host.empty())
+      requesturi = Uri.Path;
+   else
+      requesturi = Itm->Uri;
+
+   // The "+" is encoded as a workaround for a amazon S3 bug
+   // see LP bugs #1003633 and #1086997.
+   requesturi = QuoteString(requesturi, "+~ ");
+
    /* Build the request. No keep-alive is included as it is the default
       in 1.1, can cause problems with proxies, and we are an HTTP/1.1
       client anyway.
       C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
-   // see LP bugs #1003633 and #1086997. The "+" is encoded as a workaround
-   // for a amazon S3 bug
    sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
-	   QuoteString(Uri.Path,"+~ ").c_str(),ProperHost.c_str());
+	   requesturi.c_str(),ProperHost.c_str());
+
    // generate a cache control header (if needed)
    if (_config->FindB("Acquire::http::No-Cache",false) == true) 
    {
diff --git a/test/integration/test-bug-717891-abolute-uris-for-proxies b/test/integration/test-bug-717891-abolute-uris-for-proxies
new file mode 100755
index 000000000..e9c38492e
--- /dev/null
+++ b/test/integration/test-bug-717891-abolute-uris-for-proxies
@@ -0,0 +1,28 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+setupenvironment
+configarchitecture 'amd64'
+
+buildsimplenativepackage 'unrelated' 'all' '0.5~squeeze1' 'unstable'
+
+setupaptarchive
+changetowebserver --request-absolute='uri'
+
+msgtest 'Check that absolute paths are' 'not accepted'
+aptget update >/dev/null 2>&1 && msgfail || msgpass
+
+echo 'Acquire::http::Proxy "http://localhost:8080";' > rootdir/etc/apt/apt.conf.d/99proxy
+
+msgtest 'Check that requests to proxies are' 'absolute uris'
+aptget update >/dev/null 2>&1 && msgpass || msgfail
+
+testequal 'Reading package lists...
+Building dependency tree...
+The following NEW packages will be installed:
+  unrelated
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+Inst unrelated (0.5~squeeze1 unstable [all])
+Conf unrelated (0.5~squeeze1 unstable [all])' aptget install unrelated -s
diff --git a/test/interactive-helper/aptwebserver.cc b/test/interactive-helper/aptwebserver.cc
index a8d191d0e..fde95fec9 100644
--- a/test/interactive-helper/aptwebserver.cc
+++ b/test/interactive-helper/aptwebserver.cc
@@ -319,6 +319,33 @@ bool parseFirstLine(int const client, std::string const &request,	/*{{{*/
       sendError(client, 500, request, sendContent, "Filename contains an unencoded space");
       return false;
    }
+
+   std::string host = LookupTag(request, "Host", "");
+   if (host.empty() == true)
+   {
+      // RFC 2616 §14.23 requires Host
+      sendError(client, 400, request, sendContent, "Host header is required");
+      return false;
+   }
+   host = "http://" + host;
+
+   // Proxies require absolute uris, so this is a simple proxy-fake option
+   std::string const absolute = _config->Find("aptwebserver::request::absolute", "uri,path");
+   if (strncmp(host.c_str(), filename.c_str(), host.length()) == 0)
+   {
+      if (absolute.find("uri") == std::string::npos)
+      {
+	 sendError(client, 400, request, sendContent, "Request is absoluteURI, but configured to not accept that");
+	 return false;
+      }
+      // strip the host from the request to make it an absolute path
+      filename.erase(0, host.length());
+   }
+   else if (absolute.find("path") == std::string::npos)
+   {
+      sendError(client, 400, request, sendContent, "Request is absolutePath, but configured to not accept that");
+      return false;
+   }
    filename = DeQuoteString(filename);
 
    // this is not a secure server, but at least prevent the obvious …
@@ -342,6 +369,7 @@ int main(int const argc, const char * argv[])
 {
    CommandLine::Args Args[] = {
       {0, "port", "aptwebserver::port", CommandLine::HasArg},
+      {0, "request-absolute", "aptwebserver::request::absolute", CommandLine::HasArg},
       {'c',"config-file",0,CommandLine::ConfigFile},
       {'o',"option",0,CommandLine::ArbItem},
       {0,0,0,0}
@@ -447,14 +475,6 @@ int main(int const argc, const char * argv[])
 	    if (parseFirstLine(client, *m, filename, sendContent, closeConnection) == false)
 	       continue;
 
-	    std::string host = LookupTag(*m, "Host", "");
-	    if (host.empty() == true)
-	    {
-	       // RFC 2616 §14.23 requires Host
-	       sendError(client, 400, *m, sendContent, "Host header is required");
-	       continue;
-	    }
-
 	    // string replacements in the requested filename
 	    ::Configuration::Item const *Replaces = _config->Tree("aptwebserver::redirect::replace");
 	    if (Replaces != NULL)
-- 
cgit v1.2.3-70-g09d2