merged from debian-sid

19 files changed, 364 insertions, 197 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index afb3daad3..d2aca597e 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -219,19 +219,19 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
 
    if(TF.Step(Tags) == true)
    {
-      string local_sha1;
       bool found = false;
       DiffInfo d;
       string size;
 
-      string tmp = Tags.FindS("SHA1-Current");
+      string const tmp = Tags.FindS("SHA1-Current");
       std::stringstream ss(tmp);
-      ss >> ServerSha1;
+      ss >> ServerSha1 >> size;
+      unsigned long const ServerSize = atol(size.c_str());
 
       FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
       SHA1Summation SHA1;
       SHA1.AddFD(fd.Fd(), fd.Size());
-      local_sha1 = string(SHA1.Result());
+      string const local_sha1 = SHA1.Result();
 
       if(local_sha1 == ServerSha1) 
       {
@@ -248,20 +248,56 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
 	    std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
 
 	 // check the historie and see what patches we need
-	 string history = Tags.FindS("SHA1-History");     
+	 string const history = Tags.FindS("SHA1-History");
 	 std::stringstream hist(history);
-	 while(hist >> d.sha1 >> size >> d.file) 
+	 while(hist >> d.sha1 >> size >> d.file)
 	 {
-	    d.size = atoi(size.c_str());
 	    // read until the first match is found
+	    // from that point on, we probably need all diffs
 	    if(d.sha1 == local_sha1) 
 	       found=true;
-	    // from that point on, we probably need all diffs
-	    if(found) 
+	    else if (found == false)
+	       continue;
+
+	    if(Debug)
+	       std::clog << "Need to get diff: " << d.file << std::endl;
+	    available_patches.push_back(d);
+	 }
+
+	 if (available_patches.empty() == false)
+	 {
+	    // patching with too many files is rather slow compared to a fast download
+	    unsigned long const fileLimit = _config->FindI("Acquire::PDiffs::FileLimit", 0);
+	    if (fileLimit != 0 && fileLimit < available_patches.size())
+	    {
+	       if (Debug)
+		  std::clog << "Need " << available_patches.size() << " diffs (Limit is " << fileLimit
+			<< ") so fallback to complete download" << std::endl;
+	       return false;
+	    }
+
+	    // see if the patches are too big
+	    found = false; // it was true and it will be true again at the end
+	    d = *available_patches.begin();
+	    string const firstPatch = d.file;
+	    unsigned long patchesSize = 0;
+	    std::stringstream patches(Tags.FindS("SHA1-Patches"));
+	    while(patches >> d.sha1 >> size >> d.file)
+	    {
+	       if (firstPatch == d.file)
+		  found = true;
+	       else if (found == false)
+		  continue;
+
+	       patchesSize += atol(size.c_str());
+	    }
+	    unsigned long const sizeLimit = ServerSize * _config->FindI("Acquire::PDiffs::SizeLimit", 100);
+	    if (sizeLimit > 0 && (sizeLimit/100) < patchesSize)
 	    {
-	       if(Debug)
-		  std::clog << "Need to get diff: " << d.file << std::endl;
-	       available_patches.push_back(d);
+	       if (Debug)
+		  std::clog << "Need " << patchesSize << " bytes (Limit is " << sizeLimit/100
+			<< ") so fallback to complete download" << std::endl;
+	       return false;
 	    }
 	 }
       }
@@ -270,7 +306,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)		/*{{{*/
       if(found) 
       {
 	 // queue the diffs
-	 string::size_type last_space = Description.rfind(" ");
+	 string::size_type const last_space = Description.rfind(" ");
 	 if(last_space != string::npos)
 	    Description.erase(last_space, Description.size()-last_space);
 	 new pkgAcqIndexDiffs(Owner, RealURI, Description, Desc.ShortDesc,
diff --git a/apt-pkg/contrib/configuration.cc b/apt-pkg/contrib/configuration.cc
index 4e8586e83..8e3b84499 100644
--- a/apt-pkg/contrib/configuration.cc
+++ b/apt-pkg/contrib/configuration.cc
@@ -22,14 +22,8 @@
 #include <apti18n.h>
 
 #include <vector>
-#include <algorithm>
 #include <fstream>
 #include <iostream>
-    
-#include <stdio.h>
-#include <dirent.h>
-#include <sys/stat.h>
-#include <unistd.h>
 
 using namespace std;
 									/*}}}*/
@@ -835,39 +829,10 @@ bool ReadConfigFile(Configuration &Conf,const string &FName,bool AsSectional,
 // ReadConfigDir - Read a directory of config files			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool ReadConfigDir(Configuration &Conf,const string &Dir,bool AsSectional,
-		   unsigned Depth)
-{   
-   DIR *D = opendir(Dir.c_str());
-   if (D == 0)
-      return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
-
-   vector<string> List;
-   
-   for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D))
-   {
-      if (Ent->d_name[0] == '.')
-	 continue;
-      
-      // Skip bad file names ala run-parts
-      const char *C = Ent->d_name;
-      for (; *C != 0; C++)
-	 if (isalpha(*C) == 0 && isdigit(*C) == 0 && *C != '_' && *C != '-')
-	    break;
-      if (*C != 0)
-	 continue;
-      
-      // Make sure it is a file and not something else
-      string File = flCombine(Dir,Ent->d_name);
-      struct stat St;
-      if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0)
-	 continue;
-      
-      List.push_back(File);      
-   }   
-   closedir(D);
-   
-   sort(List.begin(),List.end());
+bool ReadConfigDir(Configuration &Conf,const string &Dir,
+		   bool AsSectional, unsigned Depth)
+{
+   vector<string> const List = GetListOfFilesInDir(Dir, "conf", true, true);
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); I++)
diff --git a/apt-pkg/contrib/error.h b/apt-pkg/contrib/error.h
index a3be6a575..90747ff7e 100644
--- a/apt-pkg/contrib/error.h
+++ b/apt-pkg/contrib/error.h
@@ -40,26 +40,15 @@
 #ifndef PKGLIB_ERROR_H
 #define PKGLIB_ERROR_H
 
+#include <apt-pkg/macros.h>
 
-
-#ifdef __GNUG__
-// Methods have a hidden this parameter that is visible to this attribute
-#define APT_MFORMAT1 __attribute__ ((format (printf, 2, 3)))
-#define APT_MFORMAT2 __attribute__ ((format (printf, 3, 4)))
-#else
-#define APT_MFORMAT1
-#define APT_MFORMAT2    
-#endif    
-    
 #include <string>
 
-using std::string;
-
 class GlobalError
 {
    struct Item
    {
-      string Text;
+      std::string Text;
       bool Error;
       Item *Next;
    };
@@ -71,18 +60,18 @@ class GlobalError
    public:
 
    // Call to generate an error from a library call.
-   bool Errno(const char *Function,const char *Description,...) APT_MFORMAT2;
-   bool WarningE(const char *Function,const char *Description,...) APT_MFORMAT2;
+   bool Errno(const char *Function,const char *Description,...) __like_printf_2 __cold;
+   bool WarningE(const char *Function,const char *Description,...) __like_printf_2 __cold;
 
    /* A warning should be considered less severe than an error, and may be
       ignored by the client. */
-   bool Error(const char *Description,...) APT_MFORMAT1;
-   bool Warning(const char *Description,...) APT_MFORMAT1;
+   bool Error(const char *Description,...) __like_printf_1 __cold;
+   bool Warning(const char *Description,...) __like_printf_1 __cold;
 
    // Simple accessors
    inline bool PendingError() {return PendingFlag;};
    inline bool empty() {return List == 0;};
-   bool PopMessage(string &Text);
+   bool PopMessage(std::string &Text);
    void Discard();
 
    // Usefull routine to dump to cerr
@@ -95,7 +84,4 @@ class GlobalError
 GlobalError *_GetErrorObj();
 #define _error _GetErrorObj()
 
-#undef APT_MFORMAT1
-#undef APT_MFORMAT2
-
 #endif
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index 4240d9f49..da32983f1 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -34,9 +34,11 @@
 #include <sys/types.h>
 #include <sys/time.h>
 #include <sys/wait.h>
+#include <dirent.h>
 #include <signal.h>
 #include <errno.h>
 #include <set>
+#include <algorithm>
 									/*}}}*/
 
 using namespace std;
@@ -195,6 +197,128 @@ bool FileExists(string File)
    return true;
 }
 									/*}}}*/
+// GetListOfFilesInDir - returns a vector of files in the given dir	/*{{{*/
+// ---------------------------------------------------------------------
+/* If an extension is given only files with this extension are included
+   in the returned vector, otherwise every "normal" file is included. */
+std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
+					bool const &SortList)
+{
+   return GetListOfFilesInDir(Dir, Ext, SortList, false);
+}
+std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
+					bool const &SortList, bool const &AllowNoExt)
+{
+   std::vector<string> ext;
+   ext.reserve(2);
+   if (Ext.empty() == false)
+      ext.push_back(Ext);
+   if (AllowNoExt == true && ext.empty() == false)
+      ext.push_back("");
+   return GetListOfFilesInDir(Dir, ext, SortList);
+}
+std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> const &Ext,
+					bool const &SortList)
+{
+   // Attention debuggers: need to be set with the environment config file!
+   bool const Debug = _config->FindB("Debug::GetListOfFilesInDir", false);
+   if (Debug == true)
+   {
+      std::clog << "Accept in " << Dir << " only files with the following " << Ext.size() << " extensions:" << std::endl;
+      if (Ext.empty() == true)
+	 std::clog << "\tNO extension" << std::endl;
+      else
+	 for (std::vector<string>::const_iterator e = Ext.begin();
+	      e != Ext.end(); ++e)
+	    std::clog << '\t' << (e->empty() == true ? "NO" : *e) << " extension" << std::endl;
+   }
+
+   std::vector<string> List;
+   DIR *D = opendir(Dir.c_str());
+   if (D == 0) 
+   {
+      _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
+      return List;
+   }
+
+   for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D)) 
+   {
+      // skip "hidden" files
+      if (Ent->d_name[0] == '.')
+	 continue;
+
+      // check for accepted extension:
+      // no extension given -> periods are bad as hell!
+      // extensions given -> "" extension allows no extension
+      if (Ext.empty() == false)
+      {
+	 string d_ext = flExtension(Ent->d_name);
+	 if (d_ext == Ent->d_name) // no extension
+	 {
+	    if (std::find(Ext.begin(), Ext.end(), "") == Ext.end())
+	    {
+	       if (Debug == true)
+		  std::clog << "Bad file: " << Ent->d_name << " → no extension" << std::endl;
+	       continue;
+	    }
+	 }
+	 else if (std::find(Ext.begin(), Ext.end(), d_ext) == Ext.end())
+	 {
+	    if (Debug == true)
+	       std::clog << "Bad file: " << Ent->d_name << " → bad extension »" << flExtension(Ent->d_name) << "«" << std::endl;
+	    continue;
+	 }
+      }
+
+      // Skip bad filenames ala run-parts
+      const char *C = Ent->d_name;
+      for (; *C != 0; ++C)
+	 if (isalpha(*C) == 0 && isdigit(*C) == 0
+	     && *C != '_' && *C != '-') {
+	    // no required extension -> dot is a bad character
+	    if (*C == '.' && Ext.empty() == false)
+	       continue;
+	    break;
+	 }
+
+      // we don't reach the end of the name -> bad character included
+      if (*C != 0)
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → bad character »"
+	       << *C << "« in filename (period allowed: " << (Ext.empty() ? "no" : "yes") << ")" << std::endl;
+	 continue;
+      }
+
+      // skip filenames which end with a period. These are never valid
+      if (*(C - 1) == '.')
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → Period as last character" << std::endl;
+	 continue;
+      }
+
+      // Make sure it is a file and not something else
+      string const File = flCombine(Dir,Ent->d_name);
+      struct stat St;
+      if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0)
+      {
+	 if (Debug == true)
+	    std::clog << "Bad file: " << Ent->d_name << " → stat says not a good file" << std::endl;
+	 continue;
+      }
+
+      if (Debug == true)
+	 std::clog << "Accept file: " << Ent->d_name << " in " << Dir << std::endl;
+      List.push_back(File);
+   }
+   closedir(D);
+
+   if (SortList == true)
+      std::sort(List.begin(),List.end());
+   return List;
+}
+									/*}}}*/
 // SafeGetCWD - This is a safer getcwd that returns a dynamic string	/*{{{*/
 // ---------------------------------------------------------------------
 /* We return / on failure. */
diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h
index 73b5ea3be..85a94898c 100644
--- a/apt-pkg/contrib/fileutl.h
+++ b/apt-pkg/contrib/fileutl.h
@@ -23,6 +23,7 @@
 
 
 #include <string>
+#include <vector>
 
 using std::string;
 
@@ -81,6 +82,13 @@ bool RunScripts(const char *Cnf);
 bool CopyFile(FileFd &From,FileFd &To);
 int GetLock(string File,bool Errors = true);
 bool FileExists(string File);
+// FIXME: next ABI-Break: merge the two method-headers
+std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
+					bool const &SortList);
+std::vector<string> GetListOfFilesInDir(string const &Dir, string const &Ext,
+					bool const &SortList, bool const &AllowNoExt);
+std::vector<string> GetListOfFilesInDir(string const &Dir, std::vector<string> const &Ext,
+					bool const &SortList);
 string SafeGetCWD();
 void SetCloseExec(int Fd,bool Close);
 void SetNonBlock(int Fd,bool Block);
diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc
index b43771ea7..985d89d90 100644
--- a/apt-pkg/contrib/hashes.cc
+++ b/apt-pkg/contrib/hashes.cc
@@ -14,9 +14,9 @@
 #include <apt-pkg/hashes.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/configuration.h>
-    
+#include <apt-pkg/macros.h>
+
 #include <unistd.h>    
-#include <system.h>    
 #include <string>
 #include <iostream>
 									/*}}}*/
diff --git a/apt-pkg/contrib/system.h b/apt-pkg/contrib/macros.h
index 7ec3d7feb..9aeb77b81 100644
--- a/apt-pkg/contrib/system.h
+++ b/apt-pkg/contrib/macros.h
@@ -1,9 +1,8 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: system.h,v 1.3 1999/12/10 23:40:29 jgg Exp $
 /* ######################################################################
    
-   System Header - Usefull private definitions
+   Macros Header - Various useful macro definitions
 
    This source is placed in the Public Domain, do with it what you will
    It was originally written by Brian C. White.
@@ -11,8 +10,8 @@
    ##################################################################### */
 									/*}}}*/
 // Private header
-#ifndef SYSTEM_H
-#define SYSTEM_H
+#ifndef MACROS_H
+#define MACROS_H
 
 // MIN_VAL(SINT16) will return -0x8000 and MAX_VAL(SINT16) = 0x7FFF
 #define	MIN_VAL(t)	(((t)(-1) > 0) ? (t)( 0) : (t)(((1L<<(sizeof(t)*8-1))  )))
@@ -55,4 +54,35 @@
 #define CLRFLAG(v,f)	((v) &=~FLAG(f))
 #define	CHKFLAG(v,f)	((v) &  FLAG(f) ? true : false)
 
+// some nice optional GNUC features
+#if __GNUC__ >= 3
+        #define __must_check    __attribute__ ((warn_unused_result))
+        #define __deprecated    __attribute__ ((deprecated))
+        /* likely() and unlikely() can be used to mark boolean expressions
+           as (not) likely true which will help the compiler to optimise */
+        #define likely(x)       __builtin_expect (!!(x), 1)
+        #define unlikely(x)     __builtin_expect (!!(x), 0)
+#else
+        #define __must_check    /* no warn_unused_result */
+        #define __deprecated    /* no deprecated */
+        #define likely(x)       (x)
+        #define unlikely(x)     (x)
+#endif
+
+// cold functions are unlikely() to be called
+#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) || __GNUC__ > 4
+        #define __cold  __attribute__ ((__cold__))
+#else
+        #define __cold  /* no cold marker */
+#endif
+
+#ifdef __GNUG__
+// Methods have a hidden this parameter that is visible to this attribute
+	#define __like_printf_1 __attribute__ ((format (printf, 2, 3)))
+	#define __like_printf_2 __attribute__ ((format (printf, 3, 4)))
+#else
+	#define __like_printf_1
+	#define __like_printf_2
+#endif
+
 #endif
diff --git a/apt-pkg/contrib/md5.cc b/apt-pkg/contrib/md5.cc
index 2bfd70f1b..c0fa8493d 100644
--- a/apt-pkg/contrib/md5.cc
+++ b/apt-pkg/contrib/md5.cc
@@ -37,14 +37,13 @@
 // Include Files							/*{{{*/
 #include <apt-pkg/md5.h>
 #include <apt-pkg/strutl.h>
+#include <apt-pkg/macros.h>
 
 #include <string.h>
 #include <unistd.h>
 #include <netinet/in.h>                          // For htonl
 #include <inttypes.h>
 #include <config.h>
-#include <system.h>
-
 									/*}}}*/
 
 // byteSwap - Swap bytes in a buffer					/*{{{*/
diff --git a/apt-pkg/contrib/sha1.cc b/apt-pkg/contrib/sha1.cc
index b70f31dc6..eae52d52f 100644
--- a/apt-pkg/contrib/sha1.cc
+++ b/apt-pkg/contrib/sha1.cc
@@ -31,12 +31,12 @@
 // Include Files                                                        /*{{{*/
 #include <apt-pkg/sha1.h>
 #include <apt-pkg/strutl.h>
+#include <apt-pkg/macros.h>
 
 #include <string.h>
 #include <unistd.h>
 #include <inttypes.h>
 #include <config.h>
-#include <system.h>
 									/*}}}*/
 
 // SHA1Transform - Alters an existing SHA-1 hash			/*{{{*/
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index 4c05f2df8..2913fbf44 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -43,9 +43,10 @@ bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest)
 {
   iconv_t cd;
   const char *inbuf;
-  char *inptr, *outbuf, *outptr;
-  size_t insize, outsize;
-  
+  char *inptr, *outbuf;
+  size_t insize, bufsize;
+  dest->clear();
+
   cd = iconv_open(codeset, "UTF-8");
   if (cd == (iconv_t)(-1)) {
      // Something went wrong
@@ -55,33 +56,49 @@ bool UTF8ToCodeset(const char *codeset, const string &orig, string *dest)
      else
 	perror("iconv_open");
      
-     // Clean the destination string
-     *dest = "";
-     
      return false;
   }
 
-  insize = outsize = orig.size();
+  insize = bufsize = orig.size();
   inbuf = orig.data();
   inptr = (char *)inbuf;
-  outbuf = new char[insize+1];
-  outptr = outbuf;
+  outbuf = new char[bufsize];
+  size_t lastError = -1;
 
   while (insize != 0)
   {
+     char *outptr = outbuf;
+     size_t outsize = bufsize;
      size_t const err = iconv(cd, &inptr, &insize, &outptr, &outsize);
+     dest->append(outbuf, outptr - outbuf);
      if (err == (size_t)(-1))
      {
-	insize--;
-	outsize++;
-	inptr++;
-	*outptr = '?';
-	outptr++;
+	switch (errno)
+	{
+	case EILSEQ:
+	   insize--;
+	   inptr++;
+	   // replace a series of unknown multibytes with a single "?"
+	   if (lastError != insize) {
+	      lastError = insize - 1;
+	      dest->append("?");
+	   }
+	   break;
+	case EINVAL:
+	   insize = 0;
+	   break;
+	case E2BIG:
+	   if (outptr == outbuf)
+	   {
+	      bufsize *= 2;
+	      delete[] outbuf;
+	      outbuf = new char[bufsize];
+	   }
+	   break;
+	}
      }
   }
 
-  *outptr = '\0';
-  *dest = outbuf;
   delete[] outbuf;
   
   iconv_close(cd);
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index ed7633803..be8175e31 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -306,9 +306,19 @@ pkgCache::PkgFileIterator debPackagesIndex::FindInCache(pkgCache &Cache) const
       
       struct stat St;
       if (stat(File.FileName(),&St) != 0)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "PackagesIndex::FindInCache - stat failed on " << File.FileName() << std::endl;
 	 return pkgCache::PkgFileIterator(Cache);
+      }
       if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "PackagesIndex::FindInCache - size (" << St.st_size << " <> " << File->Size
+			<< ") or mtime (" << St.st_mtime << " <> " << File->mtime
+			<< ") doesn't match for " << File.FileName() << std::endl;
 	 return pkgCache::PkgFileIterator(Cache);
+      }
       return File;
    }
    
@@ -472,9 +482,19 @@ pkgCache::PkgFileIterator debTranslationsIndex::FindInCache(pkgCache &Cache) con
 
       struct stat St;
       if (stat(File.FileName(),&St) != 0)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "TranslationIndex::FindInCache - stat failed on " << File.FileName() << std::endl;
 	 return pkgCache::PkgFileIterator(Cache);
+      }
       if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "TranslationIndex::FindInCache - size (" << St.st_size << " <> " << File->Size
+			<< ") or mtime (" << St.st_mtime << " <> " << File->mtime
+			<< ") doesn't match for " << File.FileName() << std::endl;
 	 return pkgCache::PkgFileIterator(Cache);
+      }
       return File;
    }   
    return File;
@@ -541,9 +561,19 @@ pkgCache::PkgFileIterator debStatusIndex::FindInCache(pkgCache &Cache) const
       
       struct stat St;
       if (stat(File.FileName(),&St) != 0)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "StatusIndex::FindInCache - stat failed on " << File.FileName() << std::endl;
 	 return pkgCache::PkgFileIterator(Cache);
+      }
       if ((unsigned)St.st_size != File->Size || St.st_mtime != File->mtime)
+      {
+         if (_config->FindB("Debug::pkgCacheGen", false))
+	    std::clog << "StatusIndex::FindInCache - size (" << St.st_size << " <> " << File->Size
+			<< ") or mtime (" << St.st_mtime << " <> " << File->mtime
+			<< ") doesn't match for " << File.FileName() << std::endl;
 	 return pkgCache::PkgFileIterator(Cache);
+      }
       return File;
    }   
    return File;
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index 517b771a5..25b0953e0 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -16,10 +16,9 @@
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/crc-16.h>
 #include <apt-pkg/md5.h>
+#include <apt-pkg/macros.h>
 
 #include <ctype.h>
-
-#include <system.h>
 									/*}}}*/
 
 static debListParser::WordList PrioList[] = {{"important",pkgCache::State::Important},
diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index 88098c379..9d144a227 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -580,10 +580,11 @@ bool pkgDPkgPM::OpenLog()
    if (!logfile_name.empty())
    {
       term_out = fopen(logfile_name.c_str(),"a");
+      if (term_out == NULL)
+	 return _error->WarningE(_("Could not open file '%s'"), logfile_name.c_str());
+
       chmod(logfile_name.c_str(), 0600);
-      fprintf(term_out, "\n\nLog started: ");
-      fprintf(term_out, "%s", timestr);
-      fprintf(term_out, "\n");
+      fprintf(term_out, "\nLog started: %s\n", timestr);
    }
 
    // write 
diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc
index 57c9f95ca..53eb11172 100644
--- a/apt-pkg/indexcopy.cc
+++ b/apt-pkg/indexcopy.cc
@@ -275,7 +275,7 @@ bool IndexCopy::CopyPackages(string CDROM,string Name,vector<string> &List,
       _error->Warning("No valid records were found.");
 
    if (NotFound + WrongSize > 10)
-      _error->Warning("Alot of entries were discarded, something may be wrong.\n");
+      _error->Warning("A lot of entries were discarded, something may be wrong.\n");
    
 
    return true;
@@ -847,7 +847,7 @@ bool TranslationsCopy::CopyTranslations(string CDROM,string Name,	/*{{{*/
       _error->Warning("No valid records were found.");
 
    if (NotFound + WrongSize > 10)
-      _error->Warning("Alot of entries were discarded, something may be wrong.\n");
+      _error->Warning("A lot of entries were discarded, something may be wrong.\n");
    
 
    return true;
diff --git a/apt-pkg/makefile b/apt-pkg/makefile
index 3d6209658..bdd49c089 100644
--- a/apt-pkg/makefile
+++ b/apt-pkg/makefile
@@ -24,7 +24,8 @@ SOURCE = contrib/mmap.cc contrib/error.cc contrib/strutl.cc \
 	 contrib/cdromutl.cc contrib/crc-16.cc contrib/netrc.cc \
 	 contrib/fileutl.cc 
 HEADERS = mmap.h error.h configuration.h fileutl.h  cmndline.h netrc.h\
-	  md5.h crc-16.h cdromutl.h strutl.h sptr.h sha1.h sha256.h hashes.h 
+	  md5.h crc-16.h cdromutl.h strutl.h sptr.h sha1.h sha256.h hashes.h \
+	  macros.h
 
 # Source code for the core main library
 SOURCE+= pkgcache.cc version.cc depcache.cc \
@@ -53,7 +54,4 @@ HEADERS+= debversion.h debsrcrecords.h dpkgpm.h debrecords.h \
 
 HEADERS := $(addprefix apt-pkg/,$(HEADERS))
 
-# Private header files
-HEADERS+= system.h
-
 include $(LIBRARY_H)
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index b0ce6e598..4a0f3ee58 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -35,7 +35,6 @@
 #include <unistd.h>
 
 #include <ctype.h>
-#include <system.h>
 									/*}}}*/
 
 using std::string;
@@ -298,7 +297,7 @@ const char *
 pkgCache::PkgIterator::CandVersion() const 
 {
   //TargetVer is empty, so don't use it.
-  VerIterator version = pkgPolicy::pkgPolicy(Owner).GetCandidateVer(*this);
+  VerIterator version = pkgPolicy(Owner).GetCandidateVer(*this);
   if (version.IsGood())
     return version.VerStr();
   return 0;
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 68180c702..3eeb18cae 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -21,6 +21,7 @@
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/sptr.h>
 #include <apt-pkg/pkgsystem.h>
+#include <apt-pkg/macros.h>
 
 #include <apt-pkg/tagfile.h>
 
@@ -32,7 +33,6 @@
 #include <unistd.h>
 #include <errno.h>
 #include <stdio.h>
-#include <system.h>
 									/*}}}*/
 typedef vector<pkgIndexFile *>::iterator FileIterator;
 
@@ -664,16 +664,23 @@ unsigned long pkgCacheGenerator::WriteUniqString(const char *S,
 static bool CheckValidity(const string &CacheFile, FileIterator Start, 
                           FileIterator End,MMap **OutMap = 0)
 {
+   bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
    // No file, certainly invalid
    if (CacheFile.empty() == true || FileExists(CacheFile) == false)
+   {
+      if (Debug == true)
+	 std::clog << "CacheFile doesn't exist" << std::endl;
       return false;
-   
+   }
+
    // Map it
    FileFd CacheF(CacheFile,FileFd::ReadOnly);
    SPtr<MMap> Map = new MMap(CacheF,0);
    pkgCache Cache(Map);
    if (_error->PendingError() == true || Map->Size() == 0)
    {
+      if (Debug == true)
+	 std::clog << "Errors are pending or Map is empty()" << std::endl;
       _error->Discard();
       return false;
    }
@@ -683,9 +690,15 @@ static bool CheckValidity(const string &CacheFile, FileIterator Start,
    SPtrArray<bool> Visited = new bool[Cache.HeaderP->PackageFileCount];
    memset(Visited,0,sizeof(*Visited)*Cache.HeaderP->PackageFileCount);
    for (; Start != End; Start++)
-   {      
+   {
+      if (Debug == true)
+	 std::clog << "Checking PkgFile " << (*Start)->Describe() << ": ";
       if ((*Start)->HasPackages() == false)
+      {
+         if (Debug == true)
+	    std::clog << "Has NO packages" << std::endl;
 	 continue;
+      }
     
       if ((*Start)->Exists() == false)
       {
@@ -693,23 +706,40 @@ static bool CheckValidity(const string &CacheFile, FileIterator Start,
 	 _error->WarningE("stat",_("Couldn't stat source package list %s"),
 			  (*Start)->Describe().c_str());
 #endif
+         if (Debug == true)
+	    std::clog << "file doesn't exist" << std::endl;
 	 continue;
       }
 
       // FindInCache is also expected to do an IMS check.
       pkgCache::PkgFileIterator File = (*Start)->FindInCache(Cache);
       if (File.end() == true)
+      {
+	 if (Debug == true)
+	    std::clog << "FindInCache returned end-Pointer" << std::endl;
 	 return false;
+      }
 
       Visited[File->ID] = true;
+      if (Debug == true)
+	 std::clog << "with ID " << File->ID << " is valid" << std::endl;
    }
    
    for (unsigned I = 0; I != Cache.HeaderP->PackageFileCount; I++)
       if (Visited[I] == false)
+      {
+	 if (Debug == true)
+	    std::clog << "File with ID" << I << " wasn't visited" << std::endl;
 	 return false;
+      }
    
    if (_error->PendingError() == true)
    {
+      if (Debug == true)
+      {
+	 std::clog << "Validity failed because of pending errors:" << std::endl;
+	 _error->DumpErrors();
+      }
       _error->Discard();
       return false;
    }
@@ -796,7 +826,8 @@ static bool BuildCache(pkgCacheGenerator &Gen,
 bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 			MMap **OutMap,bool AllowMem)
 {
-   unsigned long MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
+   bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
+   unsigned long const MapSize = _config->FindI("APT::Cache-Limit",24*1024*1024);
    
    vector<pkgIndexFile *> Files;
    for (vector<metaIndex *>::const_iterator i = List.begin();
@@ -810,13 +841,13 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
          Files.push_back (*j);
    }
    
-   unsigned long EndOfSource = Files.size();
+   unsigned long const EndOfSource = Files.size();
    if (_system->AddStatusFiles(Files) == false)
       return false;
 
    // Decide if we can write to the files..
-   string CacheFile = _config->FindFile("Dir::Cache::pkgcache");
-   string SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
+   string const CacheFile = _config->FindFile("Dir::Cache::pkgcache");
+   string const SrcCacheFile = _config->FindFile("Dir::Cache::srcpkgcache");
    
    // Decide if we can write to the cache
    bool Writeable = false;
@@ -825,7 +856,9 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
    else
       if (SrcCacheFile.empty() == false)
 	 Writeable = access(flNotFile(SrcCacheFile).c_str(),W_OK) == 0;
-   
+   if (Debug == true)
+      std::clog << "Do we have write-access to the cache files? " << (Writeable ? "YES" : "NO") << std::endl;
+
    if (Writeable == false && AllowMem == false && CacheFile.empty() == false)
       return _error->Error(_("Unable to write to %s"),flNotFile(CacheFile).c_str());
    
@@ -835,8 +868,12 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
    if (CheckValidity(CacheFile,Files.begin(),Files.end(),OutMap) == true)
    {
       Progress.OverallProgress(1,1,1,_("Reading package lists"));
+      if (Debug == true)
+	 std::clog << "pkgcache.bin is valid - no need to build anything" << std::endl;
       return true;
    }
+   else if (Debug == true)
+	 std::clog << "pkgcache.bin is NOT valid" << std::endl;
    
    /* At this point we know we need to reconstruct the package cache,
       begin. */
@@ -850,11 +887,15 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
       Map = new DynamicMMap(*CacheF,MMap::Public,MapSize);
       if (_error->PendingError() == true)
 	 return false;
+      if (Debug == true)
+	 std::clog << "Open filebased MMap" << std::endl;
    }
    else
    {
       // Just build it in memory..
       Map = new DynamicMMap(0,MapSize);
+      if (Debug == true)
+	 std::clog << "Open memory Map (not filebased)" << std::endl;
    }
    
    // Lets try the source cache.
@@ -863,16 +904,18 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
    if (CheckValidity(SrcCacheFile,Files.begin(),
 		     Files.begin()+EndOfSource) == true)
    {
+      if (Debug == true)
+	 std::clog << "srcpkgcache.bin is valid - populate MMap with it." << std::endl;
       // Preload the map with the source cache
       FileFd SCacheF(SrcCacheFile,FileFd::ReadOnly);
-      unsigned long alloc = Map->RawAllocate(SCacheF.Size());
+      unsigned long const alloc = Map->RawAllocate(SCacheF.Size());
       if ((alloc == 0 && _error->PendingError())
 		|| SCacheF.Read((unsigned char *)Map->Data() + alloc,
 				SCacheF.Size()) == false)
 	 return false;
 
       TotalSize = ComputeSize(Files.begin()+EndOfSource,Files.end());
-      
+
       // Build the status cache
       pkgCacheGenerator Gen(Map.Get(),&Progress);
       if (_error->PendingError() == true)
@@ -883,6 +926,8 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
    }
    else
    {
+      if (Debug == true)
+	 std::clog << "srcpkgcache.bin is NOT valid - rebuild" << std::endl;
       TotalSize = ComputeSize(Files.begin(),Files.end());
       
       // Build the source cache
@@ -921,6 +966,8 @@ bool pkgMakeStatusCache(pkgSourceList &List,OpProgress &Progress,
 		     Files.begin()+EndOfSource,Files.end()) == false)
 	 return false;
    }
+   if (Debug == true)
+      std::clog << "Caches are ready for shipping" << std::endl;
 
    if (_error->PendingError() == true)
       return false;
diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index 81fdb0431..f9901bc9a 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -27,14 +27,12 @@
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/tagfile.h>
 #include <apt-pkg/strutl.h>
+#include <apt-pkg/fileutl.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/sptr.h>
-    
+
 #include <apti18n.h>
 
-#include <dirent.h>
-#include <sys/stat.h>
-#include <algorithm>
 #include <iostream>
 #include <sstream>
 									/*}}}*/
@@ -282,36 +280,7 @@ bool ReadPinDir(pkgPolicy &Plcy,string Dir)
       return true;
    }
 
-   DIR *D = opendir(Dir.c_str());
-   if (D == 0)
-      return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
-
-   vector<string> List;
-
-   for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D))
-   {
-      if (Ent->d_name[0] == '.')
-	 continue;
-
-      // Skip bad file names ala run-parts
-      const char *C = Ent->d_name;
-      for (; *C != 0; C++)
-	 if (isalpha(*C) == 0 && isdigit(*C) == 0 && *C != '_' && *C != '-')
-	    break;
-      if (*C != 0)
-	 continue;
-
-      // Make sure it is a file and not something else
-      string File = flCombine(Dir,Ent->d_name);
-      struct stat St;
-      if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0)
-	 continue;
-
-      List.push_back(File);
-   }
-   closedir(D);
-
-   sort(List.begin(),List.end());
+   vector<string> const List = GetListOfFilesInDir(Dir, "pref", true, true);
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); I++)
diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index 4b3abe918..929259961 100644
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -17,13 +17,6 @@
 #include <apti18n.h>
 
 #include <fstream>
-
-// CNC:2003-03-03 - This is needed for ReadDir stuff.
-#include <algorithm>
-#include <stdio.h>
-#include <dirent.h>
-#include <sys/stat.h>
-#include <unistd.h>
 									/*}}}*/
 
 using namespace std;
@@ -322,41 +315,7 @@ bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 /* */
 bool pkgSourceList::ReadSourceDir(string Dir)
 {
-   DIR *D = opendir(Dir.c_str());
-   if (D == 0)
-      return _error->Errno("opendir",_("Unable to read %s"),Dir.c_str());
-
-   vector<string> List;
-   
-   for (struct dirent *Ent = readdir(D); Ent != 0; Ent = readdir(D))
-   {
-      if (Ent->d_name[0] == '.')
-	 continue;
-
-      // CNC:2003-12-02 Only accept .list files as valid sourceparts
-      if (flExtension(Ent->d_name) != "list")
-	 continue;
-      
-      // Skip bad file names ala run-parts
-      const char *C = Ent->d_name;
-      for (; *C != 0; C++)
-	 if (isalpha(*C) == 0 && isdigit(*C) == 0
-             && *C != '_' && *C != '-' && *C != '.')
-	    break;
-      if (*C != 0)
-	 continue;
-      
-      // Make sure it is a file and not something else
-      string File = flCombine(Dir,Ent->d_name);
-      struct stat St;
-      if (stat(File.c_str(),&St) != 0 || S_ISREG(St.st_mode) == 0)
-	 continue;
-      
-      List.push_back(File);      
-   }   
-   closedir(D);
-   
-   sort(List.begin(),List.end());
+   vector<string> const List = GetListOfFilesInDir(Dir, "list", true);
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); I++)