bring back deb822 sources.list entries as .sources

Having two different formats in the same file is very dirty and causes
external tools to fail hard trying to parse them. It is probably not a
good idea for them to parse them in the first place, but they do and we
shouldn't break them if there is a better way.

So we solve this issue for now by giving our deb822 format a new
filename extension ".sources" which unsupporting applications are likely
to ignore an can begin gradually moving forward rather than waiting for
the unknown applications to catch up.

Currently and for the forseeable future apt is going to support both
with the same feature set as documented in the manpage, with the
longtime plan of adopting the 'new' format as default, but that is a
long way to go and might get going more from having an easier time
setting options than from us pushing it explicitely.

5 files changed, 140 insertions, 139 deletions

diff --git a/apt-pkg/policy.cc b/apt-pkg/policy.cc
index cd48e040c..170da7c63 100644
--- a/apt-pkg/policy.cc
+++ b/apt-pkg/policy.cc
@@ -396,21 +396,6 @@ APT_PURE signed short pkgPolicy::GetPriority(pkgCache::PkgFileIterator const &Fi
    return PFPriority[File->ID];
 }
 									/*}}}*/
-// PreferenceSection class - Overriding the default TrimRecord method	/*{{{*/
-// ---------------------------------------------------------------------
-/* The preference file is a user generated file so the parser should
-   therefore be a bit more friendly by allowing comments and new lines
-   all over the place rather than forcing a special format */
-class PreferenceSection : public pkgTagSection
-{
-   void TrimRecord(bool /*BeforeRecord*/, const char* &End)
-   {
-      for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r' || Stop[0] == '#'); Stop++)
-	 if (Stop[0] == '#')
-	    Stop = (const char*) memchr(Stop,'\n',End-Stop);
-   }
-};
-									/*}}}*/
 // ReadPinDir - Load the pin files from this dir into a Policy		/*{{{*/
 // ---------------------------------------------------------------------
 /* This will load each pin file in the given dir into a Policy. If the
@@ -455,8 +440,8 @@ bool ReadPinFile(pkgPolicy &Plcy,string File)
    pkgTagFile TF(&Fd);
    if (_error->PendingError() == true)
       return false;
-   
-   PreferenceSection Tags;
+
+   pkgUserTagSection Tags;
    while (TF.Step(Tags) == true)
    {
       // can happen when there are only comments in a record
diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index 6ef99863d..69f7ac043 100644
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -82,15 +82,15 @@ bool pkgSourceList::Type::FixupURI(string &URI) const
    return true;
 }
 									/*}}}*/
-bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,
+bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,	/*{{{*/
                                       pkgTagSection &Tags,
-                                      int i,
+                                      unsigned int const i,
                                       FileFd &Fd)
 {
    map<string, string> Options;
 
    string Enabled = Tags.FindS("Enabled");
-   if (Enabled.size() > 0 && StringToBool(Enabled) == false)
+   if (Enabled.empty() == false && StringToBool(Enabled) == false)
       return true;
 
    std::map<char const * const, char const * const> mapping;
@@ -115,46 +115,63 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,
    // now create one item per suite/section
    string Suite = Tags.FindS("Suites");
    Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
-   string const Section = Tags.FindS("Sections");
-   string URIS = Tags.FindS("URIs");
+   string const Component = Tags.FindS("Components");
+   string const URIS = Tags.FindS("URIs");
+
+   std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
+   std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
+   std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
+
+   if (list_uris.empty())
+      // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
+      return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
 
-   std::vector<std::string> list_uris = StringSplit(URIS, " ");
-   std::vector<std::string> list_dist = StringSplit(Suite, " ");
-   std::vector<std::string> list_section = StringSplit(Section, " ");
-   
    for (std::vector<std::string>::const_iterator U = list_uris.begin();
         U != list_uris.end(); ++U)
    {
-      std::string URI = (*U);
-      if (!FixupURI(URI))
-      {
-         _error->Error(_("Malformed stanza %u in source list %s (URI parse)"),i,Fd.Name().c_str());
-         return false;
-      }
+      std::string URI = *U;
+      if (U->empty() || FixupURI(URI) == false)
+	 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
+
+      if (list_suite.empty())
+	 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
 
-      for (std::vector<std::string>::const_iterator I = list_dist.begin();
-           I != list_dist.end(); ++I)
+      for (std::vector<std::string>::const_iterator S = list_suite.begin();
+           S != list_suite.end(); ++S)
       {
-         for (std::vector<std::string>::const_iterator J = list_section.begin();
-              J != list_section.end(); ++J)
-         {
-            if (CreateItem(List, URI, (*I), (*J), Options) == false)
-            {
-               return false;
-            }
-         }
+	 if (S->empty() == false && (*S)[S->size() - 1] == '/')
+	 {
+	    if (list_comp.empty() == false)
+	       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
+	    if (CreateItem(List, URI, *S, "", Options) == false)
+	       return false;
+	 }
+	 else
+	 {
+	    if (list_comp.empty())
+	       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
+
+	    for (std::vector<std::string>::const_iterator C = list_comp.begin();
+		  C != list_comp.end(); ++C)
+	    {
+	       if (CreateItem(List, URI, *S, *C, Options) == false)
+	       {
+		  return false;
+	       }
+	    }
+	 }
       }
    }
    return true;
 }
-
+									/*}}}*/
 // Type::ParseLine - Parse a single line				/*{{{*/
 // ---------------------------------------------------------------------
 /* This is a generic one that is the 'usual' format for sources.list
    Weird types may override this. */
 bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 				    const char *Buffer,
-				    unsigned long const &CurLine,
+				    unsigned int const CurLine,
 				    string const &File) const
 {
    for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
@@ -171,10 +188,10 @@ bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 	 // get one option, e.g. option1=value1
 	 string option;
 	 if (ParseQuoteWord(Buffer,option) == false)
-	    return _error->Error(_("Malformed line %lu in source list %s ([option] unparseable)"),CurLine,File.c_str());
+	    return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
 
 	 if (option.length() < 3)
-	    return _error->Error(_("Malformed line %lu in source list %s ([option] too short)"),CurLine,File.c_str());
+	    return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
 
 	 // accept options even if the last has no space before the ]-end marker
 	 if (option.at(option.length()-1) == ']')
@@ -185,16 +202,16 @@ bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 
 	 size_t const needle = option.find('=');
 	 if (needle == string::npos)
-	    return _error->Error(_("Malformed line %lu in source list %s ([%s] is not an assignment)"),CurLine,File.c_str(), option.c_str());
+	    return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
 
 	 string const key = string(option, 0, needle);
 	 string const value = string(option, needle + 1, option.length());
 
 	 if (key.empty() == true)
-	    return _error->Error(_("Malformed line %lu in source list %s ([%s] has no key)"),CurLine,File.c_str(), option.c_str());
+	    return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
 
 	 if (value.empty() == true)
-	    return _error->Error(_("Malformed line %lu in source list %s ([%s] key %s has no value)"),CurLine,File.c_str(),option.c_str(),key.c_str());
+	    return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
 
 	 Options[key] = value;
       }
@@ -207,33 +224,33 @@ bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
    string Section;
 
    if (ParseQuoteWord(Buffer,URI) == false)
-      return _error->Error(_("Malformed line %lu in source list %s (URI)"),CurLine,File.c_str());
+      return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
    if (ParseQuoteWord(Buffer,Dist) == false)
-      return _error->Error(_("Malformed line %lu in source list %s (dist)"),CurLine,File.c_str());
-      
+      return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
+
    if (FixupURI(URI) == false)
-      return _error->Error(_("Malformed line %lu in source list %s (URI parse)"),CurLine,File.c_str());
+      return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
 
    // Check for an absolute dists specification.
    if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
    {
       if (ParseQuoteWord(Buffer,Section) == true)
-	 return _error->Error(_("Malformed line %lu in source list %s (absolute dist)"),CurLine,File.c_str());
+	 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
       Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
       return CreateItem(List, URI, Dist, Section, Options);
    }
-   
+
    // Grab the rest of the dists
    if (ParseQuoteWord(Buffer,Section) == false)
-      return _error->Error(_("Malformed line %lu in source list %s (dist parse)"),CurLine,File.c_str());
-   
+      return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
+
    do
    {
       if (CreateItem(List, URI, Dist, Section, Options) == false)
 	 return false;
    }
    while (ParseQuoteWord(Buffer,Section) == true);
-   
+
    return true;
 }
 									/*}}}*/
@@ -243,11 +260,6 @@ bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
 pkgSourceList::pkgSourceList() : d(NULL)
 {
 }
-
-pkgSourceList::pkgSourceList(string File) : d(NULL)
-{
-   Read(File);
-}
 									/*}}}*/
 // SourceList::~pkgSourceList - Destructor				/*{{{*/
 // ---------------------------------------------------------------------
@@ -305,7 +317,7 @@ void pkgSourceList::Reset()
 // SourceList::Read - Parse the sourcelist file				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool pkgSourceList::Read(string File)
+bool pkgSourceList::Read(string const &File)
 {
    Reset();
    return ReadAppend(File);
@@ -314,71 +326,63 @@ bool pkgSourceList::Read(string File)
 // SourceList::ReadAppend - Parse a sourcelist file			/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool pkgSourceList::ReadAppend(string File)
+bool pkgSourceList::ReadAppend(string const &File)
 {
-   if (_config->FindB("APT::Sources::Use-Deb822", false) == true)
-   {
-      int lines_parsed =ParseFileDeb822(File);
-      if (lines_parsed < 0)
-         return false;
-      else if (lines_parsed > 0)
-         return true;
-      // no lines parsed  ... fall through and use old style parser
-   }
-   return ParseFileOldStyle(File);
+   if (flExtension(File) == "sources")
+      return ParseFileDeb822(File);
+   else
+      return ParseFileOldStyle(File);
 }
 
 // SourceList::ReadFileOldStyle - Read Traditional style sources.list 	/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool pkgSourceList::ParseFileOldStyle(string File)
+bool pkgSourceList::ParseFileOldStyle(std::string const &File)
 {
    // Open the stream for reading
    ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
    if (F.fail() == true)
       return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
 
-   // CNC:2003-12-10 - 300 is too short.
-   char Buffer[1024];
-
-   int CurLine = 0;
-   while (F.eof() == false)
+   std::string Buffer;
+   for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
    {
-      F.getline(Buffer,sizeof(Buffer));
-      CurLine++;
-      _strtabexpand(Buffer,sizeof(Buffer));
-      if (F.fail() && !F.eof())
-	 return _error->Error(_("Line %u too long in source list %s."),
-			      CurLine,File.c_str());
-
-      
-      char *I;
-      // CNC:2003-02-20 - Do not break if '#' is inside [].
-      for (I = Buffer; *I != 0 && *I != '#'; I++)
-         if (*I == '[')
-         {
-	    char *b_end = strchr(I + 1, ']');
-            if (b_end != NULL)
-               I = b_end;
-         }
-      *I = 0;
-      
-      const char *C = _strstrip(Buffer);
-      
-      // Comment or blank
-      if (C[0] == '#' || C[0] == 0)
+      // remove comments
+      size_t curpos = 0;
+      while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
+      {
+	 size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
+	 size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
+	 if (openbrackets > closedbrackets)
+	 {
+	    // a # in an option, unlikely, but oh well, it was supported so stick to it
+	    ++curpos;
+	    continue;
+	 }
+	 Buffer.erase(curpos);
+	 break;
+      }
+      // remove spaces before/after
+      curpos = Buffer.find_first_not_of(" \t\r");
+      if (curpos != 0)
+	 Buffer.erase(0, curpos);
+      curpos = Buffer.find_last_not_of(" \t\r");
+      if (curpos != std::string::npos)
+	 Buffer.erase(curpos + 1);
+
+      if (Buffer.empty())
 	 continue;
-      	    
+
       // Grok it
-      string LineType;
-      if (ParseQuoteWord(C,LineType) == false)
+      std::string const LineType = Buffer.substr(0, Buffer.find(' '));
+      if (LineType.empty() || LineType == Buffer)
 	 return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
 
       Type *Parse = Type::GetType(LineType.c_str());
       if (Parse == 0)
 	 return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
-      
-      if (Parse->ParseLine(SrcList, C, CurLine, File) == false)
+
+      if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
 	 return false;
    }
    return true;
@@ -387,30 +391,25 @@ bool pkgSourceList::ParseFileOldStyle(string File)
 // SourceList::ParseFileDeb822 - Parse deb822 style sources.list 	/*{{{*/
 // ---------------------------------------------------------------------
 /* Returns: the number of stanzas parsed*/
-int pkgSourceList::ParseFileDeb822(string File)
+bool pkgSourceList::ParseFileDeb822(string const &File)
 {
-   pkgTagSection Tags;
-   unsigned int i=0;
+   pkgUserTagSection Tags;
+   unsigned int i = 1;
 
    // see if we can read the file
-   _error->PushToStack();
    FileFd Fd(File, FileFd::ReadOnly);
    pkgTagFile Sources(&Fd);
    if (_error->PendingError() == true)
-   {
-      _error->RevertToStack();
-      return 0;
-   }
-   _error->MergeWithStack();
-   
+      return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
+
    // read step by step
    while (Sources.Step(Tags) == true)
    {
-      if(!Tags.Exists("Types")) 
-         continue;
+      if(Tags.Exists("Types") == false)
+	 return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 
       string const types = Tags.FindS("Types");
-      std::vector<std::string> list_types = StringSplit(types, " ");
+      std::vector<std::string> const list_types = VectorizeString(types, ' ');
       for (std::vector<std::string>::const_iterator I = list_types.begin();
         I != list_types.end(); ++I)
       {
@@ -418,18 +417,16 @@ int pkgSourceList::ParseFileDeb822(string File)
          if (Parse == 0)
          {
             _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
-            return -1;
+            return false;
          }
-         
+
          if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
-            return -1;
+            return false;
 
-         i++;
+         ++i;
       }
    }
-
-   // we are done, return the number of stanzas read
-   return i;
+   return true;
 }
 									/*}}}*/
 // SourceList::FindIndex - Get the index associated with a file		/*{{{*/
@@ -471,9 +468,12 @@ bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
 // Based on ReadConfigDir()						/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool pkgSourceList::ReadSourceDir(string Dir)
+bool pkgSourceList::ReadSourceDir(string const &Dir)
 {
-   vector<string> const List = GetListOfFilesInDir(Dir, "list", true);
+   std::vector<std::string> ext;
+   ext.push_back("list");
+   ext.push_back("sources");
+   std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
 
    // Read the files
    for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h
index 4f42b3e91..079f3cb3d 100644
--- a/apt-pkg/sourcelist.h
+++ b/apt-pkg/sourcelist.h
@@ -72,11 +72,11 @@ class pkgSourceList
       bool FixupURI(std::string &URI) const;
       virtual bool ParseStanza(std::vector<metaIndex *> &List,
                                pkgTagSection &Tags,
-                               int stanza_n,
+                               unsigned int const stanza_n,
                                FileFd &Fd);
       virtual bool ParseLine(std::vector<metaIndex *> &List,
 			     const char *Buffer,
-			     unsigned long const &CurLine,std::string const &File) const;
+			     unsigned int const CurLine,std::string const &File) const;
       virtual bool CreateItem(std::vector<metaIndex *> &List,std::string const &URI,
 			      std::string const &Dist,std::string const &Section,
 			      std::map<std::string, std::string> const &Options) const = 0;
@@ -90,18 +90,19 @@ class pkgSourceList
 
    std::vector<metaIndex *> SrcList;
 
-   int ParseFileDeb822(std::string File);
-   bool ParseFileOldStyle(std::string File);
+   private:
+   APT_HIDDEN bool ParseFileDeb822(std::string const &File);
+   APT_HIDDEN bool ParseFileOldStyle(std::string const &File);
 
    public:
 
    bool ReadMainList();
-   bool Read(std::string File);
+   bool Read(std::string const &File);
 
    // CNC:2003-03-03
    void Reset();
-   bool ReadAppend(std::string File);
-   bool ReadSourceDir(std::string Dir);
+   bool ReadAppend(std::string const &File);
+   bool ReadSourceDir(std::string const &Dir);
    
    // List accessors
    inline const_iterator begin() const {return SrcList.begin();};
@@ -117,7 +118,6 @@ class pkgSourceList
    time_t GetLastModifiedTime();
 
    pkgSourceList();
-   explicit pkgSourceList(std::string File);
    virtual ~pkgSourceList();
 };
 
diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc
index 6d7d8185b..cc63b213f 100644
--- a/apt-pkg/tagfile.cc
+++ b/apt-pkg/tagfile.cc
@@ -775,6 +775,14 @@ bool pkgTagSection::Write(FileFd &File, char const * const * const Order, std::v
 }
 									/*}}}*/
 
+void pkgUserTagSection::TrimRecord(bool /*BeforeRecord*/, const char* &End)/*{{{*/
+{
+   for (; Stop < End && (Stop[0] == '\n' || Stop[0] == '\r' || Stop[0] == '#'); Stop++)
+      if (Stop[0] == '#')
+	 Stop = (const char*) memchr(Stop,'\n',End-Stop);
+}
+									/*}}}*/
+
 #include "tagfile-order.c"
 
 // TFRewrite - Rewrite a control record					/*{{{*/
diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h
index d0d0c7a84..81fff89f0 100644
--- a/apt-pkg/tagfile.h
+++ b/apt-pkg/tagfile.h
@@ -142,6 +142,14 @@ class pkgTagSection
    bool Write(FileFd &File, char const * const * const Order = NULL, std::vector<Tag> const &Rewrite = std::vector<Tag>()) const;
 };
 
+
+/* For user generated file the parser should be a bit more relaxed in exchange
+   for being a bit slower to allow comments and new lines all over the place */
+class pkgUserTagSection : public pkgTagSection
+{
+   virtual void TrimRecord(bool BeforeRecord, const char* &End);
+};
+
 class pkgTagFilePrivate;
 class pkgTagFile
 {