support arch:all data e.g. in separate Packages file

Based on a discussion with Niels Thykier who asked for Contents-all this
implements apt trying for all architecture dependent files to get a file
for the architecture all, which is treated internally now as an official
architecture which is always around (like native). This way arch:all
data can be shared instead of duplicated for each architecture requiring
the user to download the same information again and again.

There is one problem however: In Debian there is already a binary-all/
Packages file, but the binary-any files still include arch:all packages,
so that downloading this file now would be a waste of time, bandwidth
and diskspace. We therefore need a way to decide if it makes sense to
download the all file for Packages in Debian or not. The obvious answer
would be a special flag in the Release file indicating this, which would
need to default to 'no' and every reasonable repository would override
it to 'yes' in a few years time, but the flag would be there "forever".

Looking closer at a Release file we see the field "Architectures", which
doesn't include 'all' at the moment. With the idea outlined above that
'all' is a "proper" architecture now, we interpret this field as being
authoritative in declaring which architectures are supported by this
repository. If it says 'all', apt will try to get all, if not it will be
skipped. This gives us another interesting feature: If I configure a
source to download armel and mips, but it declares it supports only
armel apt will now print a notice saying as much. Previously this was a
very cryptic failure. If on the other hand the repository supports mips,
too, but for some reason doesn't ship mips packages at the moment, this
'missing' file is silently ignored (= that is the same as the repository
including an empty file).

The Architectures field isn't mandatory through, so if it isn't there,
we assume that every architecture is supported by this repository, which
skips the arch:all if not listed in the release file.

3 files changed, 67 insertions, 49 deletions

diff --git a/ftparchive/apt-ftparchive.cc b/ftparchive/apt-ftparchive.cc
index b31c92845..c7e38badf 100644
--- a/ftparchive/apt-ftparchive.cc
+++ b/ftparchive/apt-ftparchive.cc
@@ -68,6 +68,7 @@ struct PackageMap
 
    // We generate for this given arch
    string Arch;
+   bool IncludeArchAll;
    
    // Stuff for the Source File
    string SrcFile;
@@ -122,9 +123,9 @@ struct PackageMap
 		    vector<PackageMap>::iterator End,
 		    unsigned long &Left);
    
-   PackageMap() : LongDesc(true), TransWriter(NULL), DeLinkLimit(0), Permissions(1),
-		  ContentsDone(false), PkgDone(false), SrcDone(false),
-		  ContentsMTime(0) {};
+   PackageMap() : IncludeArchAll(true), LongDesc(true), TransWriter(NULL),
+		  DeLinkLimit(0), Permissions(1), ContentsDone(false),
+		  PkgDone(false), SrcDone(false), ContentsMTime(0) {};
 };
 									/*}}}*/
 
@@ -184,7 +185,7 @@ bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
    PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
 			   flCombine(OverrideDir,BinOverride),
 			   flCombine(OverrideDir,ExtraOverride),
-			   Arch);
+			   Arch, IncludeArchAll);
    if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
@@ -364,7 +365,7 @@ bool PackageMap::GenContents(Configuration &Setup,
    MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
 		      CntCompress,Permissions);
    Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
-   ContentsWriter Contents(&Comp.Input, "", Arch);
+   ContentsWriter Contents(&Comp.Input, "", Arch, IncludeArchAll);
    if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
       return _error->Error(_("Package extension list is too long"));
    if (_error->PendingError() == true)
@@ -487,6 +488,8 @@ static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*
    bool const LongDescription = Setup.FindB("Default::LongDescription",
 					_config->FindB("APT::FTPArchive::LongDescription", true));
    string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
+   bool const ConfIncludeArchAllExists = _config->Exists("APT::FTPArchive::IncludeArchitectureAll");
+   bool const ConfIncludeArchAll = _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true);
 
    // Process 'tree' type sections
    const Configuration::Item *Top = Setup.Tree("tree");
@@ -501,25 +504,32 @@ static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*
       string Section;
       while (ParseQuoteWord(Sections,Section) == true)
       {
-	 string Arch;
-	 struct SubstVar const Vars[] = {{"$(DIST)",&Dist},
+	 struct SubstVar Vars[] = {{"$(DIST)",&Dist},
 					 {"$(SECTION)",&Section},
-					 {"$(ARCH)",&Arch},
-					 {NULL, NULL}};
+					 {"$(ARCH)",nullptr},
+					 {nullptr, nullptr}};
 	 mode_t const Perms = Block.FindI("FileMode", Permissions);
 	 bool const LongDesc = Block.FindB("LongDescription", LongDescription);
-	 TranslationWriter *TransWriter = NULL;
-
-	 string const Tmp2 = Block.Find("Architectures");
-	 const char *Archs = Tmp2.c_str();
-	 while (ParseQuoteWord(Archs,Arch) == true)
+	 TranslationWriter *TransWriter = nullptr;
+
+	 std::string Tmp2 = Block.Find("Architectures");
+	 std::transform(Tmp2.begin(), Tmp2.end(), Tmp2.begin(), ::tolower);
+	 std::vector<std::string> const Archs = VectorizeString(Tmp2, ' ');
+	 bool IncludeArchAll;
+	 if (ConfIncludeArchAllExists == true)
+	    IncludeArchAll = ConfIncludeArchAll;
+	 else
+	    IncludeArchAll = std::find(Archs.begin(), Archs.end(), "all") == Archs.end();
+	 for (auto const& Arch: Archs)
 	 {
+	    if (Arch.empty()) continue;
+	    Vars[2].Contents = &Arch;
 	    PackageMap Itm;
 	    Itm.Permissions = Perms;
 	    Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
 	    Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
 
-	    if (stringcasecmp(Arch,"source") == 0)
+	    if (Arch == "source")
 	    {
 	       Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
 	       Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
@@ -536,6 +546,7 @@ static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*
 	       Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
 	       Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
 	       Itm.Arch = Arch;
+	       Itm.IncludeArchAll = IncludeArchAll;
 	       Itm.LongDesc = LongDesc;
 	       if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
 	       {
@@ -662,7 +673,8 @@ static bool SimpleGenPackages(CommandLine &CmdL)
    
    // Create a package writer object.
    PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
-			   Override, "", _config->Find("APT::FTPArchive::Architecture"));
+			   Override, "", _config->Find("APT::FTPArchive::Architecture"),
+			   _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true));
    if (_error->PendingError() == true)
       return false;
    
diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc
index 27d6e98e0..0bd2be566 100644
--- a/ftparchive/writer.cc
+++ b/ftparchive/writer.cc
@@ -71,7 +71,8 @@ static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
 									/*}}}*/
 
 // FTWScanner::FTWScanner - Constructor					/*{{{*/
-FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch): Arch(Arch), DoHashes(~0)
+FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch, bool const IncludeArchAll)
+   : Arch(Arch), IncludeArchAll(IncludeArchAll), DoHashes(~0)
 {
    if (GivenOutput == NULL)
    {
@@ -329,14 +330,40 @@ bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
    return true;
 }
 									/*}}}*/
+// FTWScanner::SetExts - Set extensions to support                      /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool FTWScanner::SetExts(string const &Vals)
+{
+   ClearPatterns();
+   string::size_type Start = 0;
+   while (Start <= Vals.length()-1)
+   {
+      string::size_type const Space = Vals.find(' ',Start);
+      string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
+      if ( Arch.empty() == false )
+      {
+	 AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
+	 if (IncludeArchAll == true && Arch != "all")
+	    AddPattern(string("*_all") + Vals.substr(Start, Length));
+      }
+      else
+	 AddPattern(string("*") + Vals.substr(Start, Length));
+
+      Start += Length + 1;
+   }
+
+   return true;
+}
+									/*}}}*/
 
 // PackagesWriter::PackagesWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
 PackagesWriter::PackagesWriter(FileFd * const GivenOutput, TranslationWriter * const transWriter,
       string const &DB,string const &Overrides,string const &ExtOverrides,
-      string const &Arch) :
-   FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats), TransWriter(transWriter)
+      string const &Arch, bool const IncludeArchAll) :
+   FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats), TransWriter(transWriter)
 {
    SetExts(".deb .udeb");
    DeLinkLimit = 0;
@@ -363,31 +390,6 @@ PackagesWriter::PackagesWriter(FileFd * const GivenOutput, TranslationWriter * c
    _error->DumpErrors();
 }
                                                                         /*}}}*/
-// FTWScanner::SetExts - Set extensions to support                      /*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool FTWScanner::SetExts(string const &Vals)
-{
-   ClearPatterns();
-   string::size_type Start = 0;
-   while (Start <= Vals.length()-1)
-   {
-      string::size_type const Space = Vals.find(' ',Start);
-      string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
-      if ( Arch.empty() == false )
-      {
-	 AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
-	 AddPattern(string("*_all") + Vals.substr(Start, Length));
-      }
-      else
-	 AddPattern(string("*") + Vals.substr(Start, Length));
-
-      Start += Length + 1;
-   }
-
-   return true;
-}
-									/*}}}*/
 // PackagesWriter::DoPackage - Process a single package			/*{{{*/
 // ---------------------------------------------------------------------
 /* This method takes a package and gets its control information and 
@@ -879,8 +881,9 @@ bool SourcesWriter::DoPackage(string FileName)
 // ContentsWriter::ContentsWriter - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB, string const &Arch) :
-		    FTWScanner(GivenOutput, Arch), Db(DB), Stats(Db.Stats)
+ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB,
+      string const &Arch, bool const IncludeArchAll) :
+		    FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats)
 
 {
    SetExts(".deb");
diff --git a/ftparchive/writer.h b/ftparchive/writer.h
index 98012beee..ea4c66da4 100644
--- a/ftparchive/writer.h
+++ b/ftparchive/writer.h
@@ -40,6 +40,7 @@ class FTWScanner
    protected:
    vector<string> Patterns;
    string Arch;
+   bool IncludeArchAll;
    const char *OriginalPath;
    bool ErrorPrinted;
 
@@ -79,7 +80,7 @@ class FTWScanner
    void AddPatterns(std::vector<std::string> const &patterns) { Patterns.insert(Patterns.end(), patterns.begin(), patterns.end()); };
    bool SetExts(string const &Vals);
 
-   FTWScanner(FileFd * const Output, string const &Arch = string());
+   FTWScanner(FileFd * const Output, string const &Arch = string(), bool const IncludeArchAll = true);
    virtual ~FTWScanner();
 };
 
@@ -125,7 +126,8 @@ class PackagesWriter : public FTWScanner
    PackagesWriter(FileFd * const Output, TranslationWriter * const TransWriter, string const &DB,
                   string const &Overrides,
                   string const &ExtOverrides = "",
-		  string const &Arch = "");
+		  string const &Arch = "",
+		  bool const IncludeArchAll = true);
    virtual ~PackagesWriter();
 };
 
@@ -149,7 +151,8 @@ class ContentsWriter : public FTWScanner
    void Finish() {Gen.Print(*Output);};
    inline bool ReadyDB(string const &DB) {return Db.ReadyDB(DB);};
 
-   ContentsWriter(FileFd * const Output, string const &DB, string const &Arch = string());
+   ContentsWriter(FileFd * const Output, string const &DB, string const &Arch = string(),
+	 bool const IncludeArchAll = true);
    virtual ~ContentsWriter() {};
 };