Merge branch 'variants-submit' into 'main'

Merge Architecture variants from Ubuntu

See merge request apt-team/apt!526

15 files changed, 256 insertions, 18 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index c692b5281..84f61b8bf 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -34,13 +34,13 @@
 #include <algorithm>
 #include <array>
 #include <cerrno>
-#include <ctime>
 #include <chrono>
 #include <cstddef>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
 #include <ctime>
+#include <fstream>
 #include <iostream>
 #include <memory>
 #include <numeric>
@@ -1598,6 +1598,48 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 	       componentsSeen.emplace(std::move(component));
 	 }
 
+   auto variants = APT::Configuration::getArchitectureVariants(true);
+   std::set<IndexTarget *> supplantedTargets;
+   std::set<std::string> seenVariants;
+   if (hasReleaseFile)
+   {
+      // Figure out the variants that are standalone and supplant 'lesser' variants
+      for (auto &variant : variants)
+      {
+	 // Check if the variant is standalone, that is, in the Architectures: field.
+	 // FIXME: We should support Standalone-Architecture-Variants field here
+	 if (not TransactionManager->MetaIndexParser->IsArchitectureSupported(variant.name))
+	    continue;
+
+	 seenVariants.insert(variant.name);
+
+	 // Iterate over all targets belonging to the architecture variant and then see if they
+	 // supplant any other target. For example main/binary-amd64v3/Packages supplants main/binary-amd64/Packages
+	 // but it won't supplant the contrib entry, or a Contents file.
+	 for (auto &Target : IndexTargets)
+	 {
+	    if (Target.Option(IndexTarget::ARCHITECTURE) != variant.name)
+	       continue;
+
+	    for (auto &BaseTarget : IndexTargets)
+	    {
+	       if (BaseTarget.Option(IndexTarget::IDENTIFIER) == Target.Option(IndexTarget::IDENTIFIER) &&
+		   BaseTarget.Option(IndexTarget::COMPONENT) == Target.Option(IndexTarget::COMPONENT) &&
+		   // Variants we already saw can never be supplanted here, because they are ordered by preference
+		   seenVariants.find(BaseTarget.Option(IndexTarget::ARCHITECTURE)) == seenVariants.end() &&
+		   // BaseTarget is supplanted by Target on the architecture CPU
+		   std::find(variant.supplants.begin(), variant.supplants.end(), BaseTarget.Option(IndexTarget::ARCHITECTURE)) != variant.supplants.end())
+	       {
+		  if (_config->FindB("Debug::Acquire::Variants", false))
+		     std::clog << "Variant target " << Target.Description << " supplants target " << BaseTarget.Description << std::endl;
+		  supplantedTargets.insert(&BaseTarget);
+	       }
+	    }
+	 }
+	 //
+      }
+   }
+
    for (auto&& Target: IndexTargets)
    {
       // if we have seen a target which is created-by a target this one here is declared a
@@ -1608,6 +1650,14 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 	 new CleanupItem(Owner, TransactionManager, Target);
 	 continue;
       }
+
+      // If the target has been supplanted by a variant target, skip it.
+      if (supplantedTargets.find(&Target) != supplantedTargets.end())
+      {
+	 new CleanupItem(Owner, TransactionManager, Target);
+	 continue;
+      }
+
       // all is an implementation detail. Users shouldn't use this as arch
       // We need this support trickery here as e.g. Debian has binary-all files already,
       // but arch:all packages are still in the arch:any files, so we would waste precious
@@ -3497,7 +3547,7 @@ pkgAcqArchive::pkgAcqArchive(pkgAcquire *const Owner, pkgSourceList *const Sourc
 	    all repositories containing this file */
 	 StoreFilename = QuoteString(Version.ParentPkg().Name(), "_:") + '_' +
 			 QuoteString(Version.VerStr(), "_:") + '_' +
-			 QuoteString(Version.Arch(), "_:.") +
+			 QuoteString(Version.ArchVariant().empty() ? Version.Arch() : Version.ArchVariant().data(), "_:.") +
 			 '.' += flExtension(poolfilename);
 
 	 Desc.URI = Index->ArchiveURI(poolfilename);
diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc
index 96ff8b285..a0164c6c3 100644
--- a/apt-pkg/aptconfiguration.cc
+++ b/apt-pkg/aptconfiguration.cc
@@ -24,6 +24,8 @@
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <fstream>
+#include <optional>
 #include <string>
 #include <vector>
 #include <dirent.h>
@@ -313,6 +315,117 @@ bool Configuration::checkLanguage(std::string Lang, bool const All) {
 	return (std::find(langs.begin(), langs.end(), Lang) != langs.end());
 }
 									/*}}}*/
+
+// getArchitectures - Return Vector of preferred Architectures		/*{{{*/
+
+static std::set<std::string> const &getCpuFlags()
+{
+   static std::set<std::string> flags;
+   if (not flags.empty())
+      return flags;
+
+   std::ifstream table(_config->FindFile("Dir::proc::cpuinfo", "/proc/cpuinfo"));
+   for (std::string line; std::getline(table, line);)
+   {
+      if (not APT::String::Startswith(line, "flags\t"))
+	 continue;
+
+      line = line.substr(line.find(':'));
+      line = line.substr(line.find_first_not_of(": "));
+      if (flags.empty())
+	 for (auto &&flag : APT::String::Split(line))
+	    flags.insert(std::move(flag));
+      else
+      {
+	 // Remove flags not present in this CPU
+	 auto split = APT::String::Split(line);
+	 std::set<std::string> newSet{split.begin(), split.end()};
+	 for (auto it = flags.begin(); it != flags.end();)
+	    if (newSet.find(*it) == newSet.end())
+	       it = flags.erase(it);
+	    else
+	       ++it;
+      }
+   }
+   return flags;
+}
+std::vector<Configuration::ArchitectureVariant> Configuration::getArchitectureVariants(bool cached)
+{
+   std::optional<std::vector<ArchitectureVariant>> static variants;
+   if (likely(cached) && variants)
+      return *variants;
+
+   variants.emplace();
+
+   auto const configuredVariants = _config->FindVector("APT::Architecture-Variants");
+   auto const architectures = getArchitectures(cached);
+   auto const autoDetect = configuredVariants.size() == 1 && configuredVariants.front() == "auto";
+   auto const autoCpuFlags = getCpuFlags();
+   auto tablepath = _config->FindFile("Dir::dpkg::varianttable", DPKG_DATADIR "/varianttable");
+   std::ifstream table(tablepath);
+   auto isAutoVariant = [&](auto &var)
+   {
+      bool res = true;
+      for (auto want : var.cpuflags)
+	 if (autoCpuFlags.find(want) == autoCpuFlags.end())
+	 {
+	    if (_config->FindB("Debug::Acquire::Variants", false))
+	       std::clog << "variant auto: " << var.name << ": missing: " << want << "\n";
+	    res = false;
+	 }
+      return res;
+   };
+   for (std::string line; std::getline(table, line);)
+   {
+      if (line[0] == '#' || line[0] == '\0')
+	 continue;
+      auto cols = APT::String::Split(line);
+      ArchitectureVariant var{cols[0], cols[1], VectorizeString(cols[2], ',')};
+      // Skip disabled architectures.
+      if (std::find(architectures.begin(), architectures.end(), var.base) == architectures.end())
+      {
+	 continue;
+      }
+      // See if this variant is configured or we detect it as auto
+      bool enabled = std::find(configuredVariants.begin(), configuredVariants.end(), cols[0]) != configuredVariants.end() || (autoDetect && isAutoVariant(var));
+      if (enabled)
+	 variants->push_back(std::move(var));
+   }
+
+   if (not autoDetect)
+      for (auto &var : configuredVariants)
+	 if (std::find_if(variants->begin(), variants->end(), [var](auto &v)
+			  { return v.name == var; }) == variants->end())
+	 {
+	    if (var == "auto")
+	       _error->Warning("Invalid value for APT::Architecture-Variants: auto cannot be combined with explicit variants");
+	    else
+	       _error->Warning("Invalid value for APT::Architecture-Variants: %s", var.c_str());
+	 }
+
+   // Build the supplants table
+   for (auto var = variants->begin(); var != variants->end(); ++var)
+   {
+      var->supplants.push_back(var->base);
+      for (auto var2 = var + 1; var2 != variants->end(); ++var2)
+	 if (var2->base == var->base)
+	    var->supplants.push_back(var2->name);
+   }
+
+   // Order variants by preference
+   std::sort(variants->begin(), variants->end(), [&](auto &a, auto &b)
+	     { return std::find(configuredVariants.begin(), configuredVariants.end(), a.name) < std::find(configuredVariants.begin(), configuredVariants.end(), b.name); });
+   if (_config->FindB("Debug::Acquire::Variants", false))
+   {
+      std::clog << "Variants enabled:";
+      for (auto &var : *variants)
+	 std::clog << " " << var.name;
+      std::clog << std::endl;
+   }
+
+   return *variants;
+}
+									/*}}}*/
 // getArchitectures - Return Vector of preferred Architectures		/*{{{*/
 std::vector<std::string> const Configuration::getArchitectures(bool const &Cached) {
 	std::vector<std::string> static archs;
diff --git a/apt-pkg/aptconfiguration.h b/apt-pkg/aptconfiguration.h
index 58c925b7f..c1979f3cc 100644
--- a/apt-pkg/aptconfiguration.h
+++ b/apt-pkg/aptconfiguration.h
@@ -131,6 +131,15 @@ namespace Configuration {							/*{{{*/
 	/** \return Check usr is merged or produce error. */
 	APT_PUBLIC bool checkUsrMerged();
 	APT_PUBLIC std::string color(std::string const &colorName, std::string const &content = "");
+
+	struct ArchitectureVariant
+	{
+	   std::string name;
+	   std::string base;
+	   std::vector<std::string> cpuflags;
+	   std::vector<std::string> supplants{};
+	};
+	APT_HIDDEN std::vector<ArchitectureVariant> getArchitectureVariants(bool cached);
 #endif
 
 	/*}}}*/
diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h
index 45e45cc83..cc2179bf7 100644
--- a/apt-pkg/cacheiterators.h
+++ b/apt-pkg/cacheiterators.h
@@ -281,6 +281,16 @@ class APT_PUBLIC pkgCache::VerIterator : public Iterator<Version, VerIterator> {
 	{
 	   return (static_cast<Version::Extra *>(Owner->Map.Data()) + S->d)->PhasedUpdatePercentage;
 	}
+	inline void ArchVariant(map_stringitem_t variant) const
+	{
+	   (static_cast<Version::Extra *>(Owner->Map.Data()) + S->d)->ArchVariant = variant;
+	}
+	inline std::string_view ArchVariant() const
+	{
+	   if (auto I = (static_cast<Version::Extra *>(Owner->Map.Data()) + S->d)->ArchVariant)
+	      return Owner->ViewString(I);
+	   return {};
+	}
 	inline bool PhasedUpdatePercentage(unsigned int percentage)
 	{
 	   if (percentage > 100)
diff --git a/apt-pkg/contrib/strutl.cc b/apt-pkg/contrib/strutl.cc
index 651c985bd..b5200831a 100644
--- a/apt-pkg/contrib/strutl.cc
+++ b/apt-pkg/contrib/strutl.cc
@@ -138,6 +138,17 @@ size_t DisplayLength(string_view str)
    return len;
 }
 
+// Splits by whitespace. There may be continuous spans of whitespace - they
+// will be considered as one.
+std::vector<std::string> Split(std::string const &s)
+{
+   std::vector<std::string> vec;
+   std::istringstream iss(s);
+   iss.imbue(std::locale::classic());
+   for (std::string current_s; iss >> current_s;)
+      vec.push_back(current_s);
+   return vec;
+}
 }
 }
 									/*}}}*/
diff --git a/apt-pkg/contrib/strutl.h b/apt-pkg/contrib/strutl.h
index 79b7c8bfa..c22b2dda2 100644
--- a/apt-pkg/contrib/strutl.h
+++ b/apt-pkg/contrib/strutl.h
@@ -48,6 +48,7 @@ namespace APT {
       APT_PUBLIC std::string Join(std::vector<std::string> list, const std::string_view &sep);
       // Returns string display length honoring multi-byte characters
       APT_PUBLIC size_t DisplayLength(std::string_view str);
+      APT_HIDDEN std::vector<std::string> Split(std::string const &str);
    }
 }
 
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index b62c1a84f..b98198430 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -215,6 +215,9 @@ bool debListParser::NewVersion(pkgCache::VerIterator &Ver)
    Ver->NextInSource = G->VersionsInSource;
    G->VersionsInSource = Ver.MapPointer();
 
+   if (auto ArchVar = Section.Find(pkgTagSection::Key::Architecture_Variant); not ArchVar.empty())
+      Ver.ArchVariant(StoreString(pkgCacheGenerator::MIXED, ArchVar));
+
    Ver->MultiArch = ParseMultiArch(true);
    // Archive Size
    Ver->Size = Section.FindULL(pkgTagSection::Key::Size);
@@ -1017,6 +1020,9 @@ bool debListParser::SameVersion(uint32_t Hash,		/*{{{*/
    unsigned char MultiArch = ParseMultiArch(false);
    if (MultiArch != Ver->MultiArch)
       return false;
+
+   if (ArchVariant() != Ver.ArchVariant())
+      return false;
    // for all practical proposes (we can check): same version
    return true;
 }
diff --git a/apt-pkg/deb/deblistparser.h b/apt-pkg/deb/deblistparser.h
index 2a9108775..7ab6800c6 100644
--- a/apt-pkg/deb/deblistparser.h
+++ b/apt-pkg/deb/deblistparser.h
@@ -99,6 +99,10 @@ class APT_HIDDEN debListParser : public pkgCacheListParser
    {
       return Section.Find(pkgTagSection::Key::SHA256);
    }
+   std::string_view ArchVariant() const
+   {
+      return Section.Find(pkgTagSection::Key::Architecture_Variant);
+   }
 #endif
 };
 
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index c040b76b9..b85163654 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -1068,6 +1068,11 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 	auto veryforeign = _config->FindVector("APT::BarbarianArchitectures");
 	Values.reserve(Values.size() + veryforeign.size());
 	std::move(veryforeign.begin(), veryforeign.end(), std::back_inserter(Values));
+	// Let's treat architecture variants as architectures for sources.list parsing
+	auto const &variants = APT::Configuration::getArchitectureVariants(true);
+	Values.reserve(Values.size() + variants.size());
+	for (auto const &var : variants)
+	   Values.push_back(var.name);
      }
       // all is a very special architecture users shouldn't be concerned with explicitly
       // but if the user does, do not override the choice
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index 61d45e37e..531cd258d 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -38,19 +38,6 @@ namespace APT {
    APT_HIDDEN extern std::unordered_map<std::string, std::vector<std::string>> ArchToTupleMap;
 }
 
-// Splits by whitespace. There may be continuous spans of whitespace - they
-// will be considered as one.
-static std::vector<std::string> split(std::string const & s)
-{
-   std::vector<std::string> vec;
-   std::istringstream iss(s);
-   iss.imbue(std::locale::classic());
-   for(std::string current_s; iss >> current_s; )
-      vec.push_back(current_s);
-   return vec;
-}
-
-
 // pkgInitArchTupleMap - Initialize the architecture tuple map				/*{{{*/
 // ---------------------------------------------------------------------
 /* This initializes */
@@ -67,7 +54,7 @@ static bool pkgInitArchTupleMap()
    {
       if (cpuline[0] == '#' || cpuline[0] == '\0')
          continue;
-      auto cpurow = split(cpuline);
+      auto cpurow = APT::String::Split(cpuline);
       auto cpu = APT::String::Strip(cpurow.at(0));
 
       cpus.emplace_back(cpu);
@@ -90,7 +77,7 @@ static bool pkgInitArchTupleMap()
       if (tupleline[0] == '#' || tupleline[0] == '\0')
          continue;
 
-      std::vector<std::string> tuplerow = split(tupleline);
+      std::vector<std::string> tuplerow = APT::String::Split(tupleline);
 
       auto tuple = APT::String::Strip(tuplerow.at(0));
       auto arch = APT::String::Strip(tuplerow.at(1));
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index 4101fb4e5..3377facac 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -193,6 +193,17 @@ bool pkgCache::ReMap(bool const &Errorchecks)
          list.append(",");
       list.append(arch);
    }
+   auto const variants = APT::Configuration::getArchitectureVariants(true);
+   if (not variants.empty())
+   {
+      list.append(";");
+      for (auto const &variant : variants)
+      {
+	 if (list.back() != ';')
+	    list.append(",");
+	 list.append(variant.name);
+      }
+   }
    if (_config->Find("APT::Architecture") != StrP + HeaderP->Architecture ||
 	 list != StrP + HeaderP->GetArchitectures())
       return _error->Error(_("The package cache was built for different architectures: %s vs %s"), StrP + HeaderP->GetArchitectures(), list.c_str());
diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h
index 7a9db3d20..042d928e1 100644
--- a/apt-pkg/pkgcache.h
+++ b/apt-pkg/pkgcache.h
@@ -724,6 +724,7 @@ struct pkgCache::Version
 struct pkgCache::Version::Extra
 {
    uint8_t PhasedUpdatePercentage;
+   map_stringitem_t ArchVariant;
 };
 #endif
 									/*}}}*/
diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc
index 3358441ee..f3c3c10a0 100644
--- a/apt-pkg/pkgcachegen.cc
+++ b/apt-pkg/pkgcachegen.cc
@@ -104,12 +104,23 @@ bool pkgCacheGenerator::Start()
       map_stringitem_t idxArchitectures;
 
       std::vector<std::string> archs = APT::Configuration::getArchitectures();
-      if (archs.size() > 1)
+      auto const &variants = APT::Configuration::getArchitectureVariants(true);
+      if (archs.size() > 1 || not variants.empty())
       {
 	 std::vector<std::string>::const_iterator a = archs.begin();
 	 std::string list = *a;
 	 for (++a; a != archs.end(); ++a)
 	    list.append(",").append(*a);
+	 if (not variants.empty())
+	 {
+	    list.append(";");
+	    for (auto const &variant : variants)
+	    {
+	       if (list.back() != ';')
+		  list.append(",");
+	       list.append(variant.name);
+	    }
+	 }
 	 idxArchitectures = WriteStringInMap(list);
 	 if (unlikely(idxArchitectures == 0))
 	    return false;
@@ -383,11 +394,20 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
 
    auto Hash = List.VersionHash();
    std::string_view ListSHA256;
+   std::string_view ListArchVariant;
 
    bool const Debug = _config->FindB("Debug::pkgCacheGen", false);
    auto DebList = dynamic_cast<debListParser *>(&List);
    if (DebList != nullptr)
       ListSHA256 = DebList->SHA256();
+   if (DebList != nullptr)
+      ListArchVariant = DebList->ArchVariant();
+
+   auto variants = _config->FindVector("APT::Architecture-Variants");
+   // Always add "no variant" at the back of the list. If it's already configured earlier, that wins
+   if (variants.empty() || variants.back() != "")
+      variants.push_back("");
+
    if (Ver.end() == false)
    {
       /* We know the list is sorted so we use that fact in the search.
@@ -426,6 +446,13 @@ bool pkgCacheGenerator::MergeListVersion(ListParser &List, pkgCache::PkgIterator
 	       if (VF.end() == true)
 		  break;
 	    }
+
+	    // Variants are ordered by their index in the variant list.
+	    if (std::find(variants.begin(), variants.end(), Ver.ArchVariant()) > std::find(variants.begin(), variants.end(), ListArchVariant))
+	    {
+	       Res = 1;
+	       break;
+	    }
 	 }
 	 // proceed with the next till we have either the right
 	 // or we found another version (which will be lower)
diff --git a/apt-pkg/tagfile-keys.list b/apt-pkg/tagfile-keys.list
index d198ea0a5..8f5407354 100644
--- a/apt-pkg/tagfile-keys.list
+++ b/apt-pkg/tagfile-keys.list
@@ -81,3 +81,4 @@ Vcs-Svn
 Version
 ### APPEND BELOW, sort in with next ABI break ###
 Source-Version
+Architecture-Variant
diff --git a/apt-pkg/tagfile-order.c b/apt-pkg/tagfile-order.c
index 7cf188c51..dbd45d6ac 100644
--- a/apt-pkg/tagfile-order.c
+++ b/apt-pkg/tagfile-order.c
@@ -9,6 +9,7 @@ static const char *iTFRewritePackageOrder[] = {
    "Package",
    "Package-Type",
    "Architecture",
+   "Architecture-Variant",
    "Subarchitecture", // NO_KEY: Used only by d-i
    "Version",
 // "Revision",         // Obsolete (warning in dpkg)
@@ -71,6 +72,7 @@ static const char *iTFRewriteSourceOrder[] = {
    "Format",
    "Binary",
    "Architecture",
+   "Architecture-Variant",
    "Version",
    "Priority",
 // "Class", // Obsolete alias for Priority, warning by dpkg