apt: Introduce the modernize-sources command

This handles rewriting .list entries to .sources ones; the
only options handled are

- signed-by => Signed-By
- trusted => Trusted

The options to specify architectures, check-valid-until, etc.
are not supported as they are not exposed in the code. We
probably should refactor the parser at some point so we can
actually read the parsed entry rather than the abstract objects
we create from it.

This can be improved further:

- Keyrings from /etc/apt/trusted.gpg.d should be moved to
  /etc/apt/keyrings.
- automatic detection by trying to verify using each keyring
  in trusted.gpg.d until we find the right one and then using
  that for Signed-By rather than leave some sources empty.

5 files changed, 315 insertions, 0 deletions

diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h
index 198c3f34a..e6c8b7c64 100644
--- a/apt-pkg/metaindex.h
+++ b/apt-pkg/metaindex.h
@@ -121,6 +121,10 @@ public:
    virtual bool IsArchitectureSupported(std::string const &arch) const;
    virtual bool IsArchitectureAllSupportedFor(IndexTarget const &target) const;
    virtual bool HasSupportForComponent(std::string const &component) const;
+
+#ifdef APT_COMPILING_APT
+   bool IsTrustedSet() { return Trusted == TRI_YES; }
+#endif
 };
 
 #endif
diff --git a/apt-private/private-sources.cc b/apt-private/private-sources.cc
index a7d003f08..84d82af33 100644
--- a/apt-private/private-sources.cc
+++ b/apt-private/private-sources.cc
@@ -6,6 +6,7 @@
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/hashes.h>
+#include <apt-pkg/metaindex.h>
 #include <apt-pkg/sourcelist.h>
 #include <apt-pkg/strutl.h>
 
@@ -15,6 +16,7 @@
 
 #include <cstddef>
 #include <iostream>
+#include <regex>
 #include <string>
 #include <sys/stat.h>
 #include <sys/types.h>
@@ -22,6 +24,8 @@
 
 #include <apti18n.h>
 
+using std::operator""sv;
+
 /* Interface discussion with donkult (for the future):
   apt [add-{archive,release,component}|edit|change-release|disable]-sources
  and be clever and work out stuff from the Release file
@@ -103,3 +107,223 @@ bool EditSources(CommandLine &CmdL)
    return res;
 }
 									/*}}}*/
+
+template <typename T, typename V>
+static auto contains(T const &container, V const &value) -> bool
+{
+   return std::find(container.begin(), container.end(), value) != container.end();
+}
+static auto merge(std::vector<std::string> &a, std::vector<std::string> const &b)
+{
+   for (auto const &e : b)
+      if (not contains(a, e))
+	 a.push_back(e);
+}
+
+static bool Modernize(std::string const &filename) /*{{{*/
+{
+   auto isMain = filename == _config->FindFile("Dir::Etc::SourceList");
+   auto simulate = _config->FindB("APT::Get::Simulate");
+
+   std::cerr << "Modernizing " << filename << "...\n";
+   pkgSourceList list;
+   if (not list.Read(filename))
+      return false;
+
+   struct Entry
+   {
+      std::set<std::string> types;
+      std::vector<std::string> uris;
+      std::vector<std::string> suites;
+      std::vector<std::string> components;
+      std::string signedBy;
+      std::map<std::string, std::string> options;
+      std::string origin;
+      bool merged = false;
+
+      auto Merge(Entry &other) -> bool
+      {
+	 auto howManyDifferent = (types != other.types) + (uris != other.uris) + (suites != other.suites) + (components != other.components);
+	 if (howManyDifferent > 1)
+	    return false;
+	 if (options != other.options)
+	    return false;
+	 if (origin != other.origin)
+	    return false;
+	 if (signedBy.empty() && not other.signedBy.empty() && (uris != other.uris || suites != other.suites))
+	    return false;
+	 if (not signedBy.empty() && not other.signedBy.empty() && signedBy != other.signedBy)
+	    return false;
+
+	 types.insert(other.types.begin(), other.types.end());
+	 merge(uris, other.uris);
+	 merge(suites, other.suites);
+	 merge(components, other.components);
+	 if (signedBy.empty())
+	    signedBy = other.signedBy;
+
+	 other.merged = true;
+	 return true;
+      }
+   };
+   std::vector<Entry> entries;
+   for (auto const &meta : list)
+   {
+      Entry e;
+
+      e.uris.push_back(meta->GetURI());
+      e.suites.push_back(meta->GetDist());
+      for (auto const &t : meta->GetIndexTargets())
+      {
+	 e.types.insert(t.Option(IndexTarget::TARGET_OF));
+	 if (not contains(e.components, t.Option(IndexTarget::COMPONENT)))
+	    e.components.push_back(t.Option(IndexTarget::COMPONENT));
+      }
+
+      if (meta->IsTrustedSet())
+	 e.options["Trusted"] = "yes";
+
+      std::string err;
+      e.signedBy = meta->GetSignedBy();
+      meta->Load(&err);
+      if (e.signedBy.empty() && not meta->GetOrigin().empty())
+      {
+	 std::string dir = _config->FindDir("Dir") + std::string{"usr/share/keyrings/"};
+	 std::string keyring = meta->GetOrigin() + "-archive-keyring.gpg";
+	 std::transform(keyring.begin(), keyring.end(), keyring.begin(), tolower);
+	 if (FileExists(dir + keyring))
+	    e.signedBy = dir + keyring;
+      }
+      if (auto k = _config->FindDir("Dir::Etc::trustedparts") + flNotDir(std::regex_replace(filename, std::regex("\\.list$"), ".gpg")); FileExists(k))
+	 e.signedBy = k;
+      if (auto k = _config->FindDir("Dir::Etc::trustedparts") + flNotDir(std::regex_replace(filename, std::regex("\\.list$"), ".asc")); FileExists(k))
+	 e.signedBy = k;
+
+      if (isMain && not meta->GetOrigin().empty())
+      {
+	 constexpr auto bad = "\\|{}[]<>\"^~_=!@#$%^&*"sv;
+	 e.origin = meta->GetOrigin();
+	 std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), tolower);
+	 std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), [](char c) -> char
+			{ return isspace(c) ? '-' : c; });
+	 std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), [bad](char c) -> char
+			{ return bad.find(c) != bad.npos ? '-' : c; });
+	 std::replace(e.origin.begin(), e.origin.end(), '/', '-');
+      }
+
+      entries.push_back(std::move(e));
+   }
+
+   for (bool merged = false; merged;)
+   {
+      merged = false;
+      for (auto it = entries.begin(); it != entries.end(); ++it)
+      {
+	 for (auto it2 = it + 1; it2 != entries.end(); ++it2)
+	    if (not it2->merged)
+	       merged |= it->Merge(*it2);
+      }
+   }
+
+   std::map<std::string, std::ofstream> streams;
+   for (auto const &e : entries)
+   {
+      std::string outname;
+      if (not isMain)
+	 outname = std::regex_replace(filename, std::regex("\\.list$"), ".sources");
+      else if (e.origin.empty())
+	 outname = _config->FindDir("Dir::Etc::SourceParts") + "moved-from-main.sources";
+      else
+	 outname = _config->FindDir("Dir::Etc::SourceParts") + (e.origin) + ".sources";
+
+      if (auto it = streams.find(outname); it == streams.end())
+      {
+	 if (not simulate)
+	    std::cerr << "- Writing " << outname << "\n";
+	 streams[outname].open(simulate ? "/dev/stdout" : outname, std::ios::app);
+      }
+      auto &out = streams[outname];
+      if (not out)
+	 _error->Warning("Cannot open %s for writing.", outname.c_str());
+      if (e.merged)
+	 continue;
+
+      if (out.tellp() != 0)
+	 out << "\n";
+
+      if (simulate)
+	 out << "# Would write to: " << outname << "\n";
+      if (isMain)
+	 out << "# Modernized from " << filename << "\n";
+      out << "Types:";
+      for (auto const &t : e.types)
+	 out << " " << t;
+      out << "\n";
+      out << "URIs: " << APT::String::Join(e.uris, " ") << "\n";
+      out << "Suites: " << APT::String::Join(e.suites, " ") << "\n";
+      out << "Components: " << APT::String::Join(e.components, " ") << "\n";
+      out << "Signed-By: " << e.signedBy << "\n";
+      for (auto const &[key, value] : e.options)
+	 out << key << ": " << value << "\n";
+      if (e.signedBy.empty())
+	 _error->Warning("Could not determine Signed-By for URIs: %s, Suites: %s", APT::String::Join(e.uris, " ").c_str(), APT::String::Join(e.suites, " ").c_str());
+   }
+
+   if (not simulate && rename(filename.c_str(), (filename + ".bak").c_str()) != 0)
+      _error->WarningE("rename", "Could not rename %s", filename.c_str());
+
+   _error->DumpErrors();
+   std::cerr << "\n";
+   return true;
+}
+									/*}}}*/
+bool ModernizeSources(CommandLine &) /*{{{*/
+{
+   auto main = _config->FindFile("Dir::Etc::SourceList");
+   auto parts = _config->FindDir("Dir::Etc::SourceParts");
+   std::vector<std::string> files;
+   if (FileExists(main))
+      files.push_back(main);
+   for (auto const &I : GetListOfFilesInDir(parts, std::vector<std::string>{"list", "sources"}, true))
+      if (APT::String::Endswith(I, ".list"))
+	 files.push_back(I);
+   if (files.empty())
+   {
+      std::cout << "All sources are modern.\n";
+      return true;
+   }
+
+   // TRANSLATOR: "No" answer printed for a yes/no question if --assume-no is set
+   auto no = _("N");
+
+   std::cout << "The following files need modernizing:\n";
+   for (auto const &I : files)
+      std::cout << "  - " << I << "\n";
+   std::cout << "\n"
+             << "Modernizing will replace .list files with the new .sources format,\n"
+             << "add Signed-By values where they can be determined automatically,\n"
+             << "and save the old files into .list.bak files.\n"
+             << "\n"
+             << "This command supports the 'signed-by' and 'trusted' options. If you\n"
+             << "have specified other options inside [] brackets, please transfer them\n"
+             << "manually to the output files; see sources.list(5) for a mapping.\n"
+             << "\n"
+             << "For a simulation, respond " << no << " in the following prompt.\n";
+
+   std::string prompt;
+   strprintf(prompt, _("Rewrite %zu sources?"), files.size());
+   if (YnPrompt(prompt.c_str(), true) == false)
+   {
+      _config->Set("APT::Get::Simulate", true);
+      std::cout << "Simulating only..." << std::endl;
+      // Make it visible.
+      usleep(100 * 1000);
+   }
+
+   bool good = true;
+   for (auto const &I : files)
+      good = good && Modernize(I);
+
+   return good;
+}
+									/*}}}*/
diff --git a/apt-private/private-sources.h b/apt-private/private-sources.h
index 0c421902e..0f1e89af2 100644
--- a/apt-private/private-sources.h
+++ b/apt-private/private-sources.h
@@ -6,5 +6,6 @@
 class CommandLine;
 
 APT_PUBLIC bool EditSources(CommandLine &CmdL);
+APT_PUBLIC bool ModernizeSources(CommandLine &CmdL);
 
 #endif
diff --git a/cmdline/apt.cc b/cmdline/apt.cc
index ef58bcebf..af8a95fa5 100644
--- a/cmdline/apt.cc
+++ b/cmdline/apt.cc
@@ -78,6 +78,7 @@ static std::vector<aptDispatchWithHelp> GetCommands()			/*{{{*/
 
       // misc
       {"edit-sources", &EditSources, _("edit the source information file")},
+      {"modernize-sources", &ModernizeSources, _("modernize .list files to .sources files")},
       {"moo", &DoMoo, nullptr},
       {"satisfy", &DoBuildDep, _("satisfy dependency strings")},
 
diff --git a/test/integration/test-apt-modernize-sources b/test/integration/test-apt-modernize-sources
new file mode 100755
index 000000000..26fd8055a
--- /dev/null
+++ b/test/integration/test-apt-modernize-sources
@@ -0,0 +1,85 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture 'native'
+
+echo 'deb http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list.d/rocks.list
+echo 'deb http://deb.debian.org/debian stable main' >> rootdir/etc/apt/sources.list
+echo 'deb-src http://deb.debian.org/debian stable main' >> rootdir/etc/apt/sources.list
+echo 'deb-src http://deb.debian.org/debian unstable main' >> rootdir/etc/apt/sources.list
+echo 'deb http://example.org/debian stable bananas' >> rootdir/etc/apt/sources.list
+echo 'deb [trusted=yes] http://example.org/debian trusted bananas' >> rootdir/etc/apt/sources.list
+
+
+mkdir -p  rootdir/usr/share/keyrings/
+mkdir -p rootdir/var/lib/apt/lists
+echo > rootdir/etc/apt/trusted.gpg.d/rocks.gpg
+echo > rootdir/usr/share/keyrings/debian-archive-keyring.gpg
+echo "Date: $(date -ud '-2 weeks' '+%a, %d %b %Y %H:%M:%S %Z')" > rootdir/var/lib/apt/lists/deb.debian.org_debian_dists_unstable_InRelease
+echo "Date: $(date -ud '-2 weeks' '+%a, %d %b %Y %H:%M:%S %Z')" > rootdir/var/lib/apt/lists/deb.debian.org_debian_dists_stable_InRelease
+echo "Origin: Debian" >> rootdir/var/lib/apt/lists/deb.debian.org_debian_dists_unstable_InRelease
+echo "Origin: Debian" >> rootdir/var/lib/apt/lists/deb.debian.org_debian_dists_stable_InRelease
+
+# We test that appending produces valid output here, so we already configure a source in here.
+echo "Types: deb" >> rootdir/etc/apt/sources.list.d/debian.sources
+echo "URIs: https://deb.debian.org/debian/" >> rootdir/etc/apt/sources.list.d/debian.sources
+echo "Suites: experimental" >> rootdir/etc/apt/sources.list.d/debian.sources
+echo "Components: contrib" >> rootdir/etc/apt/sources.list.d/debian.sources
+echo "Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg" >> rootdir/etc/apt/sources.list.d/debian.sources
+
+testwarning apt modernize-sources -y
+
+testsuccessequal "rootdir/etc/apt/sources.list.bak
+
+rootdir/etc/apt/sources.list.d:
+debian.sources
+moved-from-main.sources
+rocks.list.bak
+rocks.sources" ls -1 rootdir/etc/apt/sources.list*
+
+testfileequal rootdir/etc/apt/sources.list.d/debian.sources "Types: deb
+URIs: https://deb.debian.org/debian/
+Suites: experimental
+Components: contrib
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
+
+# Modernized from ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list
+Types: deb deb-src
+URIs: http://deb.debian.org/debian/
+Suites: stable
+Components: main
+Signed-By: ${TMPWORKINGDIRECTORY}/rootdir/usr/share/keyrings/debian-archive-keyring.gpg
+
+# Modernized from ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list
+Types: deb-src
+URIs: http://deb.debian.org/debian/
+Suites: unstable
+Components: main
+Signed-By: ${TMPWORKINGDIRECTORY}/rootdir/usr/share/keyrings/debian-archive-keyring.gpg"
+
+testfileequal rootdir/etc/apt/sources.list.d/rocks.sources "Types: deb
+URIs: http://example.org/debian/
+Suites: stable
+Components: rocks
+Signed-By: ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/trusted.gpg.d/rocks.gpg"
+
+testfileequal rootdir/etc/apt/sources.list.d/moved-from-main.sources "# Modernized from ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list
+Types: deb
+URIs: http://example.org/debian/
+Suites: stable
+Components: bananas
+Signed-By: 
+
+# Modernized from ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list
+Types: deb
+URIs: http://example.org/debian/
+Suites: trusted
+Components: bananas
+Signed-By: 
+Trusted: yes"
+
+testsuccessequal "All sources are modern." apt modernize-sources