Suggest using non-free-firmware in update for Debian

In an ideal world everyone would read release notes, but if the last
sources.list change is any indication a lot of people wont. This is
even more a problem in so far as apt isn't producing errors for
invalid repositories, but instead carries on as normal even through it
will not be able to install upgrades for the moved packages.

This commit implements two scenarios and prints a notice in those cases
pointing to the release notes:
a) User has 'non-free' but not 'non-free-firmware'
b) User has a firmware package which isn't available from anywhere

Both only happen if we are talking about a repository which identifies
itself as one of Debian and is for a release codenamed bookworm (or
sid). Note that as (usually) apt/oldstable is used to upgrade to the
new stable release these suggestions only show for users after they
have upgraded to bookworm on apt command line usage after that.

3 files changed, 193 insertions, 50 deletions

diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc
index affae655d..f1734fea2 100644
--- a/apt-private/private-update.cc
+++ b/apt-private/private-update.cc
@@ -21,13 +21,32 @@
 
 #include <ostream>
 #include <string>
+#include <tuple>
 
 #include <apti18n.h>
 									/*}}}*/
 
 // DoUpdate - Update the package lists					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
+static bool isDebianBookwormRelease(pkgCache::RlsFileIterator const &RlsFile)
+{
+   std::tuple<std::string_view, std::string_view, std::string_view> const affected[] = {
+      {"Debian", "Debian", "bookworm"},
+      {"Debian", "Debian", "sid"},
+   };
+   if (RlsFile.end() || RlsFile->Origin == nullptr || RlsFile->Label == nullptr || RlsFile->Codename == nullptr)
+      return false;
+   std::tuple<std::string_view, std::string_view, std::string_view> const release{RlsFile.Origin(), RlsFile.Label(), RlsFile.Codename()};
+   return std::find(std::begin(affected), std::end(affected), release) != std::end(affected);
+}
+static void suggestDebianNonFreeFirmware(char const *const repo, char const *const val,
+					 char const *const from, char const *const to)
+{
+   // Both messages are reused from the ReleaseInfoChange feature in acquire-item.cc
+   _error->Notice(_("Repository '%s' changed its '%s' value from '%s' to '%s'"), repo, val, from, to);
+   std::string notes;
+   strprintf(notes, "https://www.debian.org/releases/bookworm/%s/release-notes/ch-information.html#non-free-split", _config->Find("APT::Architecture").c_str());
+   _error->Notice(_("More information about this can be found online in the Release notes at: %s"), notes.c_str());
+}
 bool DoUpdate(CommandLine &CmdL)
 {
    if (CmdL.FileSize() != 1)
@@ -81,44 +100,135 @@ bool DoUpdate(CommandLine &CmdL)
    if (Cache.BuildCaches(false) == false)
       return false;
 
-   if (_config->FindB("APT::Get::Update::SourceListWarnings", true))
-   {
+   bool const SLWarnings = _config->FindB("APT::Get::Update::SourceListWarnings", true);
+   if (SLWarnings)
       List = Cache.GetSourceList();
-      for (pkgSourceList::const_iterator S = List->begin(); S != List->end(); ++S)
-      {
-	 if (APT::String::Startswith((*S)->GetURI(), "ftp://") == false)
-	    continue;
-	 pkgCache::RlsFileIterator const RlsFile = (*S)->FindInCache(Cache, false);
-	 if (RlsFile.end() || RlsFile->Origin == 0 || RlsFile->Label == 0)
-	    continue;
-	 char const *const affected[][2] = {
-	     {"Debian", "Debian"},
-	     {"Debian", "Debian-Security"},
-	     {"Debian Backports", "Debian Backports"},
-	 };
-	 auto const matchRelease = [&](decltype(affected[0]) a) {
-	    return strcmp(RlsFile.Origin(), a[0]) == 0 && strcmp(RlsFile.Label(), a[1]) == 0;
-	 };
-	 if (std::find_if(std::begin(affected), std::end(affected), matchRelease) != std::end(affected))
-	    _error->Warning("Debian shuts down public FTP services currently still used in your sources.list(5) as '%s'.\n"
-			    "See press release %s for details.",
-			    (*S)->GetURI().c_str(), "https://debian.org/News/2017/20170425");
-      }
-      for (pkgSourceList::const_iterator S = List->begin(); S != List->end(); ++S)
+
+   if (_config->FindB("APT::Get::Update::SourceListWarnings::APTAuth", SLWarnings))
+   {
+      constexpr std::string_view const affected_method[] = {"http", "https", "tor+http", "tor+https", "ftp"};
+      for (auto *S : *List)
       {
-	 URI uri((*S)->GetURI());
+	 URI uri(S->GetURI());
 	 if (uri.User.empty() && uri.Password.empty())
 	    continue;
 	 // we can't really predict if a +http method supports everything http does,
 	 // so we play it safe and use an allowlist here.
-	 char const *const affected[] = {"http", "https", "tor+http", "tor+https", "ftp"};
-	 if (std::find(std::begin(affected), std::end(affected), uri.Access) != std::end(affected))
+	 if (std::find(std::begin(affected_method), std::end(affected_method), uri.Access) != std::end(affected_method))
 	    // TRANSLATOR: the first two are manpage references, the last the URI from a sources.list
 	    _error->Notice(_("Usage of %s should be preferred over embedding login information directly in the %s entry for '%s'"),
 			   "apt_auth.conf(5)", "sources.list(5)", URI::ArchiveOnly(uri).c_str());
       }
    }
 
+   if (_config->FindB("APT::Get::Update::SourceListWarnings::NonFreeFirmware", SLWarnings))
+   {
+      // If a Debian source has a non-free component, suggest adding non-free-firmware
+      bool found_affected_release = false;
+      bool found_non_free = false;
+      bool found_non_free_firmware = false;
+      for (auto *S : *List)
+      {
+	 if (not isDebianBookwormRelease(S->FindInCache(Cache, false)))
+	    continue;
+
+	 for (auto PkgFile = Cache.GetPkgCache()->FileBegin(); not PkgFile.end(); ++PkgFile)
+	 {
+	    if (PkgFile.Flagged(pkgCache::Flag::NoPackages))
+	       continue;
+	    found_affected_release = true;
+	    const auto * const comp = PkgFile.Component();
+	    if (comp == nullptr)
+	      continue;
+	    if (strcmp(comp, "non-free") == 0)
+	       found_non_free = true;
+	    else if (strcmp(comp, "non-free-firmware") == 0)
+	    {
+	       found_non_free_firmware = true;
+	       break;
+	    }
+	 }
+	 if (found_non_free_firmware)
+	    break;
+      }
+      if (not found_non_free_firmware && found_non_free && found_affected_release)
+      {
+	 /* See if a well-known firmware package is installable from this codename
+	    if so, we likely operate with new apt on an old snapshot not supporting non-free-firmware */
+	 bool suggest_non_free_firmware = true;
+	 if (auto const Grp = Cache.GetPkgCache()->FindGrp("firmware-linux-nonfree"); not Grp.end())
+	 {
+	    for (auto Pkg = Grp.PackageList(); not Pkg.end() && suggest_non_free_firmware; Pkg = Grp.NextPkg(Pkg))
+	    {
+	       for (auto Ver = Pkg.VersionList(); not Ver.end(); ++Ver)
+	       {
+		  if (not Ver.Downloadable())
+		     continue;
+		  for (auto VerFile = Ver.FileList(); not VerFile.end(); ++VerFile)
+		  {
+		     auto const PkgFile = VerFile.File();
+		     if (PkgFile.end())
+			continue;
+		     if (not isDebianBookwormRelease(PkgFile.ReleaseFile()))
+			continue;
+		     suggest_non_free_firmware = false;
+		     break;
+		  }
+		  if (not suggest_non_free_firmware)
+		     break;
+	       }
+	    }
+	 }
+	 if (suggest_non_free_firmware)
+	    suggestDebianNonFreeFirmware("Debian bookworm", "non-free component", "non-free", "non-free non-free-firmware");
+      }
+
+      if (not found_non_free_firmware && not found_non_free && found_affected_release)
+      {
+	 /* Try to notify users who have installed firmware packages at some point, but
+	    have not enabled non-free currently – they might want to opt into updates now */
+	 APT::StringView const affected_pkgs[] = {
+	    "amd64-microcode", "atmel-firmware", "bluez-firmware", "dahdi-firmware-nonfree",
+	    "firmware-amd-graphics", "firmware-ast", "firmware-atheros", "firmware-bnx2",
+	    "firmware-bnx2x", "firmware-brcm80211", "firmware-cavium", "firmware-intel-sound",
+	    "firmware-intelwimax", "firmware-ipw2x00", "firmware-ivtv", "firmware-iwlwifi",
+	    "firmware-libertas", "firmware-linux", "firmware-linux-nonfree", "firmware-misc-nonfree",
+	    "firmware-myricom", "firmware-netronome", "firmware-netxen", "firmware-qcom-media",
+	    "firmware-qcom-soc", "firmware-qlogic", "firmware-realtek", "firmware-realtek-rtl8723cs-bt",
+	    "firmware-samsung", "firmware-siano", "firmware-sof-signed", "firmware-ti-connectivity",
+	    "firmware-zd1211", "intel-microcode", "midisport-firmware", "raspi-firmware",
+	 };
+	 bool suggest_non_free_firmware = false;
+	 for (auto pkgname : affected_pkgs)
+	 {
+	    auto const Grp = Cache.GetPkgCache()->FindGrp(pkgname);
+	    if (Grp.end())
+	       continue;
+	    for (auto Pkg = Grp.PackageList(); not Pkg.end(); Pkg = Grp.NextPkg(Pkg))
+	    {
+	       auto const Ver = Pkg.CurrentVer();
+	       if (Ver.end() || Ver.Downloadable())
+		  continue;
+	       bool another = false;
+	       for (auto V = Pkg.VersionList(); not V.end(); ++V)
+		  if (V.Downloadable())
+		  {
+		     another = true;
+		     break;
+		  }
+	       if (another)
+		  continue;
+	       suggest_non_free_firmware = true;
+	       break;
+	    }
+	    if (suggest_non_free_firmware)
+	       break;
+	 }
+	 if (suggest_non_free_firmware)
+	    suggestDebianNonFreeFirmware("Debian bookworm", "firmware component", "non-free", "non-free-firmware");
+      }
+   }
+
    // show basic stats (if the user whishes)
    if (_config->FindB("APT::Cmd::Show-Update-Stats", false) == true)
    {
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index b3deccaaf..beafbbcd4 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -123,7 +123,11 @@ APT
      Update
      {
         InteractiveReleaseInfoChanges "<BOOL>";
-        SourceListWarnings "<BOOL>";
+        SourceListWarnings "<BOOL>"
+        {
+           APTAuth "<BOOL>";
+           NonFreeFirmware "<BOOL>";
+        };
      };
   };
 
diff --git a/test/integration/test-apt-get-update-sourceslist-warning b/test/integration/test-apt-get-update-sourceslist-warning
index a99356b8b..60c8d7308 100755
--- a/test/integration/test-apt-get-update-sourceslist-warning
+++ b/test/integration/test-apt-get-update-sourceslist-warning
@@ -11,35 +11,64 @@ setupaptarchive --no-update
 testsuccess apt update
 testsuccess apt update --no-download
 
-echo 'deb ftp://ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list
-cat > rootdir/var/lib/apt/lists/ftp.tlh.debian.org_debian_dists_zurg_Release <<EOF
+cat > rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_Release <<EOF
 Origin: Debian
 Label: Debian
 Suite: unreleased
-Codename: zurg
-Date: Fri, 14 Jul 2017 11:34:35 +0000
+Codename: bookworm
+Date: Sun, 29 Jan 2023 20:14:10 +0000
 Architectures: amd64
-Components: main
-Description: Debian x.y Zurg - Not Released
+Components: main contrib non-free non-free-firmware
+Description: Debian x.y Bookworm - Not Released
 EOF
-chmod 644 rootdir/var/lib/apt/lists/ftp.tlh.debian.org_debian_dists_zurg_Release
+chmod 644 rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_Release
+touch rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_main_binary-amd64_Packages
+touch rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_contrib_binary-amd64_Packages
+touch rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_non-free_binary-amd64_Packages
+touch rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_non-free-firmware_binary-amd64_Packages
 
-testwarningequal "Reading package lists...
+NOTESURL='https://www.debian.org/releases/bookworm/amd64/release-notes/ch-information.html#non-free-split'
+BOILERPLATE='Reading package lists...
 Building dependency tree...
-All packages are up to date.
-W: Debian shuts down public FTP services currently still used in your sources.list(5) as 'ftp://ftp.tlh.debian.org/debian/'.
-   See press release https://debian.org/News/2017/20170425 for details." apt update --no-download
+All packages are up to date.'
 
+msgmsg 'Do not suggest new non-free-firmware component if no non-free'
+echo 'deb http://example.org/debian bookworm main' > rootdir/etc/apt/sources.list.d/example.list
+testsuccessequal "$BOILERPLATE" apt update --no-download
+echo 'deb-src http://example.org/debian bookworm main non-free' > rootdir/etc/apt/sources.list.d/example.list
+testsuccessequal "$BOILERPLATE" apt update --no-download
 
-echo 'deb http://apt:debian@ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list
-testsuccessequal "Reading package lists...
-Building dependency tree...
-All packages are up to date.
-N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'http://ftp.tlh.debian.org/debian'" apt update --no-download
+msgmsg 'Is non-free-firmware missing?'
+echo 'deb http://example.org/debian bookworm main non-free' > rootdir/etc/apt/sources.list.d/example.list
+cat >> rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_non-free_binary-amd64_Packages <<EOF
+Package: firmware-linux-nonfree
+Architecture: all
+Version: 1
+EOF
+testsuccessequal "$BOILERPLATE" apt update --no-download
+echo -n > rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_non-free_binary-amd64_Packages
+testsuccessequal "$BOILERPLATE
+N: Repository 'Debian bookworm' changed its 'non-free component' value from 'non-free' to 'non-free non-free-firmware'
+N: More information about this can be found online in the Release notes at: $NOTESURL" apt update --no-download
 
+msgmsg 'Component already present'
+echo 'deb http://example.org/debian bookworm non-free non-free-firmware' > rootdir/etc/apt/sources.list.d/example.list
+testsuccessequal "$BOILERPLATE" apt update --no-download
+echo 'deb http://example.org/debian bookworm non-free
+deb http://example.org/debian bookworm non-free-firmware' > rootdir/etc/apt/sources.list.d/example.list
+testsuccessequal "$BOILERPLATE" apt update --no-download
 
-echo 'deb tor+https://apt:debian@ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list
-testsuccessequal "Reading package lists...
-Building dependency tree...
-All packages are up to date.
-N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'tor+https://ftp.tlh.debian.org/debian'" apt update --no-download
+msgmsg 'Detect login info embedded in sources.list'
+echo 'deb http://apt:debian@example.org/debian bookworm main' > rootdir/etc/apt/sources.list.d/example.list
+testsuccessequal "$BOILERPLATE
+N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'http://example.org/debian'" apt update --no-download
+echo 'deb tor+https://apt:debian@example.org/debian bookworm main' > rootdir/etc/apt/sources.list.d/example.list
+testsuccessequal "$BOILERPLATE
+N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'tor+https://example.org/debian'" apt update --no-download
+
+msgmsg 'Firmware packages without upgrades'
+echo 'deb http://example.org/debian bookworm main' > rootdir/etc/apt/sources.list.d/example.list
+insertinstalledpackage 'firmware-linux-nonfree' 'all' '1'
+testsuccessequal "$BOILERPLATE
+N: Repository 'Debian bookworm' changed its 'firmware component' value from 'non-free' to 'non-free-firmware'
+N: More information about this can be found online in the Release notes at: $NOTESURL" apt update --no-download