From 8aeb07448c09375c730c76a6baf31303b129bb96 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 29 Jan 2023 16:54:39 +0100 Subject: Have values in Section config trees refer to them in all components Hard coding each and every component is not only boring but given that everyone is free to add or use more we end up in situations in which apt behaves differently for the same binary package just because metadata said it is in different components (e.g. non-free vs. non-free-firmware). It is also probably not what the casual user would expect. So we instead treat a value without a component as if it applies for all of them. The previous behaviour can be restored by prefixing the value with "/" as in the component is not defined. In an ideal world we would probably use "*/foo" for the new default instead of changing the behaviour for "foo", but it seems rather unlikely that the old behaviour is actually desired. All existing values were duplicated for all (previously) known components in Debian and Ubuntu. --- .../test-apt-move-and-forget-manual-sections | 45 +++++++++++++++++++++- test/integration/test-apt-never-markauto-sections | 3 +- 2 files changed, 46 insertions(+), 2 deletions(-) (limited to 'test/integration') diff --git a/test/integration/test-apt-move-and-forget-manual-sections b/test/integration/test-apt-move-and-forget-manual-sections index 4617abab3..ab90be0c0 100755 --- a/test/integration/test-apt-move-and-forget-manual-sections +++ b/test/integration/test-apt-move-and-forget-manual-sections @@ -11,15 +11,18 @@ testsuccess grep '^oldlibs$' move-autobit.sections buildsimplenativepackage 'libabc' 'native' '1' 'stable' '' '' 'libs' buildsimplenativepackage 'libabc' 'native' '2' 'unstable' 'Depends: libdef' '' 'oldlibs' +buildsimplenativepackage 'libzoo' 'native' '1' 'stable' '' '' 'libs' +buildsimplenativepackage 'libzoo' 'native' '2' 'unstable' 'Depends: libdef' '' 'non-free/oldlibs' buildsimplenativepackage 'libdef' 'native' '1' 'unstable' '' '' 'libs' setupaptarchive testmarkedauto testmarkedmanual +msgmsg 'Move bit on install of replacement' testsuccess aptget install libabc/stable -y testdpkginstalled 'libabc' -testdpkgnotinstalled 'libdef' +testdpkgnotinstalled 'libdef' 'libzoo' testmarkedmanual 'libabc' testmarkedauto @@ -29,3 +32,43 @@ testdpkginstalled 'libabc' 'libdef' testmarkedauto 'libabc' testmarkedmanual 'libdef' + +testsuccess apt autopurge -y +testdpkgnotinstalled 'libabc' 'libzoo' + +msgmsg 'Do not move bit if replacement is already installed' +testsuccess aptget install libzoo/stable -y +testdpkginstalled 'libzoo' + +testmarkedmanual 'libzoo' 'libdef' +testmarkedauto + +testsuccess aptmark auto libdef +testmarkedauto 'libdef' + +testsuccess aptget dist-upgrade -y +testdpkginstalled 'libzoo' 'libdef' + +testmarkedmanual 'libzoo' +testmarkedauto 'libdef' + +testsuccess apt autopurge -y libzoo- +testdpkgnotinstalled 'libabc' 'libzoo' 'libdef' + +msgmsg 'Move bit on install of replacement (subsection)' +testfailure grep '^non-free/oldlibs$' move-autobit.sections +testsuccess aptget install libzoo/stable -y +testdpkginstalled 'libzoo' +testdpkgnotinstalled 'libdef' 'libabc' + +testmarkedmanual 'libzoo' +testmarkedauto + +testsuccess aptget dist-upgrade -y +testdpkginstalled 'libzoo' 'libdef' + +testmarkedauto 'libzoo' +testmarkedmanual 'libdef' + +testsuccess apt autopurge -y +testdpkgnotinstalled 'libabc' 'libzoo' diff --git a/test/integration/test-apt-never-markauto-sections b/test/integration/test-apt-never-markauto-sections index a77d6b22b..b47966e75 100755 --- a/test/integration/test-apt-never-markauto-sections +++ b/test/integration/test-apt-never-markauto-sections @@ -8,12 +8,13 @@ configarchitecture 'amd64' 'i386' aptconfig dump --no-empty --format '%v%n' APT::Never-MarkAuto-Sections > nevermarkauto.sections testsuccess grep '^metapackages$' nevermarkauto.sections +testfailure grep '^universe/metapackages$' nevermarkauto.sections buildsimplenativepackage 'mydesktop' 'all' '1' 'unstable' 'Depends: mydesktop-core, foreignpkg Recommends: notavailable' '' 'metapackages' buildsimplenativepackage 'mydesktop-core' 'amd64' '1' 'unstable' 'Depends: bad-texteditor | texteditor, browser (>= 42), nosection, foreignpkg Recommends: notavailable -Multi-Arch: foreign' '' 'metapackages' +Multi-Arch: foreign' '' 'universe/metapackages' buildsimplenativepackage 'browser' 'amd64' '41' 'stable' buildsimplenativepackage 'browser' 'amd64' '42' 'unstable' buildsimplenativepackage 'texteditor' 'amd64' '1' 'stable' -- cgit v1.2.3-70-g09d2 From 9712edf6151308148518058bfbd5ccd937509143 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 29 Jan 2023 23:24:43 +0100 Subject: Suggest using non-free-firmware in update for Debian In an ideal world everyone would read release notes, but if the last sources.list change is any indication a lot of people wont. This is even more a problem in so far as apt isn't producing errors for invalid repositories, but instead carries on as normal even through it will not be able to install upgrades for the moved packages. This commit implements two scenarios and prints a notice in those cases pointing to the release notes: a) User has 'non-free' but not 'non-free-firmware' b) User has a firmware package which isn't available from anywhere Both only happen if we are talking about a repository which identifies itself as one of Debian and is for a release codenamed bookworm (or sid). Note that as (usually) apt/oldstable is used to upgrade to the new stable release these suggestions only show for users after they have upgraded to bookworm on apt command line usage after that. --- apt-private/private-update.cc | 166 +++++++++++++++++---- doc/examples/configure-index | 6 +- .../test-apt-get-update-sourceslist-warning | 71 ++++++--- 3 files changed, 193 insertions(+), 50 deletions(-) (limited to 'test/integration') diff --git a/apt-private/private-update.cc b/apt-private/private-update.cc index affae655d..f1734fea2 100644 --- a/apt-private/private-update.cc +++ b/apt-private/private-update.cc @@ -21,13 +21,32 @@ #include #include +#include #include /*}}}*/ // DoUpdate - Update the package lists /*{{{*/ -// --------------------------------------------------------------------- -/* */ +static bool isDebianBookwormRelease(pkgCache::RlsFileIterator const &RlsFile) +{ + std::tuple const affected[] = { + {"Debian", "Debian", "bookworm"}, + {"Debian", "Debian", "sid"}, + }; + if (RlsFile.end() || RlsFile->Origin == nullptr || RlsFile->Label == nullptr || RlsFile->Codename == nullptr) + return false; + std::tuple const release{RlsFile.Origin(), RlsFile.Label(), RlsFile.Codename()}; + return std::find(std::begin(affected), std::end(affected), release) != std::end(affected); +} +static void suggestDebianNonFreeFirmware(char const *const repo, char const *const val, + char const *const from, char const *const to) +{ + // Both messages are reused from the ReleaseInfoChange feature in acquire-item.cc + _error->Notice(_("Repository '%s' changed its '%s' value from '%s' to '%s'"), repo, val, from, to); + std::string notes; + strprintf(notes, "https://www.debian.org/releases/bookworm/%s/release-notes/ch-information.html#non-free-split", _config->Find("APT::Architecture").c_str()); + _error->Notice(_("More information about this can be found online in the Release notes at: %s"), notes.c_str()); +} bool DoUpdate(CommandLine &CmdL) { if (CmdL.FileSize() != 1) @@ -81,44 +100,135 @@ bool DoUpdate(CommandLine &CmdL) if (Cache.BuildCaches(false) == false) return false; - if (_config->FindB("APT::Get::Update::SourceListWarnings", true)) - { + bool const SLWarnings = _config->FindB("APT::Get::Update::SourceListWarnings", true); + if (SLWarnings) List = Cache.GetSourceList(); - for (pkgSourceList::const_iterator S = List->begin(); S != List->end(); ++S) - { - if (APT::String::Startswith((*S)->GetURI(), "ftp://") == false) - continue; - pkgCache::RlsFileIterator const RlsFile = (*S)->FindInCache(Cache, false); - if (RlsFile.end() || RlsFile->Origin == 0 || RlsFile->Label == 0) - continue; - char const *const affected[][2] = { - {"Debian", "Debian"}, - {"Debian", "Debian-Security"}, - {"Debian Backports", "Debian Backports"}, - }; - auto const matchRelease = [&](decltype(affected[0]) a) { - return strcmp(RlsFile.Origin(), a[0]) == 0 && strcmp(RlsFile.Label(), a[1]) == 0; - }; - if (std::find_if(std::begin(affected), std::end(affected), matchRelease) != std::end(affected)) - _error->Warning("Debian shuts down public FTP services currently still used in your sources.list(5) as '%s'.\n" - "See press release %s for details.", - (*S)->GetURI().c_str(), "https://debian.org/News/2017/20170425"); - } - for (pkgSourceList::const_iterator S = List->begin(); S != List->end(); ++S) + + if (_config->FindB("APT::Get::Update::SourceListWarnings::APTAuth", SLWarnings)) + { + constexpr std::string_view const affected_method[] = {"http", "https", "tor+http", "tor+https", "ftp"}; + for (auto *S : *List) { - URI uri((*S)->GetURI()); + URI uri(S->GetURI()); if (uri.User.empty() && uri.Password.empty()) continue; // we can't really predict if a +http method supports everything http does, // so we play it safe and use an allowlist here. - char const *const affected[] = {"http", "https", "tor+http", "tor+https", "ftp"}; - if (std::find(std::begin(affected), std::end(affected), uri.Access) != std::end(affected)) + if (std::find(std::begin(affected_method), std::end(affected_method), uri.Access) != std::end(affected_method)) // TRANSLATOR: the first two are manpage references, the last the URI from a sources.list _error->Notice(_("Usage of %s should be preferred over embedding login information directly in the %s entry for '%s'"), "apt_auth.conf(5)", "sources.list(5)", URI::ArchiveOnly(uri).c_str()); } } + if (_config->FindB("APT::Get::Update::SourceListWarnings::NonFreeFirmware", SLWarnings)) + { + // If a Debian source has a non-free component, suggest adding non-free-firmware + bool found_affected_release = false; + bool found_non_free = false; + bool found_non_free_firmware = false; + for (auto *S : *List) + { + if (not isDebianBookwormRelease(S->FindInCache(Cache, false))) + continue; + + for (auto PkgFile = Cache.GetPkgCache()->FileBegin(); not PkgFile.end(); ++PkgFile) + { + if (PkgFile.Flagged(pkgCache::Flag::NoPackages)) + continue; + found_affected_release = true; + const auto * const comp = PkgFile.Component(); + if (comp == nullptr) + continue; + if (strcmp(comp, "non-free") == 0) + found_non_free = true; + else if (strcmp(comp, "non-free-firmware") == 0) + { + found_non_free_firmware = true; + break; + } + } + if (found_non_free_firmware) + break; + } + if (not found_non_free_firmware && found_non_free && found_affected_release) + { + /* See if a well-known firmware package is installable from this codename + if so, we likely operate with new apt on an old snapshot not supporting non-free-firmware */ + bool suggest_non_free_firmware = true; + if (auto const Grp = Cache.GetPkgCache()->FindGrp("firmware-linux-nonfree"); not Grp.end()) + { + for (auto Pkg = Grp.PackageList(); not Pkg.end() && suggest_non_free_firmware; Pkg = Grp.NextPkg(Pkg)) + { + for (auto Ver = Pkg.VersionList(); not Ver.end(); ++Ver) + { + if (not Ver.Downloadable()) + continue; + for (auto VerFile = Ver.FileList(); not VerFile.end(); ++VerFile) + { + auto const PkgFile = VerFile.File(); + if (PkgFile.end()) + continue; + if (not isDebianBookwormRelease(PkgFile.ReleaseFile())) + continue; + suggest_non_free_firmware = false; + break; + } + if (not suggest_non_free_firmware) + break; + } + } + } + if (suggest_non_free_firmware) + suggestDebianNonFreeFirmware("Debian bookworm", "non-free component", "non-free", "non-free non-free-firmware"); + } + + if (not found_non_free_firmware && not found_non_free && found_affected_release) + { + /* Try to notify users who have installed firmware packages at some point, but + have not enabled non-free currently – they might want to opt into updates now */ + APT::StringView const affected_pkgs[] = { + "amd64-microcode", "atmel-firmware", "bluez-firmware", "dahdi-firmware-nonfree", + "firmware-amd-graphics", "firmware-ast", "firmware-atheros", "firmware-bnx2", + "firmware-bnx2x", "firmware-brcm80211", "firmware-cavium", "firmware-intel-sound", + "firmware-intelwimax", "firmware-ipw2x00", "firmware-ivtv", "firmware-iwlwifi", + "firmware-libertas", "firmware-linux", "firmware-linux-nonfree", "firmware-misc-nonfree", + "firmware-myricom", "firmware-netronome", "firmware-netxen", "firmware-qcom-media", + "firmware-qcom-soc", "firmware-qlogic", "firmware-realtek", "firmware-realtek-rtl8723cs-bt", + "firmware-samsung", "firmware-siano", "firmware-sof-signed", "firmware-ti-connectivity", + "firmware-zd1211", "intel-microcode", "midisport-firmware", "raspi-firmware", + }; + bool suggest_non_free_firmware = false; + for (auto pkgname : affected_pkgs) + { + auto const Grp = Cache.GetPkgCache()->FindGrp(pkgname); + if (Grp.end()) + continue; + for (auto Pkg = Grp.PackageList(); not Pkg.end(); Pkg = Grp.NextPkg(Pkg)) + { + auto const Ver = Pkg.CurrentVer(); + if (Ver.end() || Ver.Downloadable()) + continue; + bool another = false; + for (auto V = Pkg.VersionList(); not V.end(); ++V) + if (V.Downloadable()) + { + another = true; + break; + } + if (another) + continue; + suggest_non_free_firmware = true; + break; + } + if (suggest_non_free_firmware) + break; + } + if (suggest_non_free_firmware) + suggestDebianNonFreeFirmware("Debian bookworm", "firmware component", "non-free", "non-free-firmware"); + } + } + // show basic stats (if the user whishes) if (_config->FindB("APT::Cmd::Show-Update-Stats", false) == true) { diff --git a/doc/examples/configure-index b/doc/examples/configure-index index b3deccaaf..beafbbcd4 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -123,7 +123,11 @@ APT Update { InteractiveReleaseInfoChanges ""; - SourceListWarnings ""; + SourceListWarnings "" + { + APTAuth ""; + NonFreeFirmware ""; + }; }; }; diff --git a/test/integration/test-apt-get-update-sourceslist-warning b/test/integration/test-apt-get-update-sourceslist-warning index a99356b8b..60c8d7308 100755 --- a/test/integration/test-apt-get-update-sourceslist-warning +++ b/test/integration/test-apt-get-update-sourceslist-warning @@ -11,35 +11,64 @@ setupaptarchive --no-update testsuccess apt update testsuccess apt update --no-download -echo 'deb ftp://ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list -cat > rootdir/var/lib/apt/lists/ftp.tlh.debian.org_debian_dists_zurg_Release < rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_Release < rootdir/etc/apt/sources.list.d/example.list +testsuccessequal "$BOILERPLATE" apt update --no-download +echo 'deb-src http://example.org/debian bookworm main non-free' > rootdir/etc/apt/sources.list.d/example.list +testsuccessequal "$BOILERPLATE" apt update --no-download -echo 'deb http://apt:debian@ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list -testsuccessequal "Reading package lists... -Building dependency tree... -All packages are up to date. -N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'http://ftp.tlh.debian.org/debian'" apt update --no-download +msgmsg 'Is non-free-firmware missing?' +echo 'deb http://example.org/debian bookworm main non-free' > rootdir/etc/apt/sources.list.d/example.list +cat >> rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_non-free_binary-amd64_Packages < rootdir/var/lib/apt/lists/example.org_debian_dists_bookworm_non-free_binary-amd64_Packages +testsuccessequal "$BOILERPLATE +N: Repository 'Debian bookworm' changed its 'non-free component' value from 'non-free' to 'non-free non-free-firmware' +N: More information about this can be found online in the Release notes at: $NOTESURL" apt update --no-download +msgmsg 'Component already present' +echo 'deb http://example.org/debian bookworm non-free non-free-firmware' > rootdir/etc/apt/sources.list.d/example.list +testsuccessequal "$BOILERPLATE" apt update --no-download +echo 'deb http://example.org/debian bookworm non-free +deb http://example.org/debian bookworm non-free-firmware' > rootdir/etc/apt/sources.list.d/example.list +testsuccessequal "$BOILERPLATE" apt update --no-download -echo 'deb tor+https://apt:debian@ftp.tlh.debian.org/debian zurg main' > rootdir/etc/apt/sources.list.d/ftpshutdown.list -testsuccessequal "Reading package lists... -Building dependency tree... -All packages are up to date. -N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'tor+https://ftp.tlh.debian.org/debian'" apt update --no-download +msgmsg 'Detect login info embedded in sources.list' +echo 'deb http://apt:debian@example.org/debian bookworm main' > rootdir/etc/apt/sources.list.d/example.list +testsuccessequal "$BOILERPLATE +N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'http://example.org/debian'" apt update --no-download +echo 'deb tor+https://apt:debian@example.org/debian bookworm main' > rootdir/etc/apt/sources.list.d/example.list +testsuccessequal "$BOILERPLATE +N: Usage of apt_auth.conf(5) should be preferred over embedding login information directly in the sources.list(5) entry for 'tor+https://example.org/debian'" apt update --no-download + +msgmsg 'Firmware packages without upgrades' +echo 'deb http://example.org/debian bookworm main' > rootdir/etc/apt/sources.list.d/example.list +insertinstalledpackage 'firmware-linux-nonfree' 'all' '1' +testsuccessequal "$BOILERPLATE +N: Repository 'Debian bookworm' changed its 'firmware component' value from 'non-free' to 'non-free-firmware' +N: More information about this can be found online in the Release notes at: $NOTESURL" apt update --no-download -- cgit v1.2.3-70-g09d2