From 9d7a660611e9e134cd6d74a594bde59cad77c8d8 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Mon, 27 Jan 2025 15:51:24 +0100 Subject: apt: Introduce the modernize-sources command This handles rewriting .list entries to .sources ones; the only options handled are - signed-by => Signed-By - trusted => Trusted The options to specify architectures, check-valid-until, etc. are not supported as they are not exposed in the code. We probably should refactor the parser at some point so we can actually read the parsed entry rather than the abstract objects we create from it. This can be improved further: - Keyrings from /etc/apt/trusted.gpg.d should be moved to /etc/apt/keyrings. - automatic detection by trying to verify using each keyring in trusted.gpg.d until we find the right one and then using that for Signed-By rather than leave some sources empty. --- apt-private/private-sources.cc | 224 +++++++++++++++++++++++++++++++++++++++++ apt-private/private-sources.h | 1 + 2 files changed, 225 insertions(+) (limited to 'apt-private') diff --git a/apt-private/private-sources.cc b/apt-private/private-sources.cc index a7d003f08..84d82af33 100644 --- a/apt-private/private-sources.cc +++ b/apt-private/private-sources.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include #include +#include #include #include #include @@ -22,6 +24,8 @@ #include +using std::operator""sv; + /* Interface discussion with donkult (for the future): apt [add-{archive,release,component}|edit|change-release|disable]-sources and be clever and work out stuff from the Release file @@ -103,3 +107,223 @@ bool EditSources(CommandLine &CmdL) return res; } /*}}}*/ + +template +static auto contains(T const &container, V const &value) -> bool +{ + return std::find(container.begin(), container.end(), value) != container.end(); +} +static auto merge(std::vector &a, std::vector const &b) +{ + for (auto const &e : b) + if (not contains(a, e)) + a.push_back(e); +} + +static bool Modernize(std::string const &filename) /*{{{*/ +{ + auto isMain = filename == _config->FindFile("Dir::Etc::SourceList"); + auto simulate = _config->FindB("APT::Get::Simulate"); + + std::cerr << "Modernizing " << filename << "...\n"; + pkgSourceList list; + if (not list.Read(filename)) + return false; + + struct Entry + { + std::set types; + std::vector uris; + std::vector suites; + std::vector components; + std::string signedBy; + std::map options; + std::string origin; + bool merged = false; + + auto Merge(Entry &other) -> bool + { + auto howManyDifferent = (types != other.types) + (uris != other.uris) + (suites != other.suites) + (components != other.components); + if (howManyDifferent > 1) + return false; + if (options != other.options) + return false; + if (origin != other.origin) + return false; + if (signedBy.empty() && not other.signedBy.empty() && (uris != other.uris || suites != other.suites)) + return false; + if (not signedBy.empty() && not other.signedBy.empty() && signedBy != other.signedBy) + return false; + + types.insert(other.types.begin(), other.types.end()); + merge(uris, other.uris); + merge(suites, other.suites); + merge(components, other.components); + if (signedBy.empty()) + signedBy = other.signedBy; + + other.merged = true; + return true; + } + }; + std::vector entries; + for (auto const &meta : list) + { + Entry e; + + e.uris.push_back(meta->GetURI()); + e.suites.push_back(meta->GetDist()); + for (auto const &t : meta->GetIndexTargets()) + { + e.types.insert(t.Option(IndexTarget::TARGET_OF)); + if (not contains(e.components, t.Option(IndexTarget::COMPONENT))) + e.components.push_back(t.Option(IndexTarget::COMPONENT)); + } + + if (meta->IsTrustedSet()) + e.options["Trusted"] = "yes"; + + std::string err; + e.signedBy = meta->GetSignedBy(); + meta->Load(&err); + if (e.signedBy.empty() && not meta->GetOrigin().empty()) + { + std::string dir = _config->FindDir("Dir") + std::string{"usr/share/keyrings/"}; + std::string keyring = meta->GetOrigin() + "-archive-keyring.gpg"; + std::transform(keyring.begin(), keyring.end(), keyring.begin(), tolower); + if (FileExists(dir + keyring)) + e.signedBy = dir + keyring; + } + if (auto k = _config->FindDir("Dir::Etc::trustedparts") + flNotDir(std::regex_replace(filename, std::regex("\\.list$"), ".gpg")); FileExists(k)) + e.signedBy = k; + if (auto k = _config->FindDir("Dir::Etc::trustedparts") + flNotDir(std::regex_replace(filename, std::regex("\\.list$"), ".asc")); FileExists(k)) + e.signedBy = k; + + if (isMain && not meta->GetOrigin().empty()) + { + constexpr auto bad = "\\|{}[]<>\"^~_=!@#$%^&*"sv; + e.origin = meta->GetOrigin(); + std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), tolower); + std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), [](char c) -> char + { return isspace(c) ? '-' : c; }); + std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), [bad](char c) -> char + { return bad.find(c) != bad.npos ? '-' : c; }); + std::replace(e.origin.begin(), e.origin.end(), '/', '-'); + } + + entries.push_back(std::move(e)); + } + + for (bool merged = false; merged;) + { + merged = false; + for (auto it = entries.begin(); it != entries.end(); ++it) + { + for (auto it2 = it + 1; it2 != entries.end(); ++it2) + if (not it2->merged) + merged |= it->Merge(*it2); + } + } + + std::map streams; + for (auto const &e : entries) + { + std::string outname; + if (not isMain) + outname = std::regex_replace(filename, std::regex("\\.list$"), ".sources"); + else if (e.origin.empty()) + outname = _config->FindDir("Dir::Etc::SourceParts") + "moved-from-main.sources"; + else + outname = _config->FindDir("Dir::Etc::SourceParts") + (e.origin) + ".sources"; + + if (auto it = streams.find(outname); it == streams.end()) + { + if (not simulate) + std::cerr << "- Writing " << outname << "\n"; + streams[outname].open(simulate ? "/dev/stdout" : outname, std::ios::app); + } + auto &out = streams[outname]; + if (not out) + _error->Warning("Cannot open %s for writing.", outname.c_str()); + if (e.merged) + continue; + + if (out.tellp() != 0) + out << "\n"; + + if (simulate) + out << "# Would write to: " << outname << "\n"; + if (isMain) + out << "# Modernized from " << filename << "\n"; + out << "Types:"; + for (auto const &t : e.types) + out << " " << t; + out << "\n"; + out << "URIs: " << APT::String::Join(e.uris, " ") << "\n"; + out << "Suites: " << APT::String::Join(e.suites, " ") << "\n"; + out << "Components: " << APT::String::Join(e.components, " ") << "\n"; + out << "Signed-By: " << e.signedBy << "\n"; + for (auto const &[key, value] : e.options) + out << key << ": " << value << "\n"; + if (e.signedBy.empty()) + _error->Warning("Could not determine Signed-By for URIs: %s, Suites: %s", APT::String::Join(e.uris, " ").c_str(), APT::String::Join(e.suites, " ").c_str()); + } + + if (not simulate && rename(filename.c_str(), (filename + ".bak").c_str()) != 0) + _error->WarningE("rename", "Could not rename %s", filename.c_str()); + + _error->DumpErrors(); + std::cerr << "\n"; + return true; +} + /*}}}*/ +bool ModernizeSources(CommandLine &) /*{{{*/ +{ + auto main = _config->FindFile("Dir::Etc::SourceList"); + auto parts = _config->FindDir("Dir::Etc::SourceParts"); + std::vector files; + if (FileExists(main)) + files.push_back(main); + for (auto const &I : GetListOfFilesInDir(parts, std::vector{"list", "sources"}, true)) + if (APT::String::Endswith(I, ".list")) + files.push_back(I); + if (files.empty()) + { + std::cout << "All sources are modern.\n"; + return true; + } + + // TRANSLATOR: "No" answer printed for a yes/no question if --assume-no is set + auto no = _("N"); + + std::cout << "The following files need modernizing:\n"; + for (auto const &I : files) + std::cout << " - " << I << "\n"; + std::cout << "\n" + << "Modernizing will replace .list files with the new .sources format,\n" + << "add Signed-By values where they can be determined automatically,\n" + << "and save the old files into .list.bak files.\n" + << "\n" + << "This command supports the 'signed-by' and 'trusted' options. If you\n" + << "have specified other options inside [] brackets, please transfer them\n" + << "manually to the output files; see sources.list(5) for a mapping.\n" + << "\n" + << "For a simulation, respond " << no << " in the following prompt.\n"; + + std::string prompt; + strprintf(prompt, _("Rewrite %zu sources?"), files.size()); + if (YnPrompt(prompt.c_str(), true) == false) + { + _config->Set("APT::Get::Simulate", true); + std::cout << "Simulating only..." << std::endl; + // Make it visible. + usleep(100 * 1000); + } + + bool good = true; + for (auto const &I : files) + good = good && Modernize(I); + + return good; +} + /*}}}*/ diff --git a/apt-private/private-sources.h b/apt-private/private-sources.h index 0c421902e..0f1e89af2 100644 --- a/apt-private/private-sources.h +++ b/apt-private/private-sources.h @@ -6,5 +6,6 @@ class CommandLine; APT_PUBLIC bool EditSources(CommandLine &CmdL); +APT_PUBLIC bool ModernizeSources(CommandLine &CmdL); #endif -- cgit v1.2.3-70-g09d2