diff options
| -rw-r--r-- | apt-pkg/metaindex.h | 4 | ||||
| -rw-r--r-- | apt-private/private-sources.cc | 224 | ||||
| -rw-r--r-- | apt-private/private-sources.h | 1 | ||||
| -rw-r--r-- | cmdline/apt.cc | 1 | ||||
| -rwxr-xr-x | test/integration/test-apt-modernize-sources | 85 |
5 files changed, 315 insertions, 0 deletions
diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h index 198c3f34a..e6c8b7c64 100644 --- a/apt-pkg/metaindex.h +++ b/apt-pkg/metaindex.h @@ -121,6 +121,10 @@ public: virtual bool IsArchitectureSupported(std::string const &arch) const; virtual bool IsArchitectureAllSupportedFor(IndexTarget const &target) const; virtual bool HasSupportForComponent(std::string const &component) const; + +#ifdef APT_COMPILING_APT + bool IsTrustedSet() { return Trusted == TRI_YES; } +#endif }; #endif diff --git a/apt-private/private-sources.cc b/apt-private/private-sources.cc index a7d003f08..84d82af33 100644 --- a/apt-private/private-sources.cc +++ b/apt-private/private-sources.cc @@ -6,6 +6,7 @@ #include <apt-pkg/error.h> #include <apt-pkg/fileutl.h> #include <apt-pkg/hashes.h> +#include <apt-pkg/metaindex.h> #include <apt-pkg/sourcelist.h> #include <apt-pkg/strutl.h> @@ -15,6 +16,7 @@ #include <cstddef> #include <iostream> +#include <regex> #include <string> #include <sys/stat.h> #include <sys/types.h> @@ -22,6 +24,8 @@ #include <apti18n.h> +using std::operator""sv; + /* Interface discussion with donkult (for the future): apt [add-{archive,release,component}|edit|change-release|disable]-sources and be clever and work out stuff from the Release file @@ -103,3 +107,223 @@ bool EditSources(CommandLine &CmdL) return res; } /*}}}*/ + +template <typename T, typename V> +static auto contains(T const &container, V const &value) -> bool +{ + return std::find(container.begin(), container.end(), value) != container.end(); +} +static auto merge(std::vector<std::string> &a, std::vector<std::string> const &b) +{ + for (auto const &e : b) + if (not contains(a, e)) + a.push_back(e); +} + +static bool Modernize(std::string const &filename) /*{{{*/ +{ + auto isMain = filename == _config->FindFile("Dir::Etc::SourceList"); + auto simulate = _config->FindB("APT::Get::Simulate"); + + std::cerr << "Modernizing " << filename << "...\n"; + pkgSourceList list; + if (not list.Read(filename)) + return false; + + struct Entry + { + std::set<std::string> types; + std::vector<std::string> uris; + std::vector<std::string> suites; + std::vector<std::string> components; + std::string signedBy; + std::map<std::string, std::string> options; + std::string origin; + bool merged = false; + + auto Merge(Entry &other) -> bool + { + auto howManyDifferent = (types != other.types) + (uris != other.uris) + (suites != other.suites) + (components != other.components); + if (howManyDifferent > 1) + return false; + if (options != other.options) + return false; + if (origin != other.origin) + return false; + if (signedBy.empty() && not other.signedBy.empty() && (uris != other.uris || suites != other.suites)) + return false; + if (not signedBy.empty() && not other.signedBy.empty() && signedBy != other.signedBy) + return false; + + types.insert(other.types.begin(), other.types.end()); + merge(uris, other.uris); + merge(suites, other.suites); + merge(components, other.components); + if (signedBy.empty()) + signedBy = other.signedBy; + + other.merged = true; + return true; + } + }; + std::vector<Entry> entries; + for (auto const &meta : list) + { + Entry e; + + e.uris.push_back(meta->GetURI()); + e.suites.push_back(meta->GetDist()); + for (auto const &t : meta->GetIndexTargets()) + { + e.types.insert(t.Option(IndexTarget::TARGET_OF)); + if (not contains(e.components, t.Option(IndexTarget::COMPONENT))) + e.components.push_back(t.Option(IndexTarget::COMPONENT)); + } + + if (meta->IsTrustedSet()) + e.options["Trusted"] = "yes"; + + std::string err; + e.signedBy = meta->GetSignedBy(); + meta->Load(&err); + if (e.signedBy.empty() && not meta->GetOrigin().empty()) + { + std::string dir = _config->FindDir("Dir") + std::string{"usr/share/keyrings/"}; + std::string keyring = meta->GetOrigin() + "-archive-keyring.gpg"; + std::transform(keyring.begin(), keyring.end(), keyring.begin(), tolower); + if (FileExists(dir + keyring)) + e.signedBy = dir + keyring; + } + if (auto k = _config->FindDir("Dir::Etc::trustedparts") + flNotDir(std::regex_replace(filename, std::regex("\\.list$"), ".gpg")); FileExists(k)) + e.signedBy = k; + if (auto k = _config->FindDir("Dir::Etc::trustedparts") + flNotDir(std::regex_replace(filename, std::regex("\\.list$"), ".asc")); FileExists(k)) + e.signedBy = k; + + if (isMain && not meta->GetOrigin().empty()) + { + constexpr auto bad = "\\|{}[]<>\"^~_=!@#$%^&*"sv; + e.origin = meta->GetOrigin(); + std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), tolower); + std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), [](char c) -> char + { return isspace(c) ? '-' : c; }); + std::transform(e.origin.begin(), e.origin.end(), e.origin.begin(), [bad](char c) -> char + { return bad.find(c) != bad.npos ? '-' : c; }); + std::replace(e.origin.begin(), e.origin.end(), '/', '-'); + } + + entries.push_back(std::move(e)); + } + + for (bool merged = false; merged;) + { + merged = false; + for (auto it = entries.begin(); it != entries.end(); ++it) + { + for (auto it2 = it + 1; it2 != entries.end(); ++it2) + if (not it2->merged) + merged |= it->Merge(*it2); + } + } + + std::map<std::string, std::ofstream> streams; + for (auto const &e : entries) + { + std::string outname; + if (not isMain) + outname = std::regex_replace(filename, std::regex("\\.list$"), ".sources"); + else if (e.origin.empty()) + outname = _config->FindDir("Dir::Etc::SourceParts") + "moved-from-main.sources"; + else + outname = _config->FindDir("Dir::Etc::SourceParts") + (e.origin) + ".sources"; + + if (auto it = streams.find(outname); it == streams.end()) + { + if (not simulate) + std::cerr << "- Writing " << outname << "\n"; + streams[outname].open(simulate ? "/dev/stdout" : outname, std::ios::app); + } + auto &out = streams[outname]; + if (not out) + _error->Warning("Cannot open %s for writing.", outname.c_str()); + if (e.merged) + continue; + + if (out.tellp() != 0) + out << "\n"; + + if (simulate) + out << "# Would write to: " << outname << "\n"; + if (isMain) + out << "# Modernized from " << filename << "\n"; + out << "Types:"; + for (auto const &t : e.types) + out << " " << t; + out << "\n"; + out << "URIs: " << APT::String::Join(e.uris, " ") << "\n"; + out << "Suites: " << APT::String::Join(e.suites, " ") << "\n"; + out << "Components: " << APT::String::Join(e.components, " ") << "\n"; + out << "Signed-By: " << e.signedBy << "\n"; + for (auto const &[key, value] : e.options) + out << key << ": " << value << "\n"; + if (e.signedBy.empty()) + _error->Warning("Could not determine Signed-By for URIs: %s, Suites: %s", APT::String::Join(e.uris, " ").c_str(), APT::String::Join(e.suites, " ").c_str()); + } + + if (not simulate && rename(filename.c_str(), (filename + ".bak").c_str()) != 0) + _error->WarningE("rename", "Could not rename %s", filename.c_str()); + + _error->DumpErrors(); + std::cerr << "\n"; + return true; +} + /*}}}*/ +bool ModernizeSources(CommandLine &) /*{{{*/ +{ + auto main = _config->FindFile("Dir::Etc::SourceList"); + auto parts = _config->FindDir("Dir::Etc::SourceParts"); + std::vector<std::string> files; + if (FileExists(main)) + files.push_back(main); + for (auto const &I : GetListOfFilesInDir(parts, std::vector<std::string>{"list", "sources"}, true)) + if (APT::String::Endswith(I, ".list")) + files.push_back(I); + if (files.empty()) + { + std::cout << "All sources are modern.\n"; + return true; + } + + // TRANSLATOR: "No" answer printed for a yes/no question if --assume-no is set + auto no = _("N"); + + std::cout << "The following files need modernizing:\n"; + for (auto const &I : files) + std::cout << " - " << I << "\n"; + std::cout << "\n" + << "Modernizing will replace .list files with the new .sources format,\n" + << "add Signed-By values where they can be determined automatically,\n" + << "and save the old files into .list.bak files.\n" + << "\n" + << "This command supports the 'signed-by' and 'trusted' options. If you\n" + << "have specified other options inside [] brackets, please transfer them\n" + << "manually to the output files; see sources.list(5) for a mapping.\n" + << "\n" + << "For a simulation, respond " << no << " in the following prompt.\n"; + + std::string prompt; + strprintf(prompt, _("Rewrite %zu sources?"), files.size()); + if (YnPrompt(prompt.c_str(), true) == false) + { + _config->Set("APT::Get::Simulate", true); + std::cout << "Simulating only..." << std::endl; + // Make it visible. + usleep(100 * 1000); + } + + bool good = true; + for (auto const &I : files) + good = good && Modernize(I); + + return good; +} + /*}}}*/ diff --git a/apt-private/private-sources.h b/apt-private/private-sources.h index 0c421902e..0f1e89af2 100644 --- a/apt-private/private-sources.h +++ b/apt-private/private-sources.h @@ -6,5 +6,6 @@ class CommandLine; APT_PUBLIC bool EditSources(CommandLine &CmdL); +APT_PUBLIC bool ModernizeSources(CommandLine &CmdL); #endif diff --git a/cmdline/apt.cc b/cmdline/apt.cc index ef58bcebf..af8a95fa5 100644 --- a/cmdline/apt.cc +++ b/cmdline/apt.cc @@ -78,6 +78,7 @@ static std::vector<aptDispatchWithHelp> GetCommands() /*{{{*/ // misc {"edit-sources", &EditSources, _("edit the source information file")}, + {"modernize-sources", &ModernizeSources, _("modernize .list files to .sources files")}, {"moo", &DoMoo, nullptr}, {"satisfy", &DoBuildDep, _("satisfy dependency strings")}, diff --git a/test/integration/test-apt-modernize-sources b/test/integration/test-apt-modernize-sources new file mode 100755 index 000000000..26fd8055a --- /dev/null +++ b/test/integration/test-apt-modernize-sources @@ -0,0 +1,85 @@ +#!/bin/sh +set -e + +TESTDIR="$(readlink -f "$(dirname "$0")")" +. "$TESTDIR/framework" + +setupenvironment +configarchitecture 'native' + +echo 'deb http://example.org/debian stable rocks' > rootdir/etc/apt/sources.list.d/rocks.list +echo 'deb http://deb.debian.org/debian stable main' >> rootdir/etc/apt/sources.list +echo 'deb-src http://deb.debian.org/debian stable main' >> rootdir/etc/apt/sources.list +echo 'deb-src http://deb.debian.org/debian unstable main' >> rootdir/etc/apt/sources.list +echo 'deb http://example.org/debian stable bananas' >> rootdir/etc/apt/sources.list +echo 'deb [trusted=yes] http://example.org/debian trusted bananas' >> rootdir/etc/apt/sources.list + + +mkdir -p rootdir/usr/share/keyrings/ +mkdir -p rootdir/var/lib/apt/lists +echo > rootdir/etc/apt/trusted.gpg.d/rocks.gpg +echo > rootdir/usr/share/keyrings/debian-archive-keyring.gpg +echo "Date: $(date -ud '-2 weeks' '+%a, %d %b %Y %H:%M:%S %Z')" > rootdir/var/lib/apt/lists/deb.debian.org_debian_dists_unstable_InRelease +echo "Date: $(date -ud '-2 weeks' '+%a, %d %b %Y %H:%M:%S %Z')" > rootdir/var/lib/apt/lists/deb.debian.org_debian_dists_stable_InRelease +echo "Origin: Debian" >> rootdir/var/lib/apt/lists/deb.debian.org_debian_dists_unstable_InRelease +echo "Origin: Debian" >> rootdir/var/lib/apt/lists/deb.debian.org_debian_dists_stable_InRelease + +# We test that appending produces valid output here, so we already configure a source in here. +echo "Types: deb" >> rootdir/etc/apt/sources.list.d/debian.sources +echo "URIs: https://deb.debian.org/debian/" >> rootdir/etc/apt/sources.list.d/debian.sources +echo "Suites: experimental" >> rootdir/etc/apt/sources.list.d/debian.sources +echo "Components: contrib" >> rootdir/etc/apt/sources.list.d/debian.sources +echo "Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg" >> rootdir/etc/apt/sources.list.d/debian.sources + +testwarning apt modernize-sources -y + +testsuccessequal "rootdir/etc/apt/sources.list.bak + +rootdir/etc/apt/sources.list.d: +debian.sources +moved-from-main.sources +rocks.list.bak +rocks.sources" ls -1 rootdir/etc/apt/sources.list* + +testfileequal rootdir/etc/apt/sources.list.d/debian.sources "Types: deb +URIs: https://deb.debian.org/debian/ +Suites: experimental +Components: contrib +Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg + +# Modernized from ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list +Types: deb deb-src +URIs: http://deb.debian.org/debian/ +Suites: stable +Components: main +Signed-By: ${TMPWORKINGDIRECTORY}/rootdir/usr/share/keyrings/debian-archive-keyring.gpg + +# Modernized from ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list +Types: deb-src +URIs: http://deb.debian.org/debian/ +Suites: unstable +Components: main +Signed-By: ${TMPWORKINGDIRECTORY}/rootdir/usr/share/keyrings/debian-archive-keyring.gpg" + +testfileequal rootdir/etc/apt/sources.list.d/rocks.sources "Types: deb +URIs: http://example.org/debian/ +Suites: stable +Components: rocks +Signed-By: ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/trusted.gpg.d/rocks.gpg" + +testfileequal rootdir/etc/apt/sources.list.d/moved-from-main.sources "# Modernized from ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list +Types: deb +URIs: http://example.org/debian/ +Suites: stable +Components: bananas +Signed-By: + +# Modernized from ${TMPWORKINGDIRECTORY}/rootdir/etc/apt/sources.list +Types: deb +URIs: http://example.org/debian/ +Suites: trusted +Components: bananas +Signed-By: +Trusted: yes" + +testsuccessequal "All sources are modern." apt modernize-sources |