From b6274188312e6338e94edc45dbf9cc410d650477 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Mon, 19 Feb 2024 12:07:55 +0100 Subject: Automatically enable snapshots where supported Convert sources.list Snapshot option from opt-in to automatic. If we can find a snapshot server, Snapshot: yes is assumed if a snapshot is specified. On the implementation side, we record automatic snapshot enablement by adding a '?' suffix to the snapshot timestamp, if any is specified, this avoids introducing bugs into the code where we could end up with an empty snapshot. This has an annoying internal implementation caveat: Since we call GetDebReleaseIndexBy() with the SHADOWED option emplaced, if we do not find a server, we need to remove the SHADOWED option again, but we already have inserted a shadowed release index into the list. This will simply insert the release index a second time without the SHADOWED option which in preliminary testing works fine, but it would arguably be more correct to also remove the release index again if we have created it. FIXME: This only has one test case: A source with supported snapshot server is auto-discovered. We should also add a test case where we cannot detect a server and then don't fail in automatic mode. --- apt-pkg/deb/debmetaindex.cc | 17 ++++++++++++++++- test/integration/test-snapshot | 7 ++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 0f05cb21a..3bc7f78de 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -1299,7 +1300,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ std::string const &Dist, std::string const &Section, bool const &IsSrc, std::map Options) const { - auto Snapshot = GetSnapshotOption(Options, "snapshot"); + std::string SnapshotAptConf = _config->Find("APT::Snapshot"); + std::string Snapshot = GetSnapshotOption(Options, "snapshot", SnapshotAptConf.empty() ? "" : SnapshotAptConf + "?"); if (not Snapshot.empty()) { std::map SnapshotOptions = Options; @@ -1339,15 +1341,28 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ } if (Server.empty() || Server == "no") { + if (APT::String::Endswith(Snapshot, "?")) + { + Options.erase("SHADOWED"); + goto nosnapshot; + } if (Server != "no" && filename.empty()) return _error->Error("Cannot identify snapshot server for %s %s - run update without snapshot id first", URI.c_str(), Dist.c_str()); return _error->Error("Snapshots not supported for %s %s", URI.c_str(), Dist.c_str()); } + // We have found a server by now, so we enable snapshots for this source. + if (APT::String::Endswith(Snapshot, "?")) + { + Snapshot.pop_back(); + } + + assert(not Snapshot.empty()); auto SnapshotURI = SubstVar(SubstVar(Server, "@SNAPSHOTID@", Snapshot), "@PATH@", ArchiveURI.Path); if (not CreateItemInternalOne(List, SnapshotURI, Dist, Section, IsSrc, SnapshotOptions)) return false; } + nosnapshot: if (not CreateItemInternalOne(List, URI, Dist, Section, IsSrc, Options)) return false; diff --git a/test/integration/test-snapshot b/test/integration/test-snapshot index 2bf7d70d4..f7d7e6a05 100755 --- a/test/integration/test-snapshot +++ b/test/integration/test-snapshot @@ -24,7 +24,12 @@ testsuccessequal "'http://localhost:${APTHTTPPORT}/dists/stable/InRelease' local 'http://localhost:${APTHTTPPORT}/dists/stable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_stable_main_binary-all_Packages 0 'http://localhost:${APTHTTPPORT}/dists/stable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris -testsuccessequal "'http://localhost:${APTHTTPPORT}/dists/stable/InRelease' localhost:${APTHTTPPORT}_dists_stable_InRelease 0 +testsuccessequal "'https://snapshot.debian.org/archive/debian/OPTIONNOTSET/dists/stable/InRelease' snapshot.debian.org_archive_debian_OPTIONNOTSET_dists_stable_InRelease 0 +'https://snapshot.debian.org/archive/debian/OPTIONNOTSET/dists/stable/main/source/Sources.xz' snapshot.debian.org_archive_debian_OPTIONNOTSET_dists_stable_main_source_Sources 0 +'https://snapshot.debian.org/archive/debian/OPTIONNOTSET/dists/stable/main/binary-amd64/Packages.xz' snapshot.debian.org_archive_debian_OPTIONNOTSET_dists_stable_main_binary-amd64_Packages 0 +'https://snapshot.debian.org/archive/debian/OPTIONNOTSET/dists/stable/main/binary-all/Packages.xz' snapshot.debian.org_archive_debian_OPTIONNOTSET_dists_stable_main_binary-all_Packages 0 +'https://snapshot.debian.org/archive/debian/OPTIONNOTSET/dists/stable/main/i18n/Translation-en.xz' snapshot.debian.org_archive_debian_OPTIONNOTSET_dists_stable_main_i18n_Translation-en 0 +'http://localhost:${APTHTTPPORT}/dists/stable/InRelease' localhost:${APTHTTPPORT}_dists_stable_InRelease 0 'http://localhost:${APTHTTPPORT}/dists/stable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_stable_main_source_Sources 0 'http://localhost:${APTHTTPPORT}/dists/stable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_stable_main_binary-amd64_Packages 0 'http://localhost:${APTHTTPPORT}/dists/stable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_stable_main_binary-all_Packages 0 -- cgit v1.2.3-70-g09d2 From e2949fc463f9e087978b072c82b11860ee02fdb6 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 20 Feb 2024 09:56:06 +0100 Subject: Delete SHADOWED metaIndex if we don't actually use snapshots This adds a bit more code but avoids any surprises later on by having both the shadowed and non-shadowed meta index in the list. Gbp-Dch: ignore --- apt-pkg/deb/debmetaindex.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 3bc7f78de..515893133 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -1315,6 +1315,7 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ ArchiveURI.Path.erase(0, 1); std::string Server; + auto const PreviousDeb = List.empty() ? nullptr : List.back(); auto const Deb = GetDebReleaseIndexBy(List, URI, Dist, Options); std::string filename; @@ -1343,7 +1344,13 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type /*{{{*/ { if (APT::String::Endswith(Snapshot, "?")) { + // Erase the SHADOWED option and remove the release index from the list if we created it. Options.erase("SHADOWED"); + if (Deb && Deb != PreviousDeb) { + assert(List.back() == Deb); + List.pop_back(); + delete Deb; + } goto nosnapshot; } if (Server != "no" && filename.empty()) -- cgit v1.2.3-70-g09d2 From 1f67cf94ec2df0d30349f033c777495b80b41484 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 20 Feb 2024 19:04:03 +0100 Subject: test-snapshot: Fix a test case This was accidentally using testfailure instead of testfailureequal, hence trying to run the output string as a command :( --- test/integration/test-snapshot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test-snapshot b/test/integration/test-snapshot index f7d7e6a05..7c7725ba3 100755 --- a/test/integration/test-snapshot +++ b/test/integration/test-snapshot @@ -249,7 +249,7 @@ testsuccessequal "'https://snapshot.ubuntu.com/ubuntu/BANANA/dists/mantic/InRele 'http://archive.ubuntu.com/ubuntu/dists/mantic/main/binary-all/Packages.xz' archive.ubuntu.com_ubuntu_dists_mantic_main_binary-all_Packages 0 'http://archive.ubuntu.com/ubuntu/dists/mantic/main/i18n/Translation-en.xz' archive.ubuntu.com_ubuntu_dists_mantic_main_i18n_Translation-en 0 " aptget update --print-uris -o Dir::Etc::SourceList=archive.list -o Dir::Etc::SourceParts=/dev/null -o "Acquire::Snapshots::URI::Host::.archive.ubuntu.com"="https://cc.snapshot.ubuntu.com/@PATH@/@SNAPSHOTID@/" -testfailure "E: Snapshots not supported for http://archive.ubuntu.com/ubuntu/ mantic +testfailureequal "E: Snapshots not supported for http://archive.ubuntu.com/ubuntu/ mantic E: The list of sources could not be read." aptget update --print-uris -o Dir::Etc::SourceList=archive.list -o Dir::Etc::SourceParts=/dev/null -o "Acquire::Snapshots::URI::Host::archive.ubuntu.com"="no" msgmsg "Testing host based-seeds: PPA" -- cgit v1.2.3-70-g09d2 From 78c3ad795de7f14de3ee22e9f04fe24b5f21f0ea Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Tue, 20 Feb 2024 19:04:36 +0100 Subject: test-snapshot: Add test case for automatic snapshot 1. repository not supporting snapshots, implicit Enabled 2. repository not supporting snapshots, Enabled: yes 3. URL-based lookup, implicit Enabled --- test/integration/test-snapshot | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/test/integration/test-snapshot b/test/integration/test-snapshot index 7c7725ba3..26ef5a721 100755 --- a/test/integration/test-snapshot +++ b/test/integration/test-snapshot @@ -13,6 +13,13 @@ buildsimplenativepackage 'libbar' 'all' '1.0' 'stable' getlabelfromsuite() { echo 'Testcases'; } getoriginfromsuite() { echo 'Debian'; } +releasechanger() { + # modifying the Release files in lists… bad stuff. Good that this is only a test… + msgmsg "Changing $1 to $2" + sed -i "s#^${1}: .*#${1}: ${2}#" $(find rootdir/var/lib/apt/lists -name '*Release') + rm -f rootdir/var/cache/apt/*.bin +} + setupaptarchive --no-update changetowebserver @@ -35,9 +42,24 @@ testsuccessequal "'https://snapshot.debian.org/archive/debian/OPTIONNOTSET/dists 'http://localhost:${APTHTTPPORT}/dists/stable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_stable_main_binary-all_Packages 0 'http://localhost:${APTHTTPPORT}/dists/stable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris --snapshot OPTIONNOTSET +# repository does not support snapshots: we succeed here by silently ignoring the snapshot for the repository because it's not forced-on +releasechanger 'Origin' 'NoSnapshotsKnown' +testsuccessequal "'http://localhost:${APTHTTPPORT}/dists/stable/InRelease' localhost:${APTHTTPPORT}_dists_stable_InRelease 0 +'http://localhost:${APTHTTPPORT}/dists/stable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_stable_main_source_Sources 0 +'http://localhost:${APTHTTPPORT}/dists/stable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_stable_main_binary-amd64_Packages 0 +'http://localhost:${APTHTTPPORT}/dists/stable/main/binary-all/Packages.xz' localhost:${APTHTTPPORT}_dists_stable_main_binary-all_Packages 0 +'http://localhost:${APTHTTPPORT}/dists/stable/main/i18n/Translation-en.xz' localhost:${APTHTTPPORT}_dists_stable_main_i18n_Translation-en 0 " aptget update --print-uris --snapshot OPTION-NOT-SET-AND-RELEASE-SAYS-NO + + # Enable the snapshot feature sed -i 's/http:/[snapshot=enable] http:/' rootdir/etc/apt/sources.list.d/* +# repository does not support snapshots: we fail here because now, we actually forced the feature enabled +testfailuremsg "E: Snapshots not supported for http://localhost:${APTHTTPPORT}/ stable +E: Snapshots not supported for http://localhost:${APTHTTPPORT}/ stable +E: The list of sources could not be read." aptget update --print-uris -S OPTION-SET-AND-RELEASE-SAYS-NO + +releasechanger 'Origin' 'Debian' testsuccessequal "'http://localhost:${APTHTTPPORT}/dists/stable/InRelease' localhost:${APTHTTPPORT}_dists_stable_InRelease 0 'http://localhost:${APTHTTPPORT}/dists/stable/main/source/Sources.xz' localhost:${APTHTTPPORT}_dists_stable_main_source_Sources 0 'http://localhost:${APTHTTPPORT}/dists/stable/main/binary-amd64/Packages.xz' localhost:${APTHTTPPORT}_dists_stable_main_binary-amd64_Packages 0 @@ -100,12 +122,6 @@ testsuccessequal "'http://localhost:${APTHTTPPORT}/snapshot/BANANA/pool/awesome_ aptget install --print-uris -qq awesome -o Acquire::Snapshots::URI::Label::Testcases="http://localhost:${APTHTTPPORT}/snapshot/@SNAPSHOTID@/" -SBANANA -releasechanger() { - # modifying the Release files in lists… bad stuff. Good that this is only a test… - sed -i "s#^${1}: .*#${1}: ${2}#" $(find rootdir/var/lib/apt/lists -name '*Release') - rm -f rootdir/var/cache/apt/*.bin -} - msgmsg "Origin: Ubuntu" releasechanger 'Origin' 'Ubuntu' testsuccessequal "'https://snapshot.ubuntu.com/ubuntu/BANANA/dists/stable/InRelease' snapshot.ubuntu.com_ubuntu_BANANA_dists_stable_InRelease 0 @@ -202,6 +218,16 @@ rm rootdir/etc/apt/apt.conf.d/changelog.conf msgmsg "Testing host-based seed: Ubuntu" +echo "deb http://security.ubuntu.com/ubuntu/ mantic-security main" > rootdir/etc/apt/security.list +testsuccessequal "'https://snapshot.ubuntu.com/ubuntu/BANANA/dists/mantic-security/InRelease' snapshot.ubuntu.com_ubuntu_BANANA_dists_mantic-security_InRelease 0 +'https://snapshot.ubuntu.com/ubuntu/BANANA/dists/mantic-security/main/binary-amd64/Packages.xz' snapshot.ubuntu.com_ubuntu_BANANA_dists_mantic-security_main_binary-amd64_Packages 0 +'https://snapshot.ubuntu.com/ubuntu/BANANA/dists/mantic-security/main/binary-all/Packages.xz' snapshot.ubuntu.com_ubuntu_BANANA_dists_mantic-security_main_binary-all_Packages 0 +'https://snapshot.ubuntu.com/ubuntu/BANANA/dists/mantic-security/main/i18n/Translation-en.xz' snapshot.ubuntu.com_ubuntu_BANANA_dists_mantic-security_main_i18n_Translation-en 0 +'http://security.ubuntu.com/ubuntu/dists/mantic-security/InRelease' security.ubuntu.com_ubuntu_dists_mantic-security_InRelease 0 +'http://security.ubuntu.com/ubuntu/dists/mantic-security/main/binary-amd64/Packages.xz' security.ubuntu.com_ubuntu_dists_mantic-security_main_binary-amd64_Packages 0 +'http://security.ubuntu.com/ubuntu/dists/mantic-security/main/binary-all/Packages.xz' security.ubuntu.com_ubuntu_dists_mantic-security_main_binary-all_Packages 0 +'http://security.ubuntu.com/ubuntu/dists/mantic-security/main/i18n/Translation-en.xz' security.ubuntu.com_ubuntu_dists_mantic-security_main_i18n_Translation-en 0 " aptget update --print-uris -o Dir::Etc::SourceList=security.list -o Dir::Etc::SourceParts=/dev/null -SBANANA + echo "deb [snapshot=BANANA] http://security.ubuntu.com/ubuntu/ mantic-security main" > rootdir/etc/apt/security.list testsuccessequal "'https://snapshot.ubuntu.com/ubuntu/BANANA/dists/mantic-security/InRelease' snapshot.ubuntu.com_ubuntu_BANANA_dists_mantic-security_InRelease 0 'https://snapshot.ubuntu.com/ubuntu/BANANA/dists/mantic-security/main/binary-amd64/Packages.xz' snapshot.ubuntu.com_ubuntu_BANANA_dists_mantic-security_main_binary-amd64_Packages 0 -- cgit v1.2.3-70-g09d2