Initial support for snapshot servers, apt --snapshot option

Provide snapshot support for offical Debian and Ubuntu archives.

There are two ways to enable snapshots for sources:

1. Add Snapshot: yes to your sources file ([snapshot=yes]). This
   will allow you to specify a snapshot to use when updating or
   installing using the --snapshot,-S option.

2. Add Snapshot: ID to your sources files to request a specific
   snapshot for this source.

Snapshots are discovered using Label and Origin fields in the Release
file of the main source, hence you need to have updated the source at
least once before you can use snapshots.

The Release file may also declare a snapshots server to use, similar
to Changelogs, it can contain a Snapshots field with the values:

1. `Snapshots: https://example.com/@SNAPSHOTID@` where `@SNAPSHOTID@`
   is a placeholder that is replaced with the requested snapshot id

2. `Snapshots: no` to disable snapshot support for this source.
   Requesting snapshots for this source will result in a failure
   to load the source.

The implementation adds a SHADOWED option to deb source entries,
and marks the main entry as SHADOWED when a snapshot has been
requested, which will cause it to be updated, but not included
in the generated cache.

The concern here was that we need to keep generating the shadowed
entries because the cleanup in `apt update` deletes any files not
queued for download, so we gotta keep downloading the main source.

This design is not entirely optimal, but avoids the pitfalls of
having to reimplement list cleanup.

Gaps:

- Ubuntu Pro repositories and PPAs are not yet supported.

6 files changed, 112 insertions, 4 deletions

diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index c6005f1c8..4133540ab 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -123,6 +123,9 @@ class APT_HIDDEN debReleaseIndexPrivate					/*{{{*/
    time_t DateMaxFuture;
    time_t NotBefore;
 
+   std::string Snapshot;
+   std::string SnapshotsServer;
+
    std::vector<std::string> Architectures;
    std::vector<std::string> NoSupportForAll;
    std::vector<std::string> SupportedComponents;
@@ -479,6 +482,7 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro
    Suite = Section.FindS("Suite");
    Codename = Section.FindS("Codename");
    ReleaseNotes = Section.FindS("Release-Notes");
+   d->SnapshotsServer = Section.FindS("Snapshots");
    {
       std::string const archs = Section.FindS("Architectures");
       if (archs.empty() == false)
@@ -789,6 +793,18 @@ bool debReleaseIndex::SetDateMaxFuture(time_t const DateMaxFuture)
       return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), "Date-Max-Future", URI.c_str(), Dist.c_str());
    return true;
 }
+bool debReleaseIndex::SetSnapshot(std::string const Snapshot)
+{
+   if (d->Snapshot.empty())
+      d->Snapshot = Snapshot;
+   else if (d->Snapshot != Snapshot)
+      return _error->Error(_("Conflicting values set for option %s regarding source %s %s"), "Snapshot", URI.c_str(), Dist.c_str());
+   return true;
+}
+std::string debReleaseIndex::GetSnapshotsServer() const
+{
+   return d->SnapshotsServer;
+}
 bool debReleaseIndex::SetSignedBy(std::string const &pSignedBy)
 {
    if (SignedBy.empty() == true && pSignedBy.empty() == false)
@@ -1123,6 +1139,17 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 	 return defVal;
       return StringToBool(opt->second, defVal);
    }
+   static std::string GetSnapshotOption(std::map<std::string, std::string> const &Options, char const * const name, const std::string defVal="")
+   {
+      std::map<std::string, std::string>::const_iterator const opt = Options.find(name);
+      if (opt == Options.end())
+	 return defVal;
+      int boolVal = StringToBool(opt->second, -1);
+      if (boolVal != -1)
+	 return boolVal ? _config->Find("APT::Snapshot") : "";
+      return opt->second;
+   }
+
 
    static std::vector<std::string> GetMapKeys(std::map<std::string, std::string> const &Options)
    {
@@ -1167,7 +1194,7 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
       return true;
    }
 
-   static debReleaseIndex * GetDebReleaseIndexBy(std::vector<metaIndex *> &List, std::string const &URI,
+   static debReleaseIndex * GetDebReleaseIndexBy(std::vector<metaIndex *> &List, std::string URI,
 			   std::string const &Dist, std::map<std::string, std::string> const &Options)
    {
       std::map<std::string, std::string> ReleaseOptions{{
@@ -1182,6 +1209,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 	 ReleaseOptions.emplace("ALLOW_WEAK", "true");
       if (GetBoolOption(Options, "allow-downgrade-to-insecure", _config->FindB("Acquire::AllowDowngradeToInsecureRepositories")))
 	 ReleaseOptions.emplace("ALLOW_DOWNGRADE_TO_INSECURE", "true");
+      if (GetBoolOption(Options, "SHADOWED", false))
+	 ReleaseOptions.emplace("SHADOWED", "true");
 
       auto InReleasePath = Options.find("inrelease-path");
       if (InReleasePath != Options.end())
@@ -1221,10 +1250,78 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
    }
 
    protected:
+   // This is a duplicate of pkgAcqChangelog::URITemplate()  with some changes to work 
+   // on metaIndex instead of cache structures, and using Snapshots
+   std::string SnapshotServer(debReleaseIndex const *Rls) const
+   {
+      if (Rls->GetLabel().empty() && Rls->GetOrigin().empty())
+	 return "";
+      std::string const serverConfig = "Acquire::Snapshots::URI";
+      std::string server;
+#define APT_EMPTY_SERVER        \
+   if (server.empty() == false) \
+   {                            \
+      if (server != "no")       \
+	 return server;         \
+      return "";                \
+   }
+#define APT_CHECK_SERVER(X, Y)                                                                     \
+   if (not Rls->Get##X().empty())                                                                  \
+   {                                                                                               \
+      std::string const specialServerConfig = serverConfig + "::" + Y + #X + "::" + Rls->Get##X(); \
+      server = _config->Find(specialServerConfig);                                                 \
+      APT_EMPTY_SERVER                                                                             \
+   }
+      // this way e.g. Debian-Security can fallback to Debian
+      APT_CHECK_SERVER(Label, "Override::")
+      APT_CHECK_SERVER(Origin, "Override::")
 
-   bool CreateItemInternal(std::vector<metaIndex *> &List, std::string const &URI,
+      server = Rls->GetSnapshotsServer();
+      APT_EMPTY_SERVER
+
+      APT_CHECK_SERVER(Label, "")
+      APT_CHECK_SERVER(Origin, "")
+#undef APT_CHECK_SERVER
+#undef APT_EMPTY_SERVER
+      return "";
+   }
+   bool CreateItemInternal(std::vector<metaIndex *> &List, std::string URI,
+			   std::string const &Dist, std::string const &Section,
+			   bool const &IsSrc, std::map<std::string, std::string> Options) const
+   {
+      auto Snapshot = GetSnapshotOption(Options, "snapshot");
+      if (not Snapshot.empty()) {
+	 std::map<std::string, std::string> SnapshotOptions = Options;
+
+	 Options.emplace("SHADOWED", "true");
+
+	 auto const Deb = GetDebReleaseIndexBy(List, URI, Dist, Options);
+	 std::string filename;
+	 if (not ReleaseFileName(Deb, filename))
+	    return _error->Error("Cannot identify snapshot server for %s %s - run update without snapshot id first", URI.c_str(), Dist.c_str());
+	 auto OldDeb = dynamic_cast<debReleaseIndex *>(Deb->UnloadedClone());
+	 if (not OldDeb->Load(filename, nullptr))
+	    return _error->Error("Cannot identify snapshot server for %s %s - run update without snapshot id first", URI.c_str(), Dist.c_str());
+	 auto Server = SnapshotServer(OldDeb);
+	 delete OldDeb;
+
+	 if (Server.empty())
+	    return _error->Error("Snapshots not supported for %s %s", URI.c_str(), Dist.c_str());
+
+	 auto SnapshotURI = SubstVar(Server, "@SNAPSHOTID@", Snapshot);
+
+	 if (not CreateItemInternalOne(List, SnapshotURI, Dist, Section, IsSrc, SnapshotOptions))
+	    return false;
+      }
+      if (not CreateItemInternalOne(List, URI, Dist, Section, IsSrc, Options))
+	 return false;
+
+
+      return true;
+   }
+   bool CreateItemInternalOne(std::vector<metaIndex *> &List, std::string URI,
 			   std::string const &Dist, std::string const &Section,
-			   bool const &IsSrc, std::map<std::string, std::string> const &Options) const
+			   bool const &IsSrc, std::map<std::string, std::string> Options) const
    {
       auto const Deb = GetDebReleaseIndexBy(List, URI, Dist, Options);
       if (Deb == nullptr)
@@ -1263,7 +1360,8 @@ class APT_HIDDEN debSLTypeDebian : public pkgSourceList::Type		/*{{{*/
 	  Deb->SetValidUntilMax(GetTimeOption(Options, "valid-until-max")) == false ||
 	  Deb->SetValidUntilMin(GetTimeOption(Options, "valid-until-min")) == false ||
 	  Deb->SetCheckDate(GetTriStateOption(Options, "check-date")) == false ||
-	  Deb->SetDateMaxFuture(GetTimeOption(Options, "date-max-future")) == false)
+	  Deb->SetDateMaxFuture(GetTimeOption(Options, "date-max-future")) == false ||
+	  Deb->SetSnapshot(GetSnapshotOption(Options, "snapshot")) == false)
 	 return false;
 
       std::map<std::string, std::string>::const_iterator const signedby = Options.find("signed-by");
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 717f08e2b..a1a9c41b5 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -51,6 +51,8 @@ class APT_HIDDEN debReleaseIndex : public metaIndex
    bool SetValidUntilMax(time_t const Valid);
    bool SetCheckDate(TriState const CheckDate);
    bool SetDateMaxFuture(time_t const DateMaxFuture);
+   bool SetSnapshot(std::string Snapshot);
+   std::string GetSnapshotsServer() const; // As defined in the Release file
    bool SetSignedBy(std::string const &SignedBy);
    std::map<std::string, std::string> GetReleaseOptions();
 
diff --git a/apt-pkg/indexfile.cc b/apt-pkg/indexfile.cc
index f13b52940..1176903f4 100644
--- a/apt-pkg/indexfile.cc
+++ b/apt-pkg/indexfile.cc
@@ -121,6 +121,7 @@ std::string IndexTarget::Option(OptionKeys const EnumKey) const		/*{{{*/
       APT_CASE(ALLOW_WEAK);
       APT_CASE(ALLOW_DOWNGRADE_TO_INSECURE);
       APT_CASE(INRELEASE_PATH);
+      APT_CASE(SHADOWED);
 #undef APT_CASE
       case FILENAME:
       {
@@ -221,6 +222,8 @@ unsigned long pkgDebianIndexTargetFile::Size() const				/*{{{*/
 									/*}}}*/
 bool pkgDebianIndexTargetFile::Exists() const					/*{{{*/
 {
+   if (Target.OptionBool(IndexTarget::SHADOWED))
+      return false;
    return FileExists(IndexFileName());
 }
 									/*}}}*/
diff --git a/apt-pkg/indexfile.h b/apt-pkg/indexfile.h
index d0b045a9d..95cee5dbc 100644
--- a/apt-pkg/indexfile.h
+++ b/apt-pkg/indexfile.h
@@ -91,6 +91,7 @@ class APT_PUBLIC IndexTarget							/*{{{*/
       ALLOW_WEAK,
       ALLOW_DOWNGRADE_TO_INSECURE,
       INRELEASE_PATH,
+      SHADOWED,
    };
    std::string Option(OptionKeys const Key) const;
    bool OptionBool(OptionKeys const Key) const;
diff --git a/apt-pkg/init.cc b/apt-pkg/init.cc
index b9d9b15d2..f70dbf308 100644
--- a/apt-pkg/init.cc
+++ b/apt-pkg/init.cc
@@ -209,6 +209,9 @@ bool pkgInitConfig(Configuration &Cnf)
    Cnf.CndSet("Acquire::Changelogs::URI::Origin::Ubuntu", "https://changelogs.ubuntu.com/changelogs/pool/@CHANGEPATH@/changelog");
    Cnf.CndSet("Acquire::Changelogs::AlwaysOnline::Origin::Ubuntu", true);
 
+   Cnf.CndSet("Acquire::Snapshots::URI::Origin::Debian", "https://snapshot.debian.org/archive/debian/@SNAPSHOTID@/");
+   Cnf.CndSet("Acquire::Snapshots::URI::Override::Label::Debian-Security", "https://snapshot.debian.org/archive/debian-security/@SNAPSHOTID@/");
+   Cnf.CndSet("Acquire::Snapshots::URI::Origin::Ubuntu", "https://snapshot.ubuntu.com/ubuntu/@SNAPSHOTID@/");
 
    Cnf.CndSet("DPkg::Path", "/usr/sbin:/usr/bin:/sbin:/bin");
 
diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index 0ac59fcfc..055cf4142 100644
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -120,6 +120,7 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,	/*{{{*/
    mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
    mapping.insert(std::make_pair("Check-Date", std::make_pair("check-date", false)));
    mapping.insert(std::make_pair("Date-Max-Future", std::make_pair("date-max-future", false)));
+   mapping.insert(std::make_pair("Snapshot", std::make_pair("snapshot", false)));
    mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
    mapping.insert(std::make_pair("PDiffs", std::make_pair("pdiffs", false)));
    mapping.insert(std::make_pair("By-Hash", std::make_pair("by-hash", false)));