Merge branch 'fix/partialfilemirror' into 'main'

Support uncompressed indexes from partial file:/ mirrors

See merge request apt-team/apt!235

4 files changed, 135 insertions, 86 deletions

diff --git a/methods/aptmethod.h b/methods/aptmethod.h
index 1c24f3a98..1ea173075 100644
--- a/methods/aptmethod.h
+++ b/methods/aptmethod.h
@@ -514,6 +514,17 @@ protected:
       return part;
    }
 
+   static std::string CombineWithAlternatePath(std::string Path, std::string Change)
+   {
+      while (APT::String::Startswith(Change, "../"))
+      {
+	 Path.erase(Path.find_last_not_of('/'));
+	 Path = flNotFile(Path);
+	 Change.erase(0, 3);
+      }
+      return flCombine(Path, Change);
+   }
+
    aptMethod(std::string &&Binary, char const *const Ver, unsigned long const Flags) APT_NONNULL(3)
        : pkgAcqMethod(Ver, Flags), aptConfigWrapperForMethods(Binary), Binary(std::move(Binary)), SeccompFlags(0)
    {
diff --git a/methods/copy.cc b/methods/copy.cc
index 44205e4cd..9c9c6572d 100644
--- a/methods/copy.cc
+++ b/methods/copy.cc
@@ -26,7 +26,7 @@
 
 class CopyMethod final : public aptMethod
 {
-   virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+   bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
 
    public:
    CopyMethod() : aptMethod("copy", "1.0", SingleInstance | SendConfig | SendURIEncoded)
@@ -36,55 +36,71 @@ class CopyMethod final : public aptMethod
 };
 
 // CopyMethod::Fetch - Fetch a file					/*{{{*/
-// ---------------------------------------------------------------------
-/* */
-bool CopyMethod::Fetch(FetchItem *Itm)
+bool CopyMethod::URIAcquire(std::string const &Message, FetchItem *Itm)
 {
-   // this ensures that relative paths work in copy
-   std::string const File = DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1));
-
-   // Stat the file and send a start message
-   struct stat Buf;
-   if (stat(File.c_str(),&Buf) != 0)
-      return _error->Errno("stat",_("Failed to stat"));
+   struct FileCopyType {
+      std::string name;
+      struct stat stat{};
+      explicit FileCopyType(std::string &&file) : name{std::move(file)} {}
+   };
+   std::vector<FileCopyType> files;
+   // this ensures that relative paths work
+   files.emplace_back(DecodeSendURI(Itm->Uri.substr(Itm->Uri.find(':')+1)));
+   for (auto const &AltPath : VectorizeString(LookupTag(Message, "Alternate-Paths"), '\n'))
+      files.emplace_back(CombineWithAlternatePath(flNotFile(files[0].name), DecodeSendURI(AltPath)));
+   files.erase(std::remove_if(files.begin(), files.end(), [](auto &file)
+			      { return stat(file.name.c_str(), &file.stat) != 0; }),
+	       files.end());
+   if (files.empty())
+      return _error->Errno("copy-stat", _("Failed to stat"));
 
    // Forumulate a result and send a start message
    FetchResult Res;
-   Res.Size = Buf.st_size;
    Res.Filename = Itm->DestFile;
-   Res.LastModified = Buf.st_mtime;
    Res.IMSHit = false;
-   URIStart(Res);
 
-   // just calc the hashes if the source and destination are identical
-   if (File == Itm->DestFile || Itm->DestFile == "/dev/null")
+   for (auto const &File : files)
    {
+      Res.Size = File.stat.st_size;
+      Res.LastModified = File.stat.st_mtime;
+
+      // just calc the hashes if the source and destination are identical
+      if (Itm->DestFile == "/dev/null" || File.name == Itm->DestFile)
+      {
+	 URIStart(Res);
+	 CalculateHashes(Itm, Res);
+	 URIDone(Res);
+	 return true;
+      }
+
+      FileFd From(File.name, FileFd::ReadOnly);
+      FileFd To(Itm->DestFile, FileFd::WriteAtomic);
+      To.EraseOnFailure();
+      if (not From.IsOpen() || not To.IsOpen())
+	 continue;
+
+      // Copy the file
+      URIStart(Res);
+      if (not CopyFile(From, To))
+      {
+	 To.OpFail();
+	 continue;
+      }
+      From.Close();
+      To.Close();
+
       CalculateHashes(Itm, Res);
-      URIDone(Res);
-      return true;
-   }
+      if (not Itm->ExpectedHashes.empty() && Itm->ExpectedHashes != Res.Hashes)
+	 continue;
 
-   // See if the file exists
-   FileFd From(File,FileFd::ReadOnly);
-   FileFd To(Itm->DestFile,FileFd::WriteAtomic);
-   To.EraseOnFailure();
+      if (not TransferModificationTimes(File.name.c_str(), Res.Filename.c_str(), Res.LastModified))
+	 continue;
 
-   // Copy the file
-   if (CopyFile(From,To) == false)
-   {
-      To.OpFail();
-      return false;
+      URIDone(Res);
+      return true;
    }
 
-   From.Close();
-   To.Close();
-
-   if (TransferModificationTimes(File.c_str(), Res.Filename.c_str(), Res.LastModified) == false)
-      return false;
-
-   CalculateHashes(Itm, Res);
-   URIDone(Res);
-   return true;
+   return false;
 }
 									/*}}}*/
 
diff --git a/methods/file.cc b/methods/file.cc
index 0a76c0c94..111440ce1 100644
--- a/methods/file.cc
+++ b/methods/file.cc
@@ -29,7 +29,7 @@
 
 class FileMethod final : public aptMethod
 {
-   virtual bool Fetch(FetchItem *Itm) APT_OVERRIDE;
+   bool URIAcquire(std::string const &Message, FetchItem *Itm) APT_OVERRIDE;
 
    public:
    FileMethod() : aptMethod("file", "1.0", SingleInstance | SendConfig | LocalOnly | SendURIEncoded)
@@ -41,86 +41,101 @@ class FileMethod final : public aptMethod
 // FileMethod::Fetch - Fetch a file					/*{{{*/
 // ---------------------------------------------------------------------
 /* */
-bool FileMethod::Fetch(FetchItem *Itm)
+bool FileMethod::URIAcquire(std::string const &Message, FetchItem *Itm)
 {
    URI Get(Itm->Uri);
-   std::string const File = DecodeSendURI(Get.Path);
-   FetchResult Res;
    if (Get.Host.empty() == false)
       return _error->Error(_("Invalid URI, local URIS must not start with //"));
 
-   struct stat Buf;
+   std::vector<std::string> Files;
+   Files.emplace_back(DecodeSendURI(Get.Path));
+   for (auto const &AltPath : VectorizeString(LookupTag(Message, "Alternate-Paths"), '\n'))
+      Files.emplace_back(CombineWithAlternatePath(flNotFile(Files[0]), DecodeSendURI(AltPath)));
+
+   FetchResult Res;
    // deal with destination files which might linger around
-   if (lstat(Itm->DestFile.c_str(), &Buf) == 0)
+   struct stat Buf;
+   if (lstat(Itm->DestFile.c_str(), &Buf) == 0 && S_ISLNK(Buf.st_mode) && Buf.st_size > 0)
+   {
+      char name[Buf.st_size + 1];
+      if (ssize_t const sp = readlink(Itm->DestFile.c_str(), name, Buf.st_size); sp == -1)
+      {
+	 Itm->LastModified = 0;
+	 RemoveFile("file", Itm->DestFile);
+      }
+   }
+
+   int olderrno = 0;
+   // See if the file exists
+   for (auto const &File : Files)
    {
-      if ((Buf.st_mode & S_IFREG) != 0)
+      if (stat(File.c_str(), &Buf) == 0)
       {
+	 Res.Size = Buf.st_size;
+	 Res.Filename = File;
+	 Res.LastModified = Buf.st_mtime;
+	 Res.IMSHit = false;
 	 if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
 	 {
-	    if (Itm->ExpectedHashes.VerifyFile(File))
-	    {
-	       Res.Filename = Itm->DestFile;
+	    auto const filesize = Itm->ExpectedHashes.FileSize();
+	    if (filesize != 0 && filesize == Res.Size)
 	       Res.IMSHit = true;
-	    }
 	 }
+	 break;
       }
+      if (olderrno == 0)
+	 olderrno = errno;
    }
-   if (Res.IMSHit != true)
-      RemoveFile("file", Itm->DestFile);
 
-   int olderrno = 0;
-   // See if the file exists
-   if (stat(File.c_str(),&Buf) == 0)
+   if (not Res.IMSHit)
    {
-      Res.Size = Buf.st_size;
-      Res.Filename = File;
-      Res.LastModified = Buf.st_mtime;
-      Res.IMSHit = false;
-      if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+      RemoveFile("file", Itm->DestFile);
+      if (not Res.Filename.empty())
       {
-	 unsigned long long const filesize = Itm->ExpectedHashes.FileSize();
-	 if (filesize != 0 && filesize == Res.Size)
-	    Res.IMSHit = true;
+	 URIStart(Res);
+	 CalculateHashes(Itm, Res);
       }
-
-      CalculateHashes(Itm, Res);
    }
-   else
-      olderrno = errno;
-   if (Res.IMSHit == false)
-      URIStart(Res);
 
    // See if the uncompressed file exists and reuse it
    FetchResult AltRes;
-   AltRes.Filename.clear();
-   std::vector<std::string> extensions = APT::Configuration::getCompressorExtensions();
-   for (std::vector<std::string>::const_iterator ext = extensions.begin(); ext != extensions.end(); ++ext)
+   for (auto const &File : Files)
    {
-      if (APT::String::Endswith(File, *ext) == true)
+      for (const auto &ext : APT::Configuration::getCompressorExtensions())
       {
-	 std::string const unfile = File.substr(0, File.length() - ext->length());
-	 if (stat(unfile.c_str(),&Buf) == 0)
+	 if (APT::String::Endswith(File, ext))
 	 {
-	    AltRes.Size = Buf.st_size;
-	    AltRes.Filename = unfile;
-	    AltRes.LastModified = Buf.st_mtime;
-	    AltRes.IMSHit = false;
-	    if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
-	       AltRes.IMSHit = true;
-	    break;
+	    std::string const unfile = File.substr(0, File.length() - ext.length());
+	    if (stat(unfile.c_str(), &Buf) == 0)
+	    {
+	       AltRes.Size = Buf.st_size;
+	       AltRes.Filename = unfile;
+	       AltRes.LastModified = Buf.st_mtime;
+	       AltRes.IMSHit = false;
+	       if (Itm->LastModified == Buf.st_mtime && Itm->LastModified != 0)
+		  AltRes.IMSHit = true;
+	       if (Res.Filename.empty())
+	       {
+		  URIStart(Res);
+		  CalculateHashes(Itm, AltRes);
+	       }
+	       break;
+	    }
+	    // no break here as we could have situations similar to '.gz' vs '.tar.gz' here
 	 }
-	 // no break here as we could have situations similar to '.gz' vs '.tar.gz' here
       }
+      if (not AltRes.Filename.empty())
+	 break;
    }
 
-   if (AltRes.Filename.empty() == false)
+   if (not AltRes.Filename.empty())
       URIDone(Res,&AltRes);
-   else if (Res.Filename.empty() == false)
+   else if (not Res.Filename.empty())
       URIDone(Res);
    else
    {
       errno = olderrno;
-      return _error->Errno(File.c_str(), _("File not found"));
+      return _error->Errno(Files[0].c_str(), _("File not found"));
    }
 
    return true;
diff --git a/methods/mirror.cc b/methods/mirror.cc
index 787e4c7d5..5781ebcf0 100644
--- a/methods/mirror.cc
+++ b/methods/mirror.cc
@@ -326,7 +326,11 @@ std::string MirrorMethod::GetMirrorFileURI(std::string const &Message, FetchItem
    {
       if (APT::String::Startswith(Itm->Uri, uristr))
       {
-	 uristr.erase(uristr.length() - 1); // remove the ending '/'
+	 if (::URI uri{uristr}; uri.Path.length() > 1 && APT::String::Endswith(uri.Path, "/"))
+	 {
+	    uri.Path.erase(uri.Path.length() - 1); // remove the ending '/'
+	    uristr = uri;
+	 }
 	 auto const colon = uristr.find(':');
 	 if (unlikely(colon == std::string::npos))
 	    continue;
@@ -375,7 +379,10 @@ bool MirrorMethod::URIAcquire(std::string const &Message, FetchItem *Itm) /*{{{*
       msgCache[Itm->Uri] = Message;
       MirrorListInfo info;
       info.state = REQUESTED;
-      info.baseuri = mirrorfileuri + '/';
+      if (not APT::String::Endswith(mirrorfileuri, "/"))
+	 info.baseuri = mirrorfileuri + '/';
+      else
+	 info.baseuri = mirrorfileuri;
       auto const colon = info.baseuri.find(':');
       if (unlikely(colon == std::string::npos))
 	 return false;