Support uncompressed indexes from partial file:/ mirrors

If a file:/ mirror does not contain the requested compressed file, but
the uncompressed variant of that file the method will sent us a fitting
Done message with data only about the Alt-Filename, but the central hash
verification code in the acquire system does not expect that looking for
the non-existent compressed file to compare its hashes with the expected
result – which fails (if we are verifying at least).

That file:/ as well as the item code deals with this suggests that this
feature was broken by 448c38bdcd72b52f11ec5f326f822cf57653f81c, but I
couldn't (easily) compile that to verify, so this is only a guess.

Supporting this allows using index files from /var/lib/apt/lists –
which are likely uncompressed or can at least made so easily – to
construct a file:/ mirror.

4 files changed, 92 insertions, 38 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 12292eb28..1fc07ec05 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1024,8 +1024,7 @@ void pkgAcquire::Item::Start(string const &/*Message*/, unsigned long long const
 bool pkgAcquire::Item::VerifyDone(std::string const &Message,
 	 pkgAcquire::MethodConfig const * const /*Cnf*/)
 {
-   std::string const FileName = LookupTag(Message,"Filename");
-   if (FileName.empty() == true)
+   if (LookupTag(Message,"Filename").empty() && LookupTag(Message, "Alt-Filename").empty())
    {
       Status = StatError;
       ErrorText = "Method gave a blank filename";
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 0f22f71e8..035d8ce10 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -20,6 +20,7 @@
 #include <apt-pkg/error.h>
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/hashes.h>
+#include <apt-pkg/metaindex.h>
 #include <apt-pkg/proxy.h>
 #include <apt-pkg/strutl.h>
 
@@ -434,11 +435,18 @@ bool pkgAcquire::Worker::RunMessages()
 		  Log->Pulse((*O)->GetOwner());
 
 	    HashStringList ReceivedHashes;
+	    bool AltFile = false;
 	    {
-	       std::string const givenfilename = LookupTag(Message, "Filename");
-	       std::string const filename = givenfilename.empty() ? Itm->Owner->DestFile : givenfilename;
+	       std::string filename;
+	       if (filename = LookupTag(Message, "Filename"); filename.empty())
+	       {
+		  if (filename = LookupTag(Message, "Alt-Filename"); not filename.empty())
+		     AltFile = true;
+		  else
+		     filename = Itm->Owner->DestFile;
+	       }
 	       // see if we got hashes to verify
-	       ReceivedHashes = GetHashesFromMessage("", Message);
+	       ReceivedHashes = GetHashesFromMessage(AltFile ? "Alt-" : "", Message);
 	       // not all methods always sent Hashes our way
 	       if (ReceivedHashes.usable() == false)
 	       {
@@ -453,7 +461,7 @@ bool pkgAcquire::Worker::RunMessages()
 	       }
 
 	       // only local files can refer other filenames and counting them as fetched would be unfair
-	       if (Log != NULL && Itm->Owner->Complete == false && Itm->Owner->Local == false && givenfilename == filename)
+	       if (Log != nullptr && not Itm->Owner->Complete && not Itm->Owner->Local && not AltFile && Itm->Owner->DestFile == filename)
 		  Log->Fetched(ReceivedHashes.FileSize(),atoi(LookupTag(Message,"Resume-Point","0").c_str()));
 	    }
 
@@ -466,7 +474,19 @@ bool pkgAcquire::Worker::RunMessages()
 	    auto const forcedHash = _config->Find("Acquire::ForceHash");
 	    for (auto const Owner: ItmOwners)
 	    {
-	       HashStringList const ExpectedHashes = Owner->GetExpectedHashes();
+	       HashStringList const ExpectedHashes = [&]() {
+		  if (AltFile)
+		  {
+		     auto const * const transOwner = dynamic_cast<pkgAcqTransactionItem const * const>(Owner);
+		     if (transOwner != nullptr && transOwner->TransactionManager != nullptr && transOwner->TransactionManager->MetaIndexParser != nullptr)
+		     {
+			auto const * const hashes = transOwner->TransactionManager->MetaIndexParser->Lookup(transOwner->Target.MetaKey);
+			if (hashes != nullptr)
+			   return hashes->Hashes;
+		     }
+		  }
+		  return Owner->GetExpectedHashes();
+	       }();
 	       if(_config->FindB("Debug::pkgAcquire::Auth", false) == true)
 	       {
 		  std::clog << "201 URI Done: " << Owner->DescURI() << endl
diff --git a/test/integration/test-apt-get-update-unauth-warning b/test/integration/test-apt-get-update-unauth-warning
index 42c4e5a9d..46df14c29 100755
--- a/test/integration/test-apt-get-update-unauth-warning
+++ b/test/integration/test-apt-get-update-unauth-warning
@@ -44,46 +44,15 @@ testequal 'auxfiles
 lock
 partial' ls rootdir/var/lib/apt/lists
 
-filesize() {
-	local CREATEDBY="$1"
-	shift
-	stat -c%s "/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 2- ).gz"
-}
 # allow override
-#aptget update --allow-insecure-repositories -o Debug::pkgAcquire::worker=1
-#exit
 testwarningequal "Get:1 file:$APTARCHIVE unstable InRelease
 Ign:1 file:$APTARCHIVE unstable InRelease
 Get:2 file:$APTARCHIVE unstable Release
 Ign:2 file:$APTARCHIVE unstable Release
 Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
 Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
 Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
 Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
-Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
-Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
-Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources
-Ign:3 file:$APTARCHIVE unstable/main Sources
-Get:4 file:$APTARCHIVE unstable/main i386 Packages
-Ign:4 file:$APTARCHIVE unstable/main i386 Packages
-Get:5 file:$APTARCHIVE unstable/main all Packages
-Ign:5 file:$APTARCHIVE unstable/main all Packages
-Get:6 file:$APTARCHIVE unstable/main Translation-en
-Ign:6 file:$APTARCHIVE unstable/main Translation-en
-Get:3 file:$APTARCHIVE unstable/main Sources [$(filesize 'Sources') B]
-Get:4 file:$APTARCHIVE unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B]
-Get:5 file:$APTARCHIVE unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B]
-Get:6 file:$APTARCHIVE unstable/main Translation-en [$(filesize 'Translations') B]
 Reading package lists...
 W: The repository 'file:$APTARCHIVE unstable Release' does not have a Release file.
 N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use.
@@ -92,3 +61,38 @@ N: See apt-secure(8) manpage for repository creation and user configuration deta
 testfailureequal "WARNING: The following packages cannot be authenticated!
   foo
 E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo
+
+rm -rf rootdir/var/lib/apt/lists
+changetowebserver
+
+filesize() {
+	local CREATEDBY="$1"
+	shift
+	stat -c%s "${APTARCHIVE}/$(aptget indextargets --no-release-info --format '$(URI)' "Created-By: $CREATEDBY" "$@" | cut -d'/' -f 4- ).gz"
+}
+testwarningequal "Ign:1 http://localhost:$APTHTTPPORT unstable InRelease
+Ign:2 http://localhost:$APTHTTPPORT unstable Release
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Ign:3 http://localhost:$APTHTTPPORT unstable/main Sources
+Ign:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages
+Ign:5 http://localhost:$APTHTTPPORT unstable/main all Packages
+Ign:6 http://localhost:$APTHTTPPORT unstable/main Translation-en
+Get:3 http://localhost:$APTHTTPPORT unstable/main Sources [$(filesize 'Sources') B]
+Get:4 http://localhost:$APTHTTPPORT unstable/main i386 Packages [$(filesize 'Packages' 'Architecture: i386') B]
+Get:5 http://localhost:$APTHTTPPORT unstable/main all Packages [$(filesize 'Packages' 'Architecture: all') B]
+Get:6 http://localhost:$APTHTTPPORT unstable/main Translation-en [$(filesize 'Translations') B]
+Reading package lists...
+W: The repository 'http://localhost:$APTHTTPPORT unstable Release' does not have a Release file.
+N: Data from such a repository can't be authenticated and is therefore potentially dangerous to use.
+N: See apt-secure(8) manpage for repository creation and user configuration details." aptget update --allow-insecure-repositories
+# ensure we can not install the package
+testfailureequal "WARNING: The following packages cannot be authenticated!
+  foo
+E: There were unauthenticated packages and -y was used without --allow-unauthenticated" aptget install -qq -y foo
diff --git a/test/integration/test-apt-update-incomplete-file-mirror b/test/integration/test-apt-update-incomplete-file-mirror
new file mode 100755
index 000000000..d6894d607
--- /dev/null
+++ b/test/integration/test-apt-update-incomplete-file-mirror
@@ -0,0 +1,31 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture "amd64"
+configcompression 'gz'
+
+insertpackage 'unstable' 'foo' 'amd64' '1'
+
+setupaptarchive --no-update
+
+testfailure aptcache show foo
+
+# the Release file says gz available, but the mirror has only uncompressed files
+find aptarchive/dists -name '*.gz' -delete
+
+testsuccess apt update
+testsuccess aptcache show foo
+testfailure aptcache show bar
+
+testsuccess apt update
+
+rm -rf rootdir/var/lib/apt/lists
+
+find aptarchive/dists -name 'Packages' | while read FILE; do
+	echo 'hacked' > $FILE
+done
+testfailure apt update