warn if an expected file can't be acquired

If we couldn't find an entry for a Sources file we would generate an
error while for a Packages file we would silently skip it due to
assuming it is missing because it is empty. We can do better by checking
if the repository declares that it supports a component we want to get
the file from and if not say so and hint at the user making a typo.

An example were this helps is mozilla.debian.net which dropped the
firefox-aurora component (as upstream did) meaning no upgrades until the
user notices manually that the repository doesn't provide packages
anymore. With this commit warnings are raised hopefully causing the user
to investigate what is wrong (sooner).

7 files changed, 83 insertions, 5 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 3c85f8adf..17e4797d1 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1411,6 +1411,16 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
       {
 	 if (TransactionManager->MetaIndexParser->Exists(Target.MetaKey) == false)
 	 {
+	    auto const component = Target.Option(IndexTarget::COMPONENT);
+	    if (component.empty() == false && TransactionManager->MetaIndexParser->HasSupportForComponent(component) == false)
+	    {
+	       new CleanupItem(Owner, TransactionManager, Target);
+	       _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' doesn't have the component '%s' (component misspelt in sources.list?)"),
+		     Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str(), component.c_str());
+	       continue;
+
+	    }
+
 	    // optional targets that we do not have in the Release file are skipped
 	    if (hasHashes == true && Target.IsOptional)
 	    {
@@ -1440,9 +1450,10 @@ void pkgAcqMetaClearSig::QueueIndexes(bool const verify)			/*{{{*/
 
 	    if (hasHashes == true)
 	    {
-	       Status = StatAuthError;
-	       strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), Target.MetaKey.c_str());
-	       return;
+	       new CleanupItem(Owner, TransactionManager, Target);
+	       _error->Warning(_("Skipping acquire of configured file '%s' as repository '%s' does not seem to provide it (sources.list entry misspelt?)"),
+		     Target.MetaKey.c_str(), TransactionManager->Target.Description.c_str());
+	       continue;
 	    }
 	    else
 	    {
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index cba00aa8e..8c82414cb 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -51,6 +51,7 @@ class APT_HIDDEN debReleaseIndexPrivate					/*{{{*/
 
    std::vector<std::string> Architectures;
    std::vector<std::string> NoSupportForAll;
+   std::vector<std::string> SupportedComponents;
    std::map<std::string, std::string> const ReleaseOptions;
 
    debReleaseIndexPrivate(std::map<std::string, std::string> const &Options) : CheckValidUntil(metaIndex::TRI_UNSET), ValidUntilMin(0), ValidUntilMax(0), ReleaseOptions(Options) {}
@@ -404,6 +405,16 @@ bool debReleaseIndex::Load(std::string const &Filename, std::string * const Erro
       if (targets.empty() == false)
 	 d->NoSupportForAll = VectorizeString(targets, ' ');
    }
+   for (auto const &comp: VectorizeString(Section.FindS("Components"), ' '))
+   {
+      if (comp.empty())
+	 continue;
+      auto const pos = comp.find_last_of('/');
+      if (pos == std::string::npos)
+	 d->SupportedComponents.push_back(std::move(comp));
+      else // e.g. security.debian.org uses this style
+	 d->SupportedComponents.push_back(comp.substr(pos + 1));
+   }
 
    bool FoundHashSum = false;
    bool FoundStrongHashSum = false;
@@ -733,6 +744,13 @@ bool debReleaseIndex::IsArchitectureAllSupportedFor(IndexTarget const &target) c
    return std::find(d->NoSupportForAll.begin(), d->NoSupportForAll.end(), target.Option(IndexTarget::CREATED_BY)) == d->NoSupportForAll.end();
 }
 									/*}}}*/
+bool debReleaseIndex::HasSupportForComponent(std::string const &component) const/*{{{*/
+{
+   if (d->SupportedComponents.empty())
+      return true;
+   return std::find(d->SupportedComponents.begin(), d->SupportedComponents.end(), component) != d->SupportedComponents.end();
+}
+									/*}}}*/
 std::vector <pkgIndexFile *> *debReleaseIndex::GetIndexFiles()		/*{{{*/
 {
    if (Indexes != NULL)
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index f903617f7..450ca94c9 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -61,6 +61,7 @@ class APT_HIDDEN debReleaseIndex : public metaIndex
    virtual bool IsTrusted() const APT_OVERRIDE;
    bool IsArchitectureSupported(std::string const &arch) const;
    bool IsArchitectureAllSupportedFor(IndexTarget const &target) const;
+   bool HasSupportForComponent(std::string const &component) const;
 
    void AddComponent(std::string const &sourcesEntry,
 	 bool const isSrc, std::string const &Name,
diff --git a/apt-pkg/metaindex.cc b/apt-pkg/metaindex.cc
index 8b31051fb..624c7c9c7 100644
--- a/apt-pkg/metaindex.cc
+++ b/apt-pkg/metaindex.cc
@@ -128,3 +128,11 @@ bool metaIndex::IsArchitectureAllSupportedFor(IndexTarget const &target) const/*
    return true;
 }
 									/*}}}*/
+bool metaIndex::HasSupportForComponent(std::string const &component) const/*{{{*/
+{
+   debReleaseIndex const * const deb = dynamic_cast<debReleaseIndex const *>(this);
+   if (deb != NULL)
+      return deb->HasSupportForComponent(component);
+   return true;
+}
+									/*}}}*/
diff --git a/apt-pkg/metaindex.h b/apt-pkg/metaindex.h
index 3a624e86d..531143bcb 100644
--- a/apt-pkg/metaindex.h
+++ b/apt-pkg/metaindex.h
@@ -111,6 +111,7 @@ public:
    // FIXME: make virtual on next abi break
    bool IsArchitectureSupported(std::string const &arch) const;
    bool IsArchitectureAllSupportedFor(IndexTarget const &target) const;
+   bool HasSupportForComponent(std::string const &component) const;
 };
 
 #endif
diff --git a/test/integration/test-acquire-binary-all b/test/integration/test-acquire-binary-all
index ba47eddc2..379348e3f 100755
--- a/test/integration/test-acquire-binary-all
+++ b/test/integration/test-acquire-binary-all
@@ -114,9 +114,9 @@ testsuccess grep '^Get.* all Contents ' aptupdate.output
 testequal 'foo-1
 foo-2' aptcache pkgnames foo-
 
-# apt doesn't know supported archs, so missing a configured arch is a failure
+# apt doesn't know supported archs, so missing a configured arch is a problem
 configarchitecture 'amd64' 'i386'
-testfailure apt update
+testwarningmsg "W: Skipping acquire of configured file 'main/binary-i386/Packages' as repository 'file:$(readlink -f ./aptarchive) unstable InRelease' does not seem to provide it (sources.list entry misspelt?)" apt update
 testequal 'foo-1
 foo-2' aptcache pkgnames foo-
 
diff --git a/test/integration/test-apt-update-disappeared-component b/test/integration/test-apt-update-disappeared-component
new file mode 100755
index 000000000..d38405bc2
--- /dev/null
+++ b/test/integration/test-apt-update-disappeared-component
@@ -0,0 +1,39 @@
+#!/bin/sh
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+setupenvironment
+configarchitecture 'amd64'
+
+insertpackage 'unstable' 'foobar' 'all' '1'
+
+APTARCHIVE="file:$(readlink -f ./aptarchive) unstable InRelease"
+setupaptarchive --no-update
+sed -i -e 's#main\s*$#main contrib non-free#' rootdir/etc/apt/sources.list.d/*
+
+# if no Component info is available we assume all are supported, which means
+# that non-existent Packages files are assumed to be missing because they are
+# empty (as the repository is declaring support for them via Architectures)
+testwarningmsg "W: Skipping acquire of configured file 'contrib/source/Sources' as repository '${APTARCHIVE}' does not seem to provide it (sources.list entry misspelt?)
+W: Skipping acquire of configured file 'non-free/source/Sources' as repository '${APTARCHIVE}' does not seem to provide it (sources.list entry misspelt?)" apt update
+
+sed -i -e '/^Codename: / a\
+Components: main contrib' $(find ./aptarchive -name 'Release')
+signreleasefiles
+
+testwarningmsg "W: Skipping acquire of configured file 'contrib/source/Sources' as repository '${APTARCHIVE}' does not seem to provide it (sources.list entry misspelt?)
+W: Skipping acquire of configured file 'non-free/source/Sources' as repository '${APTARCHIVE}' doesn't have the component 'non-free' (component misspelt in sources.list?)
+W: Skipping acquire of configured file 'non-free/binary-amd64/Packages' as repository '${APTARCHIVE}' doesn't have the component 'non-free' (component misspelt in sources.list?)
+W: Skipping acquire of configured file 'non-free/binary-all/Packages' as repository '${APTARCHIVE}' doesn't have the component 'non-free' (component misspelt in sources.list?)
+W: Skipping acquire of configured file 'non-free/i18n/Translation-en' as repository '${APTARCHIVE}' doesn't have the component 'non-free' (component misspelt in sources.list?)" apt update
+
+# the field looks like this e.g. for security.debian.org sources
+sed -i -e 's#^Components:.*$#Components: updates/main updates/contrib#' $(find ./aptarchive -name 'Release')
+signreleasefiles
+
+testwarningmsg "W: Skipping acquire of configured file 'contrib/source/Sources' as repository '${APTARCHIVE}' does not seem to provide it (sources.list entry misspelt?)
+W: Skipping acquire of configured file 'non-free/source/Sources' as repository '${APTARCHIVE}' doesn't have the component 'non-free' (component misspelt in sources.list?)
+W: Skipping acquire of configured file 'non-free/binary-amd64/Packages' as repository '${APTARCHIVE}' doesn't have the component 'non-free' (component misspelt in sources.list?)
+W: Skipping acquire of configured file 'non-free/binary-all/Packages' as repository '${APTARCHIVE}' doesn't have the component 'non-free' (component misspelt in sources.list?)
+W: Skipping acquire of configured file 'non-free/i18n/Translation-en' as repository '${APTARCHIVE}' doesn't have the component 'non-free' (component misspelt in sources.list?)" apt update