allow multivalue fields in deb822 sources to be folded

The documentation said "spaces", but there is no real reason to be so
strict and only allow spaces to separate values as that only leads to
very long lines if e.g. multiple URIs are specified which are again hard
to deal with from a user PoV which the deb822 format is supposed to
avoid. It also deals with multiple consecutive spaces and strange things
like tabs users will surely end up using in the real world.

The old behviour on encountering folded lines is the generation of URIs
which end up containing all these whitespace characters which tends to
mess really bad with output and further processing.

Closes: 881875

3 files changed, 143 insertions, 47 deletions

diff --git a/apt-pkg/sourcelist.cc b/apt-pkg/sourcelist.cc
index adf598e48..fd0264ff1 100644
--- a/apt-pkg/sourcelist.cc
+++ b/apt-pkg/sourcelist.cc
@@ -43,6 +43,17 @@ static pkgSourceList::Type *ItmList[10];
 pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
 unsigned long pkgSourceList::Type::GlobalListLen = 0;
 
+static std::vector<std::string> FindMultiValue(pkgTagSection &Tags, char const *const Field) /*{{{*/
+{
+   auto values = Tags.FindS(Field);
+   // we ignore duplicate spaces by removing empty values
+   std::replace_if(values.begin(), values.end(), isspace_ascii, ' ');
+   auto vect = VectorizeString(values, ' ');
+   vect.erase(std::remove_if(vect.begin(), vect.end(), [](std::string const &s) { return s.empty(); }), vect.end());
+   return vect;
+}
+									/*}}}*/
+
 // Type::Type - Constructor						/*{{{*/
 // ---------------------------------------------------------------------
 /* Link this to the global list of items*/
@@ -115,11 +126,13 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,	/*{{{*/
    for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
       if (Tags.Exists(m->first))
       {
-	 std::string option = Tags.FindS(m->first);
-	 // for deb822 the " " is the delimiter, but the backend expects ","
-	 if (m->second.second == true)
-	    std::replace(option.begin(), option.end(), ' ', ',');
-	 Options[m->second.first] = option;
+	 if (m->second.second)
+	 {
+	    auto const values = FindMultiValue(Tags, m->first);
+	    Options[m->second.first] = APT::String::Join(values, ",");
+	 }
+	 else
+	    Options[m->second.first] = Tags.FindS(m->first);
       }
 
    {
@@ -129,37 +142,34 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,	/*{{{*/
    }
 
    // now create one item per suite/section
-   string Suite = Tags.FindS("Suites");
-   Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
-   string const Component = Tags.FindS("Components");
-   string const URIS = Tags.FindS("URIs");
-
-   std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
-   std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
-   std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
+   auto const list_uris = FindMultiValue(Tags, "URIs");
+   auto const list_comp = FindMultiValue(Tags, "Components");
+   auto list_suite = FindMultiValue(Tags, "Suites");
+   {
+      auto const nativeArch = _config->Find("APT::Architecture");
+      std::transform(list_suite.begin(), list_suite.end(), list_suite.begin(),
+		     [&](std::string const &suite) { return SubstVar(suite, "$(ARCH)", nativeArch); });
+   }
 
    if (list_uris.empty())
       // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
 
-   for (std::vector<std::string>::const_iterator U = list_uris.begin();
-        U != list_uris.end(); ++U)
+   if (list_suite.empty())
+      return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
+
+   for (auto URI : list_uris)
    {
-      std::string URI = *U;
-      if (U->empty() || FixupURI(URI) == false)
+      if (FixupURI(URI) == false)
 	 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
 
-      if (list_suite.empty())
-	 return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
-
-      for (std::vector<std::string>::const_iterator S = list_suite.begin();
-           S != list_suite.end(); ++S)
+      for (auto const &S : list_suite)
       {
-	 if (S->empty() == false && (*S)[S->size() - 1] == '/')
+	 if (likely(S.empty() == false) && S[S.size() - 1] == '/')
 	 {
 	    if (list_comp.empty() == false)
 	       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
-	    if (CreateItem(List, URI, *S, "", Options) == false)
+	    if (CreateItem(List, URI, S, "", Options) == false)
 	       return false;
 	 }
 	 else
@@ -167,14 +177,9 @@ bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List,	/*{{{*/
 	    if (list_comp.empty())
 	       return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
 
-	    for (std::vector<std::string>::const_iterator C = list_comp.begin();
-		  C != list_comp.end(); ++C)
-	    {
-	       if (CreateItem(List, URI, *S, *C, Options) == false)
-	       {
+	    for (auto const &C : list_comp)
+	       if (CreateItem(List, URI, S, C, Options) == false)
 		  return false;
-	       }
-	    }
 	 }
       }
    }
@@ -433,20 +438,17 @@ bool pkgSourceList::ParseFileDeb822(string const &File)
       if(Tags.Exists("Types") == false)
 	 return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
 
-      string const types = Tags.FindS("Types");
-      std::vector<std::string> const list_types = VectorizeString(types, ' ');
-      for (std::vector<std::string>::const_iterator I = list_types.begin();
-        I != list_types.end(); ++I)
+      for (auto const &type : FindMultiValue(Tags, "Types"))
       {
-         Type *Parse = Type::GetType((*I).c_str());
-         if (Parse == 0)
-         {
-            _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
-            return false;
-         }
-
-         if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
-            return false;
+	 Type *Parse = Type::GetType(type.c_str());
+	 if (Parse == 0)
+	 {
+	    _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), type.c_str(), i, Fd.Name().c_str());
+	    return false;
+	 }
+
+	 if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
+	    return false;
       }
    }
    return true;
diff --git a/doc/sources.list.5.xml b/doc/sources.list.5.xml
index 56f65927c..694082bea 100644
--- a/doc/sources.list.5.xml
+++ b/doc/sources.list.5.xml
@@ -101,9 +101,11 @@
 
        Options have the same syntax as every other field: A fieldname separated by
        a colon (<literal>:</literal>) and optionally spaces from its value(s).
-       Note especially that multiple values are separated by spaces, not by
-       commas as in the one-line format. Multivalue fields like <literal>Architectures</literal>
-       also have <literal>Architectures-Add</literal> and <literal>Architectures-Remove</literal>
+       Note especially that multiple values are separated by whitespaces (like spaces,
+       tabs and newlines), not by commas as in the one-line format.
+
+       Multivalue fields like <literal>Architectures</literal> also have
+       <literal>Architectures-Add</literal> and <literal>Architectures-Remove</literal>
        to modify the default value rather than replacing it.
    </para><para>
        This is a new format supported by apt itself since version 1.1. Previous
diff --git a/test/integration/test-apt-sources-deb822 b/test/integration/test-apt-sources-deb822
index f19f263d0..e2cf4d980 100755
--- a/test/integration/test-apt-sources-deb822
+++ b/test/integration/test-apt-sources-deb822
@@ -199,3 +199,95 @@ testsuccessequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease'
 'http://ftp.debian.org/debian/dists/stable/main/binary-i386/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-i386_Packages 0 
 'http://ftp.debian.org/debian/dists/stable/main/binary-armel/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-armel_Packages 0 
 'http://ftp.debian.org/debian/dists/stable/main/binary-all/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-all_Packages 0 " aptget update --print-uris
+
+msgcleantest 'Test deb822 sources with' 'multiline multivalues'
+cat > "$SOURCES" <<EOF
+Types: deb
+ deb-src
+URIs:
+ http://ftp.debian.org/debian
+ http://ftp.debian.org/debian2
+Suites: stable
+ 	  sid
+Components:
+ 	main
+ contrib	non-free
+Architectures: amd64
+ armhf	   powerpc
+EOF
+testsuccessequal --nomsg "'http://ftp.debian.org/debian/dists/stable/InRelease' ftp.debian.org_debian_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian/dists/sid/InRelease' ftp.debian.org_debian_dists_sid_InRelease 0 
+'http://ftp.debian.org/debian2/dists/stable/InRelease' ftp.debian.org_debian2_dists_stable_InRelease 0 
+'http://ftp.debian.org/debian2/dists/sid/InRelease' ftp.debian.org_debian2_dists_sid_InRelease 0 
+'http://ftp.debian.org/debian/dists/stable/main/source/Sources.xz' ftp.debian.org_debian_dists_stable_main_source_Sources 0 
+'http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.xz' ftp.debian.org_debian_dists_stable_contrib_source_Sources 0 
+'http://ftp.debian.org/debian/dists/stable/non-free/source/Sources.xz' ftp.debian.org_debian_dists_stable_non-free_source_Sources 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/main/binary-all/Packages.xz' ftp.debian.org_debian_dists_stable_main_binary-all_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/main/i18n/Translation-en.xz' ftp.debian.org_debian_dists_stable_main_i18n_Translation-en 0 
+'http://ftp.debian.org/debian/dists/stable/contrib/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_stable_contrib_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/contrib/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_stable_contrib_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/contrib/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_stable_contrib_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/contrib/binary-all/Packages.xz' ftp.debian.org_debian_dists_stable_contrib_binary-all_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/contrib/i18n/Translation-en.xz' ftp.debian.org_debian_dists_stable_contrib_i18n_Translation-en 0 
+'http://ftp.debian.org/debian/dists/stable/non-free/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_stable_non-free_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/non-free/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_stable_non-free_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/non-free/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_stable_non-free_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/non-free/binary-all/Packages.xz' ftp.debian.org_debian_dists_stable_non-free_binary-all_Packages 0 
+'http://ftp.debian.org/debian/dists/stable/non-free/i18n/Translation-en.xz' ftp.debian.org_debian_dists_stable_non-free_i18n_Translation-en 0 
+'http://ftp.debian.org/debian/dists/sid/main/source/Sources.xz' ftp.debian.org_debian_dists_sid_main_source_Sources 0 
+'http://ftp.debian.org/debian/dists/sid/contrib/source/Sources.xz' ftp.debian.org_debian_dists_sid_contrib_source_Sources 0 
+'http://ftp.debian.org/debian/dists/sid/non-free/source/Sources.xz' ftp.debian.org_debian_dists_sid_non-free_source_Sources 0 
+'http://ftp.debian.org/debian/dists/sid/main/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_sid_main_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/main/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_sid_main_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/main/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_sid_main_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/main/binary-all/Packages.xz' ftp.debian.org_debian_dists_sid_main_binary-all_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/main/i18n/Translation-en.xz' ftp.debian.org_debian_dists_sid_main_i18n_Translation-en 0 
+'http://ftp.debian.org/debian/dists/sid/contrib/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_sid_contrib_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/contrib/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_sid_contrib_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/contrib/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_sid_contrib_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/contrib/binary-all/Packages.xz' ftp.debian.org_debian_dists_sid_contrib_binary-all_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/contrib/i18n/Translation-en.xz' ftp.debian.org_debian_dists_sid_contrib_i18n_Translation-en 0 
+'http://ftp.debian.org/debian/dists/sid/non-free/binary-amd64/Packages.xz' ftp.debian.org_debian_dists_sid_non-free_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/non-free/binary-armhf/Packages.xz' ftp.debian.org_debian_dists_sid_non-free_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/non-free/binary-powerpc/Packages.xz' ftp.debian.org_debian_dists_sid_non-free_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/non-free/binary-all/Packages.xz' ftp.debian.org_debian_dists_sid_non-free_binary-all_Packages 0 
+'http://ftp.debian.org/debian/dists/sid/non-free/i18n/Translation-en.xz' ftp.debian.org_debian_dists_sid_non-free_i18n_Translation-en 0 
+'http://ftp.debian.org/debian2/dists/stable/main/source/Sources.xz' ftp.debian.org_debian2_dists_stable_main_source_Sources 0 
+'http://ftp.debian.org/debian2/dists/stable/contrib/source/Sources.xz' ftp.debian.org_debian2_dists_stable_contrib_source_Sources 0 
+'http://ftp.debian.org/debian2/dists/stable/non-free/source/Sources.xz' ftp.debian.org_debian2_dists_stable_non-free_source_Sources 0 
+'http://ftp.debian.org/debian2/dists/stable/main/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_stable_main_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/main/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_stable_main_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/main/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_stable_main_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/main/binary-all/Packages.xz' ftp.debian.org_debian2_dists_stable_main_binary-all_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/main/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_stable_main_i18n_Translation-en 0 
+'http://ftp.debian.org/debian2/dists/stable/contrib/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_stable_contrib_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/contrib/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_stable_contrib_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/contrib/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_stable_contrib_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/contrib/binary-all/Packages.xz' ftp.debian.org_debian2_dists_stable_contrib_binary-all_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/contrib/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_stable_contrib_i18n_Translation-en 0 
+'http://ftp.debian.org/debian2/dists/stable/non-free/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_stable_non-free_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/non-free/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_stable_non-free_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/non-free/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_stable_non-free_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/non-free/binary-all/Packages.xz' ftp.debian.org_debian2_dists_stable_non-free_binary-all_Packages 0 
+'http://ftp.debian.org/debian2/dists/stable/non-free/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_stable_non-free_i18n_Translation-en 0 
+'http://ftp.debian.org/debian2/dists/sid/main/source/Sources.xz' ftp.debian.org_debian2_dists_sid_main_source_Sources 0 
+'http://ftp.debian.org/debian2/dists/sid/contrib/source/Sources.xz' ftp.debian.org_debian2_dists_sid_contrib_source_Sources 0 
+'http://ftp.debian.org/debian2/dists/sid/non-free/source/Sources.xz' ftp.debian.org_debian2_dists_sid_non-free_source_Sources 0 
+'http://ftp.debian.org/debian2/dists/sid/main/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_sid_main_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/main/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_sid_main_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/main/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_sid_main_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/main/binary-all/Packages.xz' ftp.debian.org_debian2_dists_sid_main_binary-all_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/main/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_sid_main_i18n_Translation-en 0 
+'http://ftp.debian.org/debian2/dists/sid/contrib/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_sid_contrib_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/contrib/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_sid_contrib_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/contrib/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_sid_contrib_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/contrib/binary-all/Packages.xz' ftp.debian.org_debian2_dists_sid_contrib_binary-all_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/contrib/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_sid_contrib_i18n_Translation-en 0 
+'http://ftp.debian.org/debian2/dists/sid/non-free/binary-amd64/Packages.xz' ftp.debian.org_debian2_dists_sid_non-free_binary-amd64_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/non-free/binary-armhf/Packages.xz' ftp.debian.org_debian2_dists_sid_non-free_binary-armhf_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/non-free/binary-powerpc/Packages.xz' ftp.debian.org_debian2_dists_sid_non-free_binary-powerpc_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/non-free/binary-all/Packages.xz' ftp.debian.org_debian2_dists_sid_non-free_binary-all_Packages 0 
+'http://ftp.debian.org/debian2/dists/sid/non-free/i18n/Translation-en.xz' ftp.debian.org_debian2_dists_sid_non-free_i18n_Translation-en 0 " aptget update --print-uris