fix and document on the fly compressor config

libapt allows to configure compressors to be used by its system via
configuration implemented in 03bef78461c6f443187b60799402624326843396,
but that was never really documented and also only partly working, which
also explains why the tests weren't using it…

7 files changed, 102 insertions, 56 deletions

diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 7a44d8599..ff41246b8 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -466,7 +466,7 @@ void pkgAcquire::SetFds(int &Fd,fd_set *RSet,fd_set *WSet)
 void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
 {
    RunFdsSane(RSet, WSet);
-};
+}
 									/*}}}*/
 // Acquire::RunFdsSane - Deal with active FDs				/*{{{*/
 // ---------------------------------------------------------------------
diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc
index 01ae5a642..32778e273 100644
--- a/apt-pkg/aptconfiguration.cc
+++ b/apt-pkg/aptconfiguration.cc
@@ -35,6 +35,7 @@ namespace APT {
 // setDefaultConfigurationForCompressors				/*{{{*/
 static void setDefaultConfigurationForCompressors() {
 	// Set default application paths to check for optional compression types
+	_config->CndSet("Dir::Bin::gzip", "/bin/gzip");
 	_config->CndSet("Dir::Bin::bzip2", "/bin/bzip2");
 	_config->CndSet("Dir::Bin::xz", "/usr/bin/xz");
 	_config->CndSet("Dir::Bin::lz4", "/usr/bin/lz4");
@@ -60,6 +61,12 @@ static void setDefaultConfigurationForCompressors() {
 			_config->Set("APT::Compressor::lzma::UncompressArg::", "-d");
 		}
 	}
+	// setup the defaults for the compressiontypes => method mapping
+	_config->CndSet("Acquire::CompressionTypes::xz","xz");
+	_config->CndSet("Acquire::CompressionTypes::bz2","bzip2");
+	_config->CndSet("Acquire::CompressionTypes::lzma","lzma");
+	_config->CndSet("Acquire::CompressionTypes::gz","gzip");
+	_config->CndSet("Acquire::CompressionTypes::lz4","lz4");
 }
 									/*}}}*/
 // getCompressionTypes - Return Vector of usable compressiontypes	/*{{{*/
@@ -75,14 +82,6 @@ const Configuration::getCompressionTypes(bool const &Cached) {
 			types.clear();
 	}
 
-	// setup the defaults for the compressiontypes => method mapping
-	_config->CndSet("Acquire::CompressionTypes::xz","xz");
-	_config->CndSet("Acquire::CompressionTypes::bz2","bzip2");
-	_config->CndSet("Acquire::CompressionTypes::lzma","lzma");
-	_config->CndSet("Acquire::CompressionTypes::gz","gzip");
-	_config->CndSet("Acquire::CompressionTypes::lz4","lz4");
-
-	setDefaultConfigurationForCompressors();
 	std::vector<APT::Configuration::Compressor> const compressors = getCompressors();
 
 	// load the order setting into our vector
@@ -398,12 +397,12 @@ const Configuration::getCompressors(bool const Cached) {
 		compressors.push_back(Compressor("lzma",".lzma","false", NULL, NULL, 400));
 #endif
 
-	std::vector<std::string> const comp = _config->FindVector("APT::Compressor");
-	for (std::vector<std::string>::const_iterator c = comp.begin();
-	     c != comp.end(); ++c) {
-		if (c->empty() || *c == "." || *c == "gzip" || *c == "bzip2" || *c == "lzma" || *c == "xz")
+	std::vector<std::string> const comp = _config->FindVector("APT::Compressor", "", true);
+	for (auto const &c: comp)
+	{
+		if (c.empty() || std::any_of(compressors.begin(), compressors.end(), [&c](Compressor const &ac) { return ac.Name == c; }))
 			continue;
-		compressors.push_back(Compressor(c->c_str(), std::string(".").append(*c).c_str(), c->c_str(), "-9", "-d", 100));
+		compressors.push_back(Compressor(c.c_str(), std::string(".").append(c).c_str(), c.c_str(), nullptr, nullptr, 1000));
 	}
 
 	return compressors;
diff --git a/doc/apt-ftparchive.1.xml b/doc/apt-ftparchive.1.xml
index f478564d3..3dc6a6ceb 100644
--- a/doc/apt-ftparchive.1.xml
+++ b/doc/apt-ftparchive.1.xml
@@ -88,23 +88,26 @@
      <varlistentry><term><option>release</option></term>
      <listitem><para>
      The <literal>release</literal> command generates a Release file from a
-     directory tree. It recursively searches the given directory for uncompressed
-     <filename>Packages</filename> and <filename>Sources</filename> files and ones
-     compressed with <command>gzip</command>, <command>bzip2</command> or <command>lzma</command>
-     as well as <filename>Release</filename> and <filename>md5sum.txt</filename> files by default
-     (<literal>APT::FTPArchive::Release::Default-Patterns</literal>). Additional filename patterns
-     can be added by listing them in <literal>APT::FTPArchive::Release::Patterns</literal>.
-     It then writes to stdout a <filename>Release</filename> file containing (by default) an MD5,
+     directory tree. It recursively searches the given directory for
+     uncompressed and compressed <filename>Packages</filename>,
+     <filename>Sources</filename>, <filename>Contents</filename>,
+     <filename>Components</filename> and <filename>icons</filename> files as
+     well as <filename>Release</filename>, <filename>Index</filename> and
+     <filename>md5sum.txt</filename> files by default
+     (<literal>APT::FTPArchive::Release::Default-Patterns</literal>).
+     Additional filename patterns can be added by listing them in
+     <literal>APT::FTPArchive::Release::Patterns</literal>.  It then writes to
+     stdout a <filename>Release</filename> file containing (by default) an MD5,
      SHA1, SHA256 and SHA512 digest for each file.</para>
      <para>
      Values for the additional metadata fields in the Release file are
      taken from the corresponding variables under
      <literal>APT::FTPArchive::Release</literal>,
-     e.g. <literal>APT::FTPArchive::Release::Origin</literal>.  The supported fields
-     are: <literal>Origin</literal>, <literal>Label</literal>, <literal>Suite</literal>,
+     e.g. <literal>APT::FTPArchive::Release::Origin</literal>. The supported fields
+     are <literal>Origin</literal>, <literal>Label</literal>, <literal>Suite</literal>,
      <literal>Version</literal>, <literal>Codename</literal>, <literal>Date</literal>,
-     <literal>Valid-Until</literal>, <literal>Architectures</literal>,
-     <literal>Components</literal>, <literal>Description</literal>.</para></listitem>
+     <literal>Valid-Until</literal>, <literal>Signed-By</literal>, <literal>Architectures</literal>,
+     <literal>Components</literal> and <literal>Description</literal>.</para></listitem>
 
      </varlistentry>
 
@@ -177,10 +180,11 @@
      <variablelist>     
       <varlistentry><term><option>Packages::Compress</option></term>
       <listitem><para>
-      Sets the default compression schemes to use 
-      for the package index files. It is a string that contains a space 
-      separated list of at least one of: '.' (no compression), 'gzip' and 
-      'bzip2'. The default for all compression schemes is '. gzip'.</para></listitem>
+      Sets the default compression schemes to use
+      for the package index files. It is a string that contains a space
+      separated list of at least one of the compressors configured via the
+      <option>APT::Compressor</option> configuration scope.
+      The default for all compression schemes is '. gzip'.</para></listitem>
       </varlistentry>
 
       <varlistentry><term><option>Packages::Extensions</option></term>
diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml
index 48ad9c026..215634e99 100644
--- a/doc/apt.conf.5.xml
+++ b/doc/apt.conf.5.xml
@@ -173,6 +173,29 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
      </para></listitem>
      </varlistentry>
 
+     <varlistentry><term><option>Compressor</option></term>
+     <listitem><para>
+     This scope defines which compression formats are supported, how compression
+     and decompression can be performed if support for this format isn't built
+     into apt directly and a cost-value indicating how costly it is to compress
+     something in this format. As an example the following configuration stanza
+     would allow apt to download and uncompress as well as create and store
+     files with the low-cost <literal>.reversed</literal> file extension which
+     it will pass to the command <command>rev</command> without additional
+     commandline parameters for compression and uncompression:
+<informalexample><programlisting>
+APT::Compressor::rev {
+	Name "rev";
+	Extension ".reversed";
+	Binary "rev";
+	CompressArg {};
+	UncompressArg {};
+	Cost "10";
+};
+</programlisting></informalexample>
+     </para></listitem>
+     </varlistentry>
+
      <varlistentry><term><option>Build-Profiles</option></term>
      <listitem><para>
      List of all build profiles enabled for build-dependency resolution,
@@ -529,25 +552,25 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
      <varlistentry><term><option>CompressionTypes</option></term>
      <listitem><para>List of compression types which are understood by the acquire methods.
      Files like <filename>Packages</filename> can be available in various compression formats.
-     By default the acquire methods can decompress <command>bzip2</command>, <command>lzma</command>
-     and <command>gzip</command> compressed files; with this setting more formats can be added
-     on the fly or the used method can be changed. The syntax for this is:
+     By default the acquire methods can decompress and recompress many common formats like <command>xz</command> and
+     <command>gzip</command>; with this scope the supported formats can be queried, modified
+     as well as support for more formats added (see also <option>APT::Compressor</option>). The syntax for this is:
      <synopsis>Acquire::CompressionTypes::<replaceable>FileExtension</replaceable> "<replaceable>Methodname</replaceable>";</synopsis>
      </para><para>Also, the <literal>Order</literal> subgroup can be used to define in which order
      the acquire system will try to download the compressed files. The acquire system will try the first
      and proceed with the next compression type in this list on error, so to prefer one over the other type
-     simply add the preferred type first - default types not already added will be implicitly appended
+     simply add the preferred type first - types not already added will be implicitly appended
      to the end of the list, so e.g. <synopsis>Acquire::CompressionTypes::Order:: "gz";</synopsis> can
-     be used to prefer <command>gzip</command> compressed files over <command>bzip2</command> and <command>lzma</command>.
-     If <command>lzma</command> should be preferred over <command>gzip</command> and <command>bzip2</command> the
-     configure setting should look like this: <synopsis>Acquire::CompressionTypes::Order { "lzma"; "gz"; };</synopsis>
+     be used to prefer <command>gzip</command> compressed files over all other compression formats.
+     If <command>xz</command> should be preferred over <command>gzip</command> and <command>bzip2</command> the
+     configure setting should look like this: <synopsis>Acquire::CompressionTypes::Order { "xz"; "gz"; };</synopsis>
      It is not needed to add <literal>bz2</literal> to the list explicitly as it will be added automatically.</para>
      <para>Note that the
      <literal>Dir::Bin::<replaceable>Methodname</replaceable></literal>
-     will be checked at run time. If this option has been set, the
-     method will only be used if this file exists; e.g. for the
-     <literal>bzip2</literal> method (the inbuilt) setting is:
-     <literallayout>Dir::Bin::bzip2 "/bin/bzip2";</literallayout>
+     will be checked at run time. If this option has been set and support for
+     this format isn't directly built into apt, the method will only be used if
+     this file exists; e.g. for the <literal>bzip2</literal> method (the
+     inbuilt) setting is: <literallayout>Dir::Bin::bzip2 "/bin/bzip2";</literallayout>
      Note also that list entries specified on the command line will be added at the end of the list
      specified in the configuration files, but before the default entries. To prefer a type in this case
      over the ones specified in the configuration files you can set the option direct - not in list style.
diff --git a/doc/examples/apt-ftparchive.conf b/doc/examples/apt-ftparchive.conf
index 0d245c0af..0e8bcb2ce 100644
--- a/doc/examples/apt-ftparchive.conf
+++ b/doc/examples/apt-ftparchive.conf
@@ -7,11 +7,11 @@ Dir {
 	CacheDir ".";
 };
 
-// Create Packages, Packages.gz and Packages.bz2, remove what you don't need
+// Create Packages, Packages.gz and Packages.xz, remove/add as needed
 Default {
-	Packages::Compress ". gzip bzip2";
-	Sources::Compress ". gzip bzip2";
-	Contents::Compress ". gzip bzip2";
+	Packages::Compress ". gzip xz";
+	Sources::Compress ". gzip xz";
+	Contents::Compress ". gzip xz";
 };
 
 // Includes the main section. You can structure the directory tree under
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index 8e8110d94..dc579c394 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -88,6 +88,16 @@ APT
      Post-Invoke {"touch /var/lib/apt/post-update-stamp"; };
   };
 
+  // define a new supported compressor on the fly
+  APT::Compressor::rev {
+     Name "rev";
+     Extension ".reversed";
+     Binary "rev";
+     CompressArg {};
+     UncompressArg {};
+     Cost "10";
+  };
+
   Authentication
   {
      TrustCDROM "false";            // consider the CD-ROM always trusted
diff --git a/test/libapt/fileutl_test.cc b/test/libapt/fileutl_test.cc
index 607c4a195..8cd5132ce 100644
--- a/test/libapt/fileutl_test.cc
+++ b/test/libapt/fileutl_test.cc
@@ -4,7 +4,9 @@
 #include <apt-pkg/fileutl.h>
 #include <apt-pkg/strutl.h>
 #include <apt-pkg/aptconfiguration.h>
+#include <apt-pkg/configuration.h>
 
+#include <algorithm>
 #include <string>
 #include <vector>
 #include <stdlib.h>
@@ -162,26 +164,34 @@ static void TestFileFd(mode_t const a_umask, mode_t const ExpectedFilePermission
 
 static void TestFileFd(unsigned int const filemode)
 {
-   std::vector<APT::Configuration::Compressor> compressors = APT::Configuration::getCompressors();
-
-   // testing the (un)compress via pipe, as the 'real' compressors are usually built in via libraries
-   compressors.push_back(APT::Configuration::Compressor("rev", ".reversed", "rev", NULL, NULL, 42));
-   //compressors.push_back(APT::Configuration::Compressor("cat", ".ident", "cat", NULL, NULL, 42));
-
-   for (std::vector<APT::Configuration::Compressor>::const_iterator c = compressors.begin(); c != compressors.end(); ++c)
+   auto const compressors = APT::Configuration::getCompressors();
+   EXPECT_EQ(7, compressors.size());
+   bool atLeastOneWasTested = false;
+   for (auto const &c: compressors)
    {
       if ((filemode & FileFd::ReadWrite) == FileFd::ReadWrite &&
-	    (c->Name.empty() != true && c->Binary.empty() != true))
+	    (c.Name.empty() != true && c.Binary.empty() != true))
 	 continue;
-      TestFileFd(0002, 0664, filemode, *c);
-      TestFileFd(0022, 0644, filemode, *c);
-      TestFileFd(0077, 0600, filemode, *c);
-      TestFileFd(0026, 0640, filemode, *c);
+      atLeastOneWasTested = true;
+      TestFileFd(0002, 0664, filemode, c);
+      TestFileFd(0022, 0644, filemode, c);
+      TestFileFd(0077, 0600, filemode, c);
+      TestFileFd(0026, 0640, filemode, c);
    }
+   EXPECT_TRUE(atLeastOneWasTested);
 }
 
 TEST(FileUtlTest, FileFD)
 {
+   // testing the (un)compress via pipe, as the 'real' compressors are usually built in via libraries
+   _config->Set("APT::Compressor::rev::Name", "rev");
+   _config->Set("APT::Compressor::rev::Extension", ".reversed");
+   _config->Set("APT::Compressor::rev::Binary", "rev");
+   _config->Set("APT::Compressor::rev::Cost", 10);
+   auto const compressors = APT::Configuration::getCompressors(false);
+   EXPECT_EQ(7, compressors.size());
+   EXPECT_TRUE(std::any_of(compressors.begin(), compressors.end(), [](APT::Configuration::Compressor const &c) { return c.Name == "rev"; }));
+
    std::string const startdir = SafeGetCWD();
    EXPECT_FALSE(startdir.empty());
    std::string tempdir;