apt: Introduce the new terse apt output format 3.0

The key talking points here are:

1. Instead of long sentences, we use short concise messages,
   e.g. "The following NEW packages will be installed" becomes
   "Installing".

2. Dependencies are only listed once. We removed the
   "The following additional packages will be installed" section
   in favor of splitting up the "Installing" section into
   "Installing" and "Installing dependencies" (like dnf)

3. The order of the output is different:

   1. Packages to be installed manually
   2. Packages to be installed automatically
   4. Weak dependencies of new packages not installed
   3. Packages to be upgraded
   4. Packages to be downgraded
   5. Packages that have been kept back / are on hold
   6. Removals
   7. Essential removals

   i.e. we logically show you the action that is being
   done, followed by lists related to the action.

4. As requested by popey, we have colorful UI, with green for
   packages being installed and red for packages being removed.

Caveats:

- The list of recommends and suggests has not been updated yet,
  it should move to after the packages being installed (as they
  are what triggers them)

This also introduces output format versioning, configured by the
APT::Output-Format option. The default value is 0, except for the
apt(8) binary where it is 30 - which enables the new style.

6 files changed, 92 insertions, 36 deletions

diff --git a/apt-private/private-cmndline.cc b/apt-private/private-cmndline.cc
index 505f1f2b8..3d6816d57 100644
--- a/apt-private/private-cmndline.cc
+++ b/apt-private/private-cmndline.cc
@@ -489,6 +489,7 @@ static void BinarySpecificConfiguration(char const * const Binary)	/*{{{*/
    {
       if (getenv("NO_COLOR") == nullptr)
          _config->CndSet("Binary::apt::APT::Color", true);
+      _config->CndSet("Binary::apt::APT::Output-Version", 30);
       _config->CndSet("Binary::apt::APT::Cache::Show::Version", 2);
       _config->CndSet("Binary::apt::APT::Cache::AllVersions", false);
       _config->CndSet("Binary::apt::APT::Cache::ShowVirtuals", true);
diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc
index 254d93436..70356ea28 100644
--- a/apt-private/private-install.cc
+++ b/apt-private/private-install.cc
@@ -141,6 +141,7 @@ struct WarnUsrMerge {
 #endif
 bool InstallPackages(CacheFile &Cache, APT::PackageVector &HeldBackPackages, bool ShwKept, bool Ask, bool Safety, std::string const &Hook, CommandLine const &CmdL)
 {
+   auto outVer = _config->FindI("APT::Output-Version");
 #ifdef REQUIRE_MERGED_USR
    WarnUsrMerge warnUsrMerge(Cache);
 #endif
@@ -221,8 +222,11 @@ bool InstallPackages(CacheFile &Cache, APT::PackageVector &HeldBackPackages, boo
    }
 
    // Show all the various warning indicators
-   ShowDel(c1out,Cache);
+   if (_config->FindI("APT::Output-Version") < 30)
+      ShowDel(c1out,Cache);
    ShowNew(c1out,Cache);
+   if (_config->FindI("APT::Output-Version") >= 30 && _config->FindB("APT::Get::Show-Upgraded",true) == true)
+      ShowUpgraded(c1out,Cache);
    if (ShwKept == true)
    {
       ShowPhasing(c1out, Cache, PhasingPackages);
@@ -231,10 +235,13 @@ bool InstallPackages(CacheFile &Cache, APT::PackageVector &HeldBackPackages, boo
 	 _error->Notice("Some packages may have been kept back due to phasing.");
    }
    bool const Hold = not ShowHold(c1out,Cache);
-   if (_config->FindB("APT::Get::Show-Upgraded",true) == true)
+   if (_config->FindI("APT::Output-Version") < 30 && _config->FindB("APT::Get::Show-Upgraded",true) == true)
       ShowUpgraded(c1out,Cache);
    bool const Downgrade = !ShowDowngraded(c1out,Cache);
 
+   // Show removed packages last
+   if (_config->FindI("APT::Output-Version") >= 30)
+      ShowDel(c1out,Cache);
    bool Essential = false;
    if (_config->FindB("APT::Get::Download-Only",false) == false)
         Essential = !ShowEssential(c1out,Cache);
@@ -312,12 +319,12 @@ bool InstallPackages(CacheFile &Cache, APT::PackageVector &HeldBackPackages, boo
       if (DebBytes != FetchBytes)
 	 //TRANSLATOR: The required space between number and unit is already included
 	 // in the replacement strings, so %sB will be correctly translate in e.g. 1,5 MB
-	 ioprintf(c1out,_("Need to get %sB/%sB of archives.\n"),
+	 ioprintf(c1out,outVer < 30 ? _("Need to get %sB/%sB of archives.\n") : _("  Download size: %sB/%sB\n"),
 	       SizeToStr(FetchBytes).c_str(),SizeToStr(DebBytes).c_str());
       else if (DebBytes != 0)
 	 //TRANSLATOR: The required space between number and unit is already included
 	 // in the replacement string, so %sB will be correctly translate in e.g. 1,5 MB
-	 ioprintf(c1out,_("Need to get %sB of archives.\n"),
+	 ioprintf(c1out,outVer < 30 ? _("Need to get %sB of archives.\n") : _("  Download size: %sB\n"),
 	       SizeToStr(DebBytes).c_str());
    }
 
@@ -325,14 +332,16 @@ bool InstallPackages(CacheFile &Cache, APT::PackageVector &HeldBackPackages, boo
    if (Cache->UsrSize() >= 0)
       //TRANSLATOR: The required space between number and unit is already included
       // in the replacement string, so %sB will be correctly translate in e.g. 1,5 MB
-      ioprintf(c1out,_("After this operation, %sB of additional disk space will be used.\n"),
+      ioprintf(c1out,outVer < 30 ? _("After this operation, %sB of additional disk space will be used.\n") : _("  Installed size: %sB\n"),
 	       SizeToStr(Cache->UsrSize()).c_str());
    else
       //TRANSLATOR: The required space between number and unit is already included
       // in the replacement string, so %sB will be correctly translate in e.g. 1,5 MB
-      ioprintf(c1out,_("After this operation, %sB disk space will be freed.\n"),
+      ioprintf(c1out,outVer < 30 ? _("After this operation, %sB disk space will be freed.\n") : _("  Freed space: %sB\n"),
 	       SizeToStr(-1*Cache->UsrSize()).c_str());
 
+   if (outVer >= 30)
+      ioprintf(c1out,"\n");
    if (DownloadAllowed)
       if (CheckFreeSpaceBeforeDownload(_config->FindDir("Dir::Cache::Archives"), (FetchBytes - FetchPBytes)) == false)
 	 return false;
@@ -368,7 +377,7 @@ bool InstallPackages(CacheFile &Cache, APT::PackageVector &HeldBackPackages, boo
 	 if (_config->FindI("quiet",0) < 2 &&
 	     _config->FindB("APT::Get::Assume-Yes",false) == false)
 	 {
-	    if (YnPrompt(_("Do you want to continue?")) == false)
+	    if (YnPrompt(outVer < 30 ? _("Do you want to continue?") : _("Continue?")) == false)
 	    {
 	       c2out << _("Abort.") << std::endl;
 	       exit(1);
@@ -933,14 +942,16 @@ bool DoInstall(CommandLine &CmdL)
    /* Print out a list of packages that are going to be installed extra
       to what the user asked */
    SortedPackageUniverse Universe(Cache);
-   if (Cache->InstCount() != verset[MOD_INSTALL].size())
+   if (_config->FindI("APT::Output-Version") < 30 && Cache->InstCount() != verset[MOD_INSTALL].size())
       ShowList(c1out, _("The following additional packages will be installed:"), Universe,
 	    PkgIsExtraInstalled(&Cache, &verset[MOD_INSTALL]),
-	    &PrettyFullName, CandidateVersion(&Cache));
+	    &PrettyFullName, CandidateVersion(&Cache), "APT::Color::Green");
+
 
    /* Print out a list of suggested and recommended packages */
    {
       std::list<std::string> Recommends, Suggests, SingleRecommends, SingleSuggests;
+      SortedPackageUniverse Universe(Cache);
       for (auto const &Pkg: Universe)
       {
 	 /* Just look at the ones we want to install */
diff --git a/apt-private/private-output.cc b/apt-private/private-output.cc
index f04c35b8a..bffdd6f67 100644
--- a/apt-private/private-output.cc
+++ b/apt-private/private-output.cc
@@ -524,8 +524,10 @@ void ShowBroken(ostream &out, CacheFile &Cache, bool const Now)
 {
    if (Cache->BrokenCount() == 0)
       return;
-
-   out << _("The following packages have unmet dependencies:") << endl;
+   if (_config->FindI("APT::Output-Version") < 30)
+      out << _("The following packages have unmet dependencies:") << endl;
+   else
+      out << _("Unsatisfied dependencies:") << endl;
    SortedPackageUniverse Universe(Cache);
    for (auto const &Pkg: Universe)
       ShowBrokenPackage(out, &Cache, Pkg, Now);
@@ -535,7 +537,10 @@ void ShowBroken(ostream &out, pkgCacheFile &Cache, bool const Now)
    if (Cache->BrokenCount() == 0)
       return;
 
-   out << _("The following packages have unmet dependencies:") << endl;
+   if (_config->FindI("APT::Output-Version") < 30)
+      out << _("The following packages have unmet dependencies:") << endl;
+   else
+      out << _("Unsatisfied dependencies:") << endl;
    APT::PackageUniverse Universe(Cache);
    for (auto const &Pkg: Universe)
       ShowBrokenPackage(out, &Cache, Pkg, Now);
@@ -545,17 +550,33 @@ void ShowBroken(ostream &out, pkgCacheFile &Cache, bool const Now)
 void ShowNew(ostream &out,CacheFile &Cache)
 {
    SortedPackageUniverse Universe(Cache);
-   ShowList(out,_("The following NEW packages will be installed:"), Universe,
-	 [&Cache](pkgCache::PkgIterator const &Pkg) { return Cache[Pkg].NewInstall(); },
+   if (_config->FindI("APT::Output-Version") < 30) {
+      ShowList(out,_("The following NEW packages will be installed:"), Universe,
+	    [&Cache](pkgCache::PkgIterator const &Pkg) { return Cache[Pkg].NewInstall(); },
+	    &PrettyFullName,
+	    CandidateVersion(&Cache),
+	    "APT::Color::Green");
+      return;
+   }
+
+   ShowList(out,_("Installing:"), Universe,
+	 [&Cache](pkgCache::PkgIterator const &Pkg) { return Cache[Pkg].NewInstall() && (Cache[Pkg].Flags & pkgCache::Flag::Auto) == 0; },
 	 &PrettyFullName,
-	 CandidateVersion(&Cache));
+	 CandidateVersion(&Cache),
+	 "APT::Color::Green");
+   ShowList(out,_("Installing dependencies:"), Universe,
+	 [&Cache](pkgCache::PkgIterator const &Pkg) { return Cache[Pkg].NewInstall() && Cache[Pkg].Flags & pkgCache::Flag::Auto;},
+	 &PrettyFullName,
+	 CandidateVersion(&Cache),
+	 "APT::Color::Green");
 }
 									/*}}}*/
 // ShowDel - Show packages to delete					/*{{{*/
 void ShowDel(ostream &out,CacheFile &Cache)
 {
    SortedPackageUniverse Universe(Cache);
-   ShowList(out,_("The following packages will be REMOVED:"), Universe,
+   auto title = _config->FindI("APT::Output-Version") < 30 ? _("The following packages will be REMOVED:") : _("REMOVING:");
+   ShowList(out,title, Universe,
 	 [&Cache](pkgCache::PkgIterator const &Pkg) { return Cache[Pkg].Delete(); },
 	 [&Cache](pkgCache::PkgIterator const &Pkg)
 	 {
@@ -564,14 +585,18 @@ void ShowDel(ostream &out,CacheFile &Cache)
 	       str.append("*");
 	    return str;
 	 },
-	 CandidateVersion(&Cache));
+	 CandidateVersion(&Cache),
+	 "APT::Color::Red");
 }
 									/*}}}*/
 // ShowPhasing - Show packages kept due to phasing			/*{{{*/
 void ShowPhasing(ostream &out, CacheFile &Cache, APT::PackageVector const &HeldBackPackages)
 {
    SortedPackageUniverse Universe(Cache);
-   ShowList(out, _("The following upgrades have been deferred due to phasing:"), HeldBackPackages,
+   auto title = _config->FindI("APT::Output-Version") < 30
+	       ? _("The following upgrades have been deferred due to phasing:")
+	       : _("Not upgrading yet due to phasing:");
+   ShowList(out, title, HeldBackPackages,
 	    &AlwaysTrue,
 	    &PrettyFullName,
 	    CurrentToCandidateVersion(&Cache));
@@ -581,7 +606,8 @@ void ShowPhasing(ostream &out, CacheFile &Cache, APT::PackageVector const &HeldB
 void ShowKept(ostream &out,CacheFile &Cache, APT::PackageVector const &HeldBackPackages)
 {
    SortedPackageUniverse Universe(Cache);
-   ShowList(out,_("The following packages have been kept back:"), HeldBackPackages,
+   auto title = _config->FindI("APT::Output-Version") < 30 ? _("The following packages have been kept back:") : _("Not upgrading:");
+   ShowList(out, title, HeldBackPackages,
 	 &AlwaysTrue,
 	 &PrettyFullName,
 	 CurrentToCandidateVersion(&Cache));
@@ -591,13 +617,15 @@ void ShowKept(ostream &out,CacheFile &Cache, APT::PackageVector const &HeldBackP
 void ShowUpgraded(ostream &out,CacheFile &Cache)
 {
    SortedPackageUniverse Universe(Cache);
-   ShowList(out,_("The following packages will be upgraded:"), Universe,
+   auto title = _config->FindI("APT::Output-Version") < 30 ? _("The following packages will be upgraded:") : _("Upgrading:");
+   ShowList(out, title, Universe,
 	 [&Cache](pkgCache::PkgIterator const &Pkg)
 	 {
 	    return Cache[Pkg].Upgrade() == true && Cache[Pkg].NewInstall() == false;
 	 },
 	 &PrettyFullName,
-	 CurrentToCandidateVersion(&Cache));
+	 CurrentToCandidateVersion(&Cache),
+	 "APT::Color::Green");
 }
 									/*}}}*/
 // ShowDowngraded - Show downgraded packages				/*{{{*/
@@ -606,20 +634,23 @@ void ShowUpgraded(ostream &out,CacheFile &Cache)
 bool ShowDowngraded(ostream &out,CacheFile &Cache)
 {
    SortedPackageUniverse Universe(Cache);
-   return ShowList(out,_("The following packages will be DOWNGRADED:"), Universe,
+   auto title = _config->FindI("APT::Output-Version") < 30 ? _("The following packages will be DOWNGRADED:") : _("DOWNGRADING:");
+   return ShowList(out, title, Universe,
 	 [&Cache](pkgCache::PkgIterator const &Pkg)
 	 {
 	    return Cache[Pkg].Downgrade() == true && Cache[Pkg].NewInstall() == false;
 	 },
 	 &PrettyFullName,
-	 CurrentToCandidateVersion(&Cache));
+	 CurrentToCandidateVersion(&Cache),
+	 "APT::Color::Green");
 }
 									/*}}}*/
 // ShowHold - Show held but changed packages				/*{{{*/
 bool ShowHold(ostream &out,CacheFile &Cache)
 {
    SortedPackageUniverse Universe(Cache);
-   return ShowList(out,_("The following held packages will be changed:"), Universe,
+   auto title = _config->FindI("APT::Output-Version") < 30 ? _("The following held packages will be changed:") : _("Changing held packages:Changing held packages:");
+   return ShowList(out, title, Universe,
 	 [&Cache](pkgCache::PkgIterator const &Pkg)
 	 {
 	    return Pkg->SelectedState == pkgCache::State::Hold &&
@@ -710,6 +741,7 @@ void Stats(ostream &out, pkgDepCache &Dep, APT::PackageVector const &HeldBackPac
    unsigned long Downgrade = 0;
    unsigned long Install = 0;
    unsigned long ReInstall = 0;
+   auto outVer = _config->FindI("APT::Output-Version");
    for (pkgCache::PkgIterator I = Dep.PkgBegin(); I.end() == false; ++I)
    {
       if (Dep[I].NewInstall() == true)
@@ -726,18 +758,20 @@ void Stats(ostream &out, pkgDepCache &Dep, APT::PackageVector const &HeldBackPac
       if (Dep[I].Delete() == false && (Dep[I].iFlags & pkgDepCache::ReInstall) == pkgDepCache::ReInstall)
 	 ReInstall++;
    }   
-
-   ioprintf(out,_("%lu upgraded, %lu newly installed, "),
+   if (outVer >= 30)
+      ioprintf(out, _("Summary:\n"));
+   ioprintf(out,outVer < 30 ? _("%lu upgraded, %lu newly installed, ") : _("  Upgrades: %lu, Installs: %lu, "),
 	    Upgrade,Install);
    
    if (ReInstall != 0)
-      ioprintf(out,_("%lu reinstalled, "),ReInstall);
+      ioprintf(out,outVer < 30 ? _("%lu reinstalled, ") : _("Reinstalls: %lu, "),ReInstall);
    if (Downgrade != 0)
-      ioprintf(out,_("%lu downgraded, "),Downgrade);
+      ioprintf(out,outVer < 30 ? _("%lu downgraded, ") : _("Downgrades: %lu, "),Downgrade);
 
-   ioprintf(out,_("%lu to remove and %lu not upgraded.\n"),
+   ioprintf(out, outVer < 30 ? _("%lu to remove and %lu not upgraded.\n") : _("Removals: %lu, Not Upgrading: %lu\n"),
 	    Dep.DelCount(), HeldBackPackages.size());
-   
+
+   // FIXME: outVer
    if (Dep.BadCount() != 0)
       ioprintf(out,_("%lu not fully installed or removed.\n"),
 	       Dep.BadCount());
diff --git a/apt-private/private-output.h b/apt-private/private-output.h
index 70f5b9726..4cc7c01b4 100644
--- a/apt-private/private-output.h
+++ b/apt-private/private-output.h
@@ -42,15 +42,19 @@ template<class Container, class PredicateC, class DisplayP, class DisplayV> bool
       Container const &cont,
       PredicateC Predicate,
       DisplayP PkgDisplay,
-      DisplayV VerboseDisplay)
+      DisplayV VerboseDisplay,
+      std::string colorName = "APT::Color::Neutral")
 {
    size_t const ScreenWidth = (::ScreenWidth > 3) ? ::ScreenWidth - 3 : 0;
    int ScreenUsed = 0;
    bool const ShowVersions = _config->FindB("APT::Get::Show-Versions", false);
-   bool const ListColumns = _config->FindB("APT::Get::List-Columns", true);
+   bool const ListColumns = _config->FindB("APT::Get::List-Columns", _config->FindI("APT::Output-Version") >= 30);
    bool printedTitle = false;
    std::vector<std::string> PackageList;
 
+   auto setColor = _config->FindI("APT::Output-Version") >= 30 ? _config->Find(colorName) : "";
+   auto resetColor = _config->FindI("APT::Output-Version") >= 30 ? _config->Find("APT::Color::Neutral") : "";
+
    for (auto const &Pkg: cont)
    {
       if (Predicate(Pkg) == false)
@@ -64,7 +68,7 @@ template<class Container, class PredicateC, class DisplayP, class DisplayV> bool
 
       if (ShowVersions == true)
       {
-	 out << std::endl << "   " << PkgDisplay(Pkg);
+	 out << std::endl << "   " << setColor << PkgDisplay(Pkg) << resetColor;
 	 std::string const verbose = VerboseDisplay(Pkg);
 	 if (verbose.empty() == false)
 	    out << " (" << verbose << ")";
@@ -87,7 +91,7 @@ template<class Container, class PredicateC, class DisplayP, class DisplayV> bool
 	       out << " ";
 	       ++ScreenUsed;
 	    }
-	    out << PkgName;
+	    out << setColor << PkgName << resetColor;
 	    ScreenUsed += PkgName.length();
 	 }
       }
@@ -96,8 +100,13 @@ template<class Container, class PredicateC, class DisplayP, class DisplayV> bool
    if (printedTitle == true)
    {
       out << std::endl;
-      if (ListColumns && not PackageList.empty())
+      if (ListColumns && not PackageList.empty()) {
+	 out << setColor;
 	 ShowWithColumns(out, PackageList, 2, ScreenWidth);
+	 out << resetColor;
+      }
+      if (_config->FindI("APT::Output-Version") >= 30)
+	 out << std::endl;
       return false;
    }
    return true;
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index a8fc98daa..cc9a2e765 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -723,6 +723,7 @@ apt::system "<STRING>";
 apt::acquire::translation "<STRING>"; // deprecated in favor of Acquire::Languages
 apt::color::highlight "<STRING>";
 apt::color::neutral "<STRING>";
+apt::output-version "<INT>";
 
 dpkgpm::reporting-steps "<INT>";
 
diff --git a/test/integration/framework b/test/integration/framework
index 57d6849da..147de9808 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -444,7 +444,7 @@ _setupprojectenvironment() {
 	echo "Dir::Etc \"etc/apt\";" >> aptconfig.conf
 	echo "Dir::Log \"var/log/apt\";" >> aptconfig.conf
 	echo "APT::Get::Show-User-Simulation-Note \"false\";" >> aptconfig.conf
-	echo "APT::Get::List-Columns \"false\";" >> aptconfig.conf
+	echo "Binary::apt::APT::Output-Version \"0\";" >> aptconfig.conf
 	echo "Dir::Bin::Methods \"${TMPWORKINGDIRECTORY}/rootdir/usr/lib/apt/methods\";" >> aptconfig.conf
 	# either store apt-key were we can access it, even if we run it as a different user
 	#cp "${APTCMDLINEBINDIR}/apt-key" "${TMPWORKINGDIRECTORY}/rootdir/usr/bin/"