Don't expect duplicated dpkg status-fd messages

The progress reporting relies on parsing the status reports of
dpkg which used to repeat being in the same state multiple times
in the same run, but by fixing #365921 it will stop doing so.

The problem is in theory just with 'config-files' in case we do purge as
this (can) do remove + purge in one step, but we remove this also for
the unpack + configure combination althrough we handle these currently
in two independent dpkg calls.

5 files changed, 83 insertions, 71 deletions

diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index 68674fa02..da2d5e3ae 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -1154,16 +1154,14 @@ void pkgDPkgPM::BuildPackagesProgressMap()
 {
    // map the dpkg states to the operations that are performed
    // (this is sorted in the same way as Item::Ops)
-   static const std::array<std::array<DpkgState, 3>, 4> DpkgStatesOpMap = {{
+   static const std::array<std::array<DpkgState, 2>, 4> DpkgStatesOpMap = {{
       // Install operation
       {{
 	 {"half-installed", N_("Preparing %s")},
 	 {"unpacked", N_("Unpacking %s") },
-	 {nullptr, nullptr}
       }},
       // Configure operation
       {{
-	 {"unpacked",N_("Preparing to configure %s") },
 	 {"half-configured", N_("Configuring %s") },
 	 { "installed", N_("Installed %s")},
       }},
@@ -1171,13 +1169,11 @@ void pkgDPkgPM::BuildPackagesProgressMap()
       {{
 	 {"half-configured", N_("Preparing for removal of %s")},
 	 {"half-installed", N_("Removing %s")},
-	 {"config-files",  N_("Removed %s")},
       }},
       // Purge operation
       {{
 	 {"config-files", N_("Preparing to completely remove %s")},
 	 {"not-installed", N_("Completely removed %s")},
-	 {nullptr, nullptr}
       }},
    }};
    static_assert(Item::Purge == 3, "Enum item has unexpected index for mapping array");
@@ -1193,21 +1189,16 @@ void pkgDPkgPM::BuildPackagesProgressMap()
 
       string const name = I.Pkg.FullName();
       PackageOpsDone[name] = 0;
-      auto AddToPackageOps = std::back_inserter(PackageOps[name]);
-      if (I.Op == Item::Purge && I.Pkg->CurrentVer != 0)
-      {
-	 // purging a package which is installed first passes through remove states
-	 auto const DpkgOps = DpkgStatesOpMap[Item::Remove];
-	 std::copy(DpkgOps.begin(), DpkgOps.end(), AddToPackageOps);
+      auto AddToPackageOps = [&](decltype(I.Op) const Op) {
+	 auto const DpkgOps = DpkgStatesOpMap[Op];
+	 std::copy(DpkgOps.begin(), DpkgOps.end(), std::back_inserter(PackageOps[name]));
 	 PackagesTotal += DpkgOps.size();
-      }
-      auto const DpkgOps = DpkgStatesOpMap[I.Op];
-      std::copy_if(DpkgOps.begin(), DpkgOps.end(), AddToPackageOps, [&](DpkgState const &state) {
-	 if (state.state == nullptr)
-	    return false;
-	 ++PackagesTotal;
-	 return true;
-      });
+      };
+      // purging a package which is installed first passes through remove states
+      if (I.Op == Item::Purge && I.Pkg->CurrentVer != 0)
+	 AddToPackageOps(Item::Remove);
+      AddToPackageOps(I.Op);
+
       if ((I.Op == Item::Remove || I.Op == Item::Purge) && I.Pkg->CurrentVer != 0)
       {
 	 if (I.Pkg->CurrentState == pkgCache::State::UnPacked ||
@@ -1631,9 +1622,9 @@ bool pkgDPkgPM::Go(APT::Progress::PackageManager *progress)
 	    approvedStates.Remove(*Ver);
 	    Purges.erase(Ver);
 	    auto && RemOp = PackageOps[C.first->Pkg.FullName()];
-	    if (RemOp.size() == 5)
+	    if (RemOp.size() == 4)
 	    {
-	       RemOp.erase(std::next(RemOp.begin(), 3), RemOp.end());
+	       RemOp.erase(std::next(RemOp.begin(), 2), RemOp.end());
 	       PackagesTotal -= 2;
 	    }
 	    else
diff --git a/test/integration/test-apt-progress-fd b/test/integration/test-apt-progress-fd
index 57728ca81..2b9b6532a 100755
--- a/test/integration/test-apt-progress-fd
+++ b/test/integration/test-apt-progress-fd
@@ -10,7 +10,7 @@ configarchitecture 'amd64' 'i386'
 buildsimplenativepackage 'testing' 'amd64' '0.1' 'stable'
 buildsimplenativepackage 'testing' 'amd64' '0.8.15' 'stable'
 buildsimplenativepackage 'testing2' 'amd64,i386' '0.8.15' 'stable'
-setupaptarchive 
+setupaptarchive
 
 # install native
 exec 3> apt-progress.log
@@ -19,13 +19,12 @@ testfileequal './apt-progress.log' 'dlstatus:1:0.0000:Retrieving file 1 of 1
 dlstatus:1:100.0000:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0.0000:Running dpkg
 pmstatus:testing:0.0000:Installing testing (amd64)
-pmstatus:testing:16.6667:Preparing testing (amd64)
-pmstatus:testing:33.3333:Unpacking testing (amd64)
-pmstatus:testing:50.0000:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:50.0000:Running dpkg
-pmstatus:testing:50.0000:Configuring testing (amd64)
-pmstatus:testing:66.6667:Configuring testing (amd64)
-pmstatus:testing:83.3333:Installed testing (amd64)'
+pmstatus:testing:20.0000:Preparing testing (amd64)
+pmstatus:testing:40.0000:Unpacking testing (amd64)
+pmstatus:dpkg-exec:40.0000:Running dpkg
+pmstatus:testing:40.0000:Configuring testing (amd64)
+pmstatus:testing:60.0000:Configuring testing (amd64)
+pmstatus:testing:80.0000:Installed testing (amd64)'
 
 # upgrade
 exec 3> apt-progress.log
@@ -34,13 +33,12 @@ testfileequal './apt-progress.log' 'dlstatus:1:0.0000:Retrieving file 1 of 1
 dlstatus:1:100.0000:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0.0000:Running dpkg
 pmstatus:testing:0.0000:Installing testing (amd64)
-pmstatus:testing:16.6667:Preparing testing (amd64)
-pmstatus:testing:33.3333:Unpacking testing (amd64)
-pmstatus:testing:50.0000:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:50.0000:Running dpkg
-pmstatus:testing:50.0000:Configuring testing (amd64)
-pmstatus:testing:66.6667:Configuring testing (amd64)
-pmstatus:testing:83.3333:Installed testing (amd64)'
+pmstatus:testing:20.0000:Preparing testing (amd64)
+pmstatus:testing:40.0000:Unpacking testing (amd64)
+pmstatus:dpkg-exec:40.0000:Running dpkg
+pmstatus:testing:40.0000:Configuring testing (amd64)
+pmstatus:testing:60.0000:Configuring testing (amd64)
+pmstatus:testing:80.0000:Installed testing (amd64)'
 
 # reinstall
 exec 3> apt-progress.log
@@ -49,23 +47,21 @@ testfileequal './apt-progress.log' 'dlstatus:1:0.0000:Retrieving file 1 of 1
 dlstatus:1:100.0000:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0.0000:Running dpkg
 pmstatus:testing:0.0000:Installing testing (amd64)
-pmstatus:testing:16.6667:Preparing testing (amd64)
-pmstatus:testing:33.3333:Unpacking testing (amd64)
-pmstatus:testing:50.0000:Preparing to configure testing (amd64)
-pmstatus:dpkg-exec:50.0000:Running dpkg
-pmstatus:testing:50.0000:Configuring testing (amd64)
-pmstatus:testing:66.6667:Configuring testing (amd64)
-pmstatus:testing:83.3333:Installed testing (amd64)'
+pmstatus:testing:20.0000:Preparing testing (amd64)
+pmstatus:testing:40.0000:Unpacking testing (amd64)
+pmstatus:dpkg-exec:40.0000:Running dpkg
+pmstatus:testing:40.0000:Configuring testing (amd64)
+pmstatus:testing:60.0000:Configuring testing (amd64)
+pmstatus:testing:80.0000:Installed testing (amd64)'
 
 # and remove
 exec 3> apt-progress.log
 testsuccess aptget remove testing -y -o APT::Status-Fd=3
 testfileequal './apt-progress.log' 'pmstatus:dpkg-exec:0.0000:Running dpkg
 pmstatus:testing:0.0000:Removing testing (amd64)
-pmstatus:testing:25.0000:Preparing for removal of testing (amd64)
-pmstatus:testing:50.0000:Removing testing (amd64)
-pmstatus:testing:75.0000:Removed testing (amd64)
-pmstatus:dpkg-exec:75.0000:Running dpkg'
+pmstatus:testing:33.3333:Preparing for removal of testing (amd64)
+pmstatus:testing:66.6667:Removing testing (amd64)
+pmstatus:dpkg-exec:66.6667:Running dpkg'
 
 # install non-native and ensure we get proper progress info
 exec 3> apt-progress.log
@@ -74,12 +70,11 @@ testfileequal './apt-progress.log' 'dlstatus:1:0.0000:Retrieving file 1 of 1
 dlstatus:1:100.0000:Retrieving file 1 of 1
 pmstatus:dpkg-exec:0.0000:Running dpkg
 pmstatus:testing2:0.0000:Installing testing2 (i386)
-pmstatus:testing2:16.6667:Preparing testing2 (i386)
-pmstatus:testing2:33.3333:Unpacking testing2 (i386)
-pmstatus:testing2:50.0000:Preparing to configure testing2 (i386)
-pmstatus:dpkg-exec:50.0000:Running dpkg
-pmstatus:testing2:50.0000:Configuring testing2 (i386)
-pmstatus:testing2:66.6667:Configuring testing2 (i386)
-pmstatus:testing2:83.3333:Installed testing2 (i386)'
+pmstatus:testing2:20.0000:Preparing testing2 (i386)
+pmstatus:testing2:40.0000:Unpacking testing2 (i386)
+pmstatus:dpkg-exec:40.0000:Running dpkg
+pmstatus:testing2:40.0000:Configuring testing2 (i386)
+pmstatus:testing2:60.0000:Configuring testing2 (i386)
+pmstatus:testing2:80.0000:Installed testing2 (i386)'
 
 rm -f apt-progress*.log
diff --git a/test/integration/test-apt-progress-fd-deb822 b/test/integration/test-apt-progress-fd-deb822
index d9a4505d1..0c6c525ae 100755
--- a/test/integration/test-apt-progress-fd-deb822
+++ b/test/integration/test-apt-progress-fd-deb822
@@ -10,7 +10,7 @@ configarchitecture 'amd64' 'i386'
 buildsimplenativepackage 'testing' 'amd64' '0.1' 'stable'
 buildsimplenativepackage 'testing' 'all' '0.8.15' 'stable'
 buildsimplenativepackage 'testing2' 'amd64,i386' '0.8.15' 'stable'
-setupaptarchive 
+setupaptarchive
 
 # install native
 exec 3> apt-progress.log
@@ -27,36 +27,31 @@ Message: Installing testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 16.6667
+Percent: 20.0000
 Message: Preparing testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 33.3333
+Percent: 40.0000
 Message: Unpacking testing (amd64)
 
 Status: progress
-Package: testing:amd64
-Percent: 50.0000
-Message: Preparing to configure testing (amd64)
-
-Status: progress
-Percent: 50.0000
+Percent: 40.0000
 Message: Running dpkg
 
 Status: progress
 Package: testing:amd64
-Percent: 50.0000
+Percent: 40.0000
 Message: Configuring testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 66.6667
+Percent: 60.0000
 Message: Configuring testing (amd64)
 
 Status: progress
 Package: testing:amd64
-Percent: 83.3333
+Percent: 80.0000
 Message: Installed testing (amd64)
 '
 
diff --git a/test/integration/test-apt-progress-fd-error b/test/integration/test-apt-progress-fd-error
index 4439c042a..39c1053a1 100755
--- a/test/integration/test-apt-progress-fd-error
+++ b/test/integration/test-apt-progress-fd-error
@@ -18,8 +18,8 @@ setupaptarchive
 
 exec 3> apt-progress.log
 testfailure aptget install foo1 foo2 -y -o APT::Status-Fd=3
-msgtest 'Ensure correct error message'
-testsuccess --nomsg grep "foo2_0.8.15_[^.]\+.deb:36.3636:trying to overwrite '/usr/bin/file-conflict', which is also in package foo1 0.8.15" apt-progress.log
+msgtest 'Ensure correct error message' 'file-conflict'
+testsuccess --nomsg grep "foo2_0.8.15_[^.]\+.deb:[0-9.]\+:trying to overwrite '/usr/bin/file-conflict', which is also in package foo1 0.8.15" apt-progress.log
 
 testsuccess test -s rootdir/var/crash/foo2.0.crash
 testsuccess grep '^Package: foo2 0.8.15$' rootdir/var/crash/foo2.0.crash
diff --git a/test/integration/test-no-fds-leaked-to-maintainer-scripts b/test/integration/test-no-fds-leaked-to-maintainer-scripts
index 747af69ff..85b3d0ee1 100755
--- a/test/integration/test-no-fds-leaked-to-maintainer-scripts
+++ b/test/integration/test-no-fds-leaked-to-maintainer-scripts
@@ -55,7 +55,15 @@ checkinstall() {
 	tail -n +3 rootdir/var/log/apt/term.log | head -n -1 > terminal.log
 	testfileequal 'terminal.log' "$(cat terminal.output)"
 
-	testequal "startup archives unpack
+	testequalor2 "startup archives unpack
+install $PKGNAME <none> 1.0
+status half-installed $PKGNAME 1.0
+status unpacked $PKGNAME 1.0
+startup packages configure
+configure $PKGNAME 1.0 <none>
+status unpacked $PKGNAME 1.0
+status half-configured $PKGNAME 1.0
+status installed $PKGNAME 1.0" "startup archives unpack
 install $PKGNAME <none> 1.0
 status half-installed $PKGNAME 1.0
 status unpacked $PKGNAME 1.0
@@ -78,7 +86,18 @@ checkpurge() {
 	testfileequal 'terminal.log' "$(cat terminal.output)"
 
 	if dpkg --compare-versions "$(dpkg_version)" 'ge' '1.18.11'; then
-		testequal "startup packages remove
+		testequalor2 "startup packages remove
+status installed $PKGNAME 1.0
+remove $PKGNAME 1.0 <none>
+status half-configured $PKGNAME 1.0
+status half-installed $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+startup packages configure
+startup packages purge
+purge $PKGNAME 1.0 <none>
+status config-files $PKGNAME 1.0
+status not-installed $PKGNAME <none>
+startup packages configure" "startup packages remove
 status installed $PKGNAME 1.0
 remove $PKGNAME 1.0 <none>
 status half-configured $PKGNAME 1.0
@@ -96,7 +115,19 @@ status config-files $PKGNAME 1.0
 status not-installed $PKGNAME <none>
 startup packages configure" cut -f 3- -d' ' rootdir/var/log/dpkg.log
 	else
-		testequal "startup packages remove
+		testequalor2 "startup packages remove
+status installed $PKGNAME 1.0
+remove $PKGNAME 1.0 <none>
+status half-configured $PKGNAME 1.0
+status half-installed $PKGNAME 1.0
+status config-files $PKGNAME 1.0
+startup packages configure
+startup packages purge
+remove $PKGNAME 1.0 <none>
+purge $PKGNAME 1.0 <none>
+status config-files $PKGNAME 1.0
+status not-installed $PKGNAME <none>
+startup packages configure" "startup packages remove
 status installed $PKGNAME 1.0
 remove $PKGNAME 1.0 <none>
 status half-configured $PKGNAME 1.0