solver3: Assume manual packages

If we have allowed the removal of manual packages, assume them
all before starting the solver. This should ensure that as long
as there is a solution that does not remove a manually installed
package, it is found.

This requires a sweeping set of changes in the test suite, but
ensures that we get "safe" behavior from the solver. We have
in particular seen that without asserting the installed packages,
several people ended up with ubuntu-minimal and perl removed in
a situation where that was not warranted, that is, they install
winehq, and then pull in some new perl packages in a newer version
than the installed one, and the solver chose to create a mismatched
version set, which then caused the main perl package to not be
installable.

Oops: 1b55173a-3526-11f0-b7ac-fa163e171f02
Oops: dbd5149e-36b9-11f0-bb74-fa163ec44ecd

8 files changed, 98 insertions, 17 deletions

diff --git a/test/integration/test-apt-get-upgrade b/test/integration/test-apt-get-upgrade
index c9e56d97d..0eaa5a00a 100755
--- a/test/integration/test-apt-get-upgrade
+++ b/test/integration/test-apt-get-upgrade
@@ -5,7 +5,6 @@ TESTDIR="$(readlink -f "$(dirname "$0")")"
 . "$TESTDIR/framework"
 
 setupenvironment
-allowremovemanual
 configarchitecture "i386"
 
 # simple case
@@ -30,9 +29,14 @@ insertpackage 'unstable' 'sysvinit' 'all' '2'
 
 setupaptarchive
 
+# Solver 3.0 only wants to remove automatically installed packages
+testsuccess aptmark auto ~i
+echo 'APT::Get::HideAutoRemove "1";' > rootdir/etc/apt/apt.conf.d/hide-auto-remove
+
 # Test if normal upgrade works as expected
 UPGRADE='Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following packages have been kept back:
   init upgrade-with-conflict upgrade-with-new-dep
@@ -47,6 +51,7 @@ testsuccessequal "$UPGRADE" apt upgrade -s --without-new-pkgs
 # Test if apt-get upgrade --with-new-pkgs works
 UPGRADENEW='Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following NEW packages will be installed:
   new-dep
@@ -66,6 +71,7 @@ if [ "$APT_SOLVER" = "3.0" ]; then
 # but it's not clear that is reliably possible.
 UPGRADENEW="Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following NEW packages will be installed:
   new-dep sysvinit
@@ -91,6 +97,7 @@ testsuccessequal "$UPGRADENEW" apt upgrade -s
 # Test if apt-get dist-upgrade works
 testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following packages will be REMOVED:
   conflicting-dep
@@ -123,6 +130,7 @@ testsuccessequal "$UPGRADE" apt upgrade -s --without-new-pkgs
 
 UPGRADENEW='Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following NEW packages will be installed:
   new-dep sysvinit
@@ -146,6 +154,7 @@ testsuccessequal "$UPGRADENEW" apt upgrade -s
 
 testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following packages will be REMOVED:
   conflicting-dep
@@ -171,6 +180,7 @@ Conf upgrade-with-new-dep (2.0 unstable [all])' aptget -s dist-upgrade
 # --no-strict-pinning pulls in systemd due to the dependency
 testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following packages will be REMOVED:
   conflicting-dep
@@ -199,6 +209,7 @@ Pin: release unstable
 Pin-Priority: -1' > rootdir/etc/apt/preferences.d/no-strict-pinning.pref
 testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following packages will be REMOVED:
   conflicting-dep
diff --git a/test/integration/test-bug-591882-conkeror b/test/integration/test-bug-591882-conkeror
index 6d4712a25..f3ac2866e 100755
--- a/test/integration/test-bug-591882-conkeror
+++ b/test/integration/test-bug-591882-conkeror
@@ -4,7 +4,6 @@ set -e
 TESTDIR="$(readlink -f "$(dirname "$0")")"
 . "$TESTDIR/framework"
 setupenvironment
-allowremovemanual
 configarchitecture "i386"
 setupaptarchive
 
@@ -77,7 +76,7 @@ E: Trivial Only specified but this is not a trivial operation."
 
 # solver3 otherwise chooses to keep back xulrunner-1.9 rather than replace it with xulrunner-1.9.1,,
 # and keep back reverse dependencies of xulrunner-1.9 which seems correct.
-testsuccess aptmark auto xulrunner-1.9
+testsuccess aptmark auto libdatrie0 libkrb53 libxcb-xlib0 xulrunner-1.9
 # Test that the old behavior can be restored with the option
 testfailureequal "$UPGRADEFAIL" aptget dist-upgrade --trivial-only -o pkgProblemResolver::FixByInstall=0 --solver internal
 testfailureequal "$UPGRADESUCCESS" aptget dist-upgrade --trivial-only #-o pkgProblemResolver::FixByInstall=0
diff --git a/test/integration/test-bug-64141-install-dependencies-for-on-hold b/test/integration/test-bug-64141-install-dependencies-for-on-hold
index ddfad92af..cfb241614 100755
--- a/test/integration/test-bug-64141-install-dependencies-for-on-hold
+++ b/test/integration/test-bug-64141-install-dependencies-for-on-hold
@@ -36,10 +36,11 @@ E: Trivial Only specified but this is not a trivial operation.' aptget dist-upgr
 testfailure aptget dist-upgrade --trivial-only --solver 3.0 -o debug::apt::solver=2
 testsuccess grep -Fx "[0] Reject:apt:$native=0.8.10 (oldcrap:$native -> oldcrap:$native=1-1)" rootdir/tmp/testfailure.output
 
-allowremovemanual
+testsuccess aptmark auto ~i
 
 testfailureequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following packages will be REMOVED:
   oldcrap
@@ -54,6 +55,7 @@ E: Trivial Only specified but this is not a trivial operation.' aptget dist-upgr
 
 testfailureequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
 The following packages will be REMOVED:
   oldcrap
@@ -68,9 +70,13 @@ E: Trivial Only specified but this is not a trivial operation.' aptget dist-upgr
 
 testsuccess aptmark hold apt
 
-testfailureequal 'Reading package lists...
+testfailureequal "Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
+The following package was automatically installed and is no longer required:
+  oldcrap
+Use 'apt autoremove' to remove it.
 The following packages have been kept back:
   apt
 The following packages will be upgraded:
@@ -78,4 +84,4 @@ The following packages will be upgraded:
 1 upgraded, 0 newly installed, 0 to remove and 1 not upgraded.
 Need to get 0 B/42 B of archives.
 After this operation, 0 B of additional disk space will be used.
-E: Trivial Only specified but this is not a trivial operation.' aptget dist-upgrade --trivial-only -o Test='hold-back-apt'
+E: Trivial Only specified but this is not a trivial operation." aptget dist-upgrade --trivial-only -o Test='hold-back-apt'
diff --git a/test/integration/test-bug-657695-resolver-breaks-on-virtuals b/test/integration/test-bug-657695-resolver-breaks-on-virtuals
index e3ff6d1b5..20b4711dc 100755
--- a/test/integration/test-bug-657695-resolver-breaks-on-virtuals
+++ b/test/integration/test-bug-657695-resolver-breaks-on-virtuals
@@ -23,11 +23,24 @@ The following packages have been kept back:
   xserver-xorg-core
 0 upgraded, 0 newly installed, 0 to remove and 1 not upgraded.' aptget dist-upgrade --trivial-only --solver 3.0
 
+
 allowremovemanual
+testsuccessequal 'Reading package lists...
+Building dependency tree...
+Calculating upgrade...
+The following packages have been kept back:
+  xserver-xorg-core
+0 upgraded, 0 newly installed, 0 to remove and 1 not upgraded.' aptget dist-upgrade --trivial-only --solver 3.0
 
-testfailureequal 'Reading package lists...
+testsuccess aptmark auto ~i
+
+testfailureequal "Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
+The following package was automatically installed and is no longer required:
+  xserver-xorg-core
+Use 'apt autoremove' to remove it.
 The following packages will be REMOVED:
   xserver-xorg-video-driver1 xserver-xorg-video-driver10
   xserver-xorg-video-driver11 xserver-xorg-video-driver12
@@ -59,4 +72,4 @@ The following packages will be upgraded:
 1 upgraded, 0 newly installed, 50 to remove and 0 not upgraded.
 Need to get 0 B/42 B of archives.
 After this operation, 2150 kB disk space will be freed.
-E: Trivial Only specified but this is not a trivial operation.' aptget dist-upgrade --trivial-only
+E: Trivial Only specified but this is not a trivial operation." aptget dist-upgrade --trivial-only
diff --git a/test/integration/test-bug-lp1562402-nomark-removals-as-keep b/test/integration/test-bug-lp1562402-nomark-removals-as-keep
index 41a9591fa..6cc624961 100755
--- a/test/integration/test-bug-lp1562402-nomark-removals-as-keep
+++ b/test/integration/test-bug-lp1562402-nomark-removals-as-keep
@@ -25,12 +25,16 @@ The following packages have been kept back:
   maas-common maas-region-controller maas-region-controller-min
 0 upgraded, 0 newly installed, 0 to remove and 3 not upgraded.' aptget dist-upgrade -s --solver 3.0
 
-allowremovemanual
+testsuccess aptmark auto ~i
 
 testsuccess aptget dist-upgrade -s -o Debug::pkgDepCache::AutoInstall=true  -o Debug::pkgPackageManager=yes  -o Debug::pkgProblemResolver=yes
-testsuccessequal 'Reading package lists...
+testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
 Calculating upgrade...
+The following packages were automatically installed and are no longer required:
+  maas-common maas-region-api maas-region-controller
+Use 'apt autoremove' to remove them.
 The following packages will be REMOVED:
   maas-region-controller-min
 The following NEW packages will be installed:
@@ -44,5 +48,5 @@ Inst maas-common [2.0.0~alpha3+bzr4810-0ubuntu1] (2.0.0~alpha4+bzr4843-0ubuntu1~
 Inst maas-region-api (2.0.0~alpha4+bzr4843-0ubuntu1~xenial2 unstable [amd64])
 Conf maas-region-controller (2.0.0~alpha4+bzr4843-0ubuntu1~xenial2 unstable [all])
 Conf maas-common (2.0.0~alpha4+bzr4843-0ubuntu1~xenial2 unstable [all])
-Conf maas-region-api (2.0.0~alpha4+bzr4843-0ubuntu1~xenial2 unstable [amd64])' \
+Conf maas-region-api (2.0.0~alpha4+bzr4843-0ubuntu1~xenial2 unstable [amd64])" \
 aptget dist-upgrade -s
diff --git a/test/integration/test-not-upgrading-removed-depends b/test/integration/test-not-upgrading-removed-depends
index c7d67af3d..cc16bbac9 100755
--- a/test/integration/test-not-upgrading-removed-depends
+++ b/test/integration/test-not-upgrading-removed-depends
@@ -4,7 +4,6 @@ set -e
 TESTDIR="$(readlink -f "$(dirname "$0")")"
 . "$TESTDIR/framework"
 setupenvironment
-allowremovemanual
 configarchitecture 'amd64'
 
 insertpackage 'unstable' 'untouchable-for-solving' 'all' '1' 'Conflicts: main'
@@ -18,9 +17,11 @@ insertpackage 'unstable' 'else' 'all' '1'
 insertpackage 'unstable' 'meta' 'all' '1' 'Depends: main (= 2) | else'
 
 setupaptarchive
+testsuccess aptmark auto ~i
 
 testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 The following packages will be REMOVED:
   bad
 The following packages will be upgraded:
@@ -33,6 +34,7 @@ testsuccess apt install -s main -o Debug::pkgProblemResolver=1 -o Debug::pkgDepC
 testfailure grep 'untouchable-for-solving' rootdir/tmp/testsuccess.output
 testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 The following additional packages will be installed:
   main
 The following packages will be REMOVED:
diff --git a/test/integration/test-resolver-delays-remove-decisions b/test/integration/test-resolver-delays-remove-decisions
index 062a2e7e9..f3480ca08 100755
--- a/test/integration/test-resolver-delays-remove-decisions
+++ b/test/integration/test-resolver-delays-remove-decisions
@@ -4,7 +4,6 @@ set -e
 TESTDIR="$(readlink -f "$(dirname "$0")")"
 . "$TESTDIR/framework"
 setupenvironment
-allowremovemanual
 configarchitecture 'amd64' 'i386'
 
 insertinstalledpackage 'stuff' 'all' '1'
@@ -18,10 +17,16 @@ insertpackage 'unstable' 'bar' 'all' '1'
 
 setupaptarchive
 
+testsuccess aptmark auto ~i
+
 # We are needlessly removing "stuff" if we don't delay its marking here
 # as we do not question the remove later on
 testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
+The following package was automatically installed and is no longer required:
+  stuff
+Use 'apt autoremove' to remove it.
 The following additional packages will be installed:
   bar
 The following NEW packages will be installed:
@@ -34,6 +39,7 @@ Conf foobar (1 unstable [all])" apt install foobar -s
 
 testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
   MarkInstall foobar:amd64 < none -> 1 @un puN Ib > FU=1
   Installing foo:amd64 as Depends of foobar:amd64
      Delayed Removing: stuff:amd64 as upgrade is not an option for foo:amd64 (1)
@@ -46,6 +52,9 @@ Building dependency tree...
 Starting pkgProblemResolver with broken count: 0
 Starting 2 pkgProblemResolver with broken count: 0
 Done
+The following package was automatically installed and is no longer required:
+  stuff
+Use 'apt autoremove' to remove it.
 The following additional packages will be installed:
   bar
 The following NEW packages will be installed:
@@ -60,6 +69,7 @@ insertinstalledpackage 'uninstallable' 'all' '1'
 
 testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
 The following additional packages will be installed:
   foo foo-dep
 The following packages will be REMOVED:
@@ -77,13 +87,14 @@ Conf foobar (1 unstable [all])" apt install foobar -s
 
 testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
   MarkInstall foobar:amd64 < none -> 1 @un puN Ib > FU=1
   Installing foo:amd64 as Depends of foobar:amd64
      Delayed Removing: stuff:amd64 as upgrade is not an option for foo:amd64 (1)
     MarkInstall foo:amd64 < none -> 1 @un uN Ib > FU=0
     Installing foo-dep:amd64 as Depends of foo:amd64
       MarkInstall foo-dep:amd64 < none -> 1 @un uN > FU=0
-    MarkDelete stuff:amd64 < 1 @ii mK > FU=0
+    MarkDelete stuff:amd64 < 1 @ii gK > FU=0
 Starting pkgProblemResolver with broken count: 0
 Starting 2 pkgProblemResolver with broken count: 0
 Done
@@ -106,6 +117,7 @@ Conf foobar (1 unstable [all])" apt install foobar -s -o Debug::pkgProblemResolv
 # Same solution but the installs are considered protected now as there is no other solution
 testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
 Package 'bar' is not installed, so not removed
 The following additional packages will be installed:
   foo foo-dep
@@ -124,10 +136,11 @@ Conf foobar (1 unstable [all])" apt install foobar bar- -q=0 -s
 
 testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
   MarkInstall foobar:amd64 < none -> 1 @un puN Ib > FU=1
   Installing foo:amd64 as Depends of foobar:amd64
      Removing: stuff:amd64 as upgrade is not an option for foo:amd64 (1)
-      MarkDelete stuff:amd64 < 1 @ii mK > FU=0
+      MarkDelete stuff:amd64 < 1 @ii gK > FU=0
     MarkInstall foo:amd64 < none -> 1 @un puN Ib > FU=0
     Installing foo-dep:amd64 as Depends of foo:amd64
       MarkInstall foo-dep:amd64 < none -> 1 @un puN > FU=0
diff --git a/test/integration/test-resolver-provider-exchange b/test/integration/test-resolver-provider-exchange
index 2874b88e4..938d8f5b9 100755
--- a/test/integration/test-resolver-provider-exchange
+++ b/test/integration/test-resolver-provider-exchange
@@ -4,7 +4,6 @@ set -e
 TESTDIR="$(readlink -f "$(dirname "$0")")"
 . "$TESTDIR/framework"
 setupenvironment
-allowremovemanual
 configarchitecture 'amd64'
 
 insertinstalledpackage 'fuse' 'all' '2'
@@ -22,10 +21,12 @@ Conflicts: fuse3'
 insertpackage 'unstable' 'foobar-r3' 'all' '1' 'Recommends: fuse3'
 
 setupaptarchive
+testsuccess aptmark auto ~i
 
 installfoobars() {
 	testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 The following NEW packages will be installed:
   foobar-d
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
@@ -33,6 +34,7 @@ Inst foobar-d (1 unstable [all])
 Conf foobar-d (1 unstable [all])' apt install -s foobar-d
 	testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 The following NEW packages will be installed:
   foobar-d2
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
@@ -40,6 +42,7 @@ Inst foobar-d2 (1 unstable [all])
 Conf foobar-d2 (1 unstable [all])' apt install -s foobar-d2
 	testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
 The following additional packages will be installed:
   fuse3
 The following packages will be REMOVED:
@@ -55,6 +58,7 @@ Conf foobar-d3 (1 unstable [all])" apt install -s foobar-d3
 
 	testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 The following NEW packages will be installed:
   foobar-r
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
@@ -62,6 +66,7 @@ Inst foobar-r (1 unstable [all])
 Conf foobar-r (1 unstable [all])' apt install -s foobar-r
 	testsuccessequal 'Reading package lists...
 Building dependency tree...
+Reading state information...
 The following NEW packages will be installed:
   foobar-r2
 0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
@@ -70,6 +75,22 @@ Conf foobar-r2 (1 unstable [all])' apt install -s foobar-r2
 	if [ -z "$1" ]; then
 		testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
+The following additional packages will be installed:
+  fuse3
+The following packages will be REMOVED:
+  fuse
+The following NEW packages will be installed:
+  foobar-r3 fuse3
+0 upgraded, 2 newly installed, 1 to remove and 0 not upgraded.
+Remv fuse [2]$1
+Inst foobar-r3 (1 unstable [all])
+Inst fuse3 (3 unstable [all])
+Conf foobar-r3 (1 unstable [all])
+Conf fuse3 (3 unstable [all])" apt install -s foobar-r3
+		testsuccessequal "Reading package lists...
+Building dependency tree...
+Reading state information...
 The following additional packages will be installed:
   fuse3
 The following packages will be REMOVED:
@@ -85,6 +106,7 @@ Conf fuse3 (3 unstable [all])" apt install -s foobar-r3
 	else
 		testsuccessequal "Reading package lists...
 Building dependency tree...
+Reading state information...
 The following additional packages will be installed:
   fuse3
 The following packages will be REMOVED:
@@ -96,7 +118,18 @@ Remv fuse [2]$1
 Inst fuse3 (3 unstable [all])
 Inst foobar-r3 (1 unstable [all])
 Conf fuse3 (3 unstable [all])
-Conf foobar-r3 (1 unstable [all])" apt install -s foobar-r3
+Conf foobar-r3 (1 unstable [all])" apt install -s foobar-r3 --solver internal
+	testsuccessequal "Reading package lists...
+Building dependency tree...
+Reading state information...
+Solving dependencies...
+Recommended packages:
+  fuse3
+The following NEW packages will be installed:
+  foobar-r3
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+Inst foobar-r3 (1 unstable [all])
+Conf foobar-r3 (1 unstable [all])" apt install -s foobar-r3 --solver 3.0
 	fi
 }
 msgmsg 'fuse has no installed dependers'