From 109fdcac9a16ffa06b06366134dcaaef7e2b3379 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 20 Mar 2025 14:30:54 +0100 Subject: solver3: Record manual flag before discovering packages When discovering packages we look at the manual flag to avoid reordering obsolete manually installed packages last. We inadvertently initialized that flag interlaced with discovering packages so if we discovered packages depending on obsolete, but manually installed, packages first, we were not aware they were manually installed and demoted them. I am struggling to come up with a test case for this as I can't get the package objects in the cache to be in the right order for the bug to trigger. LP: #2102720 --- apt-pkg/solver3.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apt-pkg/solver3.cc b/apt-pkg/solver3.cc index 4030b9de1..4d988b523 100644 --- a/apt-pkg/solver3.cc +++ b/apt-pkg/solver3.cc @@ -1099,6 +1099,11 @@ bool APT::Solver::FromDepCache(pkgDepCache &depcache) } } + // Clause discovery depends on the manual flag, so we need to set the manual flag first before we discover any packages + for (auto P = cache.PkgBegin(); not P.end(); P++) + if (P->CurrentVer && not(depcache[P].Flags & pkgCache::Flag::Auto) && (depcache[P].Keep() || depcache[P].Install())) + (*this)[P].flags.manual = true; + for (auto P = cache.PkgBegin(); not P.end(); P++) { if (P->VersionList == nullptr) @@ -1142,8 +1147,6 @@ bool APT::Solver::FromDepCache(pkgDepCache &depcache) if (unlikely(debug >= 1)) std::cerr << "Install " << P.FullName() << " (" << (isEssential ? "E" : "") << (isAuto ? "M" : "") << (Root ? "R" : "") << ")" << "\n"; - - (*this)[P].flags.manual = not isAuto; if (not isOptional) { // Pre-empt the non-optional requests, as we don't want to queue them, we can just "unit propagate" here. -- cgit v1.2.3-70-g09d2 From a2cb87afcbc89d1f8e770c26121a75d9c1654458 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 20 Mar 2025 15:03:23 +0100 Subject: solver3: Use zstd-compressed, base64-encoded dumps Store the solver dumps with zstd compression and base64-encode them in the crash report; such that they take up significantly less space and can fit in cassandra's schema. Base64 is wrapped at 76 characters as the base64 tool does. Do some magic std::string_view conversion to avoid string copies for the substr() used to get 76 characters at a time. Note we can rely on substr(i, 76) to handle less than 76 characters just fine; it uses std::min(76, size() - i) characters. --- apt-private/private-install.cc | 19 +++++-- test/integration/test-solver3-evaluation | 96 +------------------------------- 2 files changed, 16 insertions(+), 99 deletions(-) diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index e3d176ffe..cf92f91ad 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -89,7 +89,7 @@ bool CheckNothingBroken(CacheFile &Cache) // --------------------------------------------------------------------- static void WriteApportReport(pkgCacheFile &Cache, std::string &title, std::vector const &errors, int mode, bool distUpgrade) { - auto dumpfile = flCombine(_config->FindDir("Dir::Log"), "edsp.log"); + auto dumpfile = flCombine(_config->FindDir("Dir::Log"), "edsp.log.zst"); auto crashfile = flCombine(_config->FindDir("Dir::Apport", "/var/crash"), "apt-edsp." + std::to_string(getuid()) + ".crash"); auto apt = Cache->FindPkg("apt"); if (access(flNotFile(dumpfile).c_str(), W_OK) != 0 || access(flNotFile(crashfile).c_str(), W_OK) != 0) @@ -97,7 +97,8 @@ static void WriteApportReport(pkgCacheFile &Cache, std::string &title, std::vect OpTextProgress Progress(*_config); Progress.OverallProgress(0, 100, 50, _("Writing error report")); FileFd edspDump(dumpfile, - FileFd::Exclusive | FileFd::Create | FileFd::WriteOnly | FileFd::BufferedWrite); + FileFd::Exclusive | FileFd::Create | FileFd::WriteOnly | FileFd::BufferedWrite, + FileFd::Extension); int edspFlags = 0; if (mode || distUpgrade) edspFlags |= EDSP::Request::UPGRADE_ALL; @@ -127,10 +128,18 @@ static void WriteApportReport(pkgCacheFile &Cache, std::string &title, std::vect for (auto &error : errors) crash << " " << error << "\n"; - crash << "AptSolverDump:\n"; std::ifstream toCopy(edspDump.Name()); - for (std::string line; std::getline(toCopy, line);) - crash << " " << line << "\n"; + std::ostringstream readStream; + readStream << toCopy.rdbuf(); + const auto base64 = Base64Encode(readStream.str()); + if (not base64.empty()) + { + crash << "AptSolverDump:\n"; + // Print the base64, with line wrapping, indented by one space; substr(i, 76) + // returns at most 76 characters or as many as are left. + for (size_t i=0; i < base64.size(); i += 76) + crash << " " << std::string_view{base64}.substr(i, 76) << "\n"; + } crash.close(); } diff --git a/test/integration/test-solver3-evaluation b/test/integration/test-solver3-evaluation index a873d5b21..b05e7cb9b 100755 --- a/test/integration/test-solver3-evaluation +++ b/test/integration/test-solver3-evaluation @@ -32,99 +32,7 @@ Title: Failure: The 3.0 solver produced a worse result SourcePackage: apt ErrorMessage: AptSolverDump: - Request: EDSP 0.5 - Architecture: amd64 - Architectures: amd64 - Machine-ID: 912e43bd1c1d4ba481f9f8ccab25f9ee - Install: x:amd64 y:amd64 - Solver: internal - - Package: a - Architecture: all - Version: 3 - APT-ID: 2 - Source: a - Source-Version: 3 - Priority: optional - Section: other - Size: 42 - APT-Release: - a=unstable,n=sid,c=main,b=all - APT-Pin: 500 - APT-Candidate: yes - Depends: c - - Package: b - Architecture: all - Version: 3 - APT-ID: 3 - Source: b - Source-Version: 3 - Priority: optional - Section: other - Size: 42 - APT-Release: - a=unstable,n=sid,c=main,b=all - APT-Pin: 500 - APT-Candidate: yes - - Package: c - Architecture: all - Version: 3 - APT-ID: 4 - Source: c - Source-Version: 3 - Priority: optional - Section: other - Size: 42 - APT-Release: - a=unstable,n=sid,c=main,b=all - APT-Pin: 500 - APT-Candidate: yes - - Package: x - Architecture: all - Version: 3 - APT-ID: 0 - Source: x - Source-Version: 3 - Priority: optional - Section: other - Size: 42 - APT-Release: - a=unstable,n=sid,c=main,b=all - APT-Pin: 500 - APT-Candidate: yes - Depends: a | b - - Package: y - Architecture: all - Version: 3 - APT-ID: 1 - Source: y - Source-Version: 3 - Priority: optional - Section: other - Size: 42 - APT-Release: - a=unstable,n=sid,c=main,b=all - APT-Pin: 500 - APT-Candidate: yes - Depends: b | a - - Package: apport - Architecture: all - Version: 3 - APT-ID: 6 - Source: apport - Source-Version: 3 - Priority: optional - Section: other - Installed: yes - APT-Pin: 100 - APT-Candidate: yes - " cat rootdir/var/crash/apt-edsp.$(id -u).crash - +$(cat rootdir/var/log/apt/edsp.log.zst | base64 | sed 's#^# #')" cat rootdir/var/crash/apt-edsp.$(id -u).crash testsuccessequal "Request: EDSP 0.5 Architecture: amd64 @@ -217,7 +125,7 @@ Section: other Installed: yes APT-Pin: 100 APT-Candidate: yes -" cat rootdir/var/log/apt/edsp.log +" apthelper cat-file rootdir/var/log/apt/edsp.log.zst testfailureequal "Reading package lists... Building dependency tree... -- cgit v1.2.3-70-g09d2 From 61b3b1840568b223baad08da1eb9a1fb5d6e7f91 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 20 Mar 2025 15:16:58 +0100 Subject: solver3: Avoid empty error message in evaluation crash report whoopsie does not accept empty values --- apt-private/private-install.cc | 9 ++++++--- test/integration/test-solver3-evaluation | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apt-private/private-install.cc b/apt-private/private-install.cc index cf92f91ad..f1211d25e 100644 --- a/apt-private/private-install.cc +++ b/apt-private/private-install.cc @@ -124,9 +124,12 @@ static void WriteApportReport(pkgCacheFile &Cache, std::string &title, std::vect << "Title: " << title << "\n" << "SourcePackage: apt\n"; - crash << "ErrorMessage:\n"; - for (auto &error : errors) - crash << " " << error << "\n"; + if (not errors.empty()) + { + crash << "ErrorMessage:\n"; + for (auto &error : errors) + crash << " " << error << "\n"; + } std::ifstream toCopy(edspDump.Name()); std::ostringstream readStream; diff --git a/test/integration/test-solver3-evaluation b/test/integration/test-solver3-evaluation index b05e7cb9b..c01ec9608 100755 --- a/test/integration/test-solver3-evaluation +++ b/test/integration/test-solver3-evaluation @@ -30,7 +30,6 @@ Date: Sun Mar 9 00:08:41 2025 Package: apt Title: Failure: The 3.0 solver produced a worse result SourcePackage: apt -ErrorMessage: AptSolverDump: $(cat rootdir/var/log/apt/edsp.log.zst | base64 | sed 's#^# #')" cat rootdir/var/crash/apt-edsp.$(id -u).crash -- cgit v1.2.3-70-g09d2