From 2e39d0fdeff5f9db91b3490d6750c8033aca9057 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 30 Jan 2025 15:22:14 +0000 Subject: Do not overflow ver/desc<->file relation counters The counters aren't used for anything really except for the 'stats' command that is off in its display and calculation due to 16bit not being enough for counting all the relations we have. Semi-broken more than a decade ago as I accidentally aligned it with the other file relation counters which count far less frequent ones through, so 'unsigned long' should have been changed to the id type rather than the smaller fileid type. Of course, this commit only prepares the fix as this would be an ABI break, so with the current ABI we are still overflowing. Regression-of: 4ad8619bb1f0bf777d17c568bb7a6cf7f30aac34 --- apt-pkg/pkgcache.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index 476a01514..f6d0ea097 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -341,8 +341,13 @@ struct pkgCache::Header map_id_t DependsDataCount; map_fileid_t ReleaseFileCount; map_fileid_t PackageFileCount; +#if APT_PKG_ABI <= 600 map_fileid_t VerFileCount; map_fileid_t DescFileCount; +#else + map_id_t VerFileCount; + map_id_t DescFileCount; +#endif map_id_t ProvidesCount; /** \brief index of the first PackageFile structure -- cgit v1.2.3-70-g09d2 From 1bc221a9dd22baa6e2e516d45dc858774cafddd0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 30 Jan 2025 16:39:40 +0000 Subject: Drop unused description size info from binary cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in src:apt uses this information and that isn't all to surprising as it is usually not that interesting to know how big a specific stanza of a description is as the max size is available and you are usually working with more than one description. So this information just takes up space in our binary cache… 64 bits per structure which can quickly add up to whole MBs depending on how many (and which) sources you have enabled. --- apt-pkg/pkgcache.h | 5 +++-- apt-pkg/pkgcachegen.cc | 11 ++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index f6d0ea097..d5d8fcad3 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -618,8 +618,9 @@ struct pkgCache::DescFile map_pointer NextFile; /** \brief position in the file */ map_filesize_t Offset; // File offset - /** @TODO document pkgCache::DescFile::Size */ - map_filesize_t Size; +#if APT_PKG_ABI <= 600 + [[deprecated("No usage in src:apt, try MaxDescFileSize instead")]] map_filesize_t Size; +#endif }; /*}}}*/ // Version structure /*{{{*/ diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index aaa969a8d..76dea6266 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -949,9 +949,14 @@ bool pkgCacheGenerator::NewFileDesc(pkgCache::DescIterator &Desc, *Last = DF.MapPointer(); DF->Offset = List.Offset(); - DF->Size = List.Size(); - if (Cache.HeaderP->MaxDescFileSize < DF->Size) - Cache.HeaderP->MaxDescFileSize = DF->Size; + auto const Size = List.Size(); + if (Cache.HeaderP->MaxDescFileSize < Size) + Cache.HeaderP->MaxDescFileSize = Size; +#if APT_PKG_ABI <= 600 + APT_IGNORE_DEPRECATED_PUSH + DF->Size = Size; + APT_IGNORE_DEPRECATED_POP +#endif Cache.HeaderP->DescFileCount++; return true; -- cgit v1.2.3-70-g09d2 From 1bc8dab8df60c5da7cc817bec5f53e60f7eb9817 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 30 Jan 2025 18:03:26 +0000 Subject: Drop unused version stanza size info from binary cache Same reason as with the description size before, but this one had actually one extremely dated user in src:apt left with dumpavail, which is changed to an implementation that is (in comparison) a bit slower, but should be good enough for a command that is ideally not used anymore (even if codesearch suggests otherwise). After all, what makes this a bit slower is not that we don't know the size, but that this one does additional work preparing for accessing the fields which we don't do. In exchange pkgTagFile has a better implementation for "Jitter" than the old code that we would "just" need to copy here if speed were really an issue. --- apt-pkg/pkgcache.h | 5 +++-- apt-pkg/pkgcachegen.cc | 11 ++++++++--- cmdline/apt-cache.cc | 50 +++++++++++++++++++------------------------------- 3 files changed, 30 insertions(+), 36 deletions(-) (limited to 'apt-pkg/pkgcache.h') diff --git a/apt-pkg/pkgcache.h b/apt-pkg/pkgcache.h index d5d8fcad3..0b61b0048 100644 --- a/apt-pkg/pkgcache.h +++ b/apt-pkg/pkgcache.h @@ -604,8 +604,9 @@ struct pkgCache::VerFile map_pointer NextFile; /** \brief position in the package file */ map_filesize_t Offset; // File offset - /** @TODO document pkgCache::VerFile::Size */ - map_filesize_t Size; +#if APT_PKG_ABI <= 600 + [[deprecated("No usage in src:apt, try MaxVerFileSize instead")]] map_filesize_t Size; +#endif }; /*}}}*/ // DescFile structure /*{{{*/ diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 76dea6266..055b2e08e 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -854,9 +854,14 @@ bool pkgCacheGenerator::NewFileVer(pkgCache::VerIterator &Ver, *Last = VF.MapPointer(); VF->Offset = List.Offset(); - VF->Size = List.Size(); - if (Cache.HeaderP->MaxVerFileSize < VF->Size) - Cache.HeaderP->MaxVerFileSize = VF->Size; + auto const Size = List.Size(); + if (Cache.HeaderP->MaxVerFileSize < Size) + Cache.HeaderP->MaxVerFileSize = Size; +#if APT_PKG_ABI <= 600 + APT_IGNORE_DEPRECATED_PUSH + VF->Size = Size; + APT_IGNORE_DEPRECATED_POP +#endif Cache.HeaderP->VerFileCount++; return true; diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 7fdd4dff2..4ad0b0099 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -464,7 +464,6 @@ static bool DumpAvail(CommandLine &) stdoutfd.OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false); // Iterate over all the package files and write them out. - char *Buffer = new char[Cache->HeaderP->MaxVerFileSize+10]; for (pkgCache::VerFile **J = VFList; *J != 0;) { pkgCache::PkgFileIterator File(*Cache, Cache->PkgFileP + (*J)->File); @@ -472,40 +471,23 @@ static bool DumpAvail(CommandLine &) FileFd PkgF(File.FileName(),FileFd::ReadOnly, FileFd::Extension); if (_error->PendingError() == true) break; - - /* Write all of the records from this package file, since we - already did locality sorting we can now just seek through the - file in read order. We apply 1 more optimization here, since often - there will be < 1 byte gaps between records (for the \n) we read that - into the next buffer and offset a bit.. */ - unsigned long Pos = 0; + + pkgTagFile tagsfile(&PkgF, Cache->HeaderP->MaxVerFileSize); + pkgTagSection Tags; + for (; *J != 0; J++) { if (Cache->PkgFileP + (*J)->File != File) break; - - const pkgCache::VerFile &VF = **J; - // Read the record and then write it out again. - unsigned long Jitter = VF.Offset - Pos; - if (Jitter > 8) - { - if (PkgF.Seek(VF.Offset) == false) - break; - Jitter = 0; - } - - if (PkgF.Read(Buffer,VF.Size + Jitter) == false) + const pkgCache::VerFile &VF = **J; + if (not tagsfile.Jump(Tags, VF.Offset)) break; - Buffer[VF.Size + Jitter] = '\n'; - // See above.. if ((File->Flags & pkgCache::Flag::NotSource) == pkgCache::Flag::NotSource) { - pkgTagSection Tags; - if (Tags.Scan(Buffer+Jitter,VF.Size+1) == false || - Tags.Write(stdoutfd, NULL, RW) == false || - stdoutfd.Write("\n", 1) == false) + if (not Tags.Write(stdoutfd, NULL, RW) || + not stdoutfd.Write("\n", 1)) { _error->Error("Internal Error, Unable to parse a package record"); break; @@ -513,18 +495,24 @@ static bool DumpAvail(CommandLine &) } else { - if (stdoutfd.Write(Buffer + Jitter, VF.Size + 1) == false) + char const *Start, *Stop; + Tags.GetSection(Start, Stop); + for (; Start < Stop; --Stop) + if (std::string_view{"\n\r"}.find(*(Stop - 1)) == std::string_view::npos) + break; + if (not stdoutfd.Write(Start, Stop - Start) || + not stdoutfd.Write("\n\n", 2)) + { + _error->Error("Internal Error, Unable to parse a package record"); break; + } } - - Pos = VF.Offset + VF.Size; } - if (_error->PendingError() == true) + if (_error->PendingError()) break; } - delete [] Buffer; delete [] VFList; return !_error->PendingError(); } -- cgit v1.2.3-70-g09d2