From 8e16d8c39447809506a8cd8e6f88cae3c168f82d Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 10 Dec 2011 17:35:54 +0100 Subject: * cmdline/apt-config.cc: - dump the APT::Compressor settings correctly and completely --- cmdline/apt-config.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'cmdline') diff --git a/cmdline/apt-config.cc b/cmdline/apt-config.cc index 94f6ee9b0..47bedfe3f 100644 --- a/cmdline/apt-config.cc +++ b/cmdline/apt-config.cc @@ -134,6 +134,22 @@ int main(int argc,const char *argv[]) /*{{{*/ for (std::vector::const_iterator a = archs.begin(); a != archs.end(); ++a) _config->Set("APT::Architectures::", *a); + std::vector const compressors = APT::Configuration::getCompressors(); + _config->Clear("APT::Compressor"); + string conf = "APT::Compressor::"; + for (std::vector::const_iterator c = compressors.begin(); c != compressors.end(); ++c) + { + string comp = conf + c->Name + "::"; + _config->Set(comp + "Name", c->Name); + _config->Set(comp + "Extension", c->Extension); + _config->Set(comp + "Binary", c->Binary); + _config->Set(std::string(comp + "Cost").c_str(), c->Cost); + for (std::vector::const_iterator a = c->CompressArgs.begin(); a != c->CompressArgs.end(); ++a) + _config->Set(comp + "CompressArg::", *a); + for (std::vector::const_iterator a = c->UncompressArgs.begin(); a != c->UncompressArgs.end(); ++a) + _config->Set(comp + "UncompressArg::", *a); + } + // Match the operation CmdL.DispatchArg(Cmds); -- cgit v1.2.3-70-g09d2 From 468720c59fcf48b20332cdb7b601b2b0d7cbbfbb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 10 Dec 2011 19:31:36 +0100 Subject: enable FileFd to guess the compressor based on the filename if requested or to search for compressed silbings of the given filename and use this guessing instead of hardcoding Gzip compression --- apt-pkg/contrib/fileutl.cc | 129 ++++++++++++++++++++++++++++++++++++++------ apt-pkg/contrib/fileutl.h | 14 +++-- apt-pkg/deb/debindexfile.cc | 13 ++--- apt-pkg/deb/debrecords.cc | 2 +- apt-pkg/deb/debsrcrecords.h | 2 +- cmdline/apt-cache.cc | 2 +- methods/gzip.cc | 2 +- methods/rred.cc | 2 +- 8 files changed, 135 insertions(+), 31 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index d5e7192e3..1cb3fab1e 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -725,6 +726,11 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned Close(); Flags = AutoClose; + if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) + return _error->Error("Autodetection on %s only works in ReadOnly openmode!", FileName.c_str()); + if ((Mode & WriteOnly) != WriteOnly && (Mode & (Atomic | Create | Empty | Exclusive)) != 0) + return _error->Error("ReadOnly mode for %s doesn't accept additional flags!", FileName.c_str()); + int fileflags = 0; #define if_FLAGGED_SET(FLAG, MODE) if ((Mode & FLAG) == FLAG) fileflags |= MODE if_FLAGGED_SET(ReadWrite, O_RDWR); @@ -738,6 +744,70 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned if_FLAGGED_SET(Empty, O_TRUNC); #undef if_FLAGGED_SET + // FIXME: Denote inbuilt compressors somehow - as we don't need to have the binaries for them + std::vector const compressors = APT::Configuration::getCompressors(); + std::vector::const_iterator compressor = compressors.begin(); + if (Compress == Auto) + { + Compress = None; + for (; compressor != compressors.end(); ++compressor) + { + std::string file = std::string(FileName).append(compressor->Extension); + if (FileExists(file) == false) + continue; + FileName = file; + if (compressor->Binary == ".") + Compress = None; + else + Compress = Extension; + break; + } + } + else if (Compress == Extension) + { + Compress = None; + std::string ext = flExtension(FileName); + if (ext != FileName) + { + ext = "." + ext; + for (; compressor != compressors.end(); ++compressor) + if (ext == compressor->Extension) + break; + } + } + else if (Compress != None) + { + std::string name; + switch (Compress) + { + case Gzip: name = "gzip"; break; + case Bzip2: name = "bzip2"; break; + case Lzma: name = "lzma"; break; + case Xz: name = "xz"; break; + default: return _error->Error("Can't find a match for specified compressor mode for file %s", FileName.c_str()); + } + for (; compressor != compressors.end(); ++compressor) + if (compressor->Name == name) + break; + if (compressor == compressors.end() && name != "gzip") + return _error->Error("Can't find a configured compressor %s for file %s", name.c_str(), FileName.c_str()); + } + + // if we have them, use inbuilt compressors instead of forking + if (compressor != compressors.end()) + { + if (compressor->Name == "gzip") + { + Compress = Gzip; + compressor = compressors.end(); + } + else if (compressor->Name == "." || Compress == None) + { + Compress = None; + compressor = compressors.end(); + } + } + if ((Mode & Atomic) == Atomic) { Flags |= Replace; @@ -757,18 +827,27 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned unlink(FileName.c_str()); } - if (TemporaryFileName.empty() == false) - iFd = open(TemporaryFileName.c_str(), fileflags, Perms); - else - iFd = open(FileName.c_str(), fileflags, Perms); + if (compressor != compressors.end()) + { + if ((Mode & ReadWrite) == ReadWrite) + _error->Error("External compressors like %s do not support readwrite mode for file %s", compressor->Name.c_str(), FileName.c_str()); - if (iFd != -1 && Compress == Gzip) + _error->Error("Forking external compressor %s is not implemented for %s", compressor->Name.c_str(), FileName.c_str()); + } + else { - gz = gzdopen (iFd, "r"); - if (gz == NULL) + if (TemporaryFileName.empty() == false) + iFd = open(TemporaryFileName.c_str(), fileflags, Perms); + else + iFd = open(FileName.c_str(), fileflags, Perms); + + if (iFd != -1) { - close (iFd); - iFd = -1; + if (OpenInternDescriptor(Mode, Compress) == false) + { + close (iFd); + iFd = -1; + } } } @@ -788,17 +867,33 @@ bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool A Close(); Flags = (AutoClose) ? FileFd::AutoClose : 0; iFd = Fd; - if (Mode == ReadOnlyGzip) { - gz = gzdopen (iFd, "r"); - if (gz == NULL) { - if (AutoClose) - close (iFd); - return _error->Errno("gzdopen",_("Could not open file descriptor %d"), - Fd); - } + if (OpenInternDescriptor(Mode, Compress) == false) + { + if (AutoClose) + close (iFd); + return _error->Errno("gzdopen",_("Could not open file descriptor %d"), Fd); } this->FileName = ""; return true; +} +bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) +{ + if (Compress == None) + return true; + else if (Compress == Gzip) + { + if ((Mode & ReadWrite) == ReadWrite) + gz = gzdopen(iFd, "r+"); + else if ((Mode & WriteOnly) == WriteOnly) + gz = gzdopen(iFd, "w"); + else + gz = gzdopen (iFd, "r"); + if (gz == NULL) + return false; + } + else + return false; + return true; } /*}}}*/ // FileFd::~File - Closes the file /*{{{*/ diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index fa8f92272..59a9d97e3 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -61,7 +61,7 @@ class FileFd ReadOnlyGzip, WriteAtomic = ReadWrite | Create | Atomic }; - enum CompressMode { Auto, None, Gzip, Bzip2, Lzma, Xz }; + enum CompressMode { Auto = 'A', None = 'N', Extension = 'E', Gzip = 'G', Bzip2 = 'B', Lzma = 'L', Xz = 'X' }; inline bool Read(void *To,unsigned long long Size,bool AllowEof) { @@ -94,7 +94,7 @@ class FileFd } bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long Perms = 0666); - inline bool Open(std::string const &FileName,OpenMode Mode,unsigned long Perms = 0666) { + inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long Perms = 0666) { return Open(FileName, Mode, None, Perms); }; bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false); @@ -118,11 +118,19 @@ class FileFd FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), gz(NULL) { - Open(FileName,Mode,Perms); + Open(FileName,Mode, None, Perms); + }; + FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : + iFd(-1), Flags(0), gz(NULL) + { + Open(FileName,Mode, Compress, Perms); }; FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose), gz(NULL) {}; FileFd(int Fd,bool) : iFd(Fd), Flags(0), gz(NULL) {}; virtual ~FileFd(); + + private: + bool OpenInternDescriptor(OpenMode Mode, CompressMode Compress); }; bool RunScripts(const char *Cnf); diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 27c1f7f32..d9c448598 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -159,7 +159,7 @@ unsigned long debSourcesIndex::Size() const /* we need to ignore errors here; if the lists are absent, just return 0 */ _error->PushToStack(); - FileFd f = FileFd (IndexFile("Sources"), FileFd::ReadOnlyGzip); + FileFd f = FileFd (IndexFile("Sources"), FileFd::ReadOnly, FileFd::Extension); if (!f.Failed()) size = f.Size(); @@ -288,7 +288,7 @@ unsigned long debPackagesIndex::Size() const /* we need to ignore errors here; if the lists are absent, just return 0 */ _error->PushToStack(); - FileFd f = FileFd (IndexFile("Packages"), FileFd::ReadOnlyGzip); + FileFd f = FileFd (IndexFile("Packages"), FileFd::ReadOnly, FileFd::Extension); if (!f.Failed()) size = f.Size(); @@ -305,7 +305,7 @@ unsigned long debPackagesIndex::Size() const bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const { string PackageFile = IndexFile("Packages"); - FileFd Pkg(PackageFile,FileFd::ReadOnlyGzip); + FileFd Pkg(PackageFile,FileFd::ReadOnly, FileFd::Extension); debListParser Parser(&Pkg, Architecture); if (_error->PendingError() == true) @@ -319,6 +319,7 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator File = Gen.GetCurFile(); pkgCacheGenerator::Dynamic DynFile(File); + // FIXME: Get this info from FileFd instead struct stat St; if (fstat(Pkg.Fd(),&St) != 0) return _error->Errno("fstat","Failed to stat"); @@ -489,7 +490,7 @@ unsigned long debTranslationsIndex::Size() const /* we need to ignore errors here; if the lists are absent, just return 0 */ _error->PushToStack(); - FileFd f = FileFd (IndexFile(Language), FileFd::ReadOnlyGzip); + FileFd f = FileFd (IndexFile(Language), FileFd::ReadOnly, FileFd::Extension); if (!f.Failed()) size = f.Size(); @@ -509,7 +510,7 @@ bool debTranslationsIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const string TranslationFile = IndexFile(Language); if (FileExists(TranslationFile)) { - FileFd Trans(TranslationFile,FileFd::ReadOnlyGzip); + FileFd Trans(TranslationFile,FileFd::ReadOnly, FileFd::Extension); debListParser TransParser(&Trans); if (_error->PendingError() == true) return false; @@ -590,7 +591,7 @@ unsigned long debStatusIndex::Size() const /* */ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const { - FileFd Pkg(File,FileFd::ReadOnlyGzip); + FileFd Pkg(File,FileFd::ReadOnly, FileFd::Extension); if (_error->PendingError() == true) return false; debListParser Parser(&Pkg); diff --git a/apt-pkg/deb/debrecords.cc b/apt-pkg/deb/debrecords.cc index 1afa7b74d..184c07c33 100644 --- a/apt-pkg/deb/debrecords.cc +++ b/apt-pkg/deb/debrecords.cc @@ -25,7 +25,7 @@ using std::string; // --------------------------------------------------------------------- /* */ debRecordParser::debRecordParser(string FileName,pkgCache &Cache) : - File(FileName,FileFd::ReadOnlyGzip), + File(FileName,FileFd::ReadOnly, FileFd::Extension), Tags(&File, std::max(Cache.Head().MaxVerFileSize, Cache.Head().MaxDescFileSize) + 200) { diff --git a/apt-pkg/deb/debsrcrecords.h b/apt-pkg/deb/debsrcrecords.h index 4c8d03224..5d2a67f4f 100644 --- a/apt-pkg/deb/debsrcrecords.h +++ b/apt-pkg/deb/debsrcrecords.h @@ -50,7 +50,7 @@ class debSrcRecordParser : public pkgSrcRecords::Parser virtual bool Files(std::vector &F); debSrcRecordParser(std::string const &File,pkgIndexFile const *Index) - : Parser(Index), Fd(File,FileFd::ReadOnlyGzip), Tags(&Fd,102400), + : Parser(Index), Fd(File,FileFd::ReadOnly, FileFd::Extension), Tags(&Fd,102400), Buffer(NULL) {} virtual ~debSrcRecordParser(); }; diff --git a/cmdline/apt-cache.cc b/cmdline/apt-cache.cc index 65d7b0ccd..1cd5080cc 100644 --- a/cmdline/apt-cache.cc +++ b/cmdline/apt-cache.cc @@ -1143,7 +1143,7 @@ bool DisplayRecord(pkgCacheFile &CacheFile, pkgCache::VerIterator V) return _error->Error(_("Package file %s is out of sync."),I.FileName()); FileFd PkgF; - if (PkgF.Open(I.FileName(), FileFd::ReadOnlyGzip) == false) + if (PkgF.Open(I.FileName(), FileFd::ReadOnly, FileFd::Extension) == false) return false; // Read the record diff --git a/methods/gzip.cc b/methods/gzip.cc index a51497948..6ab6548ef 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -48,7 +48,7 @@ bool GzipMethod::Fetch(FetchItem *Itm) URIStart(Res); // Open the source and destination files - FileFd From(Path,FileFd::ReadOnlyGzip); + FileFd From(Path,FileFd::ReadOnly, FileFd::Gzip); if(From.FileSize() == 0) return _error->Error(_("Empty files can't be valid archives")); diff --git a/methods/rred.cc b/methods/rred.cc index ef00fcaa3..56ad8220b 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -489,7 +489,7 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ // Open the source and destination files (the d'tor of FileFd will do // the cleanup/closing of the fds) FileFd From(Path,FileFd::ReadOnly); - FileFd Patch(Path+".ed",FileFd::ReadOnlyGzip); + FileFd Patch(Path+".ed",FileFd::ReadOnly, FileFd::Gzip); FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); if (_error->PendingError() == true) -- cgit v1.2.3-70-g09d2 From 73688d27f60b2da3889a06362ee567101e3b331e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 15 Dec 2011 09:13:21 +0100 Subject: atleast libapt should announce to itself that it is clean… (and be it if it tries to announce that…) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apt-pkg/deb/debindexfile.cc | 2 ++ apt-pkg/edsp/edspindexfile.cc | 4 ++-- apt-pkg/pkgcachegen.cc | 16 +++++++++------- apt-pkg/sourcelist.h | 4 ++-- apt-pkg/srcrecords.cc | 8 ++++---- buildlib/config.h.in | 2 ++ cmdline/apt-extracttemplates.h | 10 ++++++---- 7 files changed, 27 insertions(+), 19 deletions(-) (limited to 'cmdline') diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 2635d52c8..84791a70a 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -26,6 +26,8 @@ #include /*}}}*/ +using std::string; + // SourcesIndex::debSourcesIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ diff --git a/apt-pkg/edsp/edspindexfile.cc b/apt-pkg/edsp/edspindexfile.cc index 5d824f9cb..482581979 100644 --- a/apt-pkg/edsp/edspindexfile.cc +++ b/apt-pkg/edsp/edspindexfile.cc @@ -24,7 +24,7 @@ // edspIndex::edspIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* */ -edspIndex::edspIndex(string File) : debStatusIndex(File) +edspIndex::edspIndex(std::string File) : debStatusIndex(File) { } /*}}}*/ @@ -44,7 +44,7 @@ bool edspIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const if (Prog != NULL) Prog->SubProgress(0,File); - if (Gen.SelectFile(File,string(),*this) == false) + if (Gen.SelectFile(File,std::string(),*this) == false) return _error->Error("Problem with SelectFile %s",File.c_str()); // Store the IMS information diff --git a/apt-pkg/pkgcachegen.cc b/apt-pkg/pkgcachegen.cc index 9f999c41b..ec072fddd 100644 --- a/apt-pkg/pkgcachegen.cc +++ b/apt-pkg/pkgcachegen.cc @@ -35,12 +35,14 @@ #include /*}}}*/ -typedef vector::iterator FileIterator; +typedef std::vector::iterator FileIterator; template std::vector pkgCacheGenerator::Dynamic::toReMap; bool IsDuplicateDescription(pkgCache::DescIterator Desc, MD5SumValue const &CurMd5, std::string const &CurLang); +using std::string; + // CacheGenerator::pkgCacheGenerator - Constructor /*{{{*/ // --------------------------------------------------------------------- /* We set the dirty flag and make sure that is written to the disk */ @@ -1221,14 +1223,14 @@ bool pkgCacheGenerator::MakeStatusCache(pkgSourceList &List,OpProgress *Progress MMap **OutMap,bool AllowMem) { bool const Debug = _config->FindB("Debug::pkgCacheGen", false); - - vector Files; - for (vector::const_iterator i = List.begin(); + + std::vector Files; + for (std::vector::const_iterator i = List.begin(); i != List.end(); ++i) { - vector *Indexes = (*i)->GetIndexFiles(); - for (vector::const_iterator j = Indexes->begin(); + std::vector *Indexes = (*i)->GetIndexFiles(); + for (std::vector::const_iterator j = Indexes->begin(); j != Indexes->end(); ++j) Files.push_back (*j); @@ -1418,7 +1420,7 @@ __deprecated bool pkgMakeOnlyStatusCache(OpProgress &Progress,DynamicMMap **OutM { return pkgCacheGenerator::MakeOnlyStatusCache(&Progress, OutMap); } bool pkgCacheGenerator::MakeOnlyStatusCache(OpProgress *Progress,DynamicMMap **OutMap) { - vector Files; + std::vector Files; unsigned long EndOfSource = Files.size(); if (_system->AddStatusFiles(Files) == false) return false; diff --git a/apt-pkg/sourcelist.h b/apt-pkg/sourcelist.h index 03e29ec34..02e27101a 100644 --- a/apt-pkg/sourcelist.h +++ b/apt-pkg/sourcelist.h @@ -59,11 +59,11 @@ class pkgSourceList const char *Name; const char *Label; - bool FixupURI(string &URI) const; + bool FixupURI(std::string &URI) const; virtual bool ParseLine(std::vector &List, const char *Buffer, unsigned long const &CurLine,std::string const &File) const; - virtual bool CreateItem(vector &List,std::string const &URI, + virtual bool CreateItem(std::vector &List,std::string const &URI, std::string const &Dist,std::string const &Section, std::map const &Options) const = 0; Type(); diff --git a/apt-pkg/srcrecords.cc b/apt-pkg/srcrecords.cc index f6d2d5158..48b643eac 100644 --- a/apt-pkg/srcrecords.cc +++ b/apt-pkg/srcrecords.cc @@ -29,8 +29,8 @@ pkgSrcRecords::pkgSrcRecords(pkgSourceList &List) : Files(0), Current(0) { for (pkgSourceList::const_iterator I = List.begin(); I != List.end(); ++I) { - vector *Indexes = (*I)->GetIndexFiles(); - for (vector::const_iterator J = Indexes->begin(); + std::vector *Indexes = (*I)->GetIndexFiles(); + for (std::vector::const_iterator J = Indexes->begin(); J != Indexes->end(); ++J) { Parser* P = (*J)->CreateSrcParser(); @@ -58,7 +58,7 @@ pkgSrcRecords::pkgSrcRecords(pkgSourceList &List) : Files(0), Current(0) pkgSrcRecords::~pkgSrcRecords() { // Blow away all the parser objects - for(vector::iterator I = Files.begin(); I != Files.end(); ++I) + for(std::vector::iterator I = Files.begin(); I != Files.end(); ++I) delete *I; } /*}}}*/ @@ -68,7 +68,7 @@ pkgSrcRecords::~pkgSrcRecords() bool pkgSrcRecords::Restart() { Current = Files.begin(); - for (vector::iterator I = Files.begin(); + for (std::vector::iterator I = Files.begin(); I != Files.end(); ++I) (*I)->Restart(); diff --git a/buildlib/config.h.in b/buildlib/config.h.in index 256911231..4798fe3f5 100644 --- a/buildlib/config.h.in +++ b/buildlib/config.h.in @@ -44,3 +44,5 @@ /* The package name string */ #undef PACKAGE + +#define APT_8_CLEANER_HEADERS diff --git a/cmdline/apt-extracttemplates.h b/cmdline/apt-extracttemplates.h index d0f90d3f4..6d07a09c2 100644 --- a/cmdline/apt-extracttemplates.h +++ b/cmdline/apt-extracttemplates.h @@ -14,6 +14,8 @@ #include #include +#include + class DebFile : public pkgDirStream { FileFd File; @@ -31,11 +33,11 @@ public: bool Go(); bool ParseInfo(); - static string GetInstalledVer(const string &package); + static std::string GetInstalledVer(const std::string &package); - string Package; - string Version; - string DepVer, PreDepVer; + std::string Package; + std::string Version; + std::string DepVer, PreDepVer; unsigned int DepOp, PreDepOp; char *Config; -- cgit v1.2.3-70-g09d2