From 468720c59fcf48b20332cdb7b601b2b0d7cbbfbb Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 10 Dec 2011 19:31:36 +0100 Subject: enable FileFd to guess the compressor based on the filename if requested or to search for compressed silbings of the given filename and use this guessing instead of hardcoding Gzip compression --- methods/gzip.cc | 2 +- methods/rred.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'methods') diff --git a/methods/gzip.cc b/methods/gzip.cc index a51497948..6ab6548ef 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -48,7 +48,7 @@ bool GzipMethod::Fetch(FetchItem *Itm) URIStart(Res); // Open the source and destination files - FileFd From(Path,FileFd::ReadOnlyGzip); + FileFd From(Path,FileFd::ReadOnly, FileFd::Gzip); if(From.FileSize() == 0) return _error->Error(_("Empty files can't be valid archives")); diff --git a/methods/rred.cc b/methods/rred.cc index ef00fcaa3..56ad8220b 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -489,7 +489,7 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ // Open the source and destination files (the d'tor of FileFd will do // the cleanup/closing of the fds) FileFd From(Path,FileFd::ReadOnly); - FileFd Patch(Path+".ed",FileFd::ReadOnlyGzip); + FileFd Patch(Path+".ed",FileFd::ReadOnly, FileFd::Gzip); FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); if (_error->PendingError() == true) -- cgit v1.2.3-70-g09d2 From 032bd56ff86166fd4b6a8f69bd9d5d1bc57b886e Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 11 Dec 2011 19:46:59 +0100 Subject: - add a ReadLine method - drop the explicit export of gz-compression handling --- apt-pkg/contrib/fileutl.cc | 83 ++++++++++++++++++++++++++++++++++------------ apt-pkg/contrib/fileutl.h | 30 +++++++++++------ apt-pkg/contrib/mmap.cc | 13 +++++++- debian/changelog | 4 ++- methods/rred.cc | 42 ++++++++--------------- 5 files changed, 110 insertions(+), 62 deletions(-) (limited to 'methods') diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc index 83b68e796..58cd6dceb 100644 --- a/apt-pkg/contrib/fileutl.cc +++ b/apt-pkg/contrib/fileutl.cc @@ -44,6 +44,8 @@ #include #include +#include + #ifdef WORDS_BIGENDIAN #include #endif @@ -53,6 +55,12 @@ using namespace std; +class FileFdPrivate { + public: + gzFile gz; + FileFdPrivate() : gz(NULL) {}; +}; + // RunScripts - Run a set of scripts from a configuration subtree /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -719,11 +727,12 @@ bool ExecWait(pid_t Pid,const char *Name,bool Reap) // FileFd::Open - Open a file /*{{{*/ // --------------------------------------------------------------------- /* The most commonly used open mode combinations are given with Mode */ -bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long Perms) +bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned long const Perms) { if (Mode == ReadOnlyGzip) return Open(FileName, ReadOnly, Gzip, Perms); Close(); + d = new FileFdPrivate; Flags = AutoClose; if (Compress == Auto && (Mode & WriteOnly) == WriteOnly) @@ -865,6 +874,7 @@ bool FileFd::Open(string FileName,OpenMode Mode,CompressMode Compress, unsigned bool FileFd::OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose) { Close(); + d = new FileFdPrivate; Flags = (AutoClose) ? FileFd::AutoClose : 0; iFd = Fd; if (OpenInternDescriptor(Mode, Compress) == false) @@ -883,13 +893,14 @@ bool FileFd::OpenInternDescriptor(OpenMode Mode, CompressMode Compress) else if (Compress == Gzip) { if ((Mode & ReadWrite) == ReadWrite) - gz = gzdopen(iFd, "r+"); + d->gz = gzdopen(iFd, "r+"); else if ((Mode & WriteOnly) == WriteOnly) - gz = gzdopen(iFd, "w"); + d->gz = gzdopen(iFd, "w"); else - gz = gzdopen (iFd, "r"); - if (gz == NULL) + d->gz = gzdopen (iFd, "r"); + if (d->gz == NULL) return false; + Flags |= Compressed; } else return false; @@ -918,8 +929,8 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) do { - if (gz != NULL) - Res = gzread(gz,To,Size); + if (d->gz != NULL) + Res = gzread(d->gz,To,Size); else Res = read(iFd,To,Size); if (Res < 0 && errno == EINTR) @@ -951,6 +962,28 @@ bool FileFd::Read(void *To,unsigned long long Size,unsigned long long *Actual) return _error->Error(_("read, still have %llu to read but none left"), Size); } /*}}}*/ +// FileFd::ReadLine - Read a complete line from the file /*{{{*/ +// --------------------------------------------------------------------- +/* Beware: This method can be quiet slow for big buffers on UNcompressed + files because of the naive implementation! */ +char* FileFd::ReadLine(char *To, unsigned long long const Size) +{ + if (d->gz != NULL) + return gzgets(d->gz, To, Size); + + unsigned long long read = 0; + if (Read(To, Size, &read) == false) + return NULL; + char* c = To; + for (; *c != '\n' && *c != '\0' && read != 0; --read, ++c) + ; // find the end of the line + if (*c != '\0') + *c = '\0'; + if (read != 0) + Seek(Tell() - read); + return To; +} + /*}}}*/ // FileFd::Write - Write to the file /*{{{*/ // --------------------------------------------------------------------- /* */ @@ -960,8 +993,8 @@ bool FileFd::Write(const void *From,unsigned long long Size) errno = 0; do { - if (gz != NULL) - Res = gzwrite(gz,From,Size); + if (d->gz != NULL) + Res = gzwrite(d->gz,From,Size); else Res = write(iFd,From,Size); if (Res < 0 && errno == EINTR) @@ -990,8 +1023,8 @@ bool FileFd::Write(const void *From,unsigned long long Size) bool FileFd::Seek(unsigned long long To) { int res; - if (gz) - res = gzseek(gz,To,SEEK_SET); + if (d->gz) + res = gzseek(d->gz,To,SEEK_SET); else res = lseek(iFd,To,SEEK_SET); if (res != (signed)To) @@ -1009,8 +1042,8 @@ bool FileFd::Seek(unsigned long long To) bool FileFd::Skip(unsigned long long Over) { int res; - if (gz) - res = gzseek(gz,Over,SEEK_CUR); + if (d->gz != NULL) + res = gzseek(d->gz,Over,SEEK_CUR); else res = lseek(iFd,Over,SEEK_CUR); if (res < 0) @@ -1027,7 +1060,7 @@ bool FileFd::Skip(unsigned long long Over) /* */ bool FileFd::Truncate(unsigned long long To) { - if (gz) + if (d->gz != NULL) { Flags |= Fail; return _error->Error("Truncating gzipped files is not implemented (%s)", FileName.c_str()); @@ -1047,8 +1080,8 @@ bool FileFd::Truncate(unsigned long long To) unsigned long long FileFd::Tell() { off_t Res; - if (gz) - Res = gztell(gz); + if (d->gz != NULL) + Res = gztell(d->gz); else Res = lseek(iFd,0,SEEK_CUR); if (Res == (off_t)-1) @@ -1076,9 +1109,9 @@ unsigned long long FileFd::Size() unsigned long long size = FileSize(); // only check gzsize if we are actually a gzip file, just checking for - // "gz" is not sufficient as uncompressed files will be opened with + // "gz" is not sufficient as uncompressed files could be opened with // gzopen in "direct" mode as well - if (gz && !gzdirect(gz) && size > 0) + if (d->gz && !gzdirect(d->gz) && size > 0) { /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do * this ourselves; the original (uncompressed) file size is the last 32 @@ -1125,11 +1158,14 @@ time_t FileFd::ModificationTime() /* */ bool FileFd::Close() { + if (iFd == -1) + return true; + bool Res = true; if ((Flags & AutoClose) == AutoClose) { - if (gz != NULL) { - int const e = gzclose(gz); + if (d != NULL && d->gz != NULL) { + int const e = gzclose(d->gz); // gzdopen() on empty files always fails with "buffer error" here, ignore that if (e != 0 && e != Z_BUF_ERROR) Res &= _error->Errno("close",_("Problem closing the gzip file %s"), FileName.c_str()); @@ -1147,13 +1183,17 @@ bool FileFd::Close() } iFd = -1; - gz = NULL; if ((Flags & Fail) == Fail && (Flags & DelOnFail) == DelOnFail && FileName.empty() == false) if (unlink(FileName.c_str()) != 0) Res &= _error->WarningE("unlnk",_("Problem unlinking the file %s"), FileName.c_str()); + if (d != NULL) + { + delete d; + d = NULL; + } return Res; } @@ -1170,3 +1210,4 @@ bool FileFd::Sync() return true; } /*}}}*/ +gzFile FileFd::gzFd() {return d->gz;}; diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 8f2d7a0a0..209ca91e7 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -31,17 +31,17 @@ /* Define this for python-apt */ #define APT_HAS_GZIP 1 +class FileFdPrivate; class FileFd { protected: int iFd; enum LocalFlags {AutoClose = (1<<0),Fail = (1<<1),DelOnFail = (1<<2), - HitEof = (1<<3), Replace = (1<<4) }; + HitEof = (1<<3), Replace = (1<<4), Compressed = (1<<5) }; unsigned long Flags; std::string FileName; std::string TemporaryFileName; - gzFile gz; public: enum OpenMode { @@ -71,6 +71,7 @@ class FileFd return Read(To,Size); } bool Read(void *To,unsigned long long Size,unsigned long long *Actual = 0); + char* ReadLine(char *To, unsigned long long const Size); bool Write(const void *From,unsigned long long Size); bool Seek(unsigned long long To); bool Skip(unsigned long long To); @@ -94,8 +95,8 @@ class FileFd return T; } - bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long Perms = 0666); - inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long Perms = 0666) { + bool Open(std::string FileName,OpenMode Mode,CompressMode Compress,unsigned long const Perms = 0666); + inline bool Open(std::string const &FileName,OpenMode Mode, unsigned long const Perms = 0666) { return Open(FileName, Mode, None, Perms); }; bool OpenDescriptor(int Fd, OpenMode Mode, CompressMode Compress, bool AutoClose=false); @@ -108,29 +109,36 @@ class FileFd // Simple manipulators inline int Fd() {return iFd;}; inline void Fd(int fd) {iFd = fd;}; - inline gzFile gzFd() {return gz;}; + __deprecated gzFile gzFd(); inline bool IsOpen() {return iFd >= 0;}; inline bool Failed() {return (Flags & Fail) == Fail;}; inline void EraseOnFailure() {Flags |= DelOnFail;}; inline void OpFail() {Flags |= Fail;}; inline bool Eof() {return (Flags & HitEof) == HitEof;}; + inline bool IsCompressed() {return (Flags & Compressed) == Compressed;}; inline std::string &Name() {return FileName;}; - FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), - Flags(0), gz(NULL) + FileFd(std::string FileName,OpenMode Mode,unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL) { Open(FileName,Mode, None, Perms); }; - FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : - iFd(-1), Flags(0), gz(NULL) + FileFd(std::string FileName,OpenMode Mode, CompressMode Compress, unsigned long Perms = 0666) : iFd(-1), Flags(0), d(NULL) { Open(FileName,Mode, Compress, Perms); }; - FileFd(int Fd = -1) : iFd(Fd), Flags(AutoClose), gz(NULL) {}; - FileFd(int Fd,bool) : iFd(Fd), Flags(0), gz(NULL) {}; + FileFd() : iFd(-1), Flags(AutoClose), d(NULL) {}; + FileFd(int const Fd, OpenMode Mode = ReadWrite, CompressMode Compress = None) : iFd(-1), Flags(0), d(NULL) + { + OpenDescriptor(Fd, Mode, Compress); + }; + FileFd(int const Fd, bool const AutoClose) : iFd(-1), Flags(0), d(NULL) + { + OpenDescriptor(Fd, ReadWrite, None, AutoClose); + }; virtual ~FileFd(); private: + FileFdPrivate* d; bool OpenInternDescriptor(OpenMode Mode, CompressMode Compress); }; diff --git a/apt-pkg/contrib/mmap.cc b/apt-pkg/contrib/mmap.cc index f76169a92..1fb84b0af 100644 --- a/apt-pkg/contrib/mmap.cc +++ b/apt-pkg/contrib/mmap.cc @@ -77,7 +77,18 @@ bool MMap::Map(FileFd &Fd) if (iSize == 0) return _error->Error(_("Can't mmap an empty file")); - + + // We can't mmap compressed fd's directly, so we need to read it completely + if (Fd.IsCompressed() == true) + { + if ((Flags & ReadOnly) != ReadOnly) + return _error->Error("Compressed file %s can only be mapped readonly", Fd.Name().c_str()); + Base = new unsigned char[iSize]; + if (Fd.Seek(0L) == false || Fd.Read(Base, iSize) == false) + return false; + return true; + } + // Map it. Base = mmap(0,iSize,Prot,Map,Fd.Fd(),0); if (Base == (void *)-1) diff --git a/debian/changelog b/debian/changelog index 3e4a8fa3c..197e27a65 100644 --- a/debian/changelog +++ b/debian/changelog @@ -31,10 +31,12 @@ apt (0.8.16~exp9) UNRELEASED; urgency=low - dump the APT::Compressor settings correctly and completely * apt-pkg/contrib/fileutl.{h,cc}: - implement a ModificationTime method for FileFd + - add a ReadLine method + - drop the explicit export of gz-compression handling * apt-pkg/cdrom.cc: - support InRelease files on cdrom - -- David Kalnischkies Sun, 11 Dec 2011 01:30:12 +0100 + -- David Kalnischkies Sun, 11 Dec 2011 19:34:58 +0100 apt (0.8.16~exp8) experimental; urgency=low diff --git a/methods/rred.cc b/methods/rred.cc index 56ad8220b..2a70a9f91 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -37,13 +37,13 @@ class RredMethod : public pkgAcqMethod { // return values enum State {ED_OK, ED_ORDERING, ED_PARSER, ED_FAILURE, MMAP_FAILED}; - State applyFile(gzFile &ed_cmds, FILE *in_file, FILE *out_file, + State applyFile(FileFd &ed_cmds, FILE *in_file, FILE *out_file, unsigned long &line, char *buffer, Hashes *hash) const; void ignoreLineInFile(FILE *fin, char *buffer) const; - void ignoreLineInFile(gzFile &fin, char *buffer) const; + void ignoreLineInFile(FileFd &fin, char *buffer) const; void copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lines, Hashes *hash, char *buffer) const; - void copyLinesFromFileToFile(gzFile &fin, FILE *fout, unsigned int lines, + void copyLinesFromFileToFile(FileFd &fin, FILE *fout, unsigned int lines, Hashes *hash, char *buffer) const; State patchFile(FileFd &Patch, FileFd &From, FileFd &out_file, Hashes *hash) const; @@ -72,10 +72,10 @@ public: * \param hash the created file for correctness * \return the success State of the ed command executor */ -RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *out_file, +RredMethod::State RredMethod::applyFile(FileFd &ed_cmds, FILE *in_file, FILE *out_file, unsigned long &line, char *buffer, Hashes *hash) const { // get the current command and parse it - if (gzgets(ed_cmds, buffer, BUF_SIZE) == NULL) { + if (ed_cmds.ReadLine(buffer, BUF_SIZE) == NULL) { if (Debug == true) std::clog << "rred: encounter end of file - we can start patching now." << std::endl; line = 0; @@ -130,7 +130,7 @@ RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *ou unsigned char mode = *idx; // save the current position - unsigned const long pos = gztell(ed_cmds); + unsigned const long long pos = ed_cmds.Tell(); // if this is add or change then go to the next full stop unsigned int data_length = 0; @@ -164,7 +164,7 @@ RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *ou // include data from ed script if (mode == MODE_CHANGED || mode == MODE_ADDED) { - gzseek(ed_cmds, pos, SEEK_SET); + ed_cmds.Seek(pos); copyLinesFromFileToFile(ed_cmds, out_file, data_length, hash, buffer); } @@ -190,11 +190,11 @@ void RredMethod::copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lin } } /*}}}*/ -void RredMethod::copyLinesFromFileToFile(gzFile &fin, FILE *fout, unsigned int lines,/*{{{*/ +void RredMethod::copyLinesFromFileToFile(FileFd &fin, FILE *fout, unsigned int lines,/*{{{*/ Hashes *hash, char *buffer) const { while (0 < lines--) { do { - gzgets(fin, buffer, BUF_SIZE); + fin.ReadLine(buffer, BUF_SIZE); size_t const written = fwrite(buffer, 1, strlen(buffer), fout); hash->Add((unsigned char*)buffer, written); } while (strlen(buffer) == (BUF_SIZE - 1) && @@ -211,11 +211,11 @@ void RredMethod::ignoreLineInFile(FILE *fin, char *buffer) const { /*{{{*/ } } /*}}}*/ -void RredMethod::ignoreLineInFile(gzFile &fin, char *buffer) const { /*{{{*/ - gzgets(fin, buffer, BUF_SIZE); +void RredMethod::ignoreLineInFile(FileFd &fin, char *buffer) const { /*{{{*/ + fin.ReadLine(buffer, BUF_SIZE); while (strlen(buffer) == (BUF_SIZE - 1) && buffer[BUF_SIZE - 2] != '\n') { - gzgets(fin, buffer, BUF_SIZE); + fin.ReadLine(buffer, BUF_SIZE); buffer[0] = ' '; } } @@ -224,12 +224,11 @@ RredMethod::State RredMethod::patchFile(FileFd &Patch, FileFd &From, /*{{{*/ FileFd &out_file, Hashes *hash) const { char buffer[BUF_SIZE]; FILE* fFrom = fdopen(From.Fd(), "r"); - gzFile fPatch = Patch.gzFd(); FILE* fTo = fdopen(out_file.Fd(), "w"); /* we do a tail recursion to read the commands in the right order */ unsigned long line = -1; // assign highest possible value - State const result = applyFile(fPatch, fFrom, fTo, line, buffer, hash); + State const result = applyFile(Patch, fFrom, fTo, line, buffer, hash); /* read the rest from infile */ if (result == ED_OK) { @@ -258,20 +257,7 @@ struct EdCommand { RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From, /*{{{*/ FileFd &out_file, Hashes *hash) const { #ifdef _POSIX_MAPPED_FILES - MMap ed_cmds(MMap::ReadOnly); - if (Patch.gzFd() != NULL) { - unsigned long long mapSize = Patch.Size(); - DynamicMMap* dyn = new DynamicMMap(0, mapSize, 0); - if (dyn->validData() == false) { - delete dyn; - return MMAP_FAILED; - } - dyn->AddSize(mapSize); - gzread(Patch.gzFd(), dyn->Data(), mapSize); - ed_cmds = *dyn; - } else - ed_cmds = MMap(Patch, MMap::ReadOnly); - + MMap ed_cmds(Patch, MMap::ReadOnly); MMap in_file(From, MMap::ReadOnly); if (ed_cmds.Size() == 0 || in_file.Size() == 0) -- cgit v1.2.3-70-g09d2 From 109eb1511d0cdfa4af3196105cada30bcbb77bc8 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 17 Dec 2011 23:53:31 +0100 Subject: try to avoid direct usage of .Fd() if possible and do read()s and co on the FileFd instead --- apt-pkg/acquire-item.cc | 4 ++-- apt-pkg/contrib/hashes.cc | 40 +++++++++++++++++++++++++++++++++----- apt-pkg/contrib/hashes.h | 5 +++++ apt-pkg/contrib/hashsum.cc | 22 +++++++++++++++++++++ apt-pkg/contrib/hashsum_template.h | 3 +++ apt-pkg/deb/debindexfile.cc | 3 --- ftparchive/cachedb.cc | 8 ++++---- ftparchive/writer.cc | 2 +- methods/cdrom.cc | 2 +- methods/copy.cc | 2 +- methods/file.cc | 2 +- methods/ftp.cc | 2 +- methods/http.cc | 24 +++++++---------------- methods/https.cc | 2 +- methods/rsh.cc | 2 +- 15 files changed, 85 insertions(+), 38 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 453fce109..f231c42b4 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -438,7 +438,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile) /*{{{*/ FileFd fd(CurrentPackagesFile, FileFd::ReadOnly); SHA1Summation SHA1; - SHA1.AddFD(fd.Fd(), fd.Size()); + SHA1.AddFD(fd); string const local_sha1 = SHA1.Result(); if(local_sha1 == ServerSha1) @@ -669,7 +669,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff() /*{{{*/ FileFd fd(FinalFile, FileFd::ReadOnly); SHA1Summation SHA1; - SHA1.AddFD(fd.Fd(), fd.Size()); + SHA1.AddFD(fd); string local_sha1 = string(SHA1.Result()); if(Debug) std::clog << "QueueNextDiff: " diff --git a/apt-pkg/contrib/hashes.cc b/apt-pkg/contrib/hashes.cc index 05001f042..e1a431823 100644 --- a/apt-pkg/contrib/hashes.cc +++ b/apt-pkg/contrib/hashes.cc @@ -61,25 +61,25 @@ bool HashString::VerifyFile(std::string filename) const /*{{{*/ if(Type == "MD5Sum") { MD5Summation MD5; - MD5.AddFD(Fd.Fd(), Fd.Size()); + MD5.AddFD(Fd); fileHash = (std::string)MD5.Result(); } else if (Type == "SHA1") { SHA1Summation SHA1; - SHA1.AddFD(Fd.Fd(), Fd.Size()); + SHA1.AddFD(Fd); fileHash = (std::string)SHA1.Result(); } else if (Type == "SHA256") { SHA256Summation SHA256; - SHA256.AddFD(Fd.Fd(), Fd.Size()); + SHA256.AddFD(Fd); fileHash = (std::string)SHA256.Result(); } else if (Type == "SHA512") { SHA512Summation SHA512; - SHA512.AddFD(Fd.Fd(), Fd.Size()); + SHA512.AddFD(Fd); fileHash = (std::string)SHA512.Result(); } Fd.Close(); @@ -134,6 +134,36 @@ bool Hashes::AddFD(int const Fd,unsigned long long Size, bool const addMD5, SHA512.Add(Buf,Res); } return true; +} +bool Hashes::AddFD(FileFd &Fd,unsigned long long Size, bool const addMD5, + bool const addSHA1, bool const addSHA256, bool const addSHA512) +{ + unsigned char Buf[64*64]; + bool const ToEOF = (Size == 0); + while (Size != 0 || ToEOF) + { + unsigned long long n = sizeof(Buf); + if (!ToEOF) n = std::min(Size, n); + unsigned long long a = 0; + if (Fd.Read(Buf, n, &a) == false) // error + return false; + if (ToEOF == false) + { + if (a != n) // short read + return false; + } + else if (a == 0) // EOF + break; + Size -= a; + if (addMD5 == true) + MD5.Add(Buf, a); + if (addSHA1 == true) + SHA1.Add(Buf, a); + if (addSHA256 == true) + SHA256.Add(Buf, a); + if (addSHA512 == true) + SHA512.Add(Buf, a); + } + return true; } /*}}}*/ - diff --git a/apt-pkg/contrib/hashes.h b/apt-pkg/contrib/hashes.h index b206eccb8..0c0b6c6a7 100644 --- a/apt-pkg/contrib/hashes.h +++ b/apt-pkg/contrib/hashes.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -74,6 +75,10 @@ class Hashes { return AddFD(Fd, Size, true, true, true, true); }; bool AddFD(int const Fd, unsigned long long Size, bool const addMD5, bool const addSHA1, bool const addSHA256, bool const addSHA512); + inline bool AddFD(FileFd &Fd,unsigned long long Size = 0) + { return AddFD(Fd, Size, true, true, true, true); }; + bool AddFD(FileFd &Fd, unsigned long long Size, bool const addMD5, + bool const addSHA1, bool const addSHA256, bool const addSHA512); inline bool Add(const unsigned char *Beg,const unsigned char *End) {return Add(Beg,End-Beg);}; }; diff --git a/apt-pkg/contrib/hashsum.cc b/apt-pkg/contrib/hashsum.cc index ff3b112bb..289e43aa4 100644 --- a/apt-pkg/contrib/hashsum.cc +++ b/apt-pkg/contrib/hashsum.cc @@ -24,5 +24,27 @@ bool SummationImplementation::AddFD(int const Fd, unsigned long long Size) { Add(Buf,Res); } return true; +} +bool SummationImplementation::AddFD(FileFd &Fd, unsigned long long Size) { + unsigned char Buf[64 * 64]; + bool ToEOF = (Size == 0); + while (Size != 0 || ToEOF) + { + unsigned long long n = sizeof(Buf); + if (!ToEOF) n = std::min(Size, n); + unsigned long long a = 0; + if (Fd.Read(Buf, n, &a) == false) // error + return false; + if (ToEOF == false) + { + if (a != n) // short read + return false; + } + else if (a == 0) // EOF + break; + Size -= a; + Add(Buf, a); + } + return true; } /*}}}*/ diff --git a/apt-pkg/contrib/hashsum_template.h b/apt-pkg/contrib/hashsum_template.h index 6301ac9d0..51e3b0862 100644 --- a/apt-pkg/contrib/hashsum_template.h +++ b/apt-pkg/contrib/hashsum_template.h @@ -10,6 +10,8 @@ #ifndef APTPKG_HASHSUM_TEMPLATE_H #define APTPKG_HASHSUM_TEMPLATE_H +#include + #include #include #include @@ -108,6 +110,7 @@ class SummationImplementation { return Add((const unsigned char *)Beg, End - Beg); }; bool AddFD(int Fd, unsigned long long Size = 0); + bool AddFD(FileFd &Fd, unsigned long long Size = 0); }; #endif diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index 84791a70a..5dc2a2ac2 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -600,9 +600,6 @@ bool debStatusIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const // Store the IMS information pkgCache::PkgFileIterator CFile = Gen.GetCurFile(); - struct stat St; - if (fstat(Pkg.Fd(),&St) != 0) - return _error->Errno("fstat","Failed to stat"); CFile->Size = Pkg.FileSize(); CFile->mtime = Pkg.ModificationTime(); CFile->Archive = Gen.WriteUniqString("now"); diff --git a/ftparchive/cachedb.cc b/ftparchive/cachedb.cc index f0bfa2a6d..a8b637a80 100644 --- a/ftparchive/cachedb.cc +++ b/ftparchive/cachedb.cc @@ -351,7 +351,7 @@ bool CacheDB::GetMD5(bool const &GenOnly) return false; } MD5Summation MD5; - if (Fd->Seek(0) == false || MD5.AddFD(Fd->Fd(),CurStat.FileSize) == false) + if (Fd->Seek(0) == false || MD5.AddFD(*Fd, CurStat.FileSize) == false) return false; MD5Res = MD5.Result(); @@ -382,7 +382,7 @@ bool CacheDB::GetSHA1(bool const &GenOnly) return false; } SHA1Summation SHA1; - if (Fd->Seek(0) == false || SHA1.AddFD(Fd->Fd(),CurStat.FileSize) == false) + if (Fd->Seek(0) == false || SHA1.AddFD(*Fd, CurStat.FileSize) == false) return false; SHA1Res = SHA1.Result(); @@ -413,7 +413,7 @@ bool CacheDB::GetSHA256(bool const &GenOnly) return false; } SHA256Summation SHA256; - if (Fd->Seek(0) == false || SHA256.AddFD(Fd->Fd(),CurStat.FileSize) == false) + if (Fd->Seek(0) == false || SHA256.AddFD(*Fd, CurStat.FileSize) == false) return false; SHA256Res = SHA256.Result(); @@ -444,7 +444,7 @@ bool CacheDB::GetSHA512(bool const &GenOnly) return false; } SHA512Summation SHA512; - if (Fd->Seek(0) == false || SHA512.AddFD(Fd->Fd(),CurStat.FileSize) == false) + if (Fd->Seek(0) == false || SHA512.AddFD(*Fd, CurStat.FileSize) == false) return false; SHA512Res = SHA512.Result(); diff --git a/ftparchive/writer.cc b/ftparchive/writer.cc index 02777713c..159772991 100644 --- a/ftparchive/writer.cc +++ b/ftparchive/writer.cc @@ -1035,7 +1035,7 @@ bool ReleaseWriter::DoPackage(string FileName) CheckSums[NewFileName].size = fd.Size(); Hashes hs; - hs.AddFD(fd.Fd(), 0, DoMD5, DoSHA1, DoSHA256, DoSHA512); + hs.AddFD(fd, 0, DoMD5, DoSHA1, DoSHA256, DoSHA512); if (DoMD5 == true) CheckSums[NewFileName].MD5 = hs.MD5.Result(); if (DoSHA1 == true) diff --git a/methods/cdrom.cc b/methods/cdrom.cc index e7114b168..22d4b9164 100644 --- a/methods/cdrom.cc +++ b/methods/cdrom.cc @@ -268,7 +268,7 @@ bool CDROMMethod::Fetch(FetchItem *Itm) Hashes Hash; FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd.Fd(), Fd.Size()); + Hash.AddFD(Fd); Res.TakeHashes(Hash); URIDone(Res); diff --git a/methods/copy.cc b/methods/copy.cc index f8d58e479..e81d0022b 100644 --- a/methods/copy.cc +++ b/methods/copy.cc @@ -85,7 +85,7 @@ bool CopyMethod::Fetch(FetchItem *Itm) Hashes Hash; FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd.Fd(), Fd.Size()); + Hash.AddFD(Fd); Res.TakeHashes(Hash); URIDone(Res); diff --git a/methods/file.cc b/methods/file.cc index 5025c996d..7ed4e6f60 100644 --- a/methods/file.cc +++ b/methods/file.cc @@ -83,7 +83,7 @@ bool FileMethod::Fetch(FetchItem *Itm) Hashes Hash; FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd.Fd(), Fd.Size()); + Hash.AddFD(Fd); Res.TakeHashes(Hash); URIDone(Res); return true; diff --git a/methods/ftp.cc b/methods/ftp.cc index 2ca0ac6f7..ad8a7b828 100644 --- a/methods/ftp.cc +++ b/methods/ftp.cc @@ -868,7 +868,7 @@ bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume, if (Resume != 0) { - if (Hash.AddFD(To.Fd(),Resume) == false) + if (Hash.AddFD(To,Resume) == false) { _error->Errno("read",_("Problem hashing file")); return false; diff --git a/methods/http.cc b/methods/http.cc index 0d81c73ed..b8ed43cd2 100644 --- a/methods/http.cc +++ b/methods/http.cc @@ -1007,31 +1007,21 @@ HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv) FailFile.c_str(); // Make sure we dont do a malloc in the signal handler FailFd = File->Fd(); FailTime = Srv->Date; - - // Set the expected size - if (Srv->StartPos >= 0) - { - Res.ResumePoint = Srv->StartPos; - if (ftruncate(File->Fd(),Srv->StartPos) < 0) - _error->Errno("ftruncate", _("Failed to truncate file")); - } - - // Set the start point - lseek(File->Fd(),0,SEEK_END); delete Srv->In.Hash; Srv->In.Hash = new Hashes; - - // Fill the Hash if the file is non-empty (resume) - if (Srv->StartPos > 0) + + // Set the expected size and read file for the hashes + if (Srv->StartPos >= 0) { - lseek(File->Fd(),0,SEEK_SET); - if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false) + Res.ResumePoint = Srv->StartPos; + File->Truncate(Srv->StartPos); + + if (Srv->In.Hash->AddFD(*File,Srv->StartPos) == false) { _error->Errno("read",_("Problem hashing file")); return ERROR_NOT_FROM_SERVER; } - lseek(File->Fd(),0,SEEK_END); } SetNonBlock(File->Fd(),true); diff --git a/methods/https.cc b/methods/https.cc index 335699907..317c8a587 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -314,7 +314,7 @@ bool HttpsMethod::Fetch(FetchItem *Itm) // take hashes Hashes Hash; FileFd Fd(Res.Filename, FileFd::ReadOnly); - Hash.AddFD(Fd.Fd(), Fd.Size()); + Hash.AddFD(Fd); Res.TakeHashes(Hash); // keep apt updated diff --git a/methods/rsh.cc b/methods/rsh.cc index da9777fc4..d249ae961 100644 --- a/methods/rsh.cc +++ b/methods/rsh.cc @@ -305,7 +305,7 @@ bool RSHConn::Get(const char *Path,FileFd &To,unsigned long long Resume, return false; if (Resume != 0) { - if (Hash.AddFD(To.Fd(),Resume) == false) { + if (Hash.AddFD(To,Resume) == false) { _error->Errno("read",_("Problem hashing file")); return false; } -- cgit v1.2.3-70-g09d2 From 29966fd1cc2bab68777e85b9070dba7821a58d36 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 18 Dec 2011 01:22:04 +0100 Subject: implement the fallback method of rred by using the FileFd and the included ReadLine instead of accessing the files directly with fgets() --- methods/rred.cc | 49 ++++++++++++------------------------------------- 1 file changed, 12 insertions(+), 37 deletions(-) (limited to 'methods') diff --git a/methods/rred.cc b/methods/rred.cc index 2a70a9f91..bf9294d96 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -37,13 +37,10 @@ class RredMethod : public pkgAcqMethod { // return values enum State {ED_OK, ED_ORDERING, ED_PARSER, ED_FAILURE, MMAP_FAILED}; - State applyFile(FileFd &ed_cmds, FILE *in_file, FILE *out_file, + State applyFile(FileFd &ed_cmds, FileFd &in_file, FileFd &out_file, unsigned long &line, char *buffer, Hashes *hash) const; - void ignoreLineInFile(FILE *fin, char *buffer) const; void ignoreLineInFile(FileFd &fin, char *buffer) const; - void copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lines, - Hashes *hash, char *buffer) const; - void copyLinesFromFileToFile(FileFd &fin, FILE *fout, unsigned int lines, + void copyLinesFromFileToFile(FileFd &fin, FileFd &fout, unsigned int lines, Hashes *hash, char *buffer) const; State patchFile(FileFd &Patch, FileFd &From, FileFd &out_file, Hashes *hash) const; @@ -72,7 +69,7 @@ public: * \param hash the created file for correctness * \return the success State of the ed command executor */ -RredMethod::State RredMethod::applyFile(FileFd &ed_cmds, FILE *in_file, FILE *out_file, +RredMethod::State RredMethod::applyFile(FileFd &ed_cmds, FileFd &in_file, FileFd &out_file, unsigned long &line, char *buffer, Hashes *hash) const { // get the current command and parse it if (ed_cmds.ReadLine(buffer, BUF_SIZE) == NULL) { @@ -178,39 +175,19 @@ RredMethod::State RredMethod::applyFile(FileFd &ed_cmds, FILE *in_file, FILE *ou return ED_OK; } /*}}}*/ -void RredMethod::copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lines,/*{{{*/ - Hashes *hash, char *buffer) const { - while (0 < lines--) { - do { - fgets(buffer, BUF_SIZE, fin); - size_t const written = fwrite(buffer, 1, strlen(buffer), fout); - hash->Add((unsigned char*)buffer, written); - } while (strlen(buffer) == (BUF_SIZE - 1) && - buffer[BUF_SIZE - 2] != '\n'); - } -} - /*}}}*/ -void RredMethod::copyLinesFromFileToFile(FileFd &fin, FILE *fout, unsigned int lines,/*{{{*/ +void RredMethod::copyLinesFromFileToFile(FileFd &fin, FileFd &fout, unsigned int lines,/*{{{*/ Hashes *hash, char *buffer) const { while (0 < lines--) { do { fin.ReadLine(buffer, BUF_SIZE); - size_t const written = fwrite(buffer, 1, strlen(buffer), fout); - hash->Add((unsigned char*)buffer, written); + unsigned long long const towrite = strlen(buffer); + fout.Write(buffer, towrite); + hash->Add((unsigned char*)buffer, towrite); } while (strlen(buffer) == (BUF_SIZE - 1) && buffer[BUF_SIZE - 2] != '\n'); } } /*}}}*/ -void RredMethod::ignoreLineInFile(FILE *fin, char *buffer) const { /*{{{*/ - fgets(buffer, BUF_SIZE, fin); - while (strlen(buffer) == (BUF_SIZE - 1) && - buffer[BUF_SIZE - 2] != '\n') { - fgets(buffer, BUF_SIZE, fin); - buffer[0] = ' '; - } -} - /*}}}*/ void RredMethod::ignoreLineInFile(FileFd &fin, char *buffer) const { /*{{{*/ fin.ReadLine(buffer, BUF_SIZE); while (strlen(buffer) == (BUF_SIZE - 1) && @@ -223,20 +200,18 @@ void RredMethod::ignoreLineInFile(FileFd &fin, char *buffer) const { /*{{{*/ RredMethod::State RredMethod::patchFile(FileFd &Patch, FileFd &From, /*{{{*/ FileFd &out_file, Hashes *hash) const { char buffer[BUF_SIZE]; - FILE* fFrom = fdopen(From.Fd(), "r"); - FILE* fTo = fdopen(out_file.Fd(), "w"); /* we do a tail recursion to read the commands in the right order */ unsigned long line = -1; // assign highest possible value - State const result = applyFile(Patch, fFrom, fTo, line, buffer, hash); + State const result = applyFile(Patch, From, out_file, line, buffer, hash); /* read the rest from infile */ if (result == ED_OK) { - while (fgets(buffer, BUF_SIZE, fFrom) != NULL) { - size_t const written = fwrite(buffer, 1, strlen(buffer), fTo); - hash->Add((unsigned char*)buffer, written); + while (From.ReadLine(buffer, BUF_SIZE) != NULL) { + unsigned long long const towrite = strlen(buffer); + out_file.Write(buffer, towrite); + hash->Add((unsigned char*)buffer, towrite); } - fflush(fTo); } return result; } -- cgit v1.2.3-70-g09d2 From f23a94d53e451448b80b6aa5b52b5ba418e508a9 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Tue, 10 Jan 2012 12:05:12 +0100 Subject: as Size() can be quiet expensive for compressed files lets store the result --- methods/rred.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'methods') diff --git a/methods/rred.cc b/methods/rred.cc index bf9294d96..e37a12ed9 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -235,7 +235,9 @@ RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From, /*{{{*/ MMap ed_cmds(Patch, MMap::ReadOnly); MMap in_file(From, MMap::ReadOnly); - if (ed_cmds.Size() == 0 || in_file.Size() == 0) + unsigned long long const ed_size = ed_cmds.Size(); + unsigned long long const in_size = in_file.Size(); + if (ed_size == 0 || in_size == 0) return MMAP_FAILED; EdCommand* commands = 0; @@ -244,10 +246,10 @@ RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From, /*{{{*/ const char* begin = (char*) ed_cmds.Data(); const char* end = begin; - const char* ed_end = (char*) ed_cmds.Data() + ed_cmds.Size(); + const char* ed_end = (char*) ed_cmds.Data() + ed_size; const char* input = (char*) in_file.Data(); - const char* input_end = (char*) in_file.Data() + in_file.Size(); + const char* input_end = (char*) in_file.Data() + in_size; size_t i; -- cgit v1.2.3-70-g09d2