From 52b22cea95a1ba506ee633c1610bf241817ab529 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 12 Jan 2011 23:46:18 +0100 Subject: * methods/https.cc: - fix CURLOPT_SSL_VERIFYHOST by really passing 2 to it if enabled --- methods/https.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'methods') diff --git a/methods/https.cc b/methods/https.cc index aa6786aa8..fc649d6c2 100644 --- a/methods/https.cc +++ b/methods/https.cc @@ -143,13 +143,11 @@ bool HttpsMethod::Fetch(FetchItem *Itm) curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, peer_verify); // ... and hostname against cert CN or subjectAltName - int default_verify = 2; bool verify = _config->FindB("Acquire::https::Verify-Host",true); knob = "Acquire::https::"+remotehost+"::Verify-Host"; verify = _config->FindB(knob.c_str(),verify); - if (!verify) - default_verify = 0; - curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify); + int const default_verify = (verify == true) ? 2 : 0; + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, default_verify); // Also enforce issuer of server certificate using its cert string issuercert = _config->Find("Acquire::https::IssuerCert",""); -- cgit v1.2.3-70-g09d2 From caffd4807f612e931ecca8f7d9b0c0c10de27ce6 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sat, 15 Jan 2011 20:28:03 +0100 Subject: * methods/rred.cc: - operate optional on gzip compressed pdiffs * apt-pkg/acquire-item.cc: - don't uncompress downloaded pdiff files before feeding it to rred --- apt-pkg/acquire-item.cc | 19 +-------------- apt-pkg/acquire-item.h | 2 +- apt-pkg/contrib/fileutl.h | 1 + debian/changelog | 6 ++++- methods/rred.cc | 49 ++++++++++++++++++++++++++++++++------- test/integration/test-pdiff-usage | 1 + 6 files changed, 49 insertions(+), 29 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 2c4ce91a0..d4e90b552 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -570,24 +570,7 @@ void pkgAcqIndexDiffs::Done(string Message,unsigned long Size,string Md5Hash, /* FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(RealURI); // sucess in downloading a diff, enter ApplyDiff state - if(State == StateFetchDiff) - { - - if(Debug) - std::clog << "Sending to gzip method: " << FinalFile << std::endl; - - string FileName = LookupTag(Message,"Filename"); - State = StateUnzipDiff; - Local = true; - Desc.URI = "gzip:" + FileName; - DestFile += ".decomp"; - QueueURI(Desc); - Mode = "gzip"; - return; - } - - // sucess in downloading a diff, enter ApplyDiff state - if(State == StateUnzipDiff) + if(State == StateFetchDiff) { // rred excepts the patch as $FinalFile.ed diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 943c61876..d97a96a0f 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -449,7 +449,7 @@ class pkgAcqIndexDiffs : public pkgAcquire::Item StateFetchDiff, /** \brief The diff is currently being uncompressed. */ - StateUnzipDiff, + StateUnzipDiff, // FIXME: No longer used /** \brief The diff is currently being applied. */ StateApplyDiff diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h index 146d917d8..cde288ad2 100644 --- a/apt-pkg/contrib/fileutl.h +++ b/apt-pkg/contrib/fileutl.h @@ -72,6 +72,7 @@ class FileFd // Simple manipulators inline int Fd() {return iFd;}; inline void Fd(int fd) {iFd = fd;}; + inline gzFile gzFd() {return gz;}; inline bool IsOpen() {return iFd >= 0;}; inline bool Failed() {return (Flags & Fail) == Fail;}; inline void EraseOnFailure() {Flags |= DelOnFail;}; diff --git a/debian/changelog b/debian/changelog index 11768794d..5c8adb434 100644 --- a/debian/changelog +++ b/debian/changelog @@ -49,8 +49,12 @@ apt (0.8.11+wheezy) unstable; urgency=low - remove as it is not needed for a working 'bzr bd' * debian/{apt,apt-utils}.symbols: - ship experimental unmangled c++ symbol files + * methods/rred.cc: + - operate optional on gzip compressed pdiffs + * apt-pkg/acquire-item.cc: + - don't uncompress downloaded pdiff files before feeding it to rred - -- David Kalnischkies Sat, 15 Jan 2011 02:42:18 +0100 + -- David Kalnischkies Sat, 15 Jan 2011 20:24:03 +0100 apt (0.8.10) unstable; urgency=low diff --git a/methods/rred.cc b/methods/rred.cc index d51c45c85..1a18a381c 100644 --- a/methods/rred.cc +++ b/methods/rred.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include /*}}}*/ /** \brief RredMethod - ed-style incremential patch method {{{ @@ -33,11 +34,14 @@ class RredMethod : public pkgAcqMethod { // return values enum State {ED_OK, ED_ORDERING, ED_PARSER, ED_FAILURE, MMAP_FAILED}; - State applyFile(FILE *ed_cmds, FILE *in_file, FILE *out_file, + State applyFile(gzFile &ed_cmds, FILE *in_file, FILE *out_file, unsigned long &line, char *buffer, Hashes *hash) const; void ignoreLineInFile(FILE *fin, char *buffer) const; + void ignoreLineInFile(gzFile &fin, char *buffer) const; void copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lines, Hashes *hash, char *buffer) const; + void copyLinesFromFileToFile(gzFile &fin, FILE *fout, unsigned int lines, + Hashes *hash, char *buffer) const; State patchFile(FileFd &Patch, FileFd &From, FileFd &out_file, Hashes *hash) const; State patchMMap(FileFd &Patch, FileFd &From, FileFd &out_file, Hashes *hash) const; @@ -65,10 +69,10 @@ public: * \param hash the created file for correctness * \return the success State of the ed command executor */ -RredMethod::State RredMethod::applyFile(FILE *ed_cmds, FILE *in_file, FILE *out_file, +RredMethod::State RredMethod::applyFile(gzFile &ed_cmds, FILE *in_file, FILE *out_file, unsigned long &line, char *buffer, Hashes *hash) const { // get the current command and parse it - if (fgets(buffer, BUF_SIZE, ed_cmds) == NULL) { + if (gzgets(ed_cmds, buffer, BUF_SIZE) == NULL) { if (Debug == true) std::clog << "rred: encounter end of file - we can start patching now." << std::endl; line = 0; @@ -123,7 +127,7 @@ RredMethod::State RredMethod::applyFile(FILE *ed_cmds, FILE *in_file, FILE *out_ unsigned char mode = *idx; // save the current position - unsigned const long pos = ftell(ed_cmds); + unsigned const long pos = gztell(ed_cmds); // if this is add or change then go to the next full stop unsigned int data_length = 0; @@ -157,7 +161,7 @@ RredMethod::State RredMethod::applyFile(FILE *ed_cmds, FILE *in_file, FILE *out_ // include data from ed script if (mode == MODE_CHANGED || mode == MODE_ADDED) { - fseek(ed_cmds, pos, SEEK_SET); + gzseek(ed_cmds, pos, SEEK_SET); copyLinesFromFileToFile(ed_cmds, out_file, data_length, hash, buffer); } @@ -183,6 +187,18 @@ void RredMethod::copyLinesFromFileToFile(FILE *fin, FILE *fout, unsigned int lin } } /*}}}*/ +void RredMethod::copyLinesFromFileToFile(gzFile &fin, FILE *fout, unsigned int lines,/*{{{*/ + Hashes *hash, char *buffer) const { + while (0 < lines--) { + do { + gzgets(fin, buffer, BUF_SIZE); + size_t const written = fwrite(buffer, 1, strlen(buffer), fout); + hash->Add((unsigned char*)buffer, written); + } while (strlen(buffer) == (BUF_SIZE - 1) && + buffer[BUF_SIZE - 2] != '\n'); + } +} + /*}}}*/ void RredMethod::ignoreLineInFile(FILE *fin, char *buffer) const { /*{{{*/ fgets(buffer, BUF_SIZE, fin); while (strlen(buffer) == (BUF_SIZE - 1) && @@ -192,11 +208,20 @@ void RredMethod::ignoreLineInFile(FILE *fin, char *buffer) const { /*{{{*/ } } /*}}}*/ +void RredMethod::ignoreLineInFile(gzFile &fin, char *buffer) const { /*{{{*/ + gzgets(fin, buffer, BUF_SIZE); + while (strlen(buffer) == (BUF_SIZE - 1) && + buffer[BUF_SIZE - 2] != '\n') { + gzgets(fin, buffer, BUF_SIZE); + buffer[0] = ' '; + } +} + /*}}}*/ RredMethod::State RredMethod::patchFile(FileFd &Patch, FileFd &From, /*{{{*/ FileFd &out_file, Hashes *hash) const { char buffer[BUF_SIZE]; FILE* fFrom = fdopen(From.Fd(), "r"); - FILE* fPatch = fdopen(Patch.Fd(), "r"); + gzFile fPatch = Patch.gzFd(); FILE* fTo = fdopen(out_file.Fd(), "w"); /* we do a tail recursion to read the commands in the right order */ @@ -228,6 +253,12 @@ RredMethod::State RredMethod::patchMMap(FileFd &Patch, FileFd &From, /*{{{*/ FileFd &out_file, Hashes *hash) const { #ifdef _POSIX_MAPPED_FILES MMap ed_cmds(Patch, MMap::ReadOnly); + if (Patch.gzFd() != NULL) { + unsigned long mapSize = Patch.Size(); + DynamicMMap dyn(0, mapSize, 0); + gzread(Patch.gzFd(), dyn.Data(), mapSize); + ed_cmds = dyn; + } MMap in_file(From, MMap::ReadOnly); if (ed_cmds.Size() == 0 || in_file.Size() == 0) @@ -445,7 +476,7 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ // Open the source and destination files (the d'tor of FileFd will do // the cleanup/closing of the fds) FileFd From(Path,FileFd::ReadOnly); - FileFd Patch(Path+".ed",FileFd::ReadOnly); + FileFd Patch(Path+".ed",FileFd::ReadOnlyGzip); FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); if (_error->PendingError() == true) @@ -456,8 +487,8 @@ bool RredMethod::Fetch(FetchItem *Itm) /*{{{*/ State const result = patchMMap(Patch, From, To, &Hash); if (result == MMAP_FAILED) { // retry with patchFile - lseek(Patch.Fd(), 0, SEEK_SET); - lseek(From.Fd(), 0, SEEK_SET); + Patch.Seek(0); + From.Seek(0); To.Open(Itm->DestFile,FileFd::WriteAtomic); if (_error->PendingError() == true) return false; diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage index a70b6122c..eb1818bcd 100755 --- a/test/integration/test-pdiff-usage +++ b/test/integration/test-pdiff-usage @@ -44,6 +44,7 @@ sed -i aptarchive/Release \ -e "/^SHA256:/ a\ \ $(sha256sum $PATCHINDEX | cut -d' ' -f 1) $(stat -c%s $PATCHINDEX) Packages.diff/Index" signreleasefiles +rm -f aptarchive/Packages aptarchive/Packages.gz aptarchive/Packages.bz2 aptarchive/Packages.lzma aptget update -qq testnopackage oldstuff -- cgit v1.2.3-70-g09d2 From 2cf8c58b3f5e850d44645c23e33ce1be96bcad0b Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Sun, 16 Jan 2011 00:54:48 +0100 Subject: move file installation from rules to dh_install files --- debian/apt-transport-https.dirs | 1 + debian/apt-transport-https.install | 1 + debian/apt-utils.install | 1 + debian/apt.install | 5 +++++ debian/control | 2 +- debian/libapt-pkg-dev.install | 3 +++ debian/rules | 45 +++++++++++--------------------------- methods/makefile | 2 +- 8 files changed, 26 insertions(+), 34 deletions(-) create mode 100644 debian/apt-transport-https.dirs create mode 100644 debian/apt-transport-https.install create mode 100644 debian/apt-utils.install create mode 100644 debian/apt.install create mode 100644 debian/libapt-pkg-dev.install (limited to 'methods') diff --git a/debian/apt-transport-https.dirs b/debian/apt-transport-https.dirs new file mode 100644 index 000000000..3abb3fee8 --- /dev/null +++ b/debian/apt-transport-https.dirs @@ -0,0 +1 @@ +usr/lib/apt/methods diff --git a/debian/apt-transport-https.install b/debian/apt-transport-https.install new file mode 100644 index 000000000..848f10c78 --- /dev/null +++ b/debian/apt-transport-https.install @@ -0,0 +1 @@ +bin/methods/https usr/lib/apt/methods diff --git a/debian/apt-utils.install b/debian/apt-utils.install new file mode 100644 index 000000000..d947f26d4 --- /dev/null +++ b/debian/apt-utils.install @@ -0,0 +1 @@ +bin/libapt-inst*.so.* usr/lib/ diff --git a/debian/apt.install b/debian/apt.install new file mode 100644 index 000000000..979e04ce2 --- /dev/null +++ b/debian/apt.install @@ -0,0 +1,5 @@ +bin/apt-* usr/bin/ +bin/libapt-pkg*.so.* usr/lib/ +bin/methods/* usr/lib/apt/methods/ +scripts/dselect/* usr/lib/dpkg/methods/apt/ +locale usr/share/ diff --git a/debian/control b/debian/control index 46d19b246..7e4b798d9 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Uploaders: Michael Vogt , Otavio Salvador , Christian Perrier , Daniel Burrows , Luca Bruno , Julian Andres Klode Standards-Version: 3.9.0 -Build-Depends: dpkg-dev (>= 1.15.8), debhelper (>= 7), libdb-dev, gettext (>= 0.12), libcurl4-gnutls-dev (>= 7.19.0), zlib1g-dev | libz-dev, debiandoc-sgml, xsltproc, docbook-xsl, docbook-xml, po4a (>= 0.34-2), autotools-dev, autoconf, automake, doxygen +Build-Depends: dpkg-dev (>= 1.15.8), debhelper (>= 7.2.3~), libdb-dev, gettext (>= 0.12), libcurl4-gnutls-dev (>= 7.19.0), zlib1g-dev | libz-dev, debiandoc-sgml, xsltproc, docbook-xsl, docbook-xml, po4a (>= 0.34-2), autotools-dev, autoconf, automake, doxygen Build-Conflicts: autoconf2.13, automake1.4 Vcs-Bzr: http://bzr.debian.org/apt/debian-sid/ Vcs-Browser: http://bzr.debian.org/loggerhead/apt/debian-sid/ diff --git a/debian/libapt-pkg-dev.install b/debian/libapt-pkg-dev.install new file mode 100644 index 000000000..bd1a6c174 --- /dev/null +++ b/debian/libapt-pkg-dev.install @@ -0,0 +1,3 @@ +bin/libapt-pkg*.so usr/lib/ +bin/libapt-inst*.so usr/lib/ +include/apt-pkg/*.h usr/include/apt-pkg/ diff --git a/debian/rules b/debian/rules index 5331bea80..640900678 100755 --- a/debian/rules +++ b/debian/rules @@ -172,24 +172,6 @@ apt: build build-doc # # apt install # - cp $(BLD)/bin/apt-* debian/$@/usr/bin/ - - # Remove the bits that are in apt-utils - rm $(addprefix debian/$@/usr/bin/apt-,$(APT_UTILS)) - - # install the shared libs - find $(BLD)/bin/ -type f -name "libapt-pkg*.so.*" -exec cp -a "{}" debian/$@/usr/lib/ \; - find $(BLD)/bin/ -type l -name "libapt-pkg*.so.*" -exec cp -a "{}" debian/$@/usr/lib/ \; - - cp $(BLD)/bin/methods/* debian/$@/usr/lib/apt/methods/ - # https has its own package - rm debian/$@/usr/lib/apt/methods/https - - cp $(BLD)/scripts/dselect/* debian/$@/usr/lib/dpkg/methods/apt/ - cp -r $(BLD)/locale debian/$@/usr/share/ - - cp debian/apt.logrotate debian/$@/etc/logrotate.d/apt - cp debian/apt.conf.autoremove debian/$@/etc/apt/apt.conf.d/01autoremove # make rosetta happy and remove pot files in po/ (but leave stuff @@ -197,6 +179,14 @@ apt: build build-doc rm -f build/po/*.pot rm -f po/*.pot + dh_install -p$@ --sourcedir=$(BLD) + + # Remove the bits that are in apt-utils + rm $(addprefix debian/$@/usr/bin/apt-,$(APT_UTILS)) + + # https has its own package + rm debian/$@/usr/lib/apt/methods/https + # move the mirror failure script in place #mv debian/$@/usr/bin/apt-report-mirror-failure \ # debian/$@/usr/lib/apt/apt-report-mirror-failure \ @@ -208,6 +198,7 @@ apt: build build-doc dh_installcron -p$@ dh_installdocs -p$@ dh_installchangelogs -p$@ + dh_installlogrotate -p$@ dh_strip -p$@ dh_compress -p$@ dh_fixperms -p$@ @@ -226,13 +217,8 @@ libapt-pkg-dev: build # # libapt-pkg-dev install # - cp -a $(BLD)/bin/libapt-pkg*.so debian/libapt-pkg-dev/usr/lib/ - cp -a $(BLD)/bin/libapt-inst*.so debian/libapt-pkg-dev/usr/lib/ -# ln -s libapt-pkg.so.$(LIBAPTPKG_MAJOR) debian/libapt-pkg-dev/usr/lib/libapt-pkg.so - cp $(BLD)/include/apt-pkg/*.h debian/libapt-pkg-dev/usr/include/apt-pkg/ - + dh_install -p$@ --sourcedir=$(BLD) dh_installdocs -p$@ - dh_installchangelogs -p$@ dh_strip -p$@ dh_compress -p$@ @@ -249,11 +235,9 @@ apt-utils: build dh_prep -p$@ dh_installdirs -p$@ - # install the shared libs - find $(BLD)/bin/ -type f -name "libapt-inst*.so.*" -exec cp -a "{}" debian/$@/usr/lib/ \; - find $(BLD)/bin/ -type l -name "libapt-inst*.so.*" -exec cp -a "{}" debian/$@/usr/lib/ \; - cp $(addprefix $(BLD)/bin/apt-,$(APT_UTILS)) debian/$@/usr/bin/ + + dh_install -p$@ --sourcedir=$(BLD) dh_installdocs -p$@ dh_installexamples -p$@ @@ -277,10 +261,7 @@ apt-transport-https: build libapt-pkg-dev dh_prep -p$@ dh_installdirs -p$@ - # install the method - mkdir --parents debian/$@/usr/lib/apt/methods - cp $(BLD)/bin/methods/https debian/$@/usr/lib/apt/methods - + dh_install -p$@ --sourcedir=$(BLD) dh_installdocs -p$@ debian/apt-transport-https.README dh_installexamples -p$@ diff --git a/methods/makefile b/methods/makefile index d94a85340..4ee356cec 100644 --- a/methods/makefile +++ b/methods/makefile @@ -67,7 +67,7 @@ include $(PROGRAM_H) # The rred method PROGRAM=rred -SLIBS = -lapt-pkg $(SOCKETLIBS) $(INTLLIBS) +SLIBS = -lapt-pkg -lz $(SOCKETLIBS) $(INTLLIBS) LIB_MAKES = apt-pkg/makefile SOURCE = rred.cc include $(PROGRAM_H) -- cgit v1.2.3-70-g09d2 From 5d88572318ed7e271101b1ae8f2cc139a1a3f705 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 26 Jan 2011 16:19:30 +0100 Subject: - change the internal handling of Extensions in pkgAcqIndex - add a special uncompressed compression type to prefer those files * methods/{gzip,bzip}.cc: - print a good error message if FileSize() is zero --- apt-pkg/acquire-item.cc | 114 ++++++++++++--------- apt-pkg/acquire-item.h | 9 +- apt-pkg/aptconfiguration.cc | 8 ++ debian/changelog | 6 +- doc/apt.conf.5.xml | 9 +- doc/examples/configure-index | 2 +- methods/bzip2.cc | 3 +- methods/gzip.cc | 3 +- .../test-bug-595691-empty-and-broken-archive-files | 9 +- 9 files changed, 96 insertions(+), 67 deletions(-) (limited to 'methods') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 2cd6ab359..a603a3d70 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -622,29 +622,61 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, HashString ExpectedHash, string comprExt) : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash) { + if(comprExt.empty() == true) + { + // autoselect the compression method + std::vector types = APT::Configuration::getCompressionTypes(); + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + comprExt.append(*t).append(" "); + if (comprExt.empty() == false) + comprExt.erase(comprExt.end()-1); + } + CompressionExtension = comprExt; + + Init(URI, URIDesc, ShortDesc); +} +pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, + HashString const &ExpectedHash, indexRecords const *MetaIndexParser) + : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash) +{ + // autoselect the compression method + std::vector types = APT::Configuration::getCompressionTypes(); + CompressionExtension = ""; + if (ExpectedHash.empty() == false) + { + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true) + CompressionExtension.append(*t).append(" "); + } + else + { + for (std::vector::const_iterator t = types.begin(); t != types.end(); ++t) + CompressionExtension.append(*t).append(" "); + } + if (CompressionExtension.empty() == false) + CompressionExtension.erase(CompressionExtension.end()-1); + + Init(Target->URI, Target->Description, Target->ShortDesc); +} + /*}}}*/ +// AcqIndex::Init - defered Constructor /*{{{*/ +void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) { Decompression = false; Erase = false; DestFile = _config->FindDir("Dir::State::lists") + "partial/"; DestFile += URItoFileName(URI); - if(comprExt.empty()) - { - // autoselect the compression method - std::vector types = APT::Configuration::getCompressionTypes(); - if (types.empty() == true) - comprExt = "plain"; - else - comprExt = "." + types[0]; - } - CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt); - - Desc.URI = URI + CompressionExtension; + std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); + if (comprExt == "uncompressed") + Desc.URI = URI; + else + Desc.URI = URI + '.' + comprExt; Desc.Description = URIDesc; Desc.Owner = this; Desc.ShortDesc = ShortDesc; - + QueueURI(Desc); } /*}}}*/ @@ -666,37 +698,18 @@ string pkgAcqIndex::Custom600Headers() /*}}}*/ void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ { - std::vector types = APT::Configuration::getCompressionTypes(); - - for (std::vector::const_iterator t = types.begin(); - t != types.end(); t++) + size_t const nextExt = CompressionExtension.find(' '); + if (nextExt != std::string::npos) { - // jump over all already tried compression types - const unsigned int nameLen = Desc.URI.size() - (*t).size(); - if(Desc.URI.substr(nameLen) != *t) - continue; - - // we want to try it with the next extension (and make sure to - // not skip over the end) - t++; - if (t == types.end()) - break; - - // queue new download - Desc.URI = Desc.URI.substr(0, nameLen) + *t; - new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc, - ExpectedHash, string(".").append(*t)); - - Status = StatDone; - Complete = false; - Dequeue(); + CompressionExtension = CompressionExtension.substr(nextExt+1); + Init(RealURI, Desc.Description, Desc.ShortDesc); return; } // on decompression failure, remove bad versions in partial/ - if(Decompression && Erase) { + if (Decompression && Erase) { string s = _config->FindDir("Dir::State::lists") + "partial/"; - s += URItoFileName(RealURI); + s.append(URItoFileName(RealURI)); unlink(s.c_str()); } @@ -773,8 +786,8 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, Status = StatError; ErrorText = "Method gave a blank filename"; } - - string compExt = flExtension(flNotDir(URI(Desc.URI).Path)); + + std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' ')); // The files timestamp matches if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) { @@ -807,12 +820,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash, // get the binary name for your used compression type decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),""); if(decompProg.empty() == false); - // flExtensions returns the full name if no extension is found - // this is why we have this complicated compare operation here - // FIMXE: add a new flJustExtension() that return "" if no - // extension is found and use that above so that it can - // be tested against "" - else if(compExt == flNotDir(URI(Desc.URI).Path)) + else if(compExt == "uncompressed") decompProg = "copy"; else { _error->Error("Unsupported extension: %s", compExt.c_str()); @@ -853,6 +861,15 @@ string pkgAcqIndexTrans::Custom600Headers() /* */ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) { + size_t const nextExt = CompressionExtension.find(' '); + if (nextExt != std::string::npos) + { + CompressionExtension = CompressionExtension.substr(nextExt+1); + Init(RealURI, Desc.Description, Desc.ShortDesc); + Status = StatIdle; + return; + } + if (Cnf->LocalOnly == true || StringToBool(LookupTag(Message,"Transient-Failure"),false) == false) { @@ -862,7 +879,7 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf) Dequeue(); return; } - + Item::Failed(Message,Cnf); } /*}}}*/ @@ -1197,8 +1214,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description, (*Target)->ShortDesc, ExpectedIndexHash); else - new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, - (*Target)->ShortDesc, ExpectedIndexHash); + new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser); } } /*}}}*/ diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 581761e32..92098e3d4 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -528,8 +528,8 @@ class pkgAcqIndex : public pkgAcquire::Item /** \brief The expected hashsum of the decompressed index file. */ HashString ExpectedHash; - /** \brief The compression-related file extension that is being - * added to the downloaded file (e.g., ".gz" or ".bz2"). + /** \brief The compression-related file extensions that are being + * added to the downloaded file one by one if first fails (e.g., "gz bz2"). */ string CompressionExtension; @@ -540,7 +540,7 @@ class pkgAcqIndex : public pkgAcquire::Item virtual void Done(string Message,unsigned long Size,string Md5Hash, pkgAcquire::MethodConfig *Cnf); virtual string Custom600Headers(); - virtual string DescURI() {return RealURI + CompressionExtension;}; + virtual string DescURI() {return Desc.URI;}; virtual string HashSum() {return ExpectedHash.toStr(); }; /** \brief Create a pkgAcqIndex. @@ -565,6 +565,9 @@ class pkgAcqIndex : public pkgAcquire::Item pkgAcqIndex(pkgAcquire *Owner,string URI,string URIDesc, string ShortDesc, HashString ExpectedHash, string compressExt=""); + pkgAcqIndex(pkgAcquire *Owner, struct IndexTarget const * const Target, + HashString const &ExpectedHash, indexRecords const *MetaIndexParser); + void Init(string const &URI, string const &URIDesc, string const &ShortDesc); }; /*}}}*/ /** \brief An acquire item that is responsible for fetching a {{{ diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc index 52f54073c..e97ebfed7 100644 --- a/apt-pkg/aptconfiguration.cc +++ b/apt-pkg/aptconfiguration.cc @@ -90,6 +90,14 @@ const Configuration::getCompressionTypes(bool const &Cached) { types.push_back(Types->Tag); } + // add the special "uncompressed" type + if (std::find(types.begin(), types.end(), "uncompressed") == types.end()) + { + string const uncompr = _config->FindFile("Dir::Bin::uncompressed", ""); + if (uncompr.empty() == true || FileExists(uncompr) == true) + types.push_back("uncompressed"); + } + return types; } /*}}}*/ diff --git a/debian/changelog b/debian/changelog index 2413b9c5b..7b125fc43 100644 --- a/debian/changelog +++ b/debian/changelog @@ -54,6 +54,8 @@ apt (0.8.11+wheezy) unstable; urgency=low * apt-pkg/acquire-item.cc: - don't uncompress downloaded pdiff files before feeding it to rred - try downloading clearsigned InRelease before trying Release.gpg + - change the internal handling of Extensions in pkgAcqIndex + - add a special uncompressed compression type to prefer those files * cmdline/apt-key: - don't set trustdb-name as non-root so 'list' and 'finger' can be used without being root (Closes: #393005, #592107) @@ -62,8 +64,10 @@ apt (0.8.11+wheezy) unstable; urgency=low * ftparchive/writer.cc: - add config option to search for more patterns in release command - include Index files by default in the Release file + * methods/{gzip,bzip}.cc: + - print a good error message if FileSize() is zero - -- David Kalnischkies Mon, 24 Jan 2011 15:36:50 +0100 + -- David Kalnischkies Wed, 26 Jan 2011 16:06:10 +0100 apt (0.8.10.3) unstable; urgency=low diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml index a19d85dbc..a423dac24 100644 --- a/doc/apt.conf.5.xml +++ b/doc/apt.conf.5.xml @@ -442,12 +442,11 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";}; the bzip2 method (the inbuilt) setting is Dir::Bin::bzip2 "/bin/bzip2"; Note also that list entries specified on the command line will be added at the end of the list specified in the configuration files, but before the default entries. To prefer a type in this case - over the ones specified in in the configuration files you can set the option direct - not in list style. + over the ones specified in the configuration files you can set the option direct - not in list style. This will not override the defined list, it will only prefix the list with this type. - While it is possible to add an empty compression type to the order list, but APT in its current - version doesn't understand it correctly and will display many warnings about not downloaded files - - these warnings are most of the time false negatives. Future versions will maybe include a way to - really prefer uncompressed files to support the usage of local mirrors. + The special type uncompressed can be used to give uncompressed files a + preference, but note that most archives doesn't provide uncompressed files so this is mostly only + useable for local mirrors. GzipIndexes diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 6c078d75f..fd14d4dd7 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -281,7 +281,7 @@ Acquire lzma "lzma"; gz "gzip"; - Order { "gz"; "lzma"; "bz2"; }; + Order { "uncompressed"; "gz"; "lzma"; "bz2"; }; }; Languages diff --git a/methods/bzip2.cc b/methods/bzip2.cc index ccc3669a2..42932dded 100644 --- a/methods/bzip2.cc +++ b/methods/bzip2.cc @@ -56,9 +56,8 @@ bool Bzip2Method::Fetch(FetchItem *Itm) // Open the source and destination files FileFd From(Path,FileFd::ReadOnly); - // FIXME add an error message saying that empty files can't be valid archives if(From.FileSize() == 0) - return false; + return _error->Error(_("Empty files can't be valid archives")); int GzOut[2]; if (pipe(GzOut) < 0) diff --git a/methods/gzip.cc b/methods/gzip.cc index f1c76066e..fc4e1ecfd 100644 --- a/methods/gzip.cc +++ b/methods/gzip.cc @@ -48,9 +48,8 @@ bool GzipMethod::Fetch(FetchItem *Itm) // Open the source and destination files FileFd From(Path,FileFd::ReadOnlyGzip); - // FIXME add an error message saying that empty files can't be valid archives if(From.FileSize() == 0) - return false; + return _error->Error(_("Empty files can't be valid archives")); FileFd To(Itm->DestFile,FileFd::WriteAtomic); To.EraseOnFailure(); diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index c5379dca0..398d0cd1b 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -56,6 +56,7 @@ setupcompressor() { lzma) COMPRESS="lzma";; esac echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; }; +Dir::Bin::uncompressed \"/does/not/exist\"; Dir::Bin::gzip \"/does/not/exist\"; Dir::Bin::bzip2 \"/does/not/exist\"; Dir::Bin::lzma \"/does/not/exist\";" > rootdir/etc/apt/apt.conf.d/00compressor @@ -93,8 +94,8 @@ Reading package lists..." "empty archive Packages.$COMPRESS over file" testaptgetupdate "Ign file:$(readlink -f aptarchive)/ Translation-en Get:1 file: InRelease [] Err file: Packages - Undetermined Error -W: Failed to fetch file:$(readlink -f aptarchive/Packages.$COMPRESS) Undetermined Error + Empty files can't be valid archives +W: Failed to fetch ${COMPRESSOR}:$(readlink -f aptarchive/Packages.$COMPRESS) Empty files can't be valid archives E: Some index files failed to download, they have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over file" } @@ -127,8 +128,8 @@ Reading package lists..." "empty archive Packages.$COMPRESS over http" Ign http://localhost/ Translation-en Get:2 http://localhost Packages Err http://localhost Packages - Undetermined Error -W: Failed to fetch http://localhost:8080/Packages.$COMPRESS Undetermined Error + Empty files can't be valid archives +W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives E: Some index files failed to download, they have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http" } -- cgit v1.2.3-70-g09d2