From ed0fe654305469c1f9869f308e2f595ac8bc4c13 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Wed, 12 Jan 2011 23:31:37 +0100 Subject: remove the unused Die boolean - error reporting is done by ExecWait --- apt-pkg/deb/dpkgpm.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'apt-pkg/deb') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 9f0da3be6..95a3f173b 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -322,7 +322,6 @@ bool pkgDPkgPM::RunScriptsWithPkgs(const char *Cnf) return _error->Errno("fdopen","Faild to open new FD"); // Feed it the filenames. - bool Die = false; if (Version <= 1) { for (vector::iterator I = List.begin(); I != List.end(); I++) @@ -339,14 +338,11 @@ bool pkgDPkgPM::RunScriptsWithPkgs(const char *Cnf) into the pipe. */ fprintf(F,"%s\n",I->File.c_str()); if (ferror(F) != 0) - { - Die = true; break; - } } } else - Die = !SendV2Pkgs(F); + SendV2Pkgs(F); fclose(F); -- cgit v1.2.3-70-g09d2 From 23f3cfd036630c1c8f84159c60986f67167066e8 Mon Sep 17 00:00:00 2001 From: Petter Reinholdtsen Date: Wed, 12 Jan 2011 23:57:03 +0100 Subject: * deb/dpkgpm.cc: - fix popen/fclose mismatch reported by cppcheck. Thanks to Petter Reinholdtsen for report and patch! (Closes: #607803) --- apt-pkg/deb/dpkgpm.cc | 4 ++-- debian/changelog | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'apt-pkg/deb') diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc index 95a3f173b..3b10e1a23 100644 --- a/apt-pkg/deb/dpkgpm.cc +++ b/apt-pkg/deb/dpkgpm.cc @@ -1411,7 +1411,7 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) { while( fgets(buf, sizeof(buf), log) != NULL) fprintf(report, " %s", buf); - fclose(log); + pclose(log); } } @@ -1427,7 +1427,7 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg) { while( fgets(buf, sizeof(buf), log) != NULL) fprintf(report, " %s", buf); - fclose(log); + pclose(log); } } diff --git a/debian/changelog b/debian/changelog index 81f741d03..dde28c635 100644 --- a/debian/changelog +++ b/debian/changelog @@ -24,8 +24,11 @@ apt (0.8.11+wheezy) unstable; urgency=low - include stddefs.h to fix compile error (undefined NULL) with gcc-4.6 * methods/https.cc: - fix CURLOPT_SSL_VERIFYHOST by really passing 2 to it if enabled + * deb/dpkgpm.cc: + - fix popen/fclose mismatch reported by cppcheck. Thanks to Petter + Reinholdtsen for report and patch! (Closes: #607803) - -- David Kalnischkies Wed, 12 Jan 2011 23:46:08 +0100 + -- David Kalnischkies Wed, 12 Jan 2011 23:53:32 +0100 apt (0.8.10) unstable; urgency=low -- cgit v1.2.3-70-g09d2 From fe0f7911b650918e1d511b3453664a07f6d966d0 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Thu, 20 Jan 2011 14:53:52 +0100 Subject: - try downloading clearsigned InRelease before trying Release.gpg * apt-pkg/deb/deblistparser.cc: - rewrite LoadReleaseInfo to cope with clearsigned Releasefiles --- apt-pkg/acquire-item.cc | 54 ++++++- apt-pkg/acquire-item.h | 33 +++++ apt-pkg/deb/debindexfile.cc | 8 +- apt-pkg/deb/deblistparser.cc | 114 ++++++++++----- apt-pkg/deb/debmetaindex.cc | 34 +++-- apt-pkg/indexcopy.cc | 3 +- apt-pkg/indexrecords.cc | 17 ++- apt-pkg/tagfile.cc | 10 +- apt-pkg/tagfile.h | 2 + debian/changelog | 5 +- doc/apt-secure.8.xml | 5 +- test/integration/Packages-releasefile-verification | 18 +++ .../Packages-releasefile-verification-new | 21 +++ test/integration/framework | 7 + test/integration/marvinparanoid.pub | Bin 0 -> 629 bytes test/integration/marvinparanoid.sec | Bin 0 -> 1280 bytes .../test-bug-595691-empty-and-broken-archive-files | 32 ++--- test/integration/test-releasefile-verification | 160 +++++++++++++++++++++ 18 files changed, 430 insertions(+), 93 deletions(-) create mode 100644 test/integration/Packages-releasefile-verification create mode 100644 test/integration/Packages-releasefile-verification-new create mode 100644 test/integration/marvinparanoid.pub create mode 100644 test/integration/marvinparanoid.sec create mode 100755 test/integration/test-releasefile-verification (limited to 'apt-pkg/deb') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d4e90b552..2cd6ab359 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -1077,6 +1077,8 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash, /*{{{* { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); + if (SigFile == DestFile) + SigFile = FinalFile; Rename(DestFile,FinalFile); chmod(FinalFile.c_str(),0644); DestFile = FinalFile; @@ -1110,6 +1112,8 @@ void pkgAcqMetaIndex::RetrievalDone(string Message) /*{{{*/ { string FinalFile = _config->FindDir("Dir::State::lists"); FinalFile += URItoFileName(RealURI); + if (SigFile == DestFile) + SigFile = FinalFile; DestFile = FinalFile; } Complete = true; @@ -1141,6 +1145,10 @@ void pkgAcqMetaIndex::AuthDone(string Message) /*{{{*/ // Download further indexes with verification QueueIndexes(true); + // is it a clearsigned MetaIndex file? + if (DestFile == SigFile) + return; + // Done, move signature file into position string VerifiedSigFile = _config->FindDir("Dir::State::lists") + URItoFileName(RealURI) + ".gpg"; @@ -1300,13 +1308,20 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) if (AuthPass == true) { // gpgv method failed, if we have a good signature - string LastGoodSigFile = _config->FindDir("Dir::State::lists") + - "partial/" + URItoFileName(RealURI) + ".gpg.reverify"; + string LastGoodSigFile = _config->FindDir("Dir::State::lists"); + if (DestFile == SigFile) + LastGoodSigFile.append(URItoFileName(RealURI)); + else + LastGoodSigFile.append("partial/").append(URItoFileName(RealURI)).append(".gpg.reverify"); + if(FileExists(LastGoodSigFile)) { - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(RealURI) + ".gpg"; - Rename(LastGoodSigFile,VerifiedSigFile); + if (DestFile != SigFile) + { + string VerifiedSigFile = _config->FindDir("Dir::State::lists") + + URItoFileName(RealURI) + ".gpg"; + Rename(LastGoodSigFile,VerifiedSigFile); + } Status = StatTransientNetworkError; _error->Warning(_("A error occurred during the signature " "verification. The repository is not updated " @@ -1330,6 +1345,35 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) QueueIndexes(false); } /*}}}*/ +pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner, /*{{{*/ + string const &URI, string const &URIDesc, string const &ShortDesc, + string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, + string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, + const vector* IndexTargets, + indexRecords* MetaIndexParser) : + pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser), + MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc), + MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc) +{ + SigFile = DestFile; +} + /*}}}*/ +void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ +{ + if (AuthPass == false) + { + new pkgAcqMetaSig(Owner, + MetaSigURI, MetaSigURIDesc, MetaSigShortDesc, + MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc, + IndexTargets, MetaIndexParser); + if (Cnf->LocalOnly == true || + StringToBool(LookupTag(Message, "Transient-Failure"), false) == false) + Dequeue(); + } + else + pkgAcqMetaIndex::Failed(Message, Cnf); +} + /*}}}*/ // AcqArchive::AcqArchive - Constructor /*{{{*/ // --------------------------------------------------------------------- /* This just sets up the initial fetch environment and queues the first diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index d97a96a0f..581761e32 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -772,6 +772,39 @@ class pkgAcqMetaIndex : public pkgAcquire::Item indexRecords* MetaIndexParser); }; /*}}}*/ +/** \brief An item repsonsible for downloading clearsigned metaindexes {{{*/ +class pkgAcqMetaClearSig : public pkgAcqMetaIndex +{ + /** \brief The URI of the meta-index file for the detached signature */ + string MetaIndexURI; + + /** \brief A "URI-style" description of the meta-index file */ + string MetaIndexURIDesc; + + /** \brief A brief description of the meta-index file */ + string MetaIndexShortDesc; + + /** \brief The URI of the detached meta-signature file if the clearsigned one failed. */ + string MetaSigURI; + + /** \brief A "URI-style" description of the meta-signature file */ + string MetaSigURIDesc; + + /** \brief A brief description of the meta-signature file */ + string MetaSigShortDesc; + +public: + void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + + /** \brief Create a new pkgAcqMetaClearSig. */ + pkgAcqMetaClearSig(pkgAcquire *Owner, + string const &URI, string const &URIDesc, string const &ShortDesc, + string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc, + string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc, + const vector* IndexTargets, + indexRecords* MetaIndexParser); +}; + /*}}}*/ /** \brief An item that is responsible for fetching a package file. {{{ * * If the package file already exists in the cache, nothing will be diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc index af1209ccb..9961b5ae4 100644 --- a/apt-pkg/deb/debindexfile.cc +++ b/apt-pkg/deb/debindexfile.cc @@ -324,8 +324,14 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const return _error->Error("Problem with MergeList %s",PackageFile.c_str()); // Check the release file - string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release"); + string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease"); + bool releaseExists = false; if (FileExists(ReleaseFile) == true) + releaseExists = true; + else + ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release"); + + if (releaseExists == true || FileExists(ReleaseFile) == true) { FileFd Rel(ReleaseFile,FileFd::ReadOnly); if (_error->PendingError() == true) diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc index 1b3bfd6ae..9201e6a54 100644 --- a/apt-pkg/deb/deblistparser.cc +++ b/apt-pkg/deb/deblistparser.cc @@ -783,45 +783,89 @@ bool debListParser::Step() bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI, FileFd &File, string component) { - pkgTagFile Tags(&File, File.Size() + 256); // XXX - pkgTagSection Section; - if (Tags.Step(Section) == false) - return false; - - // FIXME: Do we need it now for multi-arch? - // mvo: I don't think we need to fill that in (it's unused since apt-0.6) -// FileI->Architecture = WriteUniqString(Arch); - // apt-secure does no longer download individual (per-section) Release // file. to provide Component pinning we use the section name now FileI->Component = WriteUniqString(component); - const char *Start; - const char *Stop; - if (Section.Find("Suite",Start,Stop) == true) - FileI->Archive = WriteUniqString(Start,Stop - Start); - if (Section.Find("Component",Start,Stop) == true) - FileI->Component = WriteUniqString(Start,Stop - Start); - if (Section.Find("Version",Start,Stop) == true) - FileI->Version = WriteUniqString(Start,Stop - Start); - if (Section.Find("Origin",Start,Stop) == true) - FileI->Origin = WriteUniqString(Start,Stop - Start); - if (Section.Find("Codename",Start,Stop) == true) - FileI->Codename = WriteUniqString(Start,Stop - Start); - if (Section.Find("Label",Start,Stop) == true) - FileI->Label = WriteUniqString(Start,Stop - Start); - if (Section.Find("Architecture",Start,Stop) == true) - FileI->Architecture = WriteUniqString(Start,Stop - Start); - - if (Section.FindFlag("NotAutomatic",FileI->Flags, - pkgCache::Flag::NotAutomatic) == false) - _error->Warning("Bad NotAutomatic flag"); - if (Section.FindFlag("ButAutomaticUpgrades",FileI->Flags, - pkgCache::Flag::ButAutomaticUpgrades) == false) - _error->Warning("Bad ButAutomaticUpgrades flag"); - // overrule the NotAutomatic setting if needed as they are both present for compatibility - else if ((FileI->Flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades) - FileI->Flags &= ~pkgCache::Flag::NotAutomatic; + FILE* release = fdopen(dup(File.Fd()), "r"); + if (release == NULL) + return false; + + char buffer[101]; + bool gpgClose = false; + while (fgets(buffer, sizeof(buffer), release) != NULL) + { + size_t len = 0; + + // Skip empty lines + for (; buffer[len] == '\r' && buffer[len] == '\n'; ++len); + if (buffer[len] == '\0') + continue; + + // only evalute the first GPG section + if (strncmp("-----", buffer, 5) == 0) + { + if (gpgClose == true) + break; + gpgClose = true; + continue; + } + + // seperate the tag from the data + for (; buffer[len] != ':' && buffer[len] != '\0'; ++len); + if (buffer[len] == '\0') + continue; + char* dataStart = buffer + len; + for (++dataStart; *dataStart == ' '; ++dataStart); + char* dataEnd = dataStart; + for (++dataEnd; *dataEnd != '\0'; ++dataEnd); + + // which datastorage need to be updated + map_ptrloc* writeTo = NULL; + if (buffer[0] == ' ') + ; + #define APT_PARSER_WRITETO(X, Y) else if (strncmp(Y, buffer, len) == 0) writeTo = &X; + APT_PARSER_WRITETO(FileI->Archive, "Suite") + APT_PARSER_WRITETO(FileI->Component, "Component") + APT_PARSER_WRITETO(FileI->Version, "Version") + APT_PARSER_WRITETO(FileI->Origin, "Origin") + APT_PARSER_WRITETO(FileI->Codename, "Codename") + APT_PARSER_WRITETO(FileI->Label, "Label") + #undef APT_PARSER_WRITETO + #define APT_PARSER_FLAGIT(X) else if (strncmp(#X, buffer, len) == 0) \ + pkgTagSection::FindFlag(FileI->Flags, pkgCache::Flag:: X, dataStart, dataEnd-1); + APT_PARSER_FLAGIT(NotAutomatic) + APT_PARSER_FLAGIT(ButAutomaticUpgrades) + #undef APT_PARSER_FLAGIT + + // load all data from the line and save it + string data; + if (writeTo != NULL) + data.append(dataStart, dataEnd); + if (sizeof(buffer) - 1 == (dataEnd - buffer)) + { + while (fgets(buffer, sizeof(buffer), release) != NULL) + { + if (writeTo != NULL) + data.append(buffer); + if (strlen(buffer) != sizeof(buffer) - 1) + break; + } + } + if (writeTo != NULL) + { + // remove spaces and stuff from the end of the data line + for (std::string::reverse_iterator s = data.rbegin(); + s != data.rend(); ++s) + { + if (*s != '\r' && *s != '\n' && *s != ' ') + break; + *s = '\0'; + } + *writeTo = WriteUniqString(data); + } + } + fclose(release); return !_error->PendingError(); } diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index 717d0bcde..e2c680b14 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -182,21 +182,15 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description, (*Target)->ShortDesc, HashString()); } - // this is normally created in pkgAcqMetaSig, but if we run - // in --print-uris mode, we add it here - new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"), - MetaIndexInfo("Release"), "Release", - MetaIndexURI("Release.gpg"), - ComputeIndexTargets(), - new indexRecords (Dist)); - } - new pkgAcqMetaSig(Owner, MetaIndexURI("Release.gpg"), - MetaIndexInfo("Release.gpg"), "Release.gpg", - MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", - ComputeIndexTargets(), - new indexRecords (Dist)); + new pkgAcqMetaClearSig(Owner, MetaIndexURI("InRelease"), + MetaIndexInfo("InRelease"), "InRelease", + MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release", + MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg", + ComputeIndexTargets(), + new indexRecords (Dist)); + // Queue the translations std::vector const lang = APT::Configuration::getLanguages(true); @@ -224,16 +218,20 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const bool debReleaseIndex::IsTrusted() const { - string VerifiedSigFile = _config->FindDir("Dir::State::lists") + - URItoFileName(MetaIndexURI("Release")) + ".gpg"; - if(_config->FindB("APT::Authentication::TrustCDROM", false)) if(URI.substr(0,strlen("cdrom:")) == "cdrom:") return true; - + + string VerifiedSigFile = _config->FindDir("Dir::State::lists") + + URItoFileName(MetaIndexURI("Release")) + ".gpg"; + if (FileExists(VerifiedSigFile)) return true; - return false; + + VerifiedSigFile = _config->FindDir("Dir::State::lists") + + URItoFileName(MetaIndexURI("InRelease")); + + return FileExists(VerifiedSigFile); } vector *debReleaseIndex::GetIndexFiles() { diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc index f88d51fc5..c2ee1c347 100644 --- a/apt-pkg/indexcopy.cc +++ b/apt-pkg/indexcopy.cc @@ -722,7 +722,8 @@ bool SigVerify::RunGPGV(std::string const &File, std::string const &FileGPG, } Args.push_back(FileGPG.c_str()); - Args.push_back(File.c_str()); + if (FileGPG != File) + Args.push_back(File.c_str()); Args.push_back(NULL); if (Debug == true) diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc index eb9a36866..10e154ad2 100644 --- a/apt-pkg/indexrecords.cc +++ b/apt-pkg/indexrecords.cc @@ -55,14 +55,17 @@ bool indexRecords::Load(const string Filename) /*{{{*/ } pkgTagSection Section; - if (TagFile.Step(Section) == false) - { - strprintf(ErrorText, _("No sections in Release file %s"), Filename.c_str()); - return false; - } - const char *Start, *End; - Section.Get (Start, End, 0); + // Skip over sections beginning with ----- as this is an idicator for clearsigns + do { + if (TagFile.Step(Section) == false) + { + strprintf(ErrorText, _("No sections in Release file %s"), Filename.c_str()); + return false; + } + + Section.Get (Start, End, 0); + } while (End - Start > 5 && strncmp(Start, "-----", 5) == 0); Suite = Section.FindS("Suite"); Dist = Section.FindS("Codename"); diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc index 96a681bec..4a2f3f7e6 100644 --- a/apt-pkg/tagfile.cc +++ b/apt-pkg/tagfile.cc @@ -399,9 +399,13 @@ bool pkgTagSection::FindFlag(const char *Tag,unsigned long &Flags, const char *Stop; if (Find(Tag,Start,Stop) == false) return true; - - switch (StringToBool(string(Start,Stop))) - { + return FindFlag(Flags, Flag, Start, Stop); +} +bool const pkgTagSection::FindFlag(unsigned long &Flags, unsigned long Flag, + char const* Start, char const* Stop) +{ + switch (StringToBool(string(Start, Stop))) + { case 0: Flags &= ~Flag; return true; diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h index 6891c1d81..61491aa04 100644 --- a/apt-pkg/tagfile.h +++ b/apt-pkg/tagfile.h @@ -60,6 +60,8 @@ class pkgTagSection unsigned long long FindULL(const char *Tag, unsigned long long const &Default = 0) const; bool FindFlag(const char *Tag,unsigned long &Flags, unsigned long Flag) const; + bool static const FindFlag(unsigned long &Flags, unsigned long Flag, + const char* Start, const char* Stop); bool Scan(const char *Start,unsigned long MaxLength); inline unsigned long size() const {return Stop - Section;}; void Trim(); diff --git a/debian/changelog b/debian/changelog index b74508804..59f516751 100644 --- a/debian/changelog +++ b/debian/changelog @@ -53,11 +53,14 @@ apt (0.8.11+wheezy) unstable; urgency=low - operate optional on gzip compressed pdiffs * apt-pkg/acquire-item.cc: - don't uncompress downloaded pdiff files before feeding it to rred + - try downloading clearsigned InRelease before trying Release.gpg * cmdline/apt-key: - don't set trustdb-name as non-root so 'list' and 'finger' can be used without being root (Closes: #393005, #592107) + * apt-pkg/deb/deblistparser.cc: + - rewrite LoadReleaseInfo to cope with clearsigned Releasefiles - -- David Kalnischkies Sun, 16 Jan 2011 17:23:28 +0100 + -- David Kalnischkies Thu, 20 Jan 2011 14:52:32 +0100 apt (0.8.10) unstable; urgency=low diff --git a/doc/apt-secure.8.xml b/doc/apt-secure.8.xml index f345c3f89..f8ff678b9 100644 --- a/doc/apt-secure.8.xml +++ b/doc/apt-secure.8.xml @@ -148,8 +148,8 @@ (you should make sure you are using a trusted communication channel when retrieving it), add it with apt-key and then run apt-get update so that apt can download - and verify the Release.gpg files from the archives you - have configured. + and verify the InRelease or Release.gpg + files from the archives you have configured. @@ -166,6 +166,7 @@ (provided in apt-utils). Sign it. You can do this by running + gpg --clearsign -o InRelease Release and gpg -abs -o Release.gpg Release. Publish the key fingerprint, diff --git a/test/integration/Packages-releasefile-verification b/test/integration/Packages-releasefile-verification new file mode 100644 index 000000000..29a385f4f --- /dev/null +++ b/test/integration/Packages-releasefile-verification @@ -0,0 +1,18 @@ +Package: apt +Version: 0.7.25.3 +Architecture: i386 +Maintainer: APT Development Team +Installed-Size: 5244 +Replaces: libapt-pkg-dev (<< 0.3.7), libapt-pkg-doc (<< 0.3.7) +Provides: libapt-pkg-libc6.9-6-4.8 +Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt +Filename: apt.deb +Size: 0 +MD5sum: d41d8cd98f00b204e9800998ecf8427e +Description: Advanced front-end for dpkg + This is Debian's next generation front-end for the dpkg package manager. + It provides the apt-get utility and APT dselect method that provides a + simpler, safer way to install and upgrade packages. + . + APT features complete installation ordering, multiple source capability + and several other unique features, see the Users Guide in apt-doc. diff --git a/test/integration/Packages-releasefile-verification-new b/test/integration/Packages-releasefile-verification-new new file mode 100644 index 000000000..e3b2edf1f --- /dev/null +++ b/test/integration/Packages-releasefile-verification-new @@ -0,0 +1,21 @@ +Package: apt +Priority: important +Section: admin +Installed-Size: 5672 +Maintainer: APT Development Team +Architecture: i386 +Version: 0.8.0~pre1 +Replaces: manpages-pl (<< 20060617-3~) +Provides: libapt-pkg4.10 +Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt +Conflicts: python-apt (<< 0.7.93.2~) +Filename: apt.deb +Size: 0 +MD5sum: d41d8cd98f00b204e9800998ecf8427e +Description: Advanced front-end for dpkg + This is Debian's next generation front-end for the dpkg package manager. + It provides the apt-get utility and APT dselect method that provides a + simpler, safer way to install and upgrade packages. + . + APT features complete installation ordering, multiple source capability + and several other unique features, see the Users Guide in apt-doc. diff --git a/test/integration/framework b/test/integration/framework index 5d54e49b6..cb3fca35d 100644 --- a/test/integration/framework +++ b/test/integration/framework @@ -383,6 +383,7 @@ buildaptarchivefromfiles() { generatereleasefiles() { msgninfo "\tGenerate Release files… " + local DATE="${1:-now}" if [ -e aptarchive/dists ]; then for dir in $(find ./aptarchive/dists -mindepth 1 -maxdepth 1 -type d); do local CODENAME="$(echo "$dir" | cut -d'/' -f 4)" @@ -395,6 +396,11 @@ NotAutomatic: yes' $dir/Release else aptftparchive -qq release ./aptarchive | sed -e '/0 Release$/ d' > aptarchive/Release # remove the self reference fi + if [ "$DATE" != "now" ]; then + for release in $(find ./aptarchive -name 'Release'); do + touch -d "$1" $release + done + fi msgdone "info" } @@ -455,6 +461,7 @@ signreleasefiles() { done for RELEASE in $(find aptarchive/ -name Release); do gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" -abs -o ${RELEASE}.gpg ${RELEASE} + gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" --clearsign -o "$(echo "${RELEASE}" | sed 's#/Release$#/InRelease#')" $RELEASE done msgdone "info" } diff --git a/test/integration/marvinparanoid.pub b/test/integration/marvinparanoid.pub new file mode 100644 index 000000000..9a59c2e40 Binary files /dev/null and b/test/integration/marvinparanoid.pub differ diff --git a/test/integration/marvinparanoid.sec b/test/integration/marvinparanoid.sec new file mode 100644 index 000000000..ff54e8680 Binary files /dev/null and b/test/integration/marvinparanoid.sec differ diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index 66792899a..c5379dca0 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -74,28 +74,24 @@ testoverfile() { setupcompressor "$1" createemptyfile 'en' - testaptgetupdate "Get:1 file: Release.gpg [] -Get:2 file: Release [] + testaptgetupdate "Get:1 file: InRelease [] Ign file:$(readlink -f aptarchive)/ Translation-en Reading package lists..." "empty file en.$COMPRESS over file" createemptyarchive 'en' - testaptgetupdate "Get:1 file: Release.gpg [] -Get:2 file: Release [] + testaptgetupdate "Get:1 file: InRelease [] Reading package lists..." "empty archive en.$COMPRESS over file" createemptyarchive 'Packages' # FIXME: Why omits the file transport the Packages Get line? #Get:3 file: Packages [] testaptgetupdate "Ign file:$(readlink -f aptarchive)/ Translation-en -Get:1 file: Release.gpg [] -Get:2 file: Release [] +Get:1 file: InRelease [] Reading package lists..." "empty archive Packages.$COMPRESS over file" createemptyfile 'Packages' testaptgetupdate "Ign file:$(readlink -f aptarchive)/ Translation-en -Get:1 file: Release.gpg [] -Get:2 file: Release [] +Get:1 file: InRelease [] Err file: Packages Undetermined Error W: Failed to fetch file:$(readlink -f aptarchive/Packages.$COMPRESS) Undetermined Error @@ -107,33 +103,29 @@ testoverhttp() { setupcompressor "$1" createemptyfile 'en' - testaptgetupdate "Get:1 http://localhost Release.gpg [] + testaptgetupdate "Get:1 http://localhost InRelease [] Get:2 http://localhost/ Translation-en -Get:3 http://localhost Release [] +Get:3 http://localhost Packages [] Ign http://localhost/ Translation-en -Get:4 http://localhost Packages [] Reading package lists..." "empty file en.$COMPRESS over http" createemptyarchive 'en' - testaptgetupdate "Get:1 http://localhost Release.gpg [] + testaptgetupdate "Get:1 http://localhost InRelease [] Get:2 http://localhost/ Translation-en [] -Get:3 http://localhost Release [] -Get:4 http://localhost Packages [] +Get:3 http://localhost Packages [] Reading package lists..." "empty archive en.$COMPRESS over http" createemptyarchive 'Packages' - testaptgetupdate "Get:1 http://localhost Release.gpg [] + testaptgetupdate "Get:1 http://localhost InRelease [] Ign http://localhost/ Translation-en -Get:2 http://localhost Release [] -Get:3 http://localhost Packages [] +Get:2 http://localhost Packages [] Reading package lists..." "empty archive Packages.$COMPRESS over http" createemptyfile 'Packages' #FIXME: we should response with a good error message instead - testaptgetupdate "Get:1 http://localhost Release.gpg [] + testaptgetupdate "Get:1 http://localhost InRelease [] Ign http://localhost/ Translation-en -Get:2 http://localhost Release [] -Get:3 http://localhost Packages +Get:2 http://localhost Packages Err http://localhost Packages Undetermined Error W: Failed to fetch http://localhost:8080/Packages.$COMPRESS Undetermined Error diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification new file mode 100755 index 000000000..961c49895 --- /dev/null +++ b/test/integration/test-releasefile-verification @@ -0,0 +1,160 @@ +#!/bin/sh +set -e + +TESTDIR=$(readlink -f $(dirname $0)) +. $TESTDIR/framework + +setupenvironment +configarchitecture "i386" + +buildaptarchive +setupflataptarchive +changetowebserver + +prepare() { + local DATE="${2:-now}" + if [ "$DATE" = 'now' -a "$1" = "${PKGFILE}-new" ]; then + DATE='now + 6 days' + fi + for release in $(find rootdir/var/lib/apt/lists 2> /dev/null); do + touch -d 'now - 6 hours' $release + done + rm -rf rootdir/var/cache/apt/archives + rm -f rootdir/var/cache/apt/*.bin + cp $1 aptarchive/Packages + find aptarchive -name 'Release' -delete + cat aptarchive/Packages | gzip > aptarchive/Packages.gz + cat aptarchive/Packages | bzip2 > aptarchive/Packages.bz2 + cat aptarchive/Packages | lzma > aptarchive/Packages.lzma + generatereleasefiles "$DATE" +} + +installaptold() { + testequal 'Reading package lists... +Building dependency tree... +Suggested packages: + aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt +The following NEW packages will be installed: + apt +0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded. +After this operation, 5370 kB of additional disk space will be used. +Get:1 http://localhost/ apt 0.7.25.3 +Download complete and in download only mode' aptget install apt -dy +} + +installaptnew() { + testequal 'Reading package lists... +Building dependency tree... +Suggested packages: + aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt +The following NEW packages will be installed: + apt +0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded. +After this operation, 5808 kB of additional disk space will be used. +Get:1 http://localhost/ apt 0.8.0~pre1 +Download complete and in download only mode' aptget install apt -dy +} + +failaptold() { + testequal 'Reading package lists... +Building dependency tree... +Suggested packages: + aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt +The following NEW packages will be installed: + apt +0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded. +After this operation, 5370 kB of additional disk space will be used. +WARNING: The following packages cannot be authenticated! + apt +E: There are problems and -y was used without --force-yes' aptget install apt -dy +} + +failaptnew() { + testequal 'Reading package lists... +Building dependency tree... +Suggested packages: + aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt +The following NEW packages will be installed: + apt +0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded. +After this operation, 5808 kB of additional disk space will be used. +WARNING: The following packages cannot be authenticated! + apt +E: There are problems and -y was used without --force-yes' aptget install apt -dy +} + +# fake our downloadable file +touch aptarchive/apt.deb + +PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')" + +runtest() { + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Cold archive signed by' 'Joe Sixpack' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass + testequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold + + prepare ${PKGFILE}-new + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Good warm archive signed by' 'Joe Sixpack' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt + installaptnew + + + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Marvin Paranoid' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Cold archive signed by' 'Marvin Paranoid' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgpass || msgfail + testequal "$(cat ${PKGFILE}) +" aptcache show apt + failaptold + + prepare ${PKGFILE}-new + # weborf doesn't support If-Range + for release in $(find rootdir/var/lib/apt/lists/partial/ -name '*Release'); do + rm $release + touch $release + done + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Bad warm archive signed by' 'Joe Sixpack' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass + testequal "$(cat ${PKGFILE}-new) +" aptcache show apt + installaptnew + + + prepare ${PKGFILE} + rm -rf rootdir/var/lib/apt/lists + signreleasefiles 'Joe Sixpack' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Cold archive signed by' 'Joe Sixpack' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass + testequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold + + prepare ${PKGFILE}-new + signreleasefiles 'Marvin Paranoid' + find aptarchive/ -name "$DELETEFILE" -delete + msgtest 'Good warm archive signed by' 'Marvin Paranoid' + aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgpass || msgfail + testequal "$(cat ${PKGFILE}) +" aptcache show apt + installaptold +} + +DELETEFILE="InRelease" +runtest +DELETEFILE="Release.gpg" +runtest -- cgit v1.2.3-70-g09d2 From ab53c018fbc7aa01c0d89586c0aa98bc944dd9e4 Mon Sep 17 00:00:00 2001 From: David Kalnischkies Date: Fri, 28 Jan 2011 12:22:37 +0100 Subject: - download and use i18n/Index to choose which Translations to download * apt-pkg/aptconfiguration.cc: - remove the inbuilt Translation files whitelist --- apt-pkg/acquire-item.cc | 218 +++++++++++++++++++-- apt-pkg/acquire-item.h | 56 ++++++ apt-pkg/aptconfiguration.cc | 51 ++--- apt-pkg/deb/debmetaindex.cc | 77 +++++--- apt-pkg/deb/debmetaindex.h | 2 + debian/changelog | 5 +- .../test-bug-595691-empty-and-broken-archive-files | 28 +-- .../test-bug-601016-description-translation | 1 + test/libapt/getlanguages_test.cc | 71 ++++--- 9 files changed, 381 insertions(+), 128 deletions(-) (limited to 'apt-pkg/deb') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index a603a3d70..776009493 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -184,6 +184,153 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode) } } /*}}}*/ +// AcqSubIndex::AcqSubIndex - Constructor /*{{{*/ +// --------------------------------------------------------------------- +/* Get the DiffIndex file first and see if there are patches availabe + * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the + * patches. If anything goes wrong in that process, it will fall back to + * the original packages file + */ +pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI, + string const &URIDesc, string const &ShortDesc, + HashString const &ExpectedHash) + : Item(Owner), ExpectedHash(ExpectedHash) +{ + Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false); + + DestFile = _config->FindDir("Dir::State::lists") + "partial/"; + DestFile += URItoFileName(URI); + + Desc.URI = URI; + Desc.Description = URIDesc; + Desc.Owner = this; + Desc.ShortDesc = ShortDesc; + + QueueURI(Desc); + + if(Debug) + std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl; +} + /*}}}*/ +// AcqSubIndex::Custom600Headers - Insert custom request headers /*{{{*/ +// --------------------------------------------------------------------- +/* The only header we use is the last-modified header. */ +string pkgAcqSubIndex::Custom600Headers() +{ + string Final = _config->FindDir("Dir::State::lists"); + Final += URItoFileName(Desc.URI); + + struct stat Buf; + if (stat(Final.c_str(),&Buf) != 0) + return "\nIndex-File: true"; + return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime); +} + /*}}}*/ +void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/ +{ + if(Debug) + std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl; + + Complete = false; + Status = StatDone; + Dequeue(); + + // No good Index is provided, so try guessing + std::vector langs = APT::Configuration::getLanguages(true); + for (std::vector::const_iterator l = langs.begin(); + l != langs.end(); ++l) + { + if (*l == "none") continue; + string const file = "Translation-" + *l; + new pkgAcqIndexTrans(Owner, Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file), + Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file), + file); + } +} + /*}}}*/ +void pkgAcqSubIndex::Done(string Message,unsigned long Size,string Md5Hash, /*{{{*/ + pkgAcquire::MethodConfig *Cnf) +{ + if(Debug) + std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl; + + string FileName = LookupTag(Message,"Filename"); + if (FileName.empty() == true) + { + Status = StatError; + ErrorText = "Method gave a blank filename"; + return; + } + + if (FileName != DestFile) + { + Local = true; + Desc.URI = "copy:" + FileName; + QueueURI(Desc); + return; + } + + Item::Done(Message,Size,Md5Hash,Cnf); + + string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI); + + // sucess in downloading the index + // rename the index + if(Debug) + std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl; + Rename(DestFile,FinalFile); + chmod(FinalFile.c_str(),0644); + DestFile = FinalFile; + + if(ParseIndex(DestFile) == false) + return Failed("", NULL); + + Complete = true; + Status = StatDone; + Dequeue(); + return; +} + /*}}}*/ +bool pkgAcqSubIndex::ParseIndex(string const &IndexFile) /*{{{*/ +{ + indexRecords SubIndexParser; + if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false) + return false; + + std::vector lang = APT::Configuration::getLanguages(true); + for (std::vector::const_iterator l = lang.begin(); + l != lang.end(); ++l) + { + if (*l == "none") + continue; + + string file = "Translation-" + *l; + indexRecords::checkSum const *Record = SubIndexParser.Lookup(file); + HashString expected; + if (Record == NULL) + { + // FIXME: the Index file provided by debian currently only includes bz2 records + Record = SubIndexParser.Lookup(file + ".bz2"); + if (Record == NULL) + continue; + } + else + { + expected = Record->Hash; + if (expected.empty() == true) + continue; + } + + IndexTarget target; + target.Description = Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file); + target.MetaKey = file; + target.ShortDesc = file; + target.URI = Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file); + new pkgAcqIndexTrans(Owner, &target, expected, &SubIndexParser); + } + return true; +} + /*}}}*/ // AcqDiffIndex::AcqDiffIndex - Constructor /*{{{*/ // --------------------------------------------------------------------- /* Get the DiffIndex file first and see if there are patches availabe @@ -841,6 +988,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, string URI,string URIDesc,string ShortDesc) : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "") { +} +pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, + HashString const &ExpectedHash, indexRecords const *MetaIndexParser) + : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser) +{ } /*}}}*/ // AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/ @@ -1182,27 +1334,41 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ HashString ExpectedIndexHash; if (verify) { - const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); - if (!Record) - { - Status = StatAuthError; - ErrorText = "Unable to find expected entry " - + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)"; - return; - } - ExpectedIndexHash = Record->Hash; - if (_config->FindB("Debug::pkgAcquire::Auth", false)) - { - std::cerr << "Queueing: " << (*Target)->URI << std::endl; - std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl; - } - if (ExpectedIndexHash.empty()) - { - Status = StatAuthError; - ErrorText = "Unable to find hash sum for " - + (*Target)->MetaKey + " in Meta-index file"; - return; - } + const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); + if (Record == NULL) + { + if ((*Target)->IsOptional() == false) + { + Status = StatAuthError; + strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str()); + return; + } + } + else + { + ExpectedIndexHash = Record->Hash; + if (_config->FindB("Debug::pkgAcquire::Auth", false)) + { + std::cerr << "Queueing: " << (*Target)->URI << std::endl; + std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl; + } + if (ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false) + { + Status = StatAuthError; + strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str()); + return; + } + } + } + + if ((*Target)->IsOptional() == true) + { + if ((*Target)->IsSubIndex() == true) + new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description, + (*Target)->ShortDesc, ExpectedIndexHash); + else + new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser); + continue; } /* Queue Packages file (either diff or full packages files, depending @@ -1836,3 +2002,13 @@ string pkgAcqFile::Custom600Headers() return ""; } /*}}}*/ +bool IndexTarget::IsOptional() const { + if (strncmp(ShortDesc.c_str(), "Translation", 11) != 0) + return false; + return true; +} +bool IndexTarget::IsSubIndex() const { + if (ShortDesc != "TranslationIndex") + return false; + return true; +} diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 92098e3d4..9bcc32f21 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -287,6 +287,50 @@ struct DiffInfo { unsigned long size; }; /*}}}*/ +/** \brief An item that is responsible for fetching a SubIndex {{{ + * + * The MetaIndex file includes only records for important indexes + * and records for these SubIndex files so these can carry records + * for addition files like PDiffs and Translations + */ +class pkgAcqSubIndex : public pkgAcquire::Item +{ + protected: + /** \brief If \b true, debugging information will be written to std::clog. */ + bool Debug; + + /** \brief The item that is currently being downloaded. */ + pkgAcquire::ItemDesc Desc; + + /** \brief The Hash that this file should have after download + */ + HashString ExpectedHash; + + public: + // Specialized action members + virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf); + virtual void Done(string Message,unsigned long Size,string Md5Hash, + pkgAcquire::MethodConfig *Cnf); + virtual string DescURI() {return Desc.URI;}; + virtual string Custom600Headers(); + virtual bool ParseIndex(string const &IndexFile); + + /** \brief Create a new pkgAcqDiffIndex. + * + * \param Owner The Acquire object that owns this item. + * + * \param URI The URI of the list file to download. + * + * \param URIDesc A long description of the list file to download. + * + * \param ShortDesc A short description of the list file to download. + * + * \param ExpectedHash The list file's MD5 signature. + */ + pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,string const &URIDesc, + string const &ShortDesc, HashString const &ExpectedHash); +}; + /*}}}*/ /** \brief An item that is responsible for fetching an index file of {{{ * package list diffs and starting the package list's download. * @@ -597,6 +641,8 @@ class pkgAcqIndexTrans : public pkgAcqIndex */ pkgAcqIndexTrans(pkgAcquire *Owner,string URI,string URIDesc, string ShortDesc); + pkgAcqIndexTrans(pkgAcquire *Owner, struct IndexTarget const * const Target, + HashString const &ExpectedHash, indexRecords const *MetaIndexParser); }; /*}}}*/ /** \brief Information about an index file. */ /*{{{*/ @@ -615,8 +661,18 @@ struct IndexTarget * looked up within the meta signature file. */ string MetaKey; + + //FIXME: We should use virtual methods here instead… + bool IsOptional() const; + bool IsSubIndex() const; }; /*}}}*/ +/** \brief Information about an optional index file. */ /*{{{*/ +struct OptionalIndexTarget : public IndexTarget +{ +}; + /*}}}*/ + /** \brief An acquire item that downloads the detached signature {{{ * of a meta-index (Release) file, then queues up the release * file itself. diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc index e97ebfed7..3cf4d2429 100644 --- a/apt-pkg/aptconfiguration.cc +++ b/apt-pkg/aptconfiguration.cc @@ -163,33 +163,6 @@ std::vector const Configuration::getLanguages(bool const &All, } closedir(D); - // get the environment language codes: LC_MESSAGES (and later LANGUAGE) - // we extract both, a long and a short code and then we will - // check if we actually need both (rare) or if the short is enough - string const envMsg = string(Locale == 0 ? std::setlocale(LC_MESSAGES, NULL) : *Locale); - size_t const lenShort = (envMsg.find('_') != string::npos) ? envMsg.find('_') : 2; - size_t const lenLong = (envMsg.find_first_of(".@") != string::npos) ? envMsg.find_first_of(".@") : (lenShort + 3); - - string envLong = envMsg.substr(0,lenLong); - string const envShort = envLong.substr(0,lenShort); - bool envLongIncluded = true; - - // to save the servers from unneeded queries, we only try also long codes - // for languages it is realistic to have a long code translation file… - // TODO: Improve translation acquire system to drop them dynamic - char const *needLong[] = { "cs", "en", "pt", "sv", "zh", NULL }; - if (envLong != envShort) { - for (char const **l = needLong; *l != NULL; l++) - if (envShort.compare(*l) == 0) { - envLongIncluded = false; - break; - } - } - - // we don't add the long code, but we allow the user to do so - if (envLongIncluded == true) - envLong.clear(); - // FIXME: Remove support for the old APT::Acquire::Translation // it was undocumented and so it should be not very widthly used string const oldAcquire = _config->Find("APT::Acquire::Translation",""); @@ -211,12 +184,22 @@ std::vector const Configuration::getLanguages(bool const &All, return codes; } - // It is very likely we will need to environment codes later, + // get the environment language codes: LC_MESSAGES (and later LANGUAGE) + // we extract both, a long and a short code and then we will + // check if we actually need both (rare) or if the short is enough + string const envMsg = string(Locale == 0 ? std::setlocale(LC_MESSAGES, NULL) : *Locale); + size_t const lenShort = (envMsg.find('_') != string::npos) ? envMsg.find('_') : 2; + size_t const lenLong = (envMsg.find_first_of(".@") != string::npos) ? envMsg.find_first_of(".@") : (lenShort + 3); + + string const envLong = envMsg.substr(0,lenLong); + string const envShort = envLong.substr(0,lenShort); + + // It is very likely we will need the environment codes later, // so let us generate them now from LC_MESSAGES and LANGUAGE std::vector environment; if (envShort != "C") { // take care of LC_MESSAGES - if (envLongIncluded == false) + if (envLong != envShort) environment.push_back(envLong); environment.push_back(envShort); // take care of LANGUAGE @@ -233,16 +216,6 @@ std::vector const Configuration::getLanguages(bool const &All, continue; if (std::find(environment.begin(), environment.end(), *e) != environment.end()) continue; - if (e->find('_') != string::npos) { - // Drop LongCodes here - ShortCodes are also included - string const shorty = e->substr(0, e->find('_')); - char const **n = needLong; - for (; *n != NULL; ++n) - if (shorty == *n) - break; - if (*n == NULL) - continue; - } ++addedLangs; environment.push_back(*e); } diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc index e2c680b14..a6edab6b9 100644 --- a/apt-pkg/deb/debmetaindex.cc +++ b/apt-pkg/deb/debmetaindex.cc @@ -119,6 +119,29 @@ string debReleaseIndex::SourceIndexURI(const char *Type, const string &Section) return URI + "dists/" + Dist + "/" + SourceIndexURISuffix(Type, Section); } +string debReleaseIndex::TranslationIndexURISuffix(const char *Type, const string &Section) const +{ + string Res =""; + if (Dist[Dist.size() - 1] != '/') + Res += Section + "/i18n/"; + return Res + Type; +} + +string debReleaseIndex::TranslationIndexURI(const char *Type, const string &Section) const +{ + string Res; + if (Dist[Dist.size() - 1] == '/') + { + if (Dist != "/") + Res = URI + Dist; + else + Res = URI; + return Res + Type; + } + else + return URI + "dists/" + Dist + "/" + TranslationIndexURISuffix(Type, Section); +} + debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) { this->URI = URI; this->Dist = Dist; @@ -155,6 +178,7 @@ vector * debReleaseIndex::ComputeIndexTargets() const { if (IndexTargets->empty() == false && ArchEntries.size() == 1) return IndexTargets; + std::set sections; for (map >::const_iterator a = ArchEntries.begin(); a != ArchEntries.end(); ++a) { if (a->first == "source") @@ -167,6 +191,37 @@ vector * debReleaseIndex::ComputeIndexTargets() const { Target->URI = IndexURI(Target->ShortDesc.c_str(), (*I)->Section, a->first); Target->Description = Info (Target->ShortDesc.c_str(), (*I)->Section, a->first); IndexTargets->push_back (Target); + sections.insert((*I)->Section); + } + } + + // get the Translations: + // - if its a dists-style repository get the i18n/Index first + // - if its flat try to acquire files by guessing + if (Dist[Dist.size() - 1] == '/') { + std::vector const lang = APT::Configuration::getLanguages(true); + for (std::set::const_iterator s = sections.begin(); + s != sections.end(); ++s) { + for (std::vector::const_iterator l = lang.begin(); + l != lang.end(); l++) { + if (*l == "none") continue; + IndexTarget * Target = new OptionalIndexTarget(); + Target->ShortDesc = "Translation-" + *l; + Target->MetaKey = TranslationIndexURISuffix(l->c_str(), *s); + Target->URI = TranslationIndexURI(l->c_str(), *s); + Target->Description = Info (Target->ShortDesc.c_str(), *s); + IndexTargets->push_back(Target); + } + } + } else { + for (std::set::const_iterator s = sections.begin(); + s != sections.end(); ++s) { + IndexTarget * Target = new OptionalIndexTarget(); + Target->ShortDesc = "TranslationIndex"; + Target->MetaKey = TranslationIndexURISuffix("Index", *s); + Target->URI = TranslationIndexURI("Index", *s); + Target->Description = Info (Target->ShortDesc.c_str(), *s); + IndexTargets->push_back (Target); } } @@ -191,28 +246,6 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const ComputeIndexTargets(), new indexRecords (Dist)); - - // Queue the translations - std::vector const lang = APT::Configuration::getLanguages(true); - map > sections; - for (map >::const_iterator a = ArchEntries.begin(); - a != ArchEntries.end(); ++a) { - if (a->first == "source") - continue; - for (vector::const_iterator I = a->second.begin(); - I != a->second.end(); I++) - sections[(*I)->Section].insert(lang.begin(), lang.end()); - } - - for (map >::const_iterator s = sections.begin(); - s != sections.end(); ++s) - for (set::const_iterator l = s->second.begin(); - l != s->second.end(); l++) { - if (*l == "none") continue; - debTranslationsIndex i = debTranslationsIndex(URI,Dist,s->first,(*l).c_str()); - i.GetIndexes(Owner); - } - return true; } diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h index 360fa5419..1561c6e00 100644 --- a/apt-pkg/deb/debmetaindex.h +++ b/apt-pkg/deb/debmetaindex.h @@ -37,6 +37,8 @@ class debReleaseIndex : public metaIndex { string IndexURISuffix(const char *Type, string const &Section, string const &Arch="native") const; string SourceIndexURI(const char *Type, const string &Section) const; string SourceIndexURISuffix(const char *Type, const string &Section) const; + string TranslationIndexURI(const char *Type, const string &Section) const; + string TranslationIndexURISuffix(const char *Type, const string &Section) const; virtual vector *GetIndexFiles(); virtual bool IsTrusted() const; diff --git a/debian/changelog b/debian/changelog index 7b125fc43..b3da32247 100644 --- a/debian/changelog +++ b/debian/changelog @@ -56,6 +56,7 @@ apt (0.8.11+wheezy) unstable; urgency=low - try downloading clearsigned InRelease before trying Release.gpg - change the internal handling of Extensions in pkgAcqIndex - add a special uncompressed compression type to prefer those files + - download and use i18n/Index to choose which Translations to download * cmdline/apt-key: - don't set trustdb-name as non-root so 'list' and 'finger' can be used without being root (Closes: #393005, #592107) @@ -66,8 +67,10 @@ apt (0.8.11+wheezy) unstable; urgency=low - include Index files by default in the Release file * methods/{gzip,bzip}.cc: - print a good error message if FileSize() is zero + * apt-pkg/aptconfiguration.cc: + - remove the inbuilt Translation files whitelist - -- David Kalnischkies Wed, 26 Jan 2011 16:06:10 +0100 + -- David Kalnischkies Fri, 28 Jan 2011 12:22:25 +0100 apt (0.8.10.3) unstable; urgency=low diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files index 398d0cd1b..684559aa7 100755 --- a/test/integration/test-bug-595691-empty-and-broken-archive-files +++ b/test/integration/test-bug-595691-empty-and-broken-archive-files @@ -32,7 +32,7 @@ createemptyarchive() { fi touch aptarchive/Packages echo -n "" | $COMPRESSOR > aptarchive/${1}.$COMPRESS - aptftparchive release aptarchive/ > aptarchive/Release + generatereleasefiles signreleasefiles rm -f aptarchive/Packages } @@ -43,7 +43,7 @@ createemptyfile() { echo -n "" | $COMPRESSOR > aptarchive/Packages.$COMPRESS fi touch aptarchive/Packages aptarchive/${1}.$COMPRESS - aptftparchive release aptarchive/ > aptarchive/Release + generatereleasefiles signreleasefiles rm -f aptarchive/Packages } @@ -76,7 +76,7 @@ testoverfile() { createemptyfile 'en' testaptgetupdate "Get:1 file: InRelease [] -Ign file:$(readlink -f aptarchive)/ Translation-en +Ign file: Translation-en Reading package lists..." "empty file en.$COMPRESS over file" createemptyarchive 'en' @@ -86,13 +86,13 @@ Reading package lists..." "empty archive en.$COMPRESS over file" createemptyarchive 'Packages' # FIXME: Why omits the file transport the Packages Get line? #Get:3 file: Packages [] - testaptgetupdate "Ign file:$(readlink -f aptarchive)/ Translation-en -Get:1 file: InRelease [] + testaptgetupdate "Get:1 file: InRelease [] +Ign file: Translation-en Reading package lists..." "empty archive Packages.$COMPRESS over file" createemptyfile 'Packages' - testaptgetupdate "Ign file:$(readlink -f aptarchive)/ Translation-en -Get:1 file: InRelease [] + testaptgetupdate "Get:1 file: InRelease [] +Ign file: Translation-en Err file: Packages Empty files can't be valid archives W: Failed to fetch ${COMPRESSOR}:$(readlink -f aptarchive/Packages.$COMPRESS) Empty files can't be valid archives @@ -105,28 +105,28 @@ testoverhttp() { createemptyfile 'en' testaptgetupdate "Get:1 http://localhost InRelease [] -Get:2 http://localhost/ Translation-en -Get:3 http://localhost Packages [] -Ign http://localhost/ Translation-en +Get:2 http://localhost Packages [] +Get:3 http://localhost Translation-en +Ign http://localhost Translation-en Reading package lists..." "empty file en.$COMPRESS over http" createemptyarchive 'en' testaptgetupdate "Get:1 http://localhost InRelease [] -Get:2 http://localhost/ Translation-en [] -Get:3 http://localhost Packages [] +Get:2 http://localhost Packages [] +Get:3 http://localhost Translation-en [] Reading package lists..." "empty archive en.$COMPRESS over http" createemptyarchive 'Packages' testaptgetupdate "Get:1 http://localhost InRelease [] -Ign http://localhost/ Translation-en Get:2 http://localhost Packages [] +Ign http://localhost Translation-en Reading package lists..." "empty archive Packages.$COMPRESS over http" createemptyfile 'Packages' #FIXME: we should response with a good error message instead testaptgetupdate "Get:1 http://localhost InRelease [] -Ign http://localhost/ Translation-en Get:2 http://localhost Packages +Ign http://localhost Translation-en Err http://localhost Packages Empty files can't be valid archives W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages) Empty files can't be valid archives diff --git a/test/integration/test-bug-601016-description-translation b/test/integration/test-bug-601016-description-translation index a31e42ee1..2a323a201 100755 --- a/test/integration/test-bug-601016-description-translation +++ b/test/integration/test-bug-601016-description-translation @@ -57,6 +57,7 @@ Description-${LOCALE}: Mächtige Oberfläche für dpkg testrun() { echo "Acquire::Languages { \"${LOCALE}\"; \"en\"; };" > rootdir/etc/apt/apt.conf.d/00languages export LC_ALL="" + rm -rf rootdir/var/lib/apt/lists rootdir/var/cache/apt/ setupaptarchive testequal "$LOCALESTANZA" aptcache show apt -o Test=File-${LOCALE} testequal "$NOLONGSTANZA" aptcache show apt -o Acquire::Languages="ww" -o Test=File-${LOCALE} diff --git a/test/libapt/getlanguages_test.cc b/test/libapt/getlanguages_test.cc index 3559aae0c..707142aef 100644 --- a/test/libapt/getlanguages_test.cc +++ b/test/libapt/getlanguages_test.cc @@ -26,16 +26,18 @@ int main(int argc,char *argv[]) env[1] = ""; std::vector vec = APT::Configuration::getLanguages(false, false, env); - equals(vec.size(), 2); - equals(vec[0], "de"); - equals(vec[1], "en"); + equals(vec.size(), 3); + equals(vec[0], "de_DE"); + equals(vec[1], "de"); + equals(vec[2], "en"); // Special: Check if the cache is actually in use env[0] = "en_GB.UTF-8"; vec = APT::Configuration::getLanguages(false, true, env); - equals(vec.size(), 2); - equals(vec[0], "de"); - equals(vec[1], "en"); + equals(vec.size(), 3); + equals(vec[0], "de_DE"); + equals(vec[1], "de"); + equals(vec[2], "en"); env[0] = "en_GB.UTF-8"; vec = APT::Configuration::getLanguages(false, false, env); @@ -52,19 +54,21 @@ int main(int argc,char *argv[]) env[0] = "tr_DE@euro"; vec = APT::Configuration::getLanguages(false, false, env); - equals(vec.size(), 2); - equals(vec[0], "tr"); - equals(vec[1], "en"); + equals(vec.size(), 3); + equals(vec[0], "tr_DE"); + equals(vec[1], "tr"); + equals(vec[2], "en"); env[0] = "de_NO"; - env[1] = "se_NO:en_GB:nb_NO:nb:no_NO:no:nn_NO:nn:da:sv:en"; + env[1] = "de_NO:en_GB:nb_NO:nb:no_NO:no:nn_NO:nn:da:sv:en"; vec = APT::Configuration::getLanguages(false, false, env); - equals(vec.size(), 5); - equals(vec[0], "de"); - equals(vec[1], "en_GB"); - equals(vec[2], "nb"); - equals(vec[3], "no"); - equals(vec[4], "en"); + equals(vec.size(), 6); + equals(vec[0], "de_NO"); + equals(vec[1], "de"); + equals(vec[2], "en_GB"); + equals(vec[3], "nb_NO"); + equals(vec[4], "nb"); + equals(vec[5], "en"); env[0] = "pt_PR.UTF-8"; env[1] = ""; @@ -76,9 +80,10 @@ int main(int argc,char *argv[]) env[0] = "ast_DE.UTF-8"; vec = APT::Configuration::getLanguages(false, false, env); // bogus, but syntactical correct - equals(vec.size(), 2); - equals(vec[0], "ast"); - equals(vec[1], "en"); + equals(vec.size(), 3); + equals(vec[0], "ast_DE"); + equals(vec[1], "ast"); + equals(vec[2], "en"); env[0] = "C"; vec = APT::Configuration::getLanguages(false, false, env); @@ -113,25 +118,28 @@ int main(int argc,char *argv[]) _config->Set("Acquire::Languages::2", "en"); env[0] = "de_DE.UTF-8"; vec = APT::Configuration::getLanguages(false, false, env); - equals(vec.size(), 2); - equals(vec[0], "de"); - equals(vec[1], "en"); + equals(vec.size(), 3); + equals(vec[0], "de_DE"); + equals(vec[1], "de"); + equals(vec[2], "en"); _config->Set("Acquire::Languages::3", "de"); env[0] = "de_DE.UTF-8"; vec = APT::Configuration::getLanguages(false, false, env); - equals(vec.size(), 2); - equals(vec[0], "de"); - equals(vec[1], "en"); + equals(vec.size(), 3); + equals(vec[0], "de_DE"); + equals(vec[1], "de"); + equals(vec[2], "en"); _config->Set("Dir::State::lists", argv[1]); vec = APT::Configuration::getLanguages(true, false, env); - equals(vec.size(), 5); - equals(vec[0], "de"); - equals(vec[1], "en"); - equals(vec[2], "none"); - equals(vec[3], "pt"); - equals(vec[4], "tr"); + equals(vec.size(), 6); + equals(vec[0], "de_DE"); + equals(vec[1], "de"); + equals(vec[2], "en"); + equals(vec[3], "none"); + equals(vec[4], "pt"); + equals(vec[5], "tr"); _config->Set("Dir::State::lists", "/non-existing-dir"); _config->Set("Acquire::Languages::1", "none"); @@ -140,6 +148,7 @@ int main(int argc,char *argv[]) equals(vec.size(), 0); env[0] = "de_DE.UTF-8"; vec = APT::Configuration::getLanguages(true, false, env); + equals(vec.size(), 2); equals(vec[0], "en"); equals(vec[1], "de"); -- cgit v1.2.3-70-g09d2