From caffd4807f612e931ecca8f7d9b0c0c10de27ce6 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Sat, 15 Jan 2011 20:28:03 +0100
Subject: * methods/rred.cc:   - operate optional on gzip compressed pdiffs *
 apt-pkg/acquire-item.cc:   - don't uncompress downloaded pdiff files before
 feeding it to rred

---
 apt-pkg/acquire-item.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'apt-pkg/acquire-item.h')

diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 943c61876..d97a96a0f 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -449,7 +449,7 @@ class pkgAcqIndexDiffs : public pkgAcquire::Item
 	 StateFetchDiff,
 	 
 	 /** \brief The diff is currently being uncompressed. */
-	 StateUnzipDiff,
+	 StateUnzipDiff, // FIXME: No longer used
 
 	 /** \brief The diff is currently being applied. */
 	 StateApplyDiff
-- 
cgit v1.2.3-70-g09d2


From fe0f7911b650918e1d511b3453664a07f6d966d0 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Thu, 20 Jan 2011 14:53:52 +0100
Subject:   - try downloading clearsigned InRelease before trying Release.gpg *
 apt-pkg/deb/deblistparser.cc:   - rewrite LoadReleaseInfo to cope with
 clearsigned Releasefiles

---
 apt-pkg/acquire-item.cc                            |  54 ++++++-
 apt-pkg/acquire-item.h                             |  33 +++++
 apt-pkg/deb/debindexfile.cc                        |   8 +-
 apt-pkg/deb/deblistparser.cc                       | 114 ++++++++++-----
 apt-pkg/deb/debmetaindex.cc                        |  34 +++--
 apt-pkg/indexcopy.cc                               |   3 +-
 apt-pkg/indexrecords.cc                            |  17 ++-
 apt-pkg/tagfile.cc                                 |  10 +-
 apt-pkg/tagfile.h                                  |   2 +
 debian/changelog                                   |   5 +-
 doc/apt-secure.8.xml                               |   5 +-
 test/integration/Packages-releasefile-verification |  18 +++
 .../Packages-releasefile-verification-new          |  21 +++
 test/integration/framework                         |   7 +
 test/integration/marvinparanoid.pub                | Bin 0 -> 629 bytes
 test/integration/marvinparanoid.sec                | Bin 0 -> 1280 bytes
 .../test-bug-595691-empty-and-broken-archive-files |  32 ++---
 test/integration/test-releasefile-verification     | 160 +++++++++++++++++++++
 18 files changed, 430 insertions(+), 93 deletions(-)
 create mode 100644 test/integration/Packages-releasefile-verification
 create mode 100644 test/integration/Packages-releasefile-verification-new
 create mode 100644 test/integration/marvinparanoid.pub
 create mode 100644 test/integration/marvinparanoid.sec
 create mode 100755 test/integration/test-releasefile-verification

(limited to 'apt-pkg/acquire-item.h')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index d4e90b552..2cd6ab359 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1077,6 +1077,8 @@ void pkgAcqMetaIndex::Done(string Message,unsigned long Size,string Hash,	/*{{{*
    {
       string FinalFile = _config->FindDir("Dir::State::lists");
       FinalFile += URItoFileName(RealURI);
+      if (SigFile == DestFile)
+	 SigFile = FinalFile;
       Rename(DestFile,FinalFile);
       chmod(FinalFile.c_str(),0644);
       DestFile = FinalFile;
@@ -1110,6 +1112,8 @@ void pkgAcqMetaIndex::RetrievalDone(string Message)			/*{{{*/
    {
       string FinalFile = _config->FindDir("Dir::State::lists");
       FinalFile += URItoFileName(RealURI);
+      if (SigFile == DestFile)
+	 SigFile = FinalFile;
       DestFile = FinalFile;
    }
    Complete = true;
@@ -1141,6 +1145,10 @@ void pkgAcqMetaIndex::AuthDone(string Message)				/*{{{*/
    // Download further indexes with verification
    QueueIndexes(true);
 
+   // is it a clearsigned MetaIndex file?
+   if (DestFile == SigFile)
+      return;
+
    // Done, move signature file into position
    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
       URItoFileName(RealURI) + ".gpg";
@@ -1300,13 +1308,20 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
    if (AuthPass == true)
    {
       // gpgv method failed, if we have a good signature 
-      string LastGoodSigFile = _config->FindDir("Dir::State::lists") +
-	 "partial/" + URItoFileName(RealURI) + ".gpg.reverify";
+      string LastGoodSigFile = _config->FindDir("Dir::State::lists");
+      if (DestFile == SigFile)
+	 LastGoodSigFile.append(URItoFileName(RealURI));
+      else
+	 LastGoodSigFile.append("partial/").append(URItoFileName(RealURI)).append(".gpg.reverify");
+
       if(FileExists(LastGoodSigFile))
       {
-	 string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
-	    URItoFileName(RealURI) + ".gpg";
-	 Rename(LastGoodSigFile,VerifiedSigFile);
+	 if (DestFile != SigFile)
+	 {
+	    string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
+					URItoFileName(RealURI) + ".gpg";
+	    Rename(LastGoodSigFile,VerifiedSigFile);
+	 }
 	 Status = StatTransientNetworkError;
 	 _error->Warning(_("A error occurred during the signature "
 			   "verification. The repository is not updated "
@@ -1330,6 +1345,35 @@ void pkgAcqMetaIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
    QueueIndexes(false);
 }
 									/*}}}*/
+pkgAcqMetaClearSig::pkgAcqMetaClearSig(pkgAcquire *Owner,		/*{{{*/
+		string const &URI, string const &URIDesc, string const &ShortDesc,
+		string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
+		string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
+		const vector<struct IndexTarget*>* IndexTargets,
+		indexRecords* MetaIndexParser) :
+	pkgAcqMetaIndex(Owner, URI, URIDesc, ShortDesc, "", IndexTargets, MetaIndexParser),
+	MetaIndexURI(MetaIndexURI), MetaIndexURIDesc(MetaIndexURIDesc), MetaIndexShortDesc(MetaIndexShortDesc),
+	MetaSigURI(MetaSigURI), MetaSigURIDesc(MetaSigURIDesc), MetaSigShortDesc(MetaSigShortDesc)
+{
+   SigFile = DestFile;
+}
+									/*}}}*/
+void pkgAcqMetaClearSig::Failed(string Message,pkgAcquire::MethodConfig *Cnf) /*{{{*/
+{
+   if (AuthPass == false)
+   {
+      new pkgAcqMetaSig(Owner,
+			MetaSigURI, MetaSigURIDesc, MetaSigShortDesc,
+			MetaIndexURI, MetaIndexURIDesc, MetaIndexShortDesc,
+			IndexTargets, MetaIndexParser);
+      if (Cnf->LocalOnly == true ||
+	  StringToBool(LookupTag(Message, "Transient-Failure"), false) == false)
+	 Dequeue();
+   }
+   else
+      pkgAcqMetaIndex::Failed(Message, Cnf);
+}
+									/*}}}*/
 // AcqArchive::AcqArchive - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
 /* This just sets up the initial fetch environment and queues the first
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index d97a96a0f..581761e32 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -772,6 +772,39 @@ class pkgAcqMetaIndex : public pkgAcquire::Item
 		   indexRecords* MetaIndexParser);
 };
 									/*}}}*/
+/** \brief An item repsonsible for downloading clearsigned metaindexes	{{{*/
+class pkgAcqMetaClearSig : public pkgAcqMetaIndex
+{
+   /** \brief The URI of the meta-index file for the detached signature */
+   string MetaIndexURI;
+
+   /** \brief A "URI-style" description of the meta-index file */
+   string MetaIndexURIDesc;
+
+   /** \brief A brief description of the meta-index file */
+   string MetaIndexShortDesc;
+
+   /** \brief The URI of the detached meta-signature file if the clearsigned one failed. */
+   string MetaSigURI;
+
+   /** \brief A "URI-style" description of the meta-signature file */
+   string MetaSigURIDesc;
+
+   /** \brief A brief description of the meta-signature file */
+   string MetaSigShortDesc;
+
+public:
+   void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
+
+   /** \brief Create a new pkgAcqMetaClearSig. */
+   pkgAcqMetaClearSig(pkgAcquire *Owner,
+		string const &URI, string const &URIDesc, string const &ShortDesc,
+		string const &MetaIndexURI, string const &MetaIndexURIDesc, string const &MetaIndexShortDesc,
+		string const &MetaSigURI, string const &MetaSigURIDesc, string const &MetaSigShortDesc,
+		const vector<struct IndexTarget*>* IndexTargets,
+		indexRecords* MetaIndexParser);
+};
+									/*}}}*/
 /** \brief An item that is responsible for fetching a package file.	{{{
  *
  *  If the package file already exists in the cache, nothing will be
diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index af1209ccb..9961b5ae4 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -324,8 +324,14 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
       return _error->Error("Problem with MergeList %s",PackageFile.c_str());
 
    // Check the release file
-   string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release");
+   string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease");
+   bool releaseExists = false;
    if (FileExists(ReleaseFile) == true)
+      releaseExists = true;
+   else
+      ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release");
+
+   if (releaseExists == true || FileExists(ReleaseFile) == true)
    {
       FileFd Rel(ReleaseFile,FileFd::ReadOnly);
       if (_error->PendingError() == true)
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index 1b3bfd6ae..9201e6a54 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -783,45 +783,89 @@ bool debListParser::Step()
 bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,
 				    FileFd &File, string component)
 {
-   pkgTagFile Tags(&File, File.Size() + 256); // XXX
-   pkgTagSection Section;
-   if (Tags.Step(Section) == false)
-      return false;
-
-   // FIXME: Do we need it now for multi-arch?
-   // mvo: I don't think we need to fill that in (it's unused since apt-0.6)
-//    FileI->Architecture = WriteUniqString(Arch);
-   
    // apt-secure does no longer download individual (per-section) Release
    // file. to provide Component pinning we use the section name now
    FileI->Component = WriteUniqString(component);
 
-   const char *Start;
-   const char *Stop;
-   if (Section.Find("Suite",Start,Stop) == true)
-      FileI->Archive = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Component",Start,Stop) == true)
-      FileI->Component = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Version",Start,Stop) == true)
-      FileI->Version = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Origin",Start,Stop) == true)
-      FileI->Origin = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Codename",Start,Stop) == true)
-      FileI->Codename = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Label",Start,Stop) == true)
-      FileI->Label = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Architecture",Start,Stop) == true)
-      FileI->Architecture = WriteUniqString(Start,Stop - Start);
-   
-   if (Section.FindFlag("NotAutomatic",FileI->Flags,
-			pkgCache::Flag::NotAutomatic) == false)
-      _error->Warning("Bad NotAutomatic flag");
-   if (Section.FindFlag("ButAutomaticUpgrades",FileI->Flags,
-			pkgCache::Flag::ButAutomaticUpgrades) == false)
-      _error->Warning("Bad ButAutomaticUpgrades flag");
-   // overrule the NotAutomatic setting if needed as they are both present for compatibility
-   else if ((FileI->Flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades)
-      FileI->Flags &= ~pkgCache::Flag::NotAutomatic;
+   FILE* release = fdopen(dup(File.Fd()), "r");
+   if (release == NULL)
+      return false;
+
+   char buffer[101];
+   bool gpgClose = false;
+   while (fgets(buffer, sizeof(buffer), release) != NULL)
+   {
+      size_t len = 0;
+
+      // Skip empty lines
+      for (; buffer[len] == '\r' && buffer[len] == '\n'; ++len);
+      if (buffer[len] == '\0')
+	 continue;
+
+      // only evalute the first GPG section
+      if (strncmp("-----", buffer, 5) == 0)
+      {
+	 if (gpgClose == true)
+	    break;
+	 gpgClose = true;
+	 continue;
+      }
+
+      // seperate the tag from the data
+      for (; buffer[len] != ':' && buffer[len] != '\0'; ++len);
+      if (buffer[len] == '\0')
+	 continue;
+      char* dataStart = buffer + len;
+      for (++dataStart; *dataStart == ' '; ++dataStart);
+      char* dataEnd = dataStart;
+      for (++dataEnd; *dataEnd != '\0'; ++dataEnd);
+
+      // which datastorage need to be updated
+      map_ptrloc* writeTo = NULL;
+      if (buffer[0] == ' ')
+	 ;
+      #define APT_PARSER_WRITETO(X, Y) else if (strncmp(Y, buffer, len) == 0) writeTo = &X;
+      APT_PARSER_WRITETO(FileI->Archive, "Suite")
+      APT_PARSER_WRITETO(FileI->Component, "Component")
+      APT_PARSER_WRITETO(FileI->Version, "Version")
+      APT_PARSER_WRITETO(FileI->Origin, "Origin")
+      APT_PARSER_WRITETO(FileI->Codename, "Codename")
+      APT_PARSER_WRITETO(FileI->Label, "Label")
+      #undef APT_PARSER_WRITETO
+      #define APT_PARSER_FLAGIT(X) else if (strncmp(#X, buffer, len) == 0) \
+	 pkgTagSection::FindFlag(FileI->Flags, pkgCache::Flag:: X, dataStart, dataEnd-1);
+      APT_PARSER_FLAGIT(NotAutomatic)
+      APT_PARSER_FLAGIT(ButAutomaticUpgrades)
+      #undef APT_PARSER_FLAGIT
+
+      // load all data from the line and save it
+      string data;
+      if (writeTo != NULL)
+	 data.append(dataStart, dataEnd);
+      if (sizeof(buffer) - 1 == (dataEnd - buffer))
+      {
+	 while (fgets(buffer, sizeof(buffer), release) != NULL)
+	 {
+	    if (writeTo != NULL)
+	       data.append(buffer);
+	    if (strlen(buffer) != sizeof(buffer) - 1)
+	       break;
+	 }
+      }
+      if (writeTo != NULL)
+      {
+	 // remove spaces and stuff from the end of the data line
+	 for (std::string::reverse_iterator s = data.rbegin();
+	      s != data.rend(); ++s)
+	 {
+	    if (*s != '\r' && *s != '\n' && *s != ' ')
+	       break;
+	    *s = '\0';
+	 }
+	 *writeTo = WriteUniqString(data);
+      }
+   }
+   fclose(release);
 
    return !_error->PendingError();
 }
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 717d0bcde..e2c680b14 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -182,21 +182,15 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 	 new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
 			 (*Target)->ShortDesc, HashString());
       }
-      // this is normally created in pkgAcqMetaSig, but if we run
-      // in --print-uris mode, we add it here
-      new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"),
-		     MetaIndexInfo("Release"), "Release",
-		     MetaIndexURI("Release.gpg"), 
-		     ComputeIndexTargets(),
-		     new indexRecords (Dist));
-
    }
 
-   new pkgAcqMetaSig(Owner, MetaIndexURI("Release.gpg"),
-		     MetaIndexInfo("Release.gpg"), "Release.gpg",
-		     MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
-		     ComputeIndexTargets(),
-		     new indexRecords (Dist));
+	new pkgAcqMetaClearSig(Owner, MetaIndexURI("InRelease"),
+		MetaIndexInfo("InRelease"), "InRelease",
+		MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
+		MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg",
+		ComputeIndexTargets(),
+		new indexRecords (Dist));
+
 
 	// Queue the translations
 	std::vector<std::string> const lang = APT::Configuration::getLanguages(true);
@@ -224,16 +218,20 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 
 bool debReleaseIndex::IsTrusted() const
 {
-   string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
-      URItoFileName(MetaIndexURI("Release")) + ".gpg";
-   
    if(_config->FindB("APT::Authentication::TrustCDROM", false))
       if(URI.substr(0,strlen("cdrom:")) == "cdrom:")
 	 return true;
-   
+
+   string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
+      URItoFileName(MetaIndexURI("Release")) + ".gpg";
+
    if (FileExists(VerifiedSigFile))
       return true;
-   return false;
+
+   VerifiedSigFile = _config->FindDir("Dir::State::lists") +
+      URItoFileName(MetaIndexURI("InRelease"));
+
+   return FileExists(VerifiedSigFile);
 }
 
 vector <pkgIndexFile *> *debReleaseIndex::GetIndexFiles() {
diff --git a/apt-pkg/indexcopy.cc b/apt-pkg/indexcopy.cc
index f88d51fc5..c2ee1c347 100644
--- a/apt-pkg/indexcopy.cc
+++ b/apt-pkg/indexcopy.cc
@@ -722,7 +722,8 @@ bool SigVerify::RunGPGV(std::string const &File, std::string const &FileGPG,
    }
 
    Args.push_back(FileGPG.c_str());
-   Args.push_back(File.c_str());
+   if (FileGPG != File)
+      Args.push_back(File.c_str());
    Args.push_back(NULL);
 
    if (Debug == true)
diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc
index eb9a36866..10e154ad2 100644
--- a/apt-pkg/indexrecords.cc
+++ b/apt-pkg/indexrecords.cc
@@ -55,14 +55,17 @@ bool indexRecords::Load(const string Filename)				/*{{{*/
    }
 
    pkgTagSection Section;
-   if (TagFile.Step(Section) == false)
-   {
-      strprintf(ErrorText, _("No sections in Release file %s"), Filename.c_str());
-      return false;
-   }
-
    const char *Start, *End;
-   Section.Get (Start, End, 0);
+   // Skip over sections beginning with ----- as this is an idicator for clearsigns
+   do {
+      if (TagFile.Step(Section) == false)
+      {
+	 strprintf(ErrorText, _("No sections in Release file %s"), Filename.c_str());
+	 return false;
+      }
+
+      Section.Get (Start, End, 0);
+   } while (End - Start > 5 && strncmp(Start, "-----", 5) == 0);
 
    Suite = Section.FindS("Suite");
    Dist = Section.FindS("Codename");
diff --git a/apt-pkg/tagfile.cc b/apt-pkg/tagfile.cc
index 96a681bec..4a2f3f7e6 100644
--- a/apt-pkg/tagfile.cc
+++ b/apt-pkg/tagfile.cc
@@ -399,9 +399,13 @@ bool pkgTagSection::FindFlag(const char *Tag,unsigned long &Flags,
    const char *Stop;
    if (Find(Tag,Start,Stop) == false)
       return true;
-   
-   switch (StringToBool(string(Start,Stop)))
-   {     
+   return FindFlag(Flags, Flag, Start, Stop);
+}
+bool const pkgTagSection::FindFlag(unsigned long &Flags, unsigned long Flag,
+					char const* Start, char const* Stop)
+{
+   switch (StringToBool(string(Start, Stop)))
+   {
       case 0:
       Flags &= ~Flag;
       return true;
diff --git a/apt-pkg/tagfile.h b/apt-pkg/tagfile.h
index 6891c1d81..61491aa04 100644
--- a/apt-pkg/tagfile.h
+++ b/apt-pkg/tagfile.h
@@ -60,6 +60,8 @@ class pkgTagSection
    unsigned long long FindULL(const char *Tag, unsigned long long const &Default = 0) const;
    bool FindFlag(const char *Tag,unsigned long &Flags,
 		 unsigned long Flag) const;
+   bool static const FindFlag(unsigned long &Flags, unsigned long Flag,
+				const char* Start, const char* Stop);
    bool Scan(const char *Start,unsigned long MaxLength);
    inline unsigned long size() const {return Stop - Section;};
    void Trim();
diff --git a/debian/changelog b/debian/changelog
index b74508804..59f516751 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -53,11 +53,14 @@ apt (0.8.11+wheezy) unstable; urgency=low
     - operate optional on gzip compressed pdiffs
   * apt-pkg/acquire-item.cc:
     - don't uncompress downloaded pdiff files before feeding it to rred
+    - try downloading clearsigned InRelease before trying Release.gpg
   * cmdline/apt-key:
     - don't set trustdb-name as non-root so 'list' and 'finger'
       can be used without being root (Closes: #393005, #592107)
+  * apt-pkg/deb/deblistparser.cc:
+    - rewrite LoadReleaseInfo to cope with clearsigned Releasefiles
 
- -- David Kalnischkies <kalnischkies@gmail.com>  Sun, 16 Jan 2011 17:23:28 +0100
+ -- David Kalnischkies <kalnischkies@gmail.com>  Thu, 20 Jan 2011 14:52:32 +0100
 
 apt (0.8.10) unstable; urgency=low
 
diff --git a/doc/apt-secure.8.xml b/doc/apt-secure.8.xml
index f345c3f89..f8ff678b9 100644
--- a/doc/apt-secure.8.xml
+++ b/doc/apt-secure.8.xml
@@ -148,8 +148,8 @@
    (you should make sure you are using a trusted communication channel
    when retrieving it), add it with <command>apt-key</command> and
    then run <command>apt-get update</command> so that apt can download
-   and verify the <filename>Release.gpg</filename> files from the archives you
-   have configured.
+   and verify the <filename>InRelease</filename> or <filename>Release.gpg</filename>
+   files from the archives you have configured.
    </para>
 </refsect1>
 
@@ -166,6 +166,7 @@
        (provided in apt-utils).</para></listitem>
    
       <listitem><para><emphasis>Sign it</emphasis>. You can do this by running
+      <command>gpg --clearsign -o InRelease Release</command> and
       <command>gpg -abs -o Release.gpg Release</command>.</para></listitem>
 
       <listitem><para><emphasis>Publish the key fingerprint</emphasis>,
diff --git a/test/integration/Packages-releasefile-verification b/test/integration/Packages-releasefile-verification
new file mode 100644
index 000000000..29a385f4f
--- /dev/null
+++ b/test/integration/Packages-releasefile-verification
@@ -0,0 +1,18 @@
+Package: apt
+Version: 0.7.25.3
+Architecture: i386
+Maintainer: APT Development Team <deity@lists.debian.org>
+Installed-Size: 5244
+Replaces: libapt-pkg-dev (<< 0.3.7), libapt-pkg-doc (<< 0.3.7)
+Provides: libapt-pkg-libc6.9-6-4.8
+Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt
+Filename: apt.deb
+Size: 0
+MD5sum: d41d8cd98f00b204e9800998ecf8427e
+Description: Advanced front-end for dpkg
+ This is Debian's next generation front-end for the dpkg package manager.
+ It provides the apt-get utility and APT dselect method that provides a
+ simpler, safer way to install and upgrade packages.
+ .
+ APT features complete installation ordering, multiple source capability
+ and several other unique features, see the Users Guide in apt-doc.
diff --git a/test/integration/Packages-releasefile-verification-new b/test/integration/Packages-releasefile-verification-new
new file mode 100644
index 000000000..e3b2edf1f
--- /dev/null
+++ b/test/integration/Packages-releasefile-verification-new
@@ -0,0 +1,21 @@
+Package: apt
+Priority: important
+Section: admin
+Installed-Size: 5672
+Maintainer: APT Development Team <deity@lists.debian.org>
+Architecture: i386
+Version: 0.8.0~pre1
+Replaces: manpages-pl (<< 20060617-3~)
+Provides: libapt-pkg4.10
+Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt
+Conflicts: python-apt (<< 0.7.93.2~)
+Filename: apt.deb
+Size: 0
+MD5sum: d41d8cd98f00b204e9800998ecf8427e
+Description: Advanced front-end for dpkg
+ This is Debian's next generation front-end for the dpkg package manager.
+ It provides the apt-get utility and APT dselect method that provides a
+ simpler, safer way to install and upgrade packages.
+ .
+ APT features complete installation ordering, multiple source capability
+ and several other unique features, see the Users Guide in apt-doc.
diff --git a/test/integration/framework b/test/integration/framework
index 5d54e49b6..cb3fca35d 100644
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -383,6 +383,7 @@ buildaptarchivefromfiles() {
 
 generatereleasefiles() {
 	msgninfo "\tGenerate Release files… "
+	local DATE="${1:-now}"
 	if [ -e aptarchive/dists ]; then
 		for dir in $(find ./aptarchive/dists -mindepth 1 -maxdepth 1 -type d); do
 			local CODENAME="$(echo "$dir" | cut -d'/' -f 4)"
@@ -395,6 +396,11 @@ NotAutomatic: yes' $dir/Release
 	else
 		aptftparchive -qq release ./aptarchive | sed -e '/0 Release$/ d' > aptarchive/Release # remove the self reference
 	fi
+	if [ "$DATE" != "now" ]; then
+		for release in $(find ./aptarchive -name 'Release'); do
+			touch -d "$1" $release
+		done
+	fi
 	msgdone "info"
 }
 
@@ -455,6 +461,7 @@ signreleasefiles() {
 	done
 	for RELEASE in $(find aptarchive/ -name Release); do
 		gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" -abs -o ${RELEASE}.gpg ${RELEASE}
+		gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" --clearsign -o "$(echo "${RELEASE}" | sed 's#/Release$#/InRelease#')" $RELEASE
 	done
 	msgdone "info"
 }
diff --git a/test/integration/marvinparanoid.pub b/test/integration/marvinparanoid.pub
new file mode 100644
index 000000000..9a59c2e40
Binary files /dev/null and b/test/integration/marvinparanoid.pub differ
diff --git a/test/integration/marvinparanoid.sec b/test/integration/marvinparanoid.sec
new file mode 100644
index 000000000..ff54e8680
Binary files /dev/null and b/test/integration/marvinparanoid.sec differ
diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index 66792899a..c5379dca0 100755
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -74,28 +74,24 @@ testoverfile() {
 	setupcompressor "$1"
 
 	createemptyfile 'en'
-	testaptgetupdate "Get:1 file:  Release.gpg []
-Get:2 file:  Release []
+	testaptgetupdate "Get:1 file:  InRelease []
 Ign file:$(readlink -f aptarchive)/  Translation-en
 Reading package lists..." "empty file en.$COMPRESS over file"
 
 	createemptyarchive 'en'
-	testaptgetupdate "Get:1 file:  Release.gpg []
-Get:2 file:  Release []
+	testaptgetupdate "Get:1 file:  InRelease []
 Reading package lists..." "empty archive en.$COMPRESS over file"
 
 	createemptyarchive 'Packages'
 	# FIXME: Why omits the file transport the Packages Get line?
 	#Get:3 file:  Packages []
 	testaptgetupdate "Ign file:$(readlink -f aptarchive)/  Translation-en
-Get:1 file:  Release.gpg []
-Get:2 file:  Release []
+Get:1 file:  InRelease []
 Reading package lists..." "empty archive Packages.$COMPRESS over file"
 
 	createemptyfile 'Packages'
 	testaptgetupdate "Ign file:$(readlink -f aptarchive)/  Translation-en
-Get:1 file:  Release.gpg []
-Get:2 file:  Release []
+Get:1 file:  InRelease []
 Err file:  Packages
   Undetermined Error
 W: Failed to fetch file:$(readlink -f aptarchive/Packages.$COMPRESS)  Undetermined Error
@@ -107,33 +103,29 @@ testoverhttp() {
 	setupcompressor "$1"
 
 	createemptyfile 'en'
-	testaptgetupdate "Get:1 http://localhost  Release.gpg []
+	testaptgetupdate "Get:1 http://localhost  InRelease []
 Get:2 http://localhost/  Translation-en
-Get:3 http://localhost  Release []
+Get:3 http://localhost  Packages []
 Ign http://localhost/  Translation-en
-Get:4 http://localhost  Packages []
 Reading package lists..." "empty file en.$COMPRESS over http"
 
 	createemptyarchive 'en'
-	testaptgetupdate "Get:1 http://localhost  Release.gpg []
+	testaptgetupdate "Get:1 http://localhost  InRelease []
 Get:2 http://localhost/  Translation-en []
-Get:3 http://localhost  Release []
-Get:4 http://localhost  Packages []
+Get:3 http://localhost  Packages []
 Reading package lists..." "empty archive en.$COMPRESS over http"
 
 	createemptyarchive 'Packages'
-	testaptgetupdate "Get:1 http://localhost  Release.gpg []
+	testaptgetupdate "Get:1 http://localhost  InRelease []
 Ign http://localhost/  Translation-en
-Get:2 http://localhost  Release []
-Get:3 http://localhost  Packages []
+Get:2 http://localhost  Packages []
 Reading package lists..." "empty archive Packages.$COMPRESS over http"
 
 	createemptyfile 'Packages'
 	#FIXME: we should response with a good error message instead
-	testaptgetupdate "Get:1 http://localhost  Release.gpg []
+	testaptgetupdate "Get:1 http://localhost  InRelease []
 Ign http://localhost/  Translation-en
-Get:2 http://localhost  Release []
-Get:3 http://localhost  Packages
+Get:2 http://localhost  Packages
 Err http://localhost  Packages
   Undetermined Error
 W: Failed to fetch http://localhost:8080/Packages.$COMPRESS  Undetermined Error
diff --git a/test/integration/test-releasefile-verification b/test/integration/test-releasefile-verification
new file mode 100755
index 000000000..961c49895
--- /dev/null
+++ b/test/integration/test-releasefile-verification
@@ -0,0 +1,160 @@
+#!/bin/sh
+set -e
+
+TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+buildaptarchive
+setupflataptarchive
+changetowebserver
+
+prepare() {
+	local DATE="${2:-now}"
+	if [ "$DATE" = 'now' -a "$1" = "${PKGFILE}-new" ]; then
+		DATE='now + 6 days'
+	fi
+	for release in $(find rootdir/var/lib/apt/lists 2> /dev/null); do
+		touch -d 'now - 6 hours' $release
+	done
+	rm -rf rootdir/var/cache/apt/archives
+	rm -f rootdir/var/cache/apt/*.bin
+	cp $1 aptarchive/Packages
+	find aptarchive -name 'Release' -delete
+	cat aptarchive/Packages | gzip > aptarchive/Packages.gz
+	cat aptarchive/Packages | bzip2 > aptarchive/Packages.bz2
+	cat aptarchive/Packages | lzma > aptarchive/Packages.lzma
+	generatereleasefiles "$DATE"
+}
+
+installaptold() {
+	testequal 'Reading package lists...
+Building dependency tree...
+Suggested packages:
+  aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt
+The following NEW packages will be installed:
+  apt
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+After this operation, 5370 kB of additional disk space will be used.
+Get:1 http://localhost/  apt 0.7.25.3
+Download complete and in download only mode' aptget install apt -dy
+}
+
+installaptnew() {
+	testequal 'Reading package lists...
+Building dependency tree...
+Suggested packages:
+  aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt
+The following NEW packages will be installed:
+  apt
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+After this operation, 5808 kB of additional disk space will be used.
+Get:1 http://localhost/  apt 0.8.0~pre1
+Download complete and in download only mode' aptget install apt -dy
+}
+
+failaptold() {
+	testequal 'Reading package lists...
+Building dependency tree...
+Suggested packages:
+  aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt
+The following NEW packages will be installed:
+  apt
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+After this operation, 5370 kB of additional disk space will be used.
+WARNING: The following packages cannot be authenticated!
+  apt
+E: There are problems and -y was used without --force-yes' aptget install apt -dy
+}
+
+failaptnew() {
+	testequal 'Reading package lists...
+Building dependency tree...
+Suggested packages:
+  aptitude synaptic wajig dpkg-dev apt-doc bzip2 lzma python-apt
+The following NEW packages will be installed:
+  apt
+0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.
+After this operation, 5808 kB of additional disk space will be used.
+WARNING: The following packages cannot be authenticated!
+  apt
+E: There are problems and -y was used without --force-yes' aptget install apt -dy
+}
+
+# fake our downloadable file
+touch aptarchive/apt.deb
+
+PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')"
+
+runtest() {
+	prepare ${PKGFILE}
+	rm -rf rootdir/var/lib/apt/lists
+	signreleasefiles 'Joe Sixpack'
+	find aptarchive/ -name "$DELETEFILE" -delete
+	msgtest 'Cold archive signed by' 'Joe Sixpack'
+	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	testequal "$(cat ${PKGFILE})
+" aptcache show apt
+	installaptold
+
+	prepare ${PKGFILE}-new
+	signreleasefiles 'Joe Sixpack'
+	find aptarchive/ -name "$DELETEFILE" -delete
+	msgtest 'Good warm archive signed by' 'Joe Sixpack'
+	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	testequal "$(cat ${PKGFILE}-new)
+" aptcache show apt
+	installaptnew
+
+
+	prepare ${PKGFILE}
+	rm -rf rootdir/var/lib/apt/lists
+	signreleasefiles 'Marvin Paranoid'
+	find aptarchive/ -name "$DELETEFILE" -delete
+	msgtest 'Cold archive signed by' 'Marvin Paranoid'
+	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgpass || msgfail
+	testequal "$(cat ${PKGFILE})
+" aptcache show apt
+	failaptold
+
+	prepare ${PKGFILE}-new
+	# weborf doesn't support If-Range
+	for release in $(find rootdir/var/lib/apt/lists/partial/ -name '*Release'); do
+		rm $release
+		touch $release
+	done
+	signreleasefiles 'Joe Sixpack'
+	find aptarchive/ -name "$DELETEFILE" -delete
+	msgtest 'Bad warm archive signed by' 'Joe Sixpack'
+	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	testequal "$(cat ${PKGFILE}-new)
+" aptcache show apt
+	installaptnew
+
+
+	prepare ${PKGFILE}
+	rm -rf rootdir/var/lib/apt/lists
+	signreleasefiles 'Joe Sixpack'
+	find aptarchive/ -name "$DELETEFILE" -delete
+	msgtest 'Cold archive signed by' 'Joe Sixpack'
+	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgfail || msgpass
+	testequal "$(cat ${PKGFILE})
+" aptcache show apt
+	installaptold
+
+	prepare ${PKGFILE}-new
+	signreleasefiles 'Marvin Paranoid'
+	find aptarchive/ -name "$DELETEFILE" -delete
+	msgtest 'Good warm archive signed by' 'Marvin Paranoid'
+	aptget update 2>&1 | grep -E '^(W|E): ' > /dev/null && msgpass || msgfail
+	testequal "$(cat ${PKGFILE})
+" aptcache show apt
+	installaptold
+}
+
+DELETEFILE="InRelease"
+runtest
+DELETEFILE="Release.gpg"
+runtest
-- 
cgit v1.2.3-70-g09d2


From 5d88572318ed7e271101b1ae8f2cc139a1a3f705 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Wed, 26 Jan 2011 16:19:30 +0100
Subject:   - change the internal handling of Extensions in pkgAcqIndex   - add
 a special uncompressed compression type to prefer those files *
 methods/{gzip,bzip}.cc:   - print a good error message if FileSize() is zero

---
 apt-pkg/acquire-item.cc                            | 114 ++++++++++++---------
 apt-pkg/acquire-item.h                             |   9 +-
 apt-pkg/aptconfiguration.cc                        |   8 ++
 debian/changelog                                   |   6 +-
 doc/apt.conf.5.xml                                 |   9 +-
 doc/examples/configure-index                       |   2 +-
 methods/bzip2.cc                                   |   3 +-
 methods/gzip.cc                                    |   3 +-
 .../test-bug-595691-empty-and-broken-archive-files |   9 +-
 9 files changed, 96 insertions(+), 67 deletions(-)

(limited to 'apt-pkg/acquire-item.h')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 2cd6ab359..a603a3d70 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -622,29 +622,61 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,
 			 HashString ExpectedHash, string comprExt)
    : Item(Owner), RealURI(URI), ExpectedHash(ExpectedHash)
 {
+   if(comprExt.empty() == true)
+   {
+      // autoselect the compression method
+      std::vector<std::string> types = APT::Configuration::getCompressionTypes();
+      for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
+	 comprExt.append(*t).append(" ");
+      if (comprExt.empty() == false)
+	 comprExt.erase(comprExt.end()-1);
+   }
+   CompressionExtension = comprExt;
+
+   Init(URI, URIDesc, ShortDesc);
+}
+pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target,
+			 HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
+   : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash)
+{
+   // autoselect the compression method
+   std::vector<std::string> types = APT::Configuration::getCompressionTypes();
+   CompressionExtension = "";
+   if (ExpectedHash.empty() == false)
+   {
+      for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
+	 if (*t == "uncompressed" || MetaIndexParser->Exists(string(Target->MetaKey).append(".").append(*t)) == true)
+	    CompressionExtension.append(*t).append(" ");
+   }
+   else
+   {
+      for (std::vector<std::string>::const_iterator t = types.begin(); t != types.end(); ++t)
+	 CompressionExtension.append(*t).append(" ");
+   }
+   if (CompressionExtension.empty() == false)
+      CompressionExtension.erase(CompressionExtension.end()-1);
+
+   Init(Target->URI, Target->Description, Target->ShortDesc);
+}
+									/*}}}*/
+// AcqIndex::Init - defered Constructor					/*{{{*/
+void pkgAcqIndex::Init(string const &URI, string const &URIDesc, string const &ShortDesc) {
    Decompression = false;
    Erase = false;
 
    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
    DestFile += URItoFileName(URI);
 
-   if(comprExt.empty()) 
-   {
-      // autoselect the compression method
-      std::vector<std::string> types = APT::Configuration::getCompressionTypes();
-      if (types.empty() == true)
-	 comprExt = "plain";
-      else
-	 comprExt = "." + types[0];
-   }
-   CompressionExtension = ((comprExt == "plain" || comprExt == ".") ? "" : comprExt);
-
-   Desc.URI = URI + CompressionExtension;
+   std::string const comprExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
+   if (comprExt == "uncompressed")
+      Desc.URI = URI;
+   else
+      Desc.URI = URI + '.' + comprExt;
 
    Desc.Description = URIDesc;
    Desc.Owner = this;
    Desc.ShortDesc = ShortDesc;
-      
+
    QueueURI(Desc);
 }
 									/*}}}*/
@@ -666,37 +698,18 @@ string pkgAcqIndex::Custom600Headers()
 									/*}}}*/
 void pkgAcqIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)	/*{{{*/
 {
-   std::vector<std::string> types = APT::Configuration::getCompressionTypes();
-
-   for (std::vector<std::string>::const_iterator t = types.begin();
-	t != types.end(); t++)
+   size_t const nextExt = CompressionExtension.find(' ');
+   if (nextExt != std::string::npos)
    {
-      // jump over all already tried compression types
-      const unsigned int nameLen = Desc.URI.size() - (*t).size();
-      if(Desc.URI.substr(nameLen) != *t)
-	 continue;
-
-      // we want to try it with the next extension (and make sure to 
-      // not skip over the end)
-      t++;
-      if (t == types.end())
-	 break;
-
-      // queue new download
-      Desc.URI = Desc.URI.substr(0, nameLen) + *t;
-      new pkgAcqIndex(Owner, RealURI, Desc.Description, Desc.ShortDesc,
-      ExpectedHash, string(".").append(*t));
-      
-      Status = StatDone;
-      Complete = false;
-      Dequeue();
+      CompressionExtension = CompressionExtension.substr(nextExt+1);
+      Init(RealURI, Desc.Description, Desc.ShortDesc);
       return;
    }
 
    // on decompression failure, remove bad versions in partial/
-   if(Decompression && Erase) {
+   if (Decompression && Erase) {
       string s = _config->FindDir("Dir::State::lists") + "partial/";
-      s += URItoFileName(RealURI);
+      s.append(URItoFileName(RealURI));
       unlink(s.c_str());
    }
 
@@ -773,8 +786,8 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
       Status = StatError;
       ErrorText = "Method gave a blank filename";
    }
-   
-   string compExt = flExtension(flNotDir(URI(Desc.URI).Path));
+
+   std::string const compExt = CompressionExtension.substr(0, CompressionExtension.find(' '));
 
    // The files timestamp matches
    if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true) {
@@ -807,12 +820,7 @@ void pkgAcqIndex::Done(string Message,unsigned long Size,string Hash,
    // get the binary name for your used compression type
    decompProg = _config->Find(string("Acquire::CompressionTypes::").append(compExt),"");
    if(decompProg.empty() == false);
-   // flExtensions returns the full name if no extension is found
-   // this is why we have this complicated compare operation here
-   // FIMXE: add a new flJustExtension() that return "" if no
-   //        extension is found and use that above so that it can
-   //        be tested against ""
-   else if(compExt == flNotDir(URI(Desc.URI).Path))
+   else if(compExt == "uncompressed")
       decompProg = "copy";
    else {
       _error->Error("Unsupported extension: %s", compExt.c_str());
@@ -853,6 +861,15 @@ string pkgAcqIndexTrans::Custom600Headers()
 /* */
 void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
 {
+   size_t const nextExt = CompressionExtension.find(' ');
+   if (nextExt != std::string::npos)
+   {
+      CompressionExtension = CompressionExtension.substr(nextExt+1);
+      Init(RealURI, Desc.Description, Desc.ShortDesc);
+      Status = StatIdle;
+      return;
+   }
+
    if (Cnf->LocalOnly == true || 
        StringToBool(LookupTag(Message,"Transient-Failure"),false) == false)
    {      
@@ -862,7 +879,7 @@ void pkgAcqIndexTrans::Failed(string Message,pkgAcquire::MethodConfig *Cnf)
       Dequeue();
       return;
    }
-   
+
    Item::Failed(Message,Cnf);
 }
 									/*}}}*/
@@ -1197,8 +1214,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)				/*{{{*/
 	 new pkgAcqDiffIndex(Owner, (*Target)->URI, (*Target)->Description,
 			     (*Target)->ShortDesc, ExpectedIndexHash);
       else
-	 new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
-			    (*Target)->ShortDesc, ExpectedIndexHash);
+	 new pkgAcqIndex(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
    }
 }
 									/*}}}*/
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 581761e32..92098e3d4 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -528,8 +528,8 @@ class pkgAcqIndex : public pkgAcquire::Item
    /** \brief The expected hashsum of the decompressed index file. */
    HashString ExpectedHash;
 
-   /** \brief The compression-related file extension that is being
-    *  added to the downloaded file (e.g., ".gz" or ".bz2").
+   /** \brief The compression-related file extensions that are being
+    *  added to the downloaded file one by one if first fails (e.g., "gz bz2").
     */
    string CompressionExtension;
 
@@ -540,7 +540,7 @@ class pkgAcqIndex : public pkgAcquire::Item
    virtual void Done(string Message,unsigned long Size,string Md5Hash,
 		     pkgAcquire::MethodConfig *Cnf);
    virtual string Custom600Headers();
-   virtual string DescURI() {return RealURI + CompressionExtension;};
+   virtual string DescURI() {return Desc.URI;};
    virtual string HashSum() {return ExpectedHash.toStr(); };
 
    /** \brief Create a pkgAcqIndex.
@@ -565,6 +565,9 @@ class pkgAcqIndex : public pkgAcquire::Item
    pkgAcqIndex(pkgAcquire *Owner,string URI,string URIDesc,
 	       string ShortDesc, HashString ExpectedHash, 
 	       string compressExt="");
+   pkgAcqIndex(pkgAcquire *Owner, struct IndexTarget const * const Target,
+			 HashString const &ExpectedHash, indexRecords const *MetaIndexParser);
+   void Init(string const &URI, string const &URIDesc, string const &ShortDesc);
 };
 									/*}}}*/
 /** \brief An acquire item that is responsible for fetching a		{{{
diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc
index 52f54073c..e97ebfed7 100644
--- a/apt-pkg/aptconfiguration.cc
+++ b/apt-pkg/aptconfiguration.cc
@@ -90,6 +90,14 @@ const Configuration::getCompressionTypes(bool const &Cached) {
 		types.push_back(Types->Tag);
 	}
 
+	// add the special "uncompressed" type
+	if (std::find(types.begin(), types.end(), "uncompressed") == types.end())
+	{
+		string const uncompr = _config->FindFile("Dir::Bin::uncompressed", "");
+		if (uncompr.empty() == true || FileExists(uncompr) == true)
+			types.push_back("uncompressed");
+	}
+
 	return types;
 }
 									/*}}}*/
diff --git a/debian/changelog b/debian/changelog
index 2413b9c5b..7b125fc43 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -54,6 +54,8 @@ apt (0.8.11+wheezy) unstable; urgency=low
   * apt-pkg/acquire-item.cc:
     - don't uncompress downloaded pdiff files before feeding it to rred
     - try downloading clearsigned InRelease before trying Release.gpg
+    - change the internal handling of Extensions in pkgAcqIndex
+    - add a special uncompressed compression type to prefer those files
   * cmdline/apt-key:
     - don't set trustdb-name as non-root so 'list' and 'finger'
       can be used without being root (Closes: #393005, #592107)
@@ -62,8 +64,10 @@ apt (0.8.11+wheezy) unstable; urgency=low
   * ftparchive/writer.cc:
     - add config option to search for more patterns in release command
     - include Index files by default in the Release file
+  * methods/{gzip,bzip}.cc:
+    - print a good error message if FileSize() is zero
 
- -- David Kalnischkies <kalnischkies@gmail.com>  Mon, 24 Jan 2011 15:36:50 +0100
+ -- David Kalnischkies <kalnischkies@gmail.com>  Wed, 26 Jan 2011 16:06:10 +0100
 
 apt (0.8.10.3) unstable; urgency=low
 
diff --git a/doc/apt.conf.5.xml b/doc/apt.conf.5.xml
index a19d85dbc..a423dac24 100644
--- a/doc/apt.conf.5.xml
+++ b/doc/apt.conf.5.xml
@@ -442,12 +442,11 @@ DPkg::Pre-Install-Pkgs {"/usr/sbin/dpkg-preconfigure --apt";};
      the bzip2 method (the inbuilt) setting is <literallayout>Dir::Bin::bzip2 "/bin/bzip2";</literallayout>
      Note also that list entries specified on the command line will be added at the end of the list
      specified in the configuration files, but before the default entries. To prefer a type in this case
-     over the ones specified in in the configuration files you can set the option direct - not in list style.
+     over the ones specified in the configuration files you can set the option direct - not in list style.
      This will not override the defined list, it will only prefix the list with this type.</para>
-     <para>While it is possible to add an empty compression type to the order list, but APT in its current
-     version doesn't understand it correctly and will display many warnings about not downloaded files -
-     these warnings are most of the time false negatives. Future versions will maybe include a way to
-     really prefer uncompressed files to support the usage of local mirrors.</para></listitem>
+     <para>The special type <literal>uncompressed</literal> can be used to give uncompressed files a
+     preference, but note that most archives doesn't provide uncompressed files so this is mostly only
+     useable for local mirrors.</para></listitem>
      </varlistentry>
 
      <varlistentry><term>GzipIndexes</term>
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index 6c078d75f..fd14d4dd7 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -281,7 +281,7 @@ Acquire
     lzma "lzma";
     gz "gzip";
 
-    Order { "gz"; "lzma"; "bz2"; };
+    Order { "uncompressed"; "gz"; "lzma"; "bz2"; };
   };
 
   Languages
diff --git a/methods/bzip2.cc b/methods/bzip2.cc
index ccc3669a2..42932dded 100644
--- a/methods/bzip2.cc
+++ b/methods/bzip2.cc
@@ -56,9 +56,8 @@ bool Bzip2Method::Fetch(FetchItem *Itm)
    // Open the source and destination files
    FileFd From(Path,FileFd::ReadOnly);
 
-   // FIXME add an error message saying that empty files can't be valid archives
    if(From.FileSize() == 0)
-      return false;
+      return _error->Error(_("Empty files can't be valid archives"));
 
    int GzOut[2];   
    if (pipe(GzOut) < 0)
diff --git a/methods/gzip.cc b/methods/gzip.cc
index f1c76066e..fc4e1ecfd 100644
--- a/methods/gzip.cc
+++ b/methods/gzip.cc
@@ -48,9 +48,8 @@ bool GzipMethod::Fetch(FetchItem *Itm)
    // Open the source and destination files
    FileFd From(Path,FileFd::ReadOnlyGzip);
 
-   // FIXME add an error message saying that empty files can't be valid archives
    if(From.FileSize() == 0)
-      return false;
+      return _error->Error(_("Empty files can't be valid archives"));
 
    FileFd To(Itm->DestFile,FileFd::WriteAtomic);   
    To.EraseOnFailure();
diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index c5379dca0..398d0cd1b 100755
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -56,6 +56,7 @@ setupcompressor() {
 	lzma) COMPRESS="lzma";;
 	esac
 	echo "Acquire::CompressionTypes::Order { \"${COMPRESS}\"; };
+Dir::Bin::uncompressed \"/does/not/exist\";
 Dir::Bin::gzip \"/does/not/exist\";
 Dir::Bin::bzip2 \"/does/not/exist\";
 Dir::Bin::lzma \"/does/not/exist\";" > rootdir/etc/apt/apt.conf.d/00compressor
@@ -93,8 +94,8 @@ Reading package lists..." "empty archive Packages.$COMPRESS over file"
 	testaptgetupdate "Ign file:$(readlink -f aptarchive)/  Translation-en
 Get:1 file:  InRelease []
 Err file:  Packages
-  Undetermined Error
-W: Failed to fetch file:$(readlink -f aptarchive/Packages.$COMPRESS)  Undetermined Error
+  Empty files can't be valid archives
+W: Failed to fetch ${COMPRESSOR}:$(readlink -f aptarchive/Packages.$COMPRESS)  Empty files can't be valid archives
 
 E: Some index files failed to download, they have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over file"
 }
@@ -127,8 +128,8 @@ Reading package lists..." "empty archive Packages.$COMPRESS over http"
 Ign http://localhost/  Translation-en
 Get:2 http://localhost  Packages
 Err http://localhost  Packages
-  Undetermined Error
-W: Failed to fetch http://localhost:8080/Packages.$COMPRESS  Undetermined Error
+  Empty files can't be valid archives
+W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages)  Empty files can't be valid archives
 
 E: Some index files failed to download, they have been ignored, or old ones used instead." "empty file Packages.$COMPRESS over http"
 }
-- 
cgit v1.2.3-70-g09d2


From ab53c018fbc7aa01c0d89586c0aa98bc944dd9e4 Mon Sep 17 00:00:00 2001
From: David Kalnischkies <kalnischkies@gmail.com>
Date: Fri, 28 Jan 2011 12:22:37 +0100
Subject:   - download and use i18n/Index to choose which Translations to
 download * apt-pkg/aptconfiguration.cc:   - remove the inbuilt Translation
 files whitelist

---
 apt-pkg/acquire-item.cc                            | 218 +++++++++++++++++++--
 apt-pkg/acquire-item.h                             |  56 ++++++
 apt-pkg/aptconfiguration.cc                        |  51 ++---
 apt-pkg/deb/debmetaindex.cc                        |  77 +++++---
 apt-pkg/deb/debmetaindex.h                         |   2 +
 debian/changelog                                   |   5 +-
 .../test-bug-595691-empty-and-broken-archive-files |  28 +--
 .../test-bug-601016-description-translation        |   1 +
 test/libapt/getlanguages_test.cc                   |  71 ++++---
 9 files changed, 381 insertions(+), 128 deletions(-)

(limited to 'apt-pkg/acquire-item.h')

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index a603a3d70..776009493 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -184,6 +184,153 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
    }
 }
 									/*}}}*/
+// AcqSubIndex::AcqSubIndex - Constructor				/*{{{*/
+// ---------------------------------------------------------------------
+/* Get the DiffIndex file first and see if there are patches availabe 
+ * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
+ * patches. If anything goes wrong in that process, it will fall back to
+ * the original packages file
+ */
+pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
+				 string const &URIDesc, string const &ShortDesc,
+				 HashString const &ExpectedHash)
+   : Item(Owner), ExpectedHash(ExpectedHash)
+{
+   Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
+
+   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+   DestFile += URItoFileName(URI);
+
+   Desc.URI = URI;
+   Desc.Description = URIDesc;
+   Desc.Owner = this;
+   Desc.ShortDesc = ShortDesc;
+
+   QueueURI(Desc);
+
+   if(Debug)
+      std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
+}
+									/*}}}*/
+// AcqSubIndex::Custom600Headers - Insert custom request headers	/*{{{*/
+// ---------------------------------------------------------------------
+/* The only header we use is the last-modified header. */
+string pkgAcqSubIndex::Custom600Headers()
+{
+   string Final = _config->FindDir("Dir::State::lists");
+   Final += URItoFileName(Desc.URI);
+
+   struct stat Buf;
+   if (stat(Final.c_str(),&Buf) != 0)
+      return "\nIndex-File: true";
+   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+}
+									/*}}}*/
+void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)	/*{{{*/
+{
+   if(Debug)
+      std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
+
+   Complete = false;
+   Status = StatDone;
+   Dequeue();
+
+   // No good Index is provided, so try guessing
+   std::vector<std::string> langs = APT::Configuration::getLanguages(true);
+   for (std::vector<std::string>::const_iterator l = langs.begin();
+	l != langs.end(); ++l)
+   {
+      if (*l == "none") continue;
+      string const file = "Translation-" + *l;
+      new pkgAcqIndexTrans(Owner, Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file),
+		Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file),
+		file);
+   }
+}
+									/*}}}*/
+void pkgAcqSubIndex::Done(string Message,unsigned long Size,string Md5Hash,	/*{{{*/
+			   pkgAcquire::MethodConfig *Cnf)
+{
+   if(Debug)
+      std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
+
+   string FileName = LookupTag(Message,"Filename");
+   if (FileName.empty() == true)
+   {
+      Status = StatError;
+      ErrorText = "Method gave a blank filename";
+      return;
+   }
+
+   if (FileName != DestFile)
+   {
+      Local = true;
+      Desc.URI = "copy:" + FileName;
+      QueueURI(Desc);
+      return;
+   }
+
+   Item::Done(Message,Size,Md5Hash,Cnf);
+
+   string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
+
+   // sucess in downloading the index
+   // rename the index
+   if(Debug)
+      std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
+   Rename(DestFile,FinalFile);
+   chmod(FinalFile.c_str(),0644);
+   DestFile = FinalFile;
+
+   if(ParseIndex(DestFile) == false)
+      return Failed("", NULL);
+
+   Complete = true;
+   Status = StatDone;
+   Dequeue();
+   return;
+}
+									/*}}}*/
+bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)		/*{{{*/
+{
+   indexRecords SubIndexParser;
+   if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
+      return false;
+
+   std::vector<std::string> lang = APT::Configuration::getLanguages(true);
+   for (std::vector<std::string>::const_iterator l = lang.begin();
+	l != lang.end(); ++l)
+   {
+      if (*l == "none")
+	 continue;
+
+      string file = "Translation-" + *l;
+      indexRecords::checkSum const *Record = SubIndexParser.Lookup(file);
+      HashString expected;
+      if (Record == NULL)
+      {
+	 // FIXME: the Index file provided by debian currently only includes bz2 records
+	 Record = SubIndexParser.Lookup(file + ".bz2");
+	 if (Record == NULL)
+	    continue;
+      }
+      else
+      {
+	 expected = Record->Hash;
+	 if (expected.empty() == true)
+	    continue;
+      }
+
+      IndexTarget target;
+      target.Description = Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file);
+      target.MetaKey = file;
+      target.ShortDesc = file;
+      target.URI = Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file);
+      new pkgAcqIndexTrans(Owner, &target, expected, &SubIndexParser);
+   }
+   return true;
+}
+									/*}}}*/
 // AcqDiffIndex::AcqDiffIndex - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* Get the DiffIndex file first and see if there are patches availabe 
@@ -841,6 +988,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
 			    string URI,string URIDesc,string ShortDesc) 
   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 {
+}
+pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
+			 HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
+  : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
+{
 }
 									/*}}}*/
 // AcqIndexTrans::Custom600Headers - Insert custom request headers	/*{{{*/
@@ -1182,27 +1334,41 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)				/*{{{*/
       HashString ExpectedIndexHash;
       if (verify)
       {
-         const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
-         if (!Record)
-         {
-            Status = StatAuthError;
-            ErrorText = "Unable to find expected entry  "
-               + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
-            return;
-         }
-         ExpectedIndexHash = Record->Hash;
-         if (_config->FindB("Debug::pkgAcquire::Auth", false))
-         {
-            std::cerr << "Queueing: " << (*Target)->URI << std::endl;
-            std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
-         }
-         if (ExpectedIndexHash.empty())
-         {
-            Status = StatAuthError;
-            ErrorText = "Unable to find hash sum for "
-               + (*Target)->MetaKey + " in Meta-index file";
-            return;
-         }
+	 const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
+	 if (Record == NULL)
+	 {
+	    if ((*Target)->IsOptional() == false)
+	    {
+	       Status = StatAuthError;
+	       strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
+	       return;
+	    }
+	 }
+	 else
+	 {
+	    ExpectedIndexHash = Record->Hash;
+	    if (_config->FindB("Debug::pkgAcquire::Auth", false))
+	    {
+	       std::cerr << "Queueing: " << (*Target)->URI << std::endl;
+	       std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
+	    }
+	    if (ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
+	    {
+	       Status = StatAuthError;
+	       strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
+	       return;
+	    }
+	 }
+      }
+
+      if ((*Target)->IsOptional() == true)
+      {
+	 if ((*Target)->IsSubIndex() == true)
+	    new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
+				(*Target)->ShortDesc, ExpectedIndexHash);
+	 else
+	    new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
+	 continue;
       }
 
       /* Queue Packages file (either diff or full packages files, depending
@@ -1836,3 +2002,13 @@ string pkgAcqFile::Custom600Headers()
    return "";
 }
 									/*}}}*/
+bool IndexTarget::IsOptional() const {
+   if (strncmp(ShortDesc.c_str(), "Translation", 11) != 0)
+      return false;
+   return true;
+}
+bool IndexTarget::IsSubIndex() const {
+   if (ShortDesc != "TranslationIndex")
+      return false;
+   return true;
+}
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 92098e3d4..9bcc32f21 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -287,6 +287,50 @@ struct DiffInfo {
    unsigned long size;
 };
 									/*}}}*/
+/** \brief An item that is responsible for fetching a SubIndex		{{{
+ *
+ *  The MetaIndex file includes only records for important indexes
+ *  and records for these SubIndex files so these can carry records
+ *  for addition files like PDiffs and Translations
+ */
+class pkgAcqSubIndex : public pkgAcquire::Item
+{
+ protected:
+   /** \brief If \b true, debugging information will be written to std::clog. */
+   bool Debug;
+
+   /** \brief The item that is currently being downloaded. */
+   pkgAcquire::ItemDesc Desc;
+
+   /** \brief The Hash that this file should have after download
+    */
+   HashString ExpectedHash;
+
+ public:
+   // Specialized action members
+   virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
+   virtual void Done(string Message,unsigned long Size,string Md5Hash,
+		     pkgAcquire::MethodConfig *Cnf);
+   virtual string DescURI() {return Desc.URI;};
+   virtual string Custom600Headers();
+   virtual bool ParseIndex(string const &IndexFile);
+
+   /** \brief Create a new pkgAcqDiffIndex.
+    *
+    *  \param Owner The Acquire object that owns this item.
+    *
+    *  \param URI The URI of the list file to download.
+    *
+    *  \param URIDesc A long description of the list file to download.
+    *
+    *  \param ShortDesc A short description of the list file to download.
+    *
+    *  \param ExpectedHash The list file's MD5 signature.
+    */
+   pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,string const &URIDesc,
+		   string const &ShortDesc, HashString const &ExpectedHash);
+};
+									/*}}}*/
 /** \brief An item that is responsible for fetching an index file of	{{{
  *  package list diffs and starting the package list's download.
  *
@@ -597,6 +641,8 @@ class pkgAcqIndexTrans : public pkgAcqIndex
     */
    pkgAcqIndexTrans(pkgAcquire *Owner,string URI,string URIDesc,
 		    string ShortDesc);
+   pkgAcqIndexTrans(pkgAcquire *Owner, struct IndexTarget const * const Target,
+		    HashString const &ExpectedHash, indexRecords const *MetaIndexParser);
 };
 									/*}}}*/
 /** \brief Information about an index file. */				/*{{{*/
@@ -615,8 +661,18 @@ struct IndexTarget
     *  looked up within the meta signature file.
     */
    string MetaKey;
+
+   //FIXME: We should use virtual methods here instead…
+   bool IsOptional() const;
+   bool IsSubIndex() const;
 };
 									/*}}}*/
+/** \brief Information about an optional index file. */			/*{{{*/
+struct OptionalIndexTarget : public IndexTarget
+{
+};
+									/*}}}*/
+
 /** \brief An acquire item that downloads the detached signature	{{{
  *  of a meta-index (Release) file, then queues up the release
  *  file itself.
diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc
index e97ebfed7..3cf4d2429 100644
--- a/apt-pkg/aptconfiguration.cc
+++ b/apt-pkg/aptconfiguration.cc
@@ -163,33 +163,6 @@ std::vector<std::string> const Configuration::getLanguages(bool const &All,
 	}
 	closedir(D);
 
-	// get the environment language codes: LC_MESSAGES (and later LANGUAGE)
-	// we extract both, a long and a short code and then we will
-	// check if we actually need both (rare) or if the short is enough
-	string const envMsg = string(Locale == 0 ? std::setlocale(LC_MESSAGES, NULL) : *Locale);
-	size_t const lenShort = (envMsg.find('_') != string::npos) ? envMsg.find('_') : 2;
-	size_t const lenLong = (envMsg.find_first_of(".@") != string::npos) ? envMsg.find_first_of(".@") : (lenShort + 3);
-
-	string envLong = envMsg.substr(0,lenLong);
-	string const envShort = envLong.substr(0,lenShort);
-	bool envLongIncluded = true;
-
-	// to save the servers from unneeded queries, we only try also long codes
-	// for languages it is realistic to have a long code translation file…
-	// TODO: Improve translation acquire system to drop them dynamic
-	char const *needLong[] = { "cs", "en", "pt", "sv", "zh", NULL };
-	if (envLong != envShort) {
-		for (char const **l = needLong; *l != NULL; l++)
-			if (envShort.compare(*l) == 0) {
-				envLongIncluded = false;
-				break;
-			}
-	}
-
-	// we don't add the long code, but we allow the user to do so
-	if (envLongIncluded == true)
-		envLong.clear();
-
 	// FIXME: Remove support for the old APT::Acquire::Translation
 	// it was undocumented and so it should be not very widthly used
 	string const oldAcquire = _config->Find("APT::Acquire::Translation","");
@@ -211,12 +184,22 @@ std::vector<std::string> const Configuration::getLanguages(bool const &All,
 			return codes;
 	}
 
-	// It is very likely we will need to environment codes later,
+	// get the environment language codes: LC_MESSAGES (and later LANGUAGE)
+	// we extract both, a long and a short code and then we will
+	// check if we actually need both (rare) or if the short is enough
+	string const envMsg = string(Locale == 0 ? std::setlocale(LC_MESSAGES, NULL) : *Locale);
+	size_t const lenShort = (envMsg.find('_') != string::npos) ? envMsg.find('_') : 2;
+	size_t const lenLong = (envMsg.find_first_of(".@") != string::npos) ? envMsg.find_first_of(".@") : (lenShort + 3);
+
+	string const envLong = envMsg.substr(0,lenLong);
+	string const envShort = envLong.substr(0,lenShort);
+
+	// It is very likely we will need the environment codes later,
 	// so let us generate them now from LC_MESSAGES and LANGUAGE
 	std::vector<string> environment;
 	if (envShort != "C") {
 		// take care of LC_MESSAGES
-		if (envLongIncluded == false)
+		if (envLong != envShort)
 			environment.push_back(envLong);
 		environment.push_back(envShort);
 		// take care of LANGUAGE
@@ -233,16 +216,6 @@ std::vector<std::string> const Configuration::getLanguages(bool const &All,
 					continue;
 				if (std::find(environment.begin(), environment.end(), *e) != environment.end())
 					continue;
-				if (e->find('_') != string::npos) {
-					// Drop LongCodes here - ShortCodes are also included
-					string const shorty = e->substr(0, e->find('_'));
-					char const **n = needLong;
-					for (; *n != NULL; ++n)
-						if (shorty == *n)
-							break;
-					if (*n == NULL)
-						continue;
-				}
 				++addedLangs;
 				environment.push_back(*e);
 			}
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index e2c680b14..a6edab6b9 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -119,6 +119,29 @@ string debReleaseIndex::SourceIndexURI(const char *Type, const string &Section)
       return URI + "dists/" + Dist + "/" + SourceIndexURISuffix(Type, Section);
 }
 
+string debReleaseIndex::TranslationIndexURISuffix(const char *Type, const string &Section) const
+{
+   string Res ="";
+   if (Dist[Dist.size() - 1] != '/')
+      Res += Section + "/i18n/";
+   return Res + Type;
+}
+
+string debReleaseIndex::TranslationIndexURI(const char *Type, const string &Section) const
+{
+   string Res;
+   if (Dist[Dist.size() - 1] == '/')
+   {
+      if (Dist != "/")
+         Res = URI + Dist;
+      else 
+         Res = URI;
+      return Res + Type;
+   }
+   else
+      return URI + "dists/" + Dist + "/" + TranslationIndexURISuffix(Type, Section);
+}
+
 debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) {
 	this->URI = URI;
 	this->Dist = Dist;
@@ -155,6 +178,7 @@ vector <struct IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
 	if (IndexTargets->empty() == false && ArchEntries.size() == 1)
 		return IndexTargets;
 
+	std::set<std::string> sections;
 	for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
 	     a != ArchEntries.end(); ++a) {
 		if (a->first == "source")
@@ -167,6 +191,37 @@ vector <struct IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
 			Target->URI = IndexURI(Target->ShortDesc.c_str(), (*I)->Section, a->first);
 			Target->Description = Info (Target->ShortDesc.c_str(), (*I)->Section, a->first);
 			IndexTargets->push_back (Target);
+			sections.insert((*I)->Section);
+		}
+	}
+
+	// get the Translations:
+	// - if its a dists-style repository get the i18n/Index first
+	// - if its flat try to acquire files by guessing
+	if (Dist[Dist.size() - 1] == '/') {
+		std::vector<std::string> const lang = APT::Configuration::getLanguages(true);
+		for (std::set<std::string>::const_iterator s = sections.begin();
+		     s != sections.end(); ++s) {
+			for (std::vector<std::string>::const_iterator l = lang.begin();
+			     l != lang.end(); l++) {
+				if (*l == "none") continue;
+				IndexTarget * Target = new OptionalIndexTarget();
+				Target->ShortDesc = "Translation-" + *l;
+				Target->MetaKey = TranslationIndexURISuffix(l->c_str(), *s);
+				Target->URI = TranslationIndexURI(l->c_str(), *s);
+				Target->Description = Info (Target->ShortDesc.c_str(), *s);
+				IndexTargets->push_back(Target);
+			}
+		}
+	} else {
+		for (std::set<std::string>::const_iterator s = sections.begin();
+		     s != sections.end(); ++s) {
+			IndexTarget * Target = new OptionalIndexTarget();
+			Target->ShortDesc = "TranslationIndex";
+			Target->MetaKey = TranslationIndexURISuffix("Index", *s);
+			Target->URI = TranslationIndexURI("Index", *s);
+			Target->Description = Info (Target->ShortDesc.c_str(), *s);
+			IndexTargets->push_back (Target);
 		}
 	}
 
@@ -191,28 +246,6 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 		ComputeIndexTargets(),
 		new indexRecords (Dist));
 
-
-	// Queue the translations
-	std::vector<std::string> const lang = APT::Configuration::getLanguages(true);
-	map<string, set<string> > sections;
-	for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
-	     a != ArchEntries.end(); ++a) {
-		if (a->first == "source")
-			continue;
-		for (vector<debSectionEntry const*>::const_iterator I = a->second.begin();
-		     I != a->second.end(); I++)
-			sections[(*I)->Section].insert(lang.begin(), lang.end());
-	}
-
-	for (map<string, set<string> >::const_iterator s = sections.begin();
-	     s != sections.end(); ++s)
-		for (set<string>::const_iterator l = s->second.begin();
-		     l != s->second.end(); l++) {
-			if (*l == "none") continue;
-			debTranslationsIndex i = debTranslationsIndex(URI,Dist,s->first,(*l).c_str());
-			i.GetIndexes(Owner);
-		}
-
 	return true;
 }
 
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 360fa5419..1561c6e00 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -37,6 +37,8 @@ class debReleaseIndex : public metaIndex {
    string IndexURISuffix(const char *Type, string const &Section, string const &Arch="native") const;
    string SourceIndexURI(const char *Type, const string &Section) const;
    string SourceIndexURISuffix(const char *Type, const string &Section) const;
+   string TranslationIndexURI(const char *Type, const string &Section) const;
+   string TranslationIndexURISuffix(const char *Type, const string &Section) const;
    virtual vector <pkgIndexFile *> *GetIndexFiles();
 
    virtual bool IsTrusted() const;
diff --git a/debian/changelog b/debian/changelog
index 7b125fc43..b3da32247 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -56,6 +56,7 @@ apt (0.8.11+wheezy) unstable; urgency=low
     - try downloading clearsigned InRelease before trying Release.gpg
     - change the internal handling of Extensions in pkgAcqIndex
     - add a special uncompressed compression type to prefer those files
+    - download and use i18n/Index to choose which Translations to download
   * cmdline/apt-key:
     - don't set trustdb-name as non-root so 'list' and 'finger'
       can be used without being root (Closes: #393005, #592107)
@@ -66,8 +67,10 @@ apt (0.8.11+wheezy) unstable; urgency=low
     - include Index files by default in the Release file
   * methods/{gzip,bzip}.cc:
     - print a good error message if FileSize() is zero
+  * apt-pkg/aptconfiguration.cc:
+    - remove the inbuilt Translation files whitelist
 
- -- David Kalnischkies <kalnischkies@gmail.com>  Wed, 26 Jan 2011 16:06:10 +0100
+ -- David Kalnischkies <kalnischkies@gmail.com>  Fri, 28 Jan 2011 12:22:25 +0100
 
 apt (0.8.10.3) unstable; urgency=low
 
diff --git a/test/integration/test-bug-595691-empty-and-broken-archive-files b/test/integration/test-bug-595691-empty-and-broken-archive-files
index 398d0cd1b..684559aa7 100755
--- a/test/integration/test-bug-595691-empty-and-broken-archive-files
+++ b/test/integration/test-bug-595691-empty-and-broken-archive-files
@@ -32,7 +32,7 @@ createemptyarchive() {
 	fi
 	touch aptarchive/Packages
 	echo -n "" | $COMPRESSOR > aptarchive/${1}.$COMPRESS
-	aptftparchive release aptarchive/ > aptarchive/Release
+	generatereleasefiles
 	signreleasefiles
 	rm -f aptarchive/Packages
 }
@@ -43,7 +43,7 @@ createemptyfile() {
 		echo -n "" | $COMPRESSOR > aptarchive/Packages.$COMPRESS
 	fi
 	touch aptarchive/Packages aptarchive/${1}.$COMPRESS
-	aptftparchive release aptarchive/ > aptarchive/Release
+	generatereleasefiles
 	signreleasefiles
 	rm -f aptarchive/Packages
 }
@@ -76,7 +76,7 @@ testoverfile() {
 
 	createemptyfile 'en'
 	testaptgetupdate "Get:1 file:  InRelease []
-Ign file:$(readlink -f aptarchive)/  Translation-en
+Ign file:  Translation-en
 Reading package lists..." "empty file en.$COMPRESS over file"
 
 	createemptyarchive 'en'
@@ -86,13 +86,13 @@ Reading package lists..." "empty archive en.$COMPRESS over file"
 	createemptyarchive 'Packages'
 	# FIXME: Why omits the file transport the Packages Get line?
 	#Get:3 file:  Packages []
-	testaptgetupdate "Ign file:$(readlink -f aptarchive)/  Translation-en
-Get:1 file:  InRelease []
+	testaptgetupdate "Get:1 file:  InRelease []
+Ign file:  Translation-en
 Reading package lists..." "empty archive Packages.$COMPRESS over file"
 
 	createemptyfile 'Packages'
-	testaptgetupdate "Ign file:$(readlink -f aptarchive)/  Translation-en
-Get:1 file:  InRelease []
+	testaptgetupdate "Get:1 file:  InRelease []
+Ign file:  Translation-en
 Err file:  Packages
   Empty files can't be valid archives
 W: Failed to fetch ${COMPRESSOR}:$(readlink -f aptarchive/Packages.$COMPRESS)  Empty files can't be valid archives
@@ -105,28 +105,28 @@ testoverhttp() {
 
 	createemptyfile 'en'
 	testaptgetupdate "Get:1 http://localhost  InRelease []
-Get:2 http://localhost/  Translation-en
-Get:3 http://localhost  Packages []
-Ign http://localhost/  Translation-en
+Get:2 http://localhost  Packages []
+Get:3 http://localhost  Translation-en
+Ign http://localhost  Translation-en
 Reading package lists..." "empty file en.$COMPRESS over http"
 
 	createemptyarchive 'en'
 	testaptgetupdate "Get:1 http://localhost  InRelease []
-Get:2 http://localhost/  Translation-en []
-Get:3 http://localhost  Packages []
+Get:2 http://localhost  Packages []
+Get:3 http://localhost  Translation-en []
 Reading package lists..." "empty archive en.$COMPRESS over http"
 
 	createemptyarchive 'Packages'
 	testaptgetupdate "Get:1 http://localhost  InRelease []
-Ign http://localhost/  Translation-en
 Get:2 http://localhost  Packages []
+Ign http://localhost  Translation-en
 Reading package lists..." "empty archive Packages.$COMPRESS over http"
 
 	createemptyfile 'Packages'
 	#FIXME: we should response with a good error message instead
 	testaptgetupdate "Get:1 http://localhost  InRelease []
-Ign http://localhost/  Translation-en
 Get:2 http://localhost  Packages
+Ign http://localhost  Translation-en
 Err http://localhost  Packages
   Empty files can't be valid archives
 W: Failed to fetch ${COMPRESSOR}:$(readlink -f rootdir/var/lib/apt/lists/partial/localhost:8080_Packages)  Empty files can't be valid archives
diff --git a/test/integration/test-bug-601016-description-translation b/test/integration/test-bug-601016-description-translation
index a31e42ee1..2a323a201 100755
--- a/test/integration/test-bug-601016-description-translation
+++ b/test/integration/test-bug-601016-description-translation
@@ -57,6 +57,7 @@ Description-${LOCALE}: Mächtige Oberfläche für dpkg
 testrun() {
 	echo "Acquire::Languages { \"${LOCALE}\"; \"en\"; };" > rootdir/etc/apt/apt.conf.d/00languages
 	export LC_ALL=""
+	rm -rf rootdir/var/lib/apt/lists rootdir/var/cache/apt/
 	setupaptarchive
 	testequal "$LOCALESTANZA" aptcache show apt -o Test=File-${LOCALE}
 	testequal "$NOLONGSTANZA" aptcache show apt -o Acquire::Languages="ww" -o Test=File-${LOCALE}
diff --git a/test/libapt/getlanguages_test.cc b/test/libapt/getlanguages_test.cc
index 3559aae0c..707142aef 100644
--- a/test/libapt/getlanguages_test.cc
+++ b/test/libapt/getlanguages_test.cc
@@ -26,16 +26,18 @@ int main(int argc,char *argv[])
 	env[1] = "";
 
 	std::vector<std::string> vec = APT::Configuration::getLanguages(false, false, env);
-	equals(vec.size(), 2);
-	equals(vec[0], "de");
-	equals(vec[1], "en");
+	equals(vec.size(), 3);
+	equals(vec[0], "de_DE");
+	equals(vec[1], "de");
+	equals(vec[2], "en");
 
 	// Special: Check if the cache is actually in use
 		env[0] = "en_GB.UTF-8";
 		vec = APT::Configuration::getLanguages(false, true, env);
-		equals(vec.size(), 2);
-		equals(vec[0], "de");
-		equals(vec[1], "en");
+		equals(vec.size(), 3);
+		equals(vec[0], "de_DE");
+		equals(vec[1], "de");
+		equals(vec[2], "en");
 
 	env[0] = "en_GB.UTF-8";
 	vec = APT::Configuration::getLanguages(false, false, env);
@@ -52,19 +54,21 @@ int main(int argc,char *argv[])
 
 	env[0] = "tr_DE@euro";
 	vec = APT::Configuration::getLanguages(false, false, env);
-	equals(vec.size(), 2);
-	equals(vec[0], "tr");
-	equals(vec[1], "en");
+	equals(vec.size(), 3);
+	equals(vec[0], "tr_DE");
+	equals(vec[1], "tr");
+	equals(vec[2], "en");
 
 	env[0] = "de_NO";
-	env[1] = "se_NO:en_GB:nb_NO:nb:no_NO:no:nn_NO:nn:da:sv:en";
+	env[1] = "de_NO:en_GB:nb_NO:nb:no_NO:no:nn_NO:nn:da:sv:en";
 	vec = APT::Configuration::getLanguages(false, false, env);
-	equals(vec.size(), 5);
-	equals(vec[0], "de");
-	equals(vec[1], "en_GB");
-	equals(vec[2], "nb");
-	equals(vec[3], "no");
-	equals(vec[4], "en");
+	equals(vec.size(), 6);
+	equals(vec[0], "de_NO");
+	equals(vec[1], "de");
+	equals(vec[2], "en_GB");
+	equals(vec[3], "nb_NO");
+	equals(vec[4], "nb");
+	equals(vec[5], "en");
 
 	env[0] = "pt_PR.UTF-8";
 	env[1] = "";
@@ -76,9 +80,10 @@ int main(int argc,char *argv[])
 
 	env[0] = "ast_DE.UTF-8";
 	vec = APT::Configuration::getLanguages(false, false, env); // bogus, but syntactical correct
-	equals(vec.size(), 2);
-	equals(vec[0], "ast");
-	equals(vec[1], "en");
+	equals(vec.size(), 3);
+	equals(vec[0], "ast_DE");
+	equals(vec[1], "ast");
+	equals(vec[2], "en");
 
 	env[0] = "C";
 	vec = APT::Configuration::getLanguages(false, false, env);
@@ -113,25 +118,28 @@ int main(int argc,char *argv[])
 	_config->Set("Acquire::Languages::2", "en");
 	env[0] = "de_DE.UTF-8";
 	vec = APT::Configuration::getLanguages(false, false, env);
-	equals(vec.size(), 2);
-	equals(vec[0], "de");
-	equals(vec[1], "en");
+	equals(vec.size(), 3);
+	equals(vec[0], "de_DE");
+	equals(vec[1], "de");
+	equals(vec[2], "en");
 
 	_config->Set("Acquire::Languages::3", "de");
 	env[0] = "de_DE.UTF-8";
 	vec = APT::Configuration::getLanguages(false, false, env);
-	equals(vec.size(), 2);
-	equals(vec[0], "de");
-	equals(vec[1], "en");
+	equals(vec.size(), 3);
+	equals(vec[0], "de_DE");
+	equals(vec[1], "de");
+	equals(vec[2], "en");
 
 	_config->Set("Dir::State::lists", argv[1]);
 	vec = APT::Configuration::getLanguages(true, false, env);
-	equals(vec.size(), 5);
-	equals(vec[0], "de");
-	equals(vec[1], "en");
-	equals(vec[2], "none");
-	equals(vec[3], "pt");
-	equals(vec[4], "tr");
+	equals(vec.size(), 6);
+	equals(vec[0], "de_DE");
+	equals(vec[1], "de");
+	equals(vec[2], "en");
+	equals(vec[3], "none");
+	equals(vec[4], "pt");
+	equals(vec[5], "tr");
 
 	_config->Set("Dir::State::lists", "/non-existing-dir");
 	_config->Set("Acquire::Languages::1", "none");
@@ -140,6 +148,7 @@ int main(int argc,char *argv[])
 	equals(vec.size(), 0);
 	env[0] = "de_DE.UTF-8";
 	vec = APT::Configuration::getLanguages(true, false, env);
+	equals(vec.size(), 2);
 	equals(vec[0], "en");
 	equals(vec[1], "de");
 
-- 
cgit v1.2.3-70-g09d2