merge 'after squeeze release'-stuff

[ David Kalnischkies ]
* apt-pkg/depcache.cc:
  - add SetCandidateRelease() to set a candidate version and
    the candidates of dependencies if needed to a specified
    release (Closes: #572709)
* cmdline/apt-get.cc:
  - if --print-uris is used don't setup downloader as we don't need
    progress, lock nor the directories it would create otherwise
  - show dependencies of essential packages which are going to remove
    only if they cause the remove of this essential (Closes: #601961)
  - keep not installed garbage packages uninstalled instead of showing
    in the autoremove section and installing those (Closes: #604222)
  - change pkg/release behavior to use the new SetCandidateRelease
    so installing packages from experimental or backports is easier
  - really do not show packages in the extra section if they were
    requested on the commandline, e.g. with a modifier (Closes: #184730)
* debian/control:
  - add Vcs-Browser now that loggerhead works again (Closes: #511168)
  - depend on debhelper 7 to raise compat level
  - depend on dpkg-dev (>= 1.15.8) to have c++ symbol mangling
* apt-pkg/contrib/fileutl.cc:
  - add a RealFileExists method and check that your configuration files
    are real files to avoid endless loops if not (Closes: #604401)
  - ignore non-regular files in GetListOfFilesInDir (Closes: #594694)
* apt-pkg/contrib/weakptr.h:
  - include stddefs.h to fix compile error (undefined NULL) with gcc-4.6
* methods/https.cc:
  - fix CURLOPT_SSL_VERIFYHOST by really passing 2 to it if enabled
* deb/dpkgpm.cc:
  - fix popen/fclose mismatch reported by cppcheck. Thanks to Petter
    Reinholdtsen for report and patch! (Closes: #607803)
* doc/apt.conf.5.xml:
  - fix multipl{y,e} spelling error reported by Jakub Wilk (Closes: #607636)
* apt-inst/contrib/extracttar.cc:
  - let apt-utils work with encoded tar headers if uid/gid are large.
    Thanks to Nobuhiro Hayashi for the patch! (Closes: #330162)
* apt-pkg/cacheiterator.h:
  - do not segfault if cache is not build (Closes: #254770)
* doc/apt-get.8.xml:
  - remove duplicated mentioning of --install-recommends
* doc/sources.list.5.xml:
  - remove obsolete references to non-us (Closes: #594495)
* debian/rules:
  - use -- instead of deprecated -u for dh_gencontrol
  - remove shlibs.local creation and usage
  - show differences in the symbol files, but never fail
* pre-build.sh:
  - remove as it is not needed for a working 'bzr bd'
* debian/{apt,apt-utils}.symbols:
  - ship experimental unmangled c++ symbol files
* methods/rred.cc:
  - operate optional on gzip compressed pdiffs
* apt-pkg/acquire-item.cc:
  - don't uncompress downloaded pdiff files before feeding it to rred
  - try downloading clearsigned InRelease before trying Release.gpg
  - change the internal handling of Extensions in pkgAcqIndex
  - add a special uncompressed compression type to prefer those files
  - download and use i18n/Index to choose which Translations to download
* cmdline/apt-key:
  - don't set trustdb-name as non-root so 'list' and 'finger'
    can be used without being root (Closes: #393005, #592107)
* apt-pkg/deb/deblistparser.cc:
  - rewrite LoadReleaseInfo to cope with clearsigned Releasefiles
* ftparchive/writer.cc:
  - add config option to search for more patterns in release command
  - include Index files by default in the Release file
* methods/{gzip,bzip}.cc:
  - print a good error message if FileSize() is zero
* apt-pkg/aptconfiguration.cc:
  - remove the inbuilt Translation files whitelist

5 files changed, 161 insertions, 82 deletions

diff --git a/apt-pkg/deb/debindexfile.cc b/apt-pkg/deb/debindexfile.cc
index af1209ccb..9961b5ae4 100644
--- a/apt-pkg/deb/debindexfile.cc
+++ b/apt-pkg/deb/debindexfile.cc
@@ -324,8 +324,14 @@ bool debPackagesIndex::Merge(pkgCacheGenerator &Gen,OpProgress *Prog) const
       return _error->Error("Problem with MergeList %s",PackageFile.c_str());
 
    // Check the release file
-   string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release");
+   string ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("InRelease");
+   bool releaseExists = false;
    if (FileExists(ReleaseFile) == true)
+      releaseExists = true;
+   else
+      ReleaseFile = debReleaseIndex(URI,Dist).MetaIndexFile("Release");
+
+   if (releaseExists == true || FileExists(ReleaseFile) == true)
    {
       FileFd Rel(ReleaseFile,FileFd::ReadOnly);
       if (_error->PendingError() == true)
diff --git a/apt-pkg/deb/deblistparser.cc b/apt-pkg/deb/deblistparser.cc
index 1b3bfd6ae..9201e6a54 100644
--- a/apt-pkg/deb/deblistparser.cc
+++ b/apt-pkg/deb/deblistparser.cc
@@ -783,45 +783,89 @@ bool debListParser::Step()
 bool debListParser::LoadReleaseInfo(pkgCache::PkgFileIterator &FileI,
 				    FileFd &File, string component)
 {
-   pkgTagFile Tags(&File, File.Size() + 256); // XXX
-   pkgTagSection Section;
-   if (Tags.Step(Section) == false)
-      return false;
-
-   // FIXME: Do we need it now for multi-arch?
-   // mvo: I don't think we need to fill that in (it's unused since apt-0.6)
-//    FileI->Architecture = WriteUniqString(Arch);
-   
    // apt-secure does no longer download individual (per-section) Release
    // file. to provide Component pinning we use the section name now
    FileI->Component = WriteUniqString(component);
 
-   const char *Start;
-   const char *Stop;
-   if (Section.Find("Suite",Start,Stop) == true)
-      FileI->Archive = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Component",Start,Stop) == true)
-      FileI->Component = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Version",Start,Stop) == true)
-      FileI->Version = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Origin",Start,Stop) == true)
-      FileI->Origin = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Codename",Start,Stop) == true)
-      FileI->Codename = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Label",Start,Stop) == true)
-      FileI->Label = WriteUniqString(Start,Stop - Start);
-   if (Section.Find("Architecture",Start,Stop) == true)
-      FileI->Architecture = WriteUniqString(Start,Stop - Start);
-   
-   if (Section.FindFlag("NotAutomatic",FileI->Flags,
-			pkgCache::Flag::NotAutomatic) == false)
-      _error->Warning("Bad NotAutomatic flag");
-   if (Section.FindFlag("ButAutomaticUpgrades",FileI->Flags,
-			pkgCache::Flag::ButAutomaticUpgrades) == false)
-      _error->Warning("Bad ButAutomaticUpgrades flag");
-   // overrule the NotAutomatic setting if needed as they are both present for compatibility
-   else if ((FileI->Flags & pkgCache::Flag::ButAutomaticUpgrades) == pkgCache::Flag::ButAutomaticUpgrades)
-      FileI->Flags &= ~pkgCache::Flag::NotAutomatic;
+   FILE* release = fdopen(dup(File.Fd()), "r");
+   if (release == NULL)
+      return false;
+
+   char buffer[101];
+   bool gpgClose = false;
+   while (fgets(buffer, sizeof(buffer), release) != NULL)
+   {
+      size_t len = 0;
+
+      // Skip empty lines
+      for (; buffer[len] == '\r' && buffer[len] == '\n'; ++len);
+      if (buffer[len] == '\0')
+	 continue;
+
+      // only evalute the first GPG section
+      if (strncmp("-----", buffer, 5) == 0)
+      {
+	 if (gpgClose == true)
+	    break;
+	 gpgClose = true;
+	 continue;
+      }
+
+      // seperate the tag from the data
+      for (; buffer[len] != ':' && buffer[len] != '\0'; ++len);
+      if (buffer[len] == '\0')
+	 continue;
+      char* dataStart = buffer + len;
+      for (++dataStart; *dataStart == ' '; ++dataStart);
+      char* dataEnd = dataStart;
+      for (++dataEnd; *dataEnd != '\0'; ++dataEnd);
+
+      // which datastorage need to be updated
+      map_ptrloc* writeTo = NULL;
+      if (buffer[0] == ' ')
+	 ;
+      #define APT_PARSER_WRITETO(X, Y) else if (strncmp(Y, buffer, len) == 0) writeTo = &X;
+      APT_PARSER_WRITETO(FileI->Archive, "Suite")
+      APT_PARSER_WRITETO(FileI->Component, "Component")
+      APT_PARSER_WRITETO(FileI->Version, "Version")
+      APT_PARSER_WRITETO(FileI->Origin, "Origin")
+      APT_PARSER_WRITETO(FileI->Codename, "Codename")
+      APT_PARSER_WRITETO(FileI->Label, "Label")
+      #undef APT_PARSER_WRITETO
+      #define APT_PARSER_FLAGIT(X) else if (strncmp(#X, buffer, len) == 0) \
+	 pkgTagSection::FindFlag(FileI->Flags, pkgCache::Flag:: X, dataStart, dataEnd-1);
+      APT_PARSER_FLAGIT(NotAutomatic)
+      APT_PARSER_FLAGIT(ButAutomaticUpgrades)
+      #undef APT_PARSER_FLAGIT
+
+      // load all data from the line and save it
+      string data;
+      if (writeTo != NULL)
+	 data.append(dataStart, dataEnd);
+      if (sizeof(buffer) - 1 == (dataEnd - buffer))
+      {
+	 while (fgets(buffer, sizeof(buffer), release) != NULL)
+	 {
+	    if (writeTo != NULL)
+	       data.append(buffer);
+	    if (strlen(buffer) != sizeof(buffer) - 1)
+	       break;
+	 }
+      }
+      if (writeTo != NULL)
+      {
+	 // remove spaces and stuff from the end of the data line
+	 for (std::string::reverse_iterator s = data.rbegin();
+	      s != data.rend(); ++s)
+	 {
+	    if (*s != '\r' && *s != '\n' && *s != ' ')
+	       break;
+	    *s = '\0';
+	 }
+	 *writeTo = WriteUniqString(data);
+      }
+   }
+   fclose(release);
 
    return !_error->PendingError();
 }
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index 717d0bcde..a6edab6b9 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -119,6 +119,29 @@ string debReleaseIndex::SourceIndexURI(const char *Type, const string &Section)
       return URI + "dists/" + Dist + "/" + SourceIndexURISuffix(Type, Section);
 }
 
+string debReleaseIndex::TranslationIndexURISuffix(const char *Type, const string &Section) const
+{
+   string Res ="";
+   if (Dist[Dist.size() - 1] != '/')
+      Res += Section + "/i18n/";
+   return Res + Type;
+}
+
+string debReleaseIndex::TranslationIndexURI(const char *Type, const string &Section) const
+{
+   string Res;
+   if (Dist[Dist.size() - 1] == '/')
+   {
+      if (Dist != "/")
+         Res = URI + Dist;
+      else 
+         Res = URI;
+      return Res + Type;
+   }
+   else
+      return URI + "dists/" + Dist + "/" + TranslationIndexURISuffix(Type, Section);
+}
+
 debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) {
 	this->URI = URI;
 	this->Dist = Dist;
@@ -155,6 +178,7 @@ vector <struct IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
 	if (IndexTargets->empty() == false && ArchEntries.size() == 1)
 		return IndexTargets;
 
+	std::set<std::string> sections;
 	for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
 	     a != ArchEntries.end(); ++a) {
 		if (a->first == "source")
@@ -167,6 +191,37 @@ vector <struct IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
 			Target->URI = IndexURI(Target->ShortDesc.c_str(), (*I)->Section, a->first);
 			Target->Description = Info (Target->ShortDesc.c_str(), (*I)->Section, a->first);
 			IndexTargets->push_back (Target);
+			sections.insert((*I)->Section);
+		}
+	}
+
+	// get the Translations:
+	// - if its a dists-style repository get the i18n/Index first
+	// - if its flat try to acquire files by guessing
+	if (Dist[Dist.size() - 1] == '/') {
+		std::vector<std::string> const lang = APT::Configuration::getLanguages(true);
+		for (std::set<std::string>::const_iterator s = sections.begin();
+		     s != sections.end(); ++s) {
+			for (std::vector<std::string>::const_iterator l = lang.begin();
+			     l != lang.end(); l++) {
+				if (*l == "none") continue;
+				IndexTarget * Target = new OptionalIndexTarget();
+				Target->ShortDesc = "Translation-" + *l;
+				Target->MetaKey = TranslationIndexURISuffix(l->c_str(), *s);
+				Target->URI = TranslationIndexURI(l->c_str(), *s);
+				Target->Description = Info (Target->ShortDesc.c_str(), *s);
+				IndexTargets->push_back(Target);
+			}
+		}
+	} else {
+		for (std::set<std::string>::const_iterator s = sections.begin();
+		     s != sections.end(); ++s) {
+			IndexTarget * Target = new OptionalIndexTarget();
+			Target->ShortDesc = "TranslationIndex";
+			Target->MetaKey = TranslationIndexURISuffix("Index", *s);
+			Target->URI = TranslationIndexURI("Index", *s);
+			Target->Description = Info (Target->ShortDesc.c_str(), *s);
+			IndexTargets->push_back (Target);
 		}
 	}
 
@@ -182,58 +237,34 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 	 new pkgAcqIndex(Owner, (*Target)->URI, (*Target)->Description,
 			 (*Target)->ShortDesc, HashString());
       }
-      // this is normally created in pkgAcqMetaSig, but if we run
-      // in --print-uris mode, we add it here
-      new pkgAcqMetaIndex(Owner, MetaIndexURI("Release"),
-		     MetaIndexInfo("Release"), "Release",
-		     MetaIndexURI("Release.gpg"), 
-		     ComputeIndexTargets(),
-		     new indexRecords (Dist));
-
    }
 
-   new pkgAcqMetaSig(Owner, MetaIndexURI("Release.gpg"),
-		     MetaIndexInfo("Release.gpg"), "Release.gpg",
-		     MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
-		     ComputeIndexTargets(),
-		     new indexRecords (Dist));
-
-	// Queue the translations
-	std::vector<std::string> const lang = APT::Configuration::getLanguages(true);
-	map<string, set<string> > sections;
-	for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
-	     a != ArchEntries.end(); ++a) {
-		if (a->first == "source")
-			continue;
-		for (vector<debSectionEntry const*>::const_iterator I = a->second.begin();
-		     I != a->second.end(); I++)
-			sections[(*I)->Section].insert(lang.begin(), lang.end());
-	}
-
-	for (map<string, set<string> >::const_iterator s = sections.begin();
-	     s != sections.end(); ++s)
-		for (set<string>::const_iterator l = s->second.begin();
-		     l != s->second.end(); l++) {
-			if (*l == "none") continue;
-			debTranslationsIndex i = debTranslationsIndex(URI,Dist,s->first,(*l).c_str());
-			i.GetIndexes(Owner);
-		}
+	new pkgAcqMetaClearSig(Owner, MetaIndexURI("InRelease"),
+		MetaIndexInfo("InRelease"), "InRelease",
+		MetaIndexURI("Release"), MetaIndexInfo("Release"), "Release",
+		MetaIndexURI("Release.gpg"), MetaIndexInfo("Release.gpg"), "Release.gpg",
+		ComputeIndexTargets(),
+		new indexRecords (Dist));
 
 	return true;
 }
 
 bool debReleaseIndex::IsTrusted() const
 {
-   string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
-      URItoFileName(MetaIndexURI("Release")) + ".gpg";
-   
    if(_config->FindB("APT::Authentication::TrustCDROM", false))
       if(URI.substr(0,strlen("cdrom:")) == "cdrom:")
 	 return true;
-   
+
+   string VerifiedSigFile = _config->FindDir("Dir::State::lists") +
+      URItoFileName(MetaIndexURI("Release")) + ".gpg";
+
    if (FileExists(VerifiedSigFile))
       return true;
-   return false;
+
+   VerifiedSigFile = _config->FindDir("Dir::State::lists") +
+      URItoFileName(MetaIndexURI("InRelease"));
+
+   return FileExists(VerifiedSigFile);
 }
 
 vector <pkgIndexFile *> *debReleaseIndex::GetIndexFiles() {
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 360fa5419..1561c6e00 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -37,6 +37,8 @@ class debReleaseIndex : public metaIndex {
    string IndexURISuffix(const char *Type, string const &Section, string const &Arch="native") const;
    string SourceIndexURI(const char *Type, const string &Section) const;
    string SourceIndexURISuffix(const char *Type, const string &Section) const;
+   string TranslationIndexURI(const char *Type, const string &Section) const;
+   string TranslationIndexURISuffix(const char *Type, const string &Section) const;
    virtual vector <pkgIndexFile *> *GetIndexFiles();
 
    virtual bool IsTrusted() const;
diff --git a/apt-pkg/deb/dpkgpm.cc b/apt-pkg/deb/dpkgpm.cc
index 7e8ee5b05..7d0d34a46 100644
--- a/apt-pkg/deb/dpkgpm.cc
+++ b/apt-pkg/deb/dpkgpm.cc
@@ -322,7 +322,6 @@ bool pkgDPkgPM::RunScriptsWithPkgs(const char *Cnf)
 	 return _error->Errno("fdopen","Faild to open new FD");
       
       // Feed it the filenames.
-      bool Die = false;
       if (Version <= 1)
       {
 	 for (vector<Item>::iterator I = List.begin(); I != List.end(); I++)
@@ -339,14 +338,11 @@ bool pkgDPkgPM::RunScriptsWithPkgs(const char *Cnf)
 	       into the pipe. */
 	    fprintf(F,"%s\n",I->File.c_str());
 	    if (ferror(F) != 0)
-	    {
-	       Die = true;
 	       break;
-	    }
 	 }
       }
       else
-	 Die = !SendV2Pkgs(F);
+	 SendV2Pkgs(F);
 
       fclose(F);
       
@@ -1415,7 +1411,7 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg)
       {
 	 while( fgets(buf, sizeof(buf), log) != NULL)
 	    fprintf(report, " %s", buf);
-	 fclose(log);
+	 pclose(log);
       }
    }
 
@@ -1431,7 +1427,7 @@ void pkgDPkgPM::WriteApportReport(const char *pkgpath, const char *errormsg)
       {
 	 while( fgets(buf, sizeof(buf), log) != NULL)
 	    fprintf(report, " %s", buf);
-	 fclose(log);
+	 pclose(log);
       }
    }