- download and use i18n/Index to choose which Translations to download

* apt-pkg/aptconfiguration.cc:
  - remove the inbuilt Translation files whitelist

5 files changed, 322 insertions, 82 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index a603a3d70..776009493 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -184,6 +184,153 @@ void pkgAcquire::Item::ReportMirrorFailure(string FailCode)
    }
 }
 									/*}}}*/
+// AcqSubIndex::AcqSubIndex - Constructor				/*{{{*/
+// ---------------------------------------------------------------------
+/* Get the DiffIndex file first and see if there are patches availabe 
+ * If so, create a pkgAcqIndexDiffs fetcher that will get and apply the
+ * patches. If anything goes wrong in that process, it will fall back to
+ * the original packages file
+ */
+pkgAcqSubIndex::pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,
+				 string const &URIDesc, string const &ShortDesc,
+				 HashString const &ExpectedHash)
+   : Item(Owner), ExpectedHash(ExpectedHash)
+{
+   Debug = _config->FindB("Debug::pkgAcquire::SubIndex",false);
+
+   DestFile = _config->FindDir("Dir::State::lists") + "partial/";
+   DestFile += URItoFileName(URI);
+
+   Desc.URI = URI;
+   Desc.Description = URIDesc;
+   Desc.Owner = this;
+   Desc.ShortDesc = ShortDesc;
+
+   QueueURI(Desc);
+
+   if(Debug)
+      std::clog << "pkgAcqSubIndex: " << Desc.URI << std::endl;
+}
+									/*}}}*/
+// AcqSubIndex::Custom600Headers - Insert custom request headers	/*{{{*/
+// ---------------------------------------------------------------------
+/* The only header we use is the last-modified header. */
+string pkgAcqSubIndex::Custom600Headers()
+{
+   string Final = _config->FindDir("Dir::State::lists");
+   Final += URItoFileName(Desc.URI);
+
+   struct stat Buf;
+   if (stat(Final.c_str(),&Buf) != 0)
+      return "\nIndex-File: true";
+   return "\nIndex-File: true\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
+}
+									/*}}}*/
+void pkgAcqSubIndex::Failed(string Message,pkgAcquire::MethodConfig *Cnf)	/*{{{*/
+{
+   if(Debug)
+      std::clog << "pkgAcqSubIndex failed: " << Desc.URI << std::endl;
+
+   Complete = false;
+   Status = StatDone;
+   Dequeue();
+
+   // No good Index is provided, so try guessing
+   std::vector<std::string> langs = APT::Configuration::getLanguages(true);
+   for (std::vector<std::string>::const_iterator l = langs.begin();
+	l != langs.end(); ++l)
+   {
+      if (*l == "none") continue;
+      string const file = "Translation-" + *l;
+      new pkgAcqIndexTrans(Owner, Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file),
+		Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file),
+		file);
+   }
+}
+									/*}}}*/
+void pkgAcqSubIndex::Done(string Message,unsigned long Size,string Md5Hash,	/*{{{*/
+			   pkgAcquire::MethodConfig *Cnf)
+{
+   if(Debug)
+      std::clog << "pkgAcqSubIndex::Done(): " << Desc.URI << std::endl;
+
+   string FileName = LookupTag(Message,"Filename");
+   if (FileName.empty() == true)
+   {
+      Status = StatError;
+      ErrorText = "Method gave a blank filename";
+      return;
+   }
+
+   if (FileName != DestFile)
+   {
+      Local = true;
+      Desc.URI = "copy:" + FileName;
+      QueueURI(Desc);
+      return;
+   }
+
+   Item::Done(Message,Size,Md5Hash,Cnf);
+
+   string FinalFile = _config->FindDir("Dir::State::lists")+URItoFileName(Desc.URI);
+
+   // sucess in downloading the index
+   // rename the index
+   if(Debug)
+      std::clog << "Renaming: " << DestFile << " -> " << FinalFile << std::endl;
+   Rename(DestFile,FinalFile);
+   chmod(FinalFile.c_str(),0644);
+   DestFile = FinalFile;
+
+   if(ParseIndex(DestFile) == false)
+      return Failed("", NULL);
+
+   Complete = true;
+   Status = StatDone;
+   Dequeue();
+   return;
+}
+									/*}}}*/
+bool pkgAcqSubIndex::ParseIndex(string const &IndexFile)		/*{{{*/
+{
+   indexRecords SubIndexParser;
+   if (FileExists(IndexFile) == false || SubIndexParser.Load(IndexFile) == false)
+      return false;
+
+   std::vector<std::string> lang = APT::Configuration::getLanguages(true);
+   for (std::vector<std::string>::const_iterator l = lang.begin();
+	l != lang.end(); ++l)
+   {
+      if (*l == "none")
+	 continue;
+
+      string file = "Translation-" + *l;
+      indexRecords::checkSum const *Record = SubIndexParser.Lookup(file);
+      HashString expected;
+      if (Record == NULL)
+      {
+	 // FIXME: the Index file provided by debian currently only includes bz2 records
+	 Record = SubIndexParser.Lookup(file + ".bz2");
+	 if (Record == NULL)
+	    continue;
+      }
+      else
+      {
+	 expected = Record->Hash;
+	 if (expected.empty() == true)
+	    continue;
+      }
+
+      IndexTarget target;
+      target.Description = Desc.Description.erase(Desc.Description.rfind(' ')+1).append(file);
+      target.MetaKey = file;
+      target.ShortDesc = file;
+      target.URI = Desc.URI.substr(0, Desc.URI.rfind('/')+1).append(file);
+      new pkgAcqIndexTrans(Owner, &target, expected, &SubIndexParser);
+   }
+   return true;
+}
+									/*}}}*/
 // AcqDiffIndex::AcqDiffIndex - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* Get the DiffIndex file first and see if there are patches availabe 
@@ -842,6 +989,11 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner,
   : pkgAcqIndex(Owner, URI, URIDesc, ShortDesc, HashString(), "")
 {
 }
+pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target,
+			 HashString const &ExpectedHash, indexRecords const *MetaIndexParser)
+  : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser)
+{
+}
 									/*}}}*/
 // AcqIndexTrans::Custom600Headers - Insert custom request headers	/*{{{*/
 // ---------------------------------------------------------------------
@@ -1182,27 +1334,41 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify)				/*{{{*/
       HashString ExpectedIndexHash;
       if (verify)
       {
-         const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
-         if (!Record)
-         {
-            Status = StatAuthError;
-            ErrorText = "Unable to find expected entry  "
-               + (*Target)->MetaKey + " in Meta-index file (malformed Release file?)";
-            return;
-         }
-         ExpectedIndexHash = Record->Hash;
-         if (_config->FindB("Debug::pkgAcquire::Auth", false))
-         {
-            std::cerr << "Queueing: " << (*Target)->URI << std::endl;
-            std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
-         }
-         if (ExpectedIndexHash.empty())
-         {
-            Status = StatAuthError;
-            ErrorText = "Unable to find hash sum for "
-               + (*Target)->MetaKey + " in Meta-index file";
-            return;
-         }
+	 const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey);
+	 if (Record == NULL)
+	 {
+	    if ((*Target)->IsOptional() == false)
+	    {
+	       Status = StatAuthError;
+	       strprintf(ErrorText, _("Unable to find expected entry '%s' in Release file (Wrong sources.list entry or malformed file)"), (*Target)->MetaKey.c_str());
+	       return;
+	    }
+	 }
+	 else
+	 {
+	    ExpectedIndexHash = Record->Hash;
+	    if (_config->FindB("Debug::pkgAcquire::Auth", false))
+	    {
+	       std::cerr << "Queueing: " << (*Target)->URI << std::endl;
+	       std::cerr << "Expected Hash: " << ExpectedIndexHash.toStr() << std::endl;
+	    }
+	    if (ExpectedIndexHash.empty() == true && (*Target)->IsOptional() == false)
+	    {
+	       Status = StatAuthError;
+	       strprintf(ErrorText, _("Unable to find hash sum for '%s' in Release file"), (*Target)->MetaKey.c_str());
+	       return;
+	    }
+	 }
+      }
+
+      if ((*Target)->IsOptional() == true)
+      {
+	 if ((*Target)->IsSubIndex() == true)
+	    new pkgAcqSubIndex(Owner, (*Target)->URI, (*Target)->Description,
+				(*Target)->ShortDesc, ExpectedIndexHash);
+	 else
+	    new pkgAcqIndexTrans(Owner, *Target, ExpectedIndexHash, MetaIndexParser);
+	 continue;
       }
 
       /* Queue Packages file (either diff or full packages files, depending
@@ -1836,3 +2002,13 @@ string pkgAcqFile::Custom600Headers()
    return "";
 }
 									/*}}}*/
+bool IndexTarget::IsOptional() const {
+   if (strncmp(ShortDesc.c_str(), "Translation", 11) != 0)
+      return false;
+   return true;
+}
+bool IndexTarget::IsSubIndex() const {
+   if (ShortDesc != "TranslationIndex")
+      return false;
+   return true;
+}
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 92098e3d4..9bcc32f21 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -287,6 +287,50 @@ struct DiffInfo {
    unsigned long size;
 };
 									/*}}}*/
+/** \brief An item that is responsible for fetching a SubIndex		{{{
+ *
+ *  The MetaIndex file includes only records for important indexes
+ *  and records for these SubIndex files so these can carry records
+ *  for addition files like PDiffs and Translations
+ */
+class pkgAcqSubIndex : public pkgAcquire::Item
+{
+ protected:
+   /** \brief If \b true, debugging information will be written to std::clog. */
+   bool Debug;
+
+   /** \brief The item that is currently being downloaded. */
+   pkgAcquire::ItemDesc Desc;
+
+   /** \brief The Hash that this file should have after download
+    */
+   HashString ExpectedHash;
+
+ public:
+   // Specialized action members
+   virtual void Failed(string Message,pkgAcquire::MethodConfig *Cnf);
+   virtual void Done(string Message,unsigned long Size,string Md5Hash,
+		     pkgAcquire::MethodConfig *Cnf);
+   virtual string DescURI() {return Desc.URI;};
+   virtual string Custom600Headers();
+   virtual bool ParseIndex(string const &IndexFile);
+
+   /** \brief Create a new pkgAcqDiffIndex.
+    *
+    *  \param Owner The Acquire object that owns this item.
+    *
+    *  \param URI The URI of the list file to download.
+    *
+    *  \param URIDesc A long description of the list file to download.
+    *
+    *  \param ShortDesc A short description of the list file to download.
+    *
+    *  \param ExpectedHash The list file's MD5 signature.
+    */
+   pkgAcqSubIndex(pkgAcquire *Owner, string const &URI,string const &URIDesc,
+		   string const &ShortDesc, HashString const &ExpectedHash);
+};
+									/*}}}*/
 /** \brief An item that is responsible for fetching an index file of	{{{
  *  package list diffs and starting the package list's download.
  *
@@ -597,6 +641,8 @@ class pkgAcqIndexTrans : public pkgAcqIndex
     */
    pkgAcqIndexTrans(pkgAcquire *Owner,string URI,string URIDesc,
 		    string ShortDesc);
+   pkgAcqIndexTrans(pkgAcquire *Owner, struct IndexTarget const * const Target,
+		    HashString const &ExpectedHash, indexRecords const *MetaIndexParser);
 };
 									/*}}}*/
 /** \brief Information about an index file. */				/*{{{*/
@@ -615,8 +661,18 @@ struct IndexTarget
     *  looked up within the meta signature file.
     */
    string MetaKey;
+
+   //FIXME: We should use virtual methods here instead…
+   bool IsOptional() const;
+   bool IsSubIndex() const;
 };
 									/*}}}*/
+/** \brief Information about an optional index file. */			/*{{{*/
+struct OptionalIndexTarget : public IndexTarget
+{
+};
+									/*}}}*/
+
 /** \brief An acquire item that downloads the detached signature	{{{
  *  of a meta-index (Release) file, then queues up the release
  *  file itself.
diff --git a/apt-pkg/aptconfiguration.cc b/apt-pkg/aptconfiguration.cc
index e97ebfed7..3cf4d2429 100644
--- a/apt-pkg/aptconfiguration.cc
+++ b/apt-pkg/aptconfiguration.cc
@@ -163,33 +163,6 @@ std::vector<std::string> const Configuration::getLanguages(bool const &All,
 	}
 	closedir(D);
 
-	// get the environment language codes: LC_MESSAGES (and later LANGUAGE)
-	// we extract both, a long and a short code and then we will
-	// check if we actually need both (rare) or if the short is enough
-	string const envMsg = string(Locale == 0 ? std::setlocale(LC_MESSAGES, NULL) : *Locale);
-	size_t const lenShort = (envMsg.find('_') != string::npos) ? envMsg.find('_') : 2;
-	size_t const lenLong = (envMsg.find_first_of(".@") != string::npos) ? envMsg.find_first_of(".@") : (lenShort + 3);
-
-	string envLong = envMsg.substr(0,lenLong);
-	string const envShort = envLong.substr(0,lenShort);
-	bool envLongIncluded = true;
-
-	// to save the servers from unneeded queries, we only try also long codes
-	// for languages it is realistic to have a long code translation file…
-	// TODO: Improve translation acquire system to drop them dynamic
-	char const *needLong[] = { "cs", "en", "pt", "sv", "zh", NULL };
-	if (envLong != envShort) {
-		for (char const **l = needLong; *l != NULL; l++)
-			if (envShort.compare(*l) == 0) {
-				envLongIncluded = false;
-				break;
-			}
-	}
-
-	// we don't add the long code, but we allow the user to do so
-	if (envLongIncluded == true)
-		envLong.clear();
-
 	// FIXME: Remove support for the old APT::Acquire::Translation
 	// it was undocumented and so it should be not very widthly used
 	string const oldAcquire = _config->Find("APT::Acquire::Translation","");
@@ -211,12 +184,22 @@ std::vector<std::string> const Configuration::getLanguages(bool const &All,
 			return codes;
 	}
 
-	// It is very likely we will need to environment codes later,
+	// get the environment language codes: LC_MESSAGES (and later LANGUAGE)
+	// we extract both, a long and a short code and then we will
+	// check if we actually need both (rare) or if the short is enough
+	string const envMsg = string(Locale == 0 ? std::setlocale(LC_MESSAGES, NULL) : *Locale);
+	size_t const lenShort = (envMsg.find('_') != string::npos) ? envMsg.find('_') : 2;
+	size_t const lenLong = (envMsg.find_first_of(".@") != string::npos) ? envMsg.find_first_of(".@") : (lenShort + 3);
+
+	string const envLong = envMsg.substr(0,lenLong);
+	string const envShort = envLong.substr(0,lenShort);
+
+	// It is very likely we will need the environment codes later,
 	// so let us generate them now from LC_MESSAGES and LANGUAGE
 	std::vector<string> environment;
 	if (envShort != "C") {
 		// take care of LC_MESSAGES
-		if (envLongIncluded == false)
+		if (envLong != envShort)
 			environment.push_back(envLong);
 		environment.push_back(envShort);
 		// take care of LANGUAGE
@@ -233,16 +216,6 @@ std::vector<std::string> const Configuration::getLanguages(bool const &All,
 					continue;
 				if (std::find(environment.begin(), environment.end(), *e) != environment.end())
 					continue;
-				if (e->find('_') != string::npos) {
-					// Drop LongCodes here - ShortCodes are also included
-					string const shorty = e->substr(0, e->find('_'));
-					char const **n = needLong;
-					for (; *n != NULL; ++n)
-						if (shorty == *n)
-							break;
-					if (*n == NULL)
-						continue;
-				}
 				++addedLangs;
 				environment.push_back(*e);
 			}
diff --git a/apt-pkg/deb/debmetaindex.cc b/apt-pkg/deb/debmetaindex.cc
index e2c680b14..a6edab6b9 100644
--- a/apt-pkg/deb/debmetaindex.cc
+++ b/apt-pkg/deb/debmetaindex.cc
@@ -119,6 +119,29 @@ string debReleaseIndex::SourceIndexURI(const char *Type, const string &Section)
       return URI + "dists/" + Dist + "/" + SourceIndexURISuffix(Type, Section);
 }
 
+string debReleaseIndex::TranslationIndexURISuffix(const char *Type, const string &Section) const
+{
+   string Res ="";
+   if (Dist[Dist.size() - 1] != '/')
+      Res += Section + "/i18n/";
+   return Res + Type;
+}
+
+string debReleaseIndex::TranslationIndexURI(const char *Type, const string &Section) const
+{
+   string Res;
+   if (Dist[Dist.size() - 1] == '/')
+   {
+      if (Dist != "/")
+         Res = URI + Dist;
+      else 
+         Res = URI;
+      return Res + Type;
+   }
+   else
+      return URI + "dists/" + Dist + "/" + TranslationIndexURISuffix(Type, Section);
+}
+
 debReleaseIndex::debReleaseIndex(string const &URI, string const &Dist) {
 	this->URI = URI;
 	this->Dist = Dist;
@@ -155,6 +178,7 @@ vector <struct IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
 	if (IndexTargets->empty() == false && ArchEntries.size() == 1)
 		return IndexTargets;
 
+	std::set<std::string> sections;
 	for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
 	     a != ArchEntries.end(); ++a) {
 		if (a->first == "source")
@@ -167,6 +191,37 @@ vector <struct IndexTarget *>* debReleaseIndex::ComputeIndexTargets() const {
 			Target->URI = IndexURI(Target->ShortDesc.c_str(), (*I)->Section, a->first);
 			Target->Description = Info (Target->ShortDesc.c_str(), (*I)->Section, a->first);
 			IndexTargets->push_back (Target);
+			sections.insert((*I)->Section);
+		}
+	}
+
+	// get the Translations:
+	// - if its a dists-style repository get the i18n/Index first
+	// - if its flat try to acquire files by guessing
+	if (Dist[Dist.size() - 1] == '/') {
+		std::vector<std::string> const lang = APT::Configuration::getLanguages(true);
+		for (std::set<std::string>::const_iterator s = sections.begin();
+		     s != sections.end(); ++s) {
+			for (std::vector<std::string>::const_iterator l = lang.begin();
+			     l != lang.end(); l++) {
+				if (*l == "none") continue;
+				IndexTarget * Target = new OptionalIndexTarget();
+				Target->ShortDesc = "Translation-" + *l;
+				Target->MetaKey = TranslationIndexURISuffix(l->c_str(), *s);
+				Target->URI = TranslationIndexURI(l->c_str(), *s);
+				Target->Description = Info (Target->ShortDesc.c_str(), *s);
+				IndexTargets->push_back(Target);
+			}
+		}
+	} else {
+		for (std::set<std::string>::const_iterator s = sections.begin();
+		     s != sections.end(); ++s) {
+			IndexTarget * Target = new OptionalIndexTarget();
+			Target->ShortDesc = "TranslationIndex";
+			Target->MetaKey = TranslationIndexURISuffix("Index", *s);
+			Target->URI = TranslationIndexURI("Index", *s);
+			Target->Description = Info (Target->ShortDesc.c_str(), *s);
+			IndexTargets->push_back (Target);
 		}
 	}
 
@@ -191,28 +246,6 @@ bool debReleaseIndex::GetIndexes(pkgAcquire *Owner, bool const &GetAll) const
 		ComputeIndexTargets(),
 		new indexRecords (Dist));
 
-
-	// Queue the translations
-	std::vector<std::string> const lang = APT::Configuration::getLanguages(true);
-	map<string, set<string> > sections;
-	for (map<string, vector<debSectionEntry const*> >::const_iterator a = ArchEntries.begin();
-	     a != ArchEntries.end(); ++a) {
-		if (a->first == "source")
-			continue;
-		for (vector<debSectionEntry const*>::const_iterator I = a->second.begin();
-		     I != a->second.end(); I++)
-			sections[(*I)->Section].insert(lang.begin(), lang.end());
-	}
-
-	for (map<string, set<string> >::const_iterator s = sections.begin();
-	     s != sections.end(); ++s)
-		for (set<string>::const_iterator l = s->second.begin();
-		     l != s->second.end(); l++) {
-			if (*l == "none") continue;
-			debTranslationsIndex i = debTranslationsIndex(URI,Dist,s->first,(*l).c_str());
-			i.GetIndexes(Owner);
-		}
-
 	return true;
 }
 
diff --git a/apt-pkg/deb/debmetaindex.h b/apt-pkg/deb/debmetaindex.h
index 360fa5419..1561c6e00 100644
--- a/apt-pkg/deb/debmetaindex.h
+++ b/apt-pkg/deb/debmetaindex.h
@@ -37,6 +37,8 @@ class debReleaseIndex : public metaIndex {
    string IndexURISuffix(const char *Type, string const &Section, string const &Arch="native") const;
    string SourceIndexURI(const char *Type, const string &Section) const;
    string SourceIndexURISuffix(const char *Type, const string &Section) const;
+   string TranslationIndexURI(const char *Type, const string &Section) const;
+   string TranslationIndexURISuffix(const char *Type, const string &Section) const;
    virtual vector <pkgIndexFile *> *GetIndexFiles();
 
    virtual bool IsTrusted() const;