Stable acquire code

Author: jgg
Date: 1998-10-26 07:11:43 GMT
Stable acquire code

8 files changed, 141 insertions, 25 deletions

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index e6eee197b..4435e2553 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-item.cc,v 1.4 1998/10/24 04:57:56 jgg Exp $
+// $Id: acquire-item.cc,v 1.5 1998/10/26 07:11:43 jgg Exp $
 /* ######################################################################
 
    Acquire Item - Item to acquire
@@ -65,6 +65,22 @@ void pkgAcquire::Item::Done(string,unsigned long,string)
    Owner->Dequeue(this);
 }
 									/*}}}*/
+// Acquire::Item::Rename - Rename a file				/*{{{*/
+// ---------------------------------------------------------------------
+/* This helper function is used by alot of item methods as thier final
+   step */
+void pkgAcquire::Item::Rename(string From,string To)
+{
+   if (rename(From.c_str(),To.c_str()) != 0)
+   {
+      char S[300];
+      sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno),
+	      From.c_str(),To.c_str());
+      Status = StatError;
+      ErrorText = S;
+   }      
+}
+									/*}}}*/
 
 // AcqIndex::AcqIndex - Constructor					/*{{{*/
 // ---------------------------------------------------------------------
@@ -73,6 +89,8 @@ void pkgAcquire::Item::Done(string,unsigned long,string)
 pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location) :
              Item(Owner), Location(Location)
 {
+   Decompression = false;
+   
    DestFile = _config->FindDir("Dir::State::lists") + "partial/";
    DestFile += URItoFileName(Location->PackagesURI());
    
@@ -97,6 +115,57 @@ string pkgAcqIndex::Custom600Headers()
    return "\nLast-Modified: " + TimeRFC1123(Buf.st_mtime);
 }
 									/*}}}*/
+// AcqIndex::Done - Finished a fetch					/*{{{*/
+// ---------------------------------------------------------------------
+/* This goes through a number of states.. On the initial fetch the
+   method could possibly return an alternate filename which points
+   to the uncompressed version of the file. If this is so the file
+   is copied into the partial directory. In all other cases the file
+   is decompressed with a gzip uri. */
+void pkgAcqIndex::Done(string Message,unsigned long Size,string MD5)
+{
+   Item::Done(Message,Size,MD5);
+
+   if (Decompression == true)
+   {
+      // Done, move it into position
+      string FinalFile = _config->FindDir("Dir::State::lists");
+      FinalFile += URItoFileName(Location->PackagesURI());
+      Rename(DestFile,FinalFile);
+      return;
+   }
+      
+   // Handle the unzipd case
+   string FileName = LookupTag(Message,"Alt-Filename");
+   if (FileName.empty() == false)
+   {
+      // The files timestamp matches
+      if (StringToBool(LookupTag(Message,"Alt-IMS-Hit"),false) == true)
+	 return;
+      
+      Decompression = true;
+      DestFile += ".decomp";
+      QueueURI("copy:" + FileName,string());
+      return;
+   }
+
+   FileName = LookupTag(Message,"Filename");
+   if (FileName.empty() == true)
+   {
+      Status = StatError;
+      ErrorText = "Method gave a blank filename";
+   }
+   
+   // The files timestamp matches
+   if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
+      return;
+   
+   Decompression = true;
+   DestFile += ".decomp";
+   QueueURI("gzip:" + FileName,string());
+}
+									/*}}}*/
+
 // AcqIndexRel::pkgAcqIndexRel - Constructor				/*{{{*/
 // ---------------------------------------------------------------------
 /* The Release file is added to the queue */
@@ -139,7 +208,12 @@ void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5)
    {
       Status = StatError;
       ErrorText = "Method gave a blank filename";
+      return;
    }
+
+   // The files timestamp matches
+   if (StringToBool(LookupTag(Message,"IMS-Hit"),false) == true)
+      return;
    
    // We have to copy it into place
    if (FileName != DestFile)
@@ -151,14 +225,6 @@ void pkgAcqIndexRel::Done(string Message,unsigned long Size,string MD5)
    // Done, move it into position
    string FinalFile = _config->FindDir("Dir::State::lists");
    FinalFile += URItoFileName(Location->ReleaseURI());
-   
-   if (rename(DestFile.c_str(),FinalFile.c_str()) != 0)
-   {
-      char S[300];
-      sprintf(S,"rename failed, %s (%s -> %s).",strerror(errno),
-	      DestFile.c_str(),FinalFile.c_str());
-      Status = StatError;
-      ErrorText = S;
-   }
+   Rename(DestFile,FinalFile);
 }
 									/*}}}*/
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 352fc3e24..bdd4d3581 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-item.h,v 1.3 1998/10/24 04:57:57 jgg Exp $
+// $Id: acquire-item.h,v 1.4 1998/10/26 07:11:44 jgg Exp $
 /* ######################################################################
 
    Acquire Item - Item to acquire
@@ -33,6 +33,8 @@ class pkgAcquire::Item
    inline void QueueURI(string URI,string Description) 
                  {Owner->Enqueue(this,URI,Description);};
    
+   void Rename(string From,string To);
+   
    public:
 
    // State of the item
@@ -60,9 +62,11 @@ class pkgAcqIndex : public pkgAcquire::Item
    protected:
    
    const pkgSourceList::Item *Location;
+   bool Decompression;
    
    public:
    
+   virtual void Done(string Message,unsigned long Size,string Md5Hash);   
    virtual string Custom600Headers();
 
    pkgAcqIndex(pkgAcquire *Owner,const pkgSourceList::Item *Location);
@@ -77,8 +81,7 @@ class pkgAcqIndexRel : public pkgAcquire::Item
    
    public:
    
-   virtual void Done(string Message,unsigned long Size,string Md5Hash);
-   
+   virtual void Done(string Message,unsigned long Size,string Md5Hash);   
    virtual string Custom600Headers();
    
    pkgAcqIndexRel(pkgAcquire *Owner,const pkgSourceList::Item *Location);
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 936d469c3..392c8ca47 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-worker.cc,v 1.6 1998/10/24 04:57:58 jgg Exp $
+// $Id: acquire-worker.cc,v 1.7 1998/10/26 07:11:45 jgg Exp $
 /* ######################################################################
 
    Acquire Worker 
@@ -106,7 +106,9 @@ bool pkgAcquire::Worker::Start()
 	 close(Pipes[I]);
       return false;
    }
-      
+   for (int I = 0; I != 4; I++)
+      SetCloseExec(Pipes[0],true);
+   
    // Fork off the process
    Process = fork();
    if (Process < 0)
@@ -122,8 +124,6 @@ bool pkgAcquire::Worker::Start()
       dup2(Pipes[1],STDOUT_FILENO);
       dup2(Pipes[2],STDIN_FILENO);
       dup2(((filebuf *)clog.rdbuf())->fd(),STDERR_FILENO);
-      for (int I = 0; I != 4; I++)
-	 close(Pipes[I]);
       SetCloseExec(STDOUT_FILENO,false);
       SetCloseExec(STDIN_FILENO,false);      
       SetCloseExec(STDERR_FILENO,false);
@@ -152,7 +152,8 @@ bool pkgAcquire::Worker::Start()
       return _error->Error("Method %s did not start correctly",Method.c_str());
 
    RunMessages();
-   SendConfiguration();
+   if (OwnerQ != 0)
+      SendConfiguration();
    
    return true;
 }
@@ -416,3 +417,35 @@ bool pkgAcquire::Worker::MethodFailure()
    return false;
 }
 									/*}}}*/
+
+// InjectConfiguration - Configuration aid for methods			/*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool pkgInjectConfiguration(string &Message,Configuration &Cnf)
+{
+   const char *I = Message.begin();
+   
+   unsigned int Length = strlen("Config-Item");
+   for (; I + Length < Message.end(); I++)
+   {
+      // Not a config item
+      if (I[Length] != ':' || stringcasecmp(I,I+Length,"Config-Item") != 0)
+	 continue;
+      
+      I += Length + 1;
+      
+      for (; I < Message.end() && *I == ' '; I++);
+      const char *Equals = I;
+      for (; Equals < Message.end() && *Equals != '='; Equals++);
+      const char *End = Equals;
+      for (; End < Message.end() && *End != '\n'; End++);
+      if (End == Equals)
+	 return false;
+      
+      Cnf.Set(string(I,Equals-I),string(Equals+1,End-Equals-1));
+      I = End;
+   }
+   
+   return true;
+}
+									/*}}}*/
diff --git a/apt-pkg/acquire-worker.h b/apt-pkg/acquire-worker.h
index eb04485b9..b0acde3e3 100644
--- a/apt-pkg/acquire-worker.h
+++ b/apt-pkg/acquire-worker.h
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire-worker.h,v 1.4 1998/10/24 04:57:59 jgg Exp $
+// $Id: acquire-worker.h,v 1.5 1998/10/26 07:11:46 jgg Exp $
 /* ######################################################################
 
    Acquire Worker - Worker process manager
@@ -13,6 +13,7 @@
 #define PKGLIB_ACQUIRE_WORKER_H
 
 #include <apt-pkg/acquire.h>
+#include <apt-pkg/configuration.h>
 
 #ifdef __GNUG__
 #pragma interface "apt-pkg/acquire-worker.h"
@@ -80,4 +81,6 @@ class pkgAcquire::Worker
    ~Worker();
 };
 
+bool pkgInjectConfiguration(string &Message,Configuration &Cnf);
+
 #endif
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 21aade75d..4ed21831d 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire.cc,v 1.4 1998/10/24 04:58:01 jgg Exp $
+// $Id: acquire.cc,v 1.5 1998/10/26 07:11:47 jgg Exp $
 /* ######################################################################
 
    Acquire - File Acquiration
@@ -33,6 +33,7 @@ pkgAcquire::pkgAcquire()
    Configs = 0;
    Workers = 0;
    ToFetch = 0;
+   Running = false;
    
    string Mode = _config->Find("Acquire::Queue-Mode","host");
    if (strcasecmp(Mode.c_str(),"host") == 0)
@@ -133,6 +134,9 @@ void pkgAcquire::Enqueue(Item *Itm,string URI,string Description)
    // Queue it into the named queue
    I->Enqueue(Itm,URI,Description);
    ToFetch++;
+   
+   if (Running == true)
+      I->Startup();
       
    // Some trace stuff
    if (Debug == true)
@@ -249,6 +253,8 @@ void pkgAcquire::RunFds(fd_set *RSet,fd_set *WSet)
    manage the actual fetch. */
 bool pkgAcquire::Run()
 {
+   Running = true;
+   
    for (Queue *I = Queues; I != 0; I = I->Next)
       I->Startup();
    
@@ -263,7 +269,10 @@ bool pkgAcquire::Run()
       SetFds(Highest,&RFds,&WFds);
       
       if (select(Highest+1,&RFds,&WFds,0,0) <= 0)
+      {
+	 Running = false;
 	 return _error->Errno("select","Select has failed");
+      }
       
       RunFds(&RFds,&WFds);
    }   
@@ -271,6 +280,7 @@ bool pkgAcquire::Run()
    for (Queue *I = Queues; I != 0; I = I->Next)
       I->Shutdown();
 
+   Running = false;
    return true;
 }
 									/*}}}*/
diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index d4dbed5f6..2e111cccd 100644
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: acquire.h,v 1.4 1998/10/24 04:58:02 jgg Exp $
+// $Id: acquire.h,v 1.5 1998/10/26 07:11:48 jgg Exp $
 /* ######################################################################
 
    Acquire - File Acquiration
@@ -66,6 +66,7 @@ class pkgAcquire
    // Configurable parameters for the schedular
    enum {QueueHost,QueueAccess} QueueMode;
    bool Debug;
+   bool Running;
    
    void Add(Item *Item);
    void Remove(Item *Item);
diff --git a/apt-pkg/contrib/fileutl.cc b/apt-pkg/contrib/fileutl.cc
index ad1fcf184..77e846117 100644
--- a/apt-pkg/contrib/fileutl.cc
+++ b/apt-pkg/contrib/fileutl.cc
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: fileutl.cc,v 1.12 1998/10/24 22:15:41 jgg Exp $
+// $Id: fileutl.cc,v 1.13 1998/10/26 07:11:49 jgg Exp $
 /* ######################################################################
    
    File Utilities
@@ -29,7 +29,7 @@
 // CopyFile - Buffered copy of a file					/*{{{*/
 // ---------------------------------------------------------------------
 /* The caller is expected to set things so that failure causes erasure */
-bool CopyFile(FileFd From,FileFd To)
+bool CopyFile(FileFd &From,FileFd &To)
 {
    if (From.IsOpen() == false || To.IsOpen() == false)
       return false;
diff --git a/apt-pkg/contrib/fileutl.h b/apt-pkg/contrib/fileutl.h
index 7a72df09f..01dc46b13 100644
--- a/apt-pkg/contrib/fileutl.h
+++ b/apt-pkg/contrib/fileutl.h
@@ -1,6 +1,6 @@
 // -*- mode: cpp; mode: fold -*-
 // Description								/*{{{*/
-// $Id: fileutl.h,v 1.7 1998/10/20 02:39:29 jgg Exp $
+// $Id: fileutl.h,v 1.8 1998/10/26 07:11:50 jgg Exp $
 /* ######################################################################
    
    File Utilities
@@ -59,7 +59,7 @@ class FileFd
    virtual ~FileFd();
 };
 
-bool CopyFile(FileFd From,FileFd To);
+bool CopyFile(FileFd &From,FileFd &To);
 int GetLock(string File,bool Errors = true);
 bool FileExists(string File);
 string SafeGetCWD();