From d0cfa8adbdd74ad7e363019739c77e713dc982e5 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 16 Apr 2014 14:43:08 +0200 Subject: make the TotalFiles more reliable in apt-get update --- apt-pkg/acquire.cc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'apt-pkg/acquire.cc') diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index a187a00ae..2b427ccd3 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -832,6 +832,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) if ((*I)->Local == true) continue; + // see if the method tells us to expect more + TotalItems += (*I)->ExpectedAdditionalItems; + TotalBytes += (*I)->FileSize; if ((*I)->Complete == true) CurrentBytes += (*I)->FileSize; @@ -901,12 +904,17 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems); - + // calculate the percentage, if we have too little data assume 0% + int Percent; + if (TotalBytes < 1*1024) + Percent = 0; + else + Percent = (CurrentBytes/float(TotalBytes)*100.0); // build the status str status << "dlstatus:" << i - << ":" << (CurrentBytes/float(TotalBytes)*100.0) - << ":" << msg - << endl; + << ":" << Percent + << ":" << msg + << endl; std::string const dlstatus = status.str(); FileFd::Write(fd, dlstatus.c_str(), dlstatus.size()); -- cgit v1.2.3-70-g09d2 From 1dca8dc55c1fcf4bda07a7e8285de7f225448697 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 16 Apr 2014 15:28:23 +0200 Subject: load the size from the metaindex into the fetcher to have even more accurate progress information --- apt-pkg/acquire-item.cc | 15 ++++++++++++--- apt-pkg/acquire-item.h | 4 ++-- apt-pkg/acquire.cc | 5 ++++- apt-pkg/indexrecords.cc | 2 +- apt-pkg/indexrecords.h | 2 +- 5 files changed, 20 insertions(+), 8 deletions(-) (limited to 'apt-pkg/acquire.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 6f6b3d59f..d5f8b0cc9 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -936,7 +936,7 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, Init(URI, URIDesc, ShortDesc); } pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser) + HashString const &ExpectedHash, indexRecords *MetaIndexParser) : Item(Owner), RealURI(Target->URI), ExpectedHash(ExpectedHash) { // autoselect the compression method @@ -963,6 +963,11 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, else Verify = true; + // load the filesize + indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); + if(Record) + FileSize = Record->Size; + Init(Target->URI, Target->Description, Target->ShortDesc); } /*}}}*/ @@ -1176,9 +1181,13 @@ pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, { } pkgAcqIndexTrans::pkgAcqIndexTrans(pkgAcquire *Owner, IndexTarget const *Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser) + HashString const &ExpectedHash, indexRecords *MetaIndexParser) : pkgAcqIndex(Owner, Target, ExpectedHash, MetaIndexParser) { + // load the filesize + indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); + if(Record) + FileSize = Record->Size; } /*}}}*/ // AcqIndexTrans::Custom600Headers - Insert custom request headers /*{{{*/ @@ -1575,7 +1584,7 @@ void pkgAcqMetaIndex::QueueIndexes(bool verify) /*{{{*/ ++Target) { HashString ExpectedIndexHash; - const indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); + indexRecords::checkSum *Record = MetaIndexParser->Lookup((*Target)->MetaKey); bool compressedAvailable = false; if (Record == NULL) { diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index c12d8d63a..bf5cb09c8 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -756,7 +756,7 @@ class pkgAcqIndex : public pkgAcquire::Item std::string ShortDesc, HashString ExpectedHash, std::string compressExt=""); pkgAcqIndex(pkgAcquire *Owner, struct IndexTarget const * const Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser); + HashString const &ExpectedHash, indexRecords *MetaIndexParser); void Init(std::string const &URI, std::string const &URIDesc, std::string const &ShortDesc); }; /*}}}*/ @@ -788,7 +788,7 @@ class pkgAcqIndexTrans : public pkgAcqIndex pkgAcqIndexTrans(pkgAcquire *Owner,std::string URI,std::string URIDesc, std::string ShortDesc); pkgAcqIndexTrans(pkgAcquire *Owner, struct IndexTarget const * const Target, - HashString const &ExpectedHash, indexRecords const *MetaIndexParser); + HashString const &ExpectedHash, indexRecords *MetaIndexParser); }; /*}}}*/ /** \brief Information about an index file. */ /*{{{*/ diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 2b427ccd3..9fc40752f 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -905,8 +905,11 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) // calculate the percentage, if we have too little data assume 0% + // FIXME: the 5k is totally arbitrary + // FIXME2: instead, use a algorithm where 50% is based on total bytes + // and the other 50% on total files int Percent; - if (TotalBytes < 1*1024) + if (TotalBytes < 5*1024) Percent = 0; else Percent = (CurrentBytes/float(TotalBytes)*100.0); diff --git a/apt-pkg/indexrecords.cc b/apt-pkg/indexrecords.cc index 5353d1098..68ebdce08 100644 --- a/apt-pkg/indexrecords.cc +++ b/apt-pkg/indexrecords.cc @@ -53,7 +53,7 @@ APT_PURE time_t indexRecords::GetValidUntil() const return this->ValidUntil; } -APT_PURE const indexRecords::checkSum *indexRecords::Lookup(const string MetaKey) +APT_PURE indexRecords::checkSum *indexRecords::Lookup(const string MetaKey) { std::map::const_iterator sum = Entries.find(MetaKey); if (sum == Entries.end()) diff --git a/apt-pkg/indexrecords.h b/apt-pkg/indexrecords.h index e31f889ad..2260a4ae1 100644 --- a/apt-pkg/indexrecords.h +++ b/apt-pkg/indexrecords.h @@ -41,7 +41,7 @@ class indexRecords indexRecords(const std::string ExpectedDist); // Lookup function - virtual const checkSum *Lookup(const std::string MetaKey); + virtual checkSum *Lookup(const std::string MetaKey); /** \brief tests if a checksum for this file is available */ bool Exists(std::string const &MetaKey) const; std::vector MetaKeys(); -- cgit v1.2.3-70-g09d2 From c62f7898b6f37b8e4b076ae3c40d56d22a884efe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 16 Apr 2014 16:34:01 +0200 Subject: add Debug::acquire::progress debug option and fixme for index file loading with the correct extension --- apt-pkg/acquire-item.cc | 1 + apt-pkg/acquire.cc | 8 ++++++++ 2 files changed, 9 insertions(+) (limited to 'apt-pkg/acquire.cc') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index d5f8b0cc9..834362b84 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -963,6 +963,7 @@ pkgAcqIndex::pkgAcqIndex(pkgAcquire *Owner, IndexTarget const *Target, else Verify = true; + // FIXME: use the appropriate compression Extension // load the filesize indexRecords::checkSum *Record = MetaIndexParser->Lookup(string(Target->MetaKey)); if(Record) diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 9fc40752f..46d5e6386 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -846,6 +846,7 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) unsigned long long ResumeSize = 0; for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0; I = Owner->WorkerStep(I)) + { if (I->CurrentItem != 0 && I->CurrentItem->Owner->Complete == false) { CurrentBytes += I->CurrentSize; @@ -856,7 +857,14 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) I->CurrentItem->Owner->Complete == false) TotalBytes += I->CurrentSize; } + } + // debug + if (_config->FindB("Debug::acquire::progress", false) == true) + std::clog << " Bytes: " + << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) + << std::endl; + // Normalize the figures and account for unknown size downloads if (TotalBytes <= 0) TotalBytes = 1; -- cgit v1.2.3-70-g09d2 From 96c6cab1fd5a5de19fbd552b42e1dd1c08417946 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 16 Apr 2014 17:41:11 +0200 Subject: calculate Percent as part of pkgAcquireStatus to provide a weighted percent for both items and bytes --- apt-pkg/acquire-item.h | 5 +++++ apt-pkg/acquire.cc | 34 +++++++++++++++++----------------- apt-pkg/acquire.h | 4 ++++ apt-private/acqprogress.cc | 2 +- 4 files changed, 27 insertions(+), 18 deletions(-) (limited to 'apt-pkg/acquire.cc') diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index bf5cb09c8..ab4a69c13 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -723,6 +723,11 @@ class pkgAcqIndex : public pkgAcquire::Item */ std::string CompressionExtension; + /** \brief Pointer to the IndexTarget data + */ + const struct IndexTarget * Target; + indexRecords *MetaIndexParser; + public: // Specialized action members diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 46d5e6386..37964c943 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -859,12 +860,6 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) } } - // debug - if (_config->FindB("Debug::acquire::progress", false) == true) - std::clog << " Bytes: " - << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) - << std::endl; - // Normalize the figures and account for unknown size downloads if (TotalBytes <= 0) TotalBytes = 1; @@ -874,6 +869,12 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) // Wha?! Is not supposed to happen. if (CurrentBytes > TotalBytes) CurrentBytes = TotalBytes; + + // debug + if (_config->FindB("Debug::acquire::progress", false) == true) + std::clog << " Bytes: " + << SizeToStr(CurrentBytes) << " / " << SizeToStr(TotalBytes) + << std::endl; // Compute the CPS struct timeval NewTime; @@ -894,6 +895,15 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) Time = NewTime; } + // calculate the percentage, if we have too little data assume 0% + // FIXME: the 5k is totally arbitrary + if (TotalBytes < 5*1024) + Percent = 0; + else + // use both files and bytes because bytes can be unreliable + Percent = (0.8 * (CurrentBytes/float(TotalBytes)*100.0) + + 0.2 * (CurrentItems/float(TotalItems)*100.0)); + int fd = _config->FindI("APT::Status-Fd",-1); if(fd > 0) { @@ -911,19 +921,9 @@ bool pkgAcquireStatus::Pulse(pkgAcquire *Owner) else snprintf(msg,sizeof(msg), _("Retrieving file %li of %li"), i, TotalItems); - - // calculate the percentage, if we have too little data assume 0% - // FIXME: the 5k is totally arbitrary - // FIXME2: instead, use a algorithm where 50% is based on total bytes - // and the other 50% on total files - int Percent; - if (TotalBytes < 5*1024) - Percent = 0; - else - Percent = (CurrentBytes/float(TotalBytes)*100.0); // build the status str status << "dlstatus:" << i - << ":" << Percent + << ":" << std::setprecision(3) << Percent << ":" << msg << endl; diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index ef16d8556..0113021b2 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -714,6 +714,10 @@ class pkgAcquireStatus /** \brief The number of items that have been successfully downloaded. */ unsigned long CurrentItems; + /** \brief The estimated percentage of the download (0-100) + */ + double Percent; + public: /** \brief If \b true, the download scheduler should call Pulse() diff --git a/apt-private/acqprogress.cc b/apt-private/acqprogress.cc index fe7a45e12..adec37405 100644 --- a/apt-private/acqprogress.cc +++ b/apt-private/acqprogress.cc @@ -170,7 +170,7 @@ bool AcqTextStatus::Pulse(pkgAcquire *Owner) ScreenWidth = sizeof(Buffer)-1; // Put in the percent done - sprintf(S,"%.0f%%",((CurrentBytes + CurrentItems)*100.0)/(TotalBytes+TotalItems)); + sprintf(S,"%.0f%%", Percent); bool Shown = false; for (pkgAcquire::Worker *I = Owner->WorkersBegin(); I != 0; -- cgit v1.2.3-70-g09d2