From 121ccd0e0c2612bab9ba5383d5599b54e29e4643 Mon Sep 17 00:00:00 2001 From: Julian Andres Klode Date: Thu, 8 Jul 2021 16:49:20 +0200 Subject: Implement exponential delay between retries Add a new Item field called FetchAfter, which determines the earliest time the item should be fetched at. Adjust insertion into queue to take it into account alongside priority, and only fill pipelines with items that are ready. --- apt-pkg/acquire-item.cc | 10 ++++++++++ apt-pkg/acquire-item.h | 3 +++ apt-pkg/acquire-worker.cc | 11 +++++++++++ apt-pkg/acquire.cc | 25 ++++++++++++++++++++++--- apt-pkg/acquire.h | 2 ++ 5 files changed, 48 insertions(+), 3 deletions(-) (limited to 'apt-pkg') diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7c23434ae..20d80edc0 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -290,6 +290,7 @@ public: std::vector BadAlternativeSites; std::vector PastRedirections; std::unordered_map CustomFields; + time_t FetchAfter = 0; Private() { @@ -1120,6 +1121,15 @@ std::string pkgAcquire::Item::HashSum() const /*{{{*/ return hs != NULL ? hs->toStr() : ""; } /*}}}*/ +void pkgAcquire::Item::FetchAfter(time_t FetchAfter) /*{{{*/ +{ + d->FetchAfter = FetchAfter; +} +time_t pkgAcquire::Item::FetchAfter() +{ + return d->FetchAfter; +} + /*}}}*/ bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/ { // store can fail due to permission errors and the item will "loop" then diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 3be8a9c62..33106226f 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -301,6 +301,9 @@ class APT_PUBLIC pkgAcquire::Item : public WeakPointable /*{{{*/ /** \brief The priority of the item, used for queuing */ int APT_HIDDEN Priority(); + void APT_HIDDEN FetchAfter(time_t FetchAfter); + time_t APT_HIDDEN FetchAfter(); + protected: /** \brief The acquire object with which this item is associated. */ pkgAcquire * const Owner; diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 3fbc5c09f..5272ae0cb 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -632,6 +632,7 @@ void pkgAcquire::Worker::HandleFailure(std::vector const &It pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log, std::string const &Message, bool const errTransient, bool const errAuthErr) { + time_t currentTime = time(nullptr); for (auto const Owner : ItmOwners) { std::string NewURI; @@ -640,6 +641,16 @@ void pkgAcquire::Worker::HandleFailure(std::vector const &It --Owner->Retries; Owner->FailMessage(Message); auto SavedDesc = Owner->GetItemDesc(); + if (_config->FindB("Acquire::Retries::Delay", true)) + { + auto Iter = _config->FindI("Acquire::Retries", 3) - Owner->Retries - 1; + Owner->FetchAfter(currentTime + (1 << Iter)); + } + else + { + Owner->FetchAfter(currentTime); + } + if (Log != nullptr) Log->Fail(SavedDesc); if (isDoomedItem(Owner) == false) diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 8693de930..806f8e212 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -24,14 +24,15 @@ #include #include +#include #include #include #include #include #include #include +#include #include -#include #include #include @@ -962,6 +963,7 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) }; QItem **OptimalI = &Items; QItem **I = &Items; + auto insertLocation = std::make_tuple(Item.Owner->FetchAfter(), -Item.Owner->Priority()); // move to the end of the queue and check for duplicates here for (; *I != 0; ) { if (Item.URI == (*I)->URI && MetaKeysMatch(Item, *I)) @@ -974,10 +976,12 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) } // Determine the optimal position to insert: before anything with a // higher priority. - int priority = (*I)->GetPriority(); + auto queueLocation = std::make_tuple((*I)->GetFetchAfter(), + -(*I)->GetPriority()); I = &(*I)->Next; - if (priority >= Item.Owner->Priority()) { + if (queueLocation <= insertLocation) + { OptimalI = I; } } @@ -1153,6 +1157,7 @@ bool pkgAcquire::Queue::Cycle() // Look for a queable item QItem *I = Items; int ActivePriority = 0; + time_t currentTime = time(nullptr); while (PipeDepth < static_cast(MaxPipeDepth)) { for (; I != 0; I = I->Next) { @@ -1170,6 +1175,11 @@ bool pkgAcquire::Queue::Cycle() // the queue is idle if (I->GetPriority() < ActivePriority) return true; + + // Item is not ready yet, delay + if (I->GetFetchAfter() > currentTime) + return true; + I->Worker = Workers; for (auto const &O: I->Owners) O->Status = pkgAcquire::Item::StatFetching; @@ -1237,6 +1247,15 @@ APT_PURE int pkgAcquire::Queue::QItem::GetPriority() const /*{{{*/ return Priority; } /*}}}*/ +APT_PURE time_t pkgAcquire::Queue::QItem::GetFetchAfter() const /*{{{*/ +{ + time_t FetchAfter = 0; + for (auto const &O : Owners) + FetchAfter = std::max(FetchAfter, O->FetchAfter()); + + return FetchAfter; +} + /*}}}*/ void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/ { /* ensure that the first owner has the best partial file of all and diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index a2c4fbc67..303554041 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -458,6 +458,8 @@ class APT_PUBLIC pkgAcquire::Queue std::string Custom600Headers() const; /** @return the maximum priority of this item */ int APT_HIDDEN GetPriority() const; + /** @return the maximum time to fetch this item at */ + time_t APT_HIDDEN GetFetchAfter() const; }; /** \brief The name of this queue. */ -- cgit v1.2.3-70-g09d2