diff options
author | Julian Andres Klode <julian.klode@canonical.com> | 2021-07-08 16:49:20 +0200 |
---|---|---|
committer | Julian Andres Klode <julian.klode@canonical.com> | 2021-07-28 13:04:10 +0200 |
commit | 121ccd0e0c2612bab9ba5383d5599b54e29e4643 (patch) | |
tree | 2ea80ee1db0f6878e53068961f6b68e1bb0d0840 | |
parent | 363e3a4c91da353d99cf1af3d5089c9455d461de (diff) |
Implement exponential delay between retries
Add a new Item field called FetchAfter, which determines the earliest
time the item should be fetched at. Adjust insertion into queue to
take it into account alongside priority, and only fill pipelines
with items that are ready.
-rw-r--r-- | apt-pkg/acquire-item.cc | 10 | ||||
-rw-r--r-- | apt-pkg/acquire-item.h | 3 | ||||
-rw-r--r-- | apt-pkg/acquire-worker.cc | 11 | ||||
-rw-r--r-- | apt-pkg/acquire.cc | 25 | ||||
-rw-r--r-- | apt-pkg/acquire.h | 2 | ||||
-rw-r--r-- | doc/examples/configure-index | 4 |
6 files changed, 51 insertions, 4 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc index 7c23434ae..20d80edc0 100644 --- a/apt-pkg/acquire-item.cc +++ b/apt-pkg/acquire-item.cc @@ -290,6 +290,7 @@ public: std::vector<std::string> BadAlternativeSites; std::vector<std::string> PastRedirections; std::unordered_map<std::string, std::string> CustomFields; + time_t FetchAfter = 0; Private() { @@ -1120,6 +1121,15 @@ std::string pkgAcquire::Item::HashSum() const /*{{{*/ return hs != NULL ? hs->toStr() : ""; } /*}}}*/ +void pkgAcquire::Item::FetchAfter(time_t FetchAfter) /*{{{*/ +{ + d->FetchAfter = FetchAfter; +} +time_t pkgAcquire::Item::FetchAfter() +{ + return d->FetchAfter; +} + /*}}}*/ bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/ { // store can fail due to permission errors and the item will "loop" then diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h index 3be8a9c62..33106226f 100644 --- a/apt-pkg/acquire-item.h +++ b/apt-pkg/acquire-item.h @@ -301,6 +301,9 @@ class APT_PUBLIC pkgAcquire::Item : public WeakPointable /*{{{*/ /** \brief The priority of the item, used for queuing */ int APT_HIDDEN Priority(); + void APT_HIDDEN FetchAfter(time_t FetchAfter); + time_t APT_HIDDEN FetchAfter(); + protected: /** \brief The acquire object with which this item is associated. */ pkgAcquire * const Owner; diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 3fbc5c09f..5272ae0cb 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -632,6 +632,7 @@ void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &It pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log, std::string const &Message, bool const errTransient, bool const errAuthErr) { + time_t currentTime = time(nullptr); for (auto const Owner : ItmOwners) { std::string NewURI; @@ -640,6 +641,16 @@ void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &It --Owner->Retries; Owner->FailMessage(Message); auto SavedDesc = Owner->GetItemDesc(); + if (_config->FindB("Acquire::Retries::Delay", true)) + { + auto Iter = _config->FindI("Acquire::Retries", 3) - Owner->Retries - 1; + Owner->FetchAfter(currentTime + (1 << Iter)); + } + else + { + Owner->FetchAfter(currentTime); + } + if (Log != nullptr) Log->Fail(SavedDesc); if (isDoomedItem(Owner) == false) diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc index 8693de930..806f8e212 100644 --- a/apt-pkg/acquire.cc +++ b/apt-pkg/acquire.cc @@ -24,14 +24,15 @@ #include <algorithm> #include <chrono> +#include <cmath> #include <iomanip> #include <iostream> #include <memory> #include <numeric> #include <sstream> #include <string> +#include <tuple> #include <vector> -#include <cmath> #include <dirent.h> #include <errno.h> @@ -962,6 +963,7 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) }; QItem **OptimalI = &Items; QItem **I = &Items; + auto insertLocation = std::make_tuple(Item.Owner->FetchAfter(), -Item.Owner->Priority()); // move to the end of the queue and check for duplicates here for (; *I != 0; ) { if (Item.URI == (*I)->URI && MetaKeysMatch(Item, *I)) @@ -974,10 +976,12 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item) } // Determine the optimal position to insert: before anything with a // higher priority. - int priority = (*I)->GetPriority(); + auto queueLocation = std::make_tuple((*I)->GetFetchAfter(), + -(*I)->GetPriority()); I = &(*I)->Next; - if (priority >= Item.Owner->Priority()) { + if (queueLocation <= insertLocation) + { OptimalI = I; } } @@ -1153,6 +1157,7 @@ bool pkgAcquire::Queue::Cycle() // Look for a queable item QItem *I = Items; int ActivePriority = 0; + time_t currentTime = time(nullptr); while (PipeDepth < static_cast<decltype(PipeDepth)>(MaxPipeDepth)) { for (; I != 0; I = I->Next) { @@ -1170,6 +1175,11 @@ bool pkgAcquire::Queue::Cycle() // the queue is idle if (I->GetPriority() < ActivePriority) return true; + + // Item is not ready yet, delay + if (I->GetFetchAfter() > currentTime) + return true; + I->Worker = Workers; for (auto const &O: I->Owners) O->Status = pkgAcquire::Item::StatFetching; @@ -1237,6 +1247,15 @@ APT_PURE int pkgAcquire::Queue::QItem::GetPriority() const /*{{{*/ return Priority; } /*}}}*/ +APT_PURE time_t pkgAcquire::Queue::QItem::GetFetchAfter() const /*{{{*/ +{ + time_t FetchAfter = 0; + for (auto const &O : Owners) + FetchAfter = std::max(FetchAfter, O->FetchAfter()); + + return FetchAfter; +} + /*}}}*/ void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/ { /* ensure that the first owner has the best partial file of all and diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h index a2c4fbc67..303554041 100644 --- a/apt-pkg/acquire.h +++ b/apt-pkg/acquire.h @@ -458,6 +458,8 @@ class APT_PUBLIC pkgAcquire::Queue std::string Custom600Headers() const; /** @return the maximum priority of this item */ int APT_HIDDEN GetPriority() const; + /** @return the maximum time to fetch this item at */ + time_t APT_HIDDEN GetFetchAfter() const; }; /** \brief The name of this queue. */ diff --git a/doc/examples/configure-index b/doc/examples/configure-index index 312ee27d6..e74ebc3d7 100644 --- a/doc/examples/configure-index +++ b/doc/examples/configure-index @@ -230,7 +230,9 @@ APT Acquire { Queue-Mode "<STRING>"; // host or access - Retries "<INT>"; + Retries "<INT>" { + Delay "<BOOL>"; // whether to backoff between retries using the delay: method + }; Source-Symlinks "<BOOL>"; ForceHash "<STRING>"; // hashmethod used for expected hash: sha256, sha1 or md5sum Send-URI-Encoded "<BOOL>"; // false does the old encode/decode dance even if we could avoid it |