summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJulian Andres Klode <julian.klode@canonical.com>2021-07-08 16:49:20 +0200
committerJulian Andres Klode <julian.klode@canonical.com>2021-07-28 13:04:10 +0200
commit121ccd0e0c2612bab9ba5383d5599b54e29e4643 (patch)
tree2ea80ee1db0f6878e53068961f6b68e1bb0d0840
parent363e3a4c91da353d99cf1af3d5089c9455d461de (diff)
Implement exponential delay between retries
Add a new Item field called FetchAfter, which determines the earliest time the item should be fetched at. Adjust insertion into queue to take it into account alongside priority, and only fill pipelines with items that are ready.
-rw-r--r--apt-pkg/acquire-item.cc10
-rw-r--r--apt-pkg/acquire-item.h3
-rw-r--r--apt-pkg/acquire-worker.cc11
-rw-r--r--apt-pkg/acquire.cc25
-rw-r--r--apt-pkg/acquire.h2
-rw-r--r--doc/examples/configure-index4
6 files changed, 51 insertions, 4 deletions
diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index 7c23434ae..20d80edc0 100644
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -290,6 +290,7 @@ public:
std::vector<std::string> BadAlternativeSites;
std::vector<std::string> PastRedirections;
std::unordered_map<std::string, std::string> CustomFields;
+ time_t FetchAfter = 0;
Private()
{
@@ -1120,6 +1121,15 @@ std::string pkgAcquire::Item::HashSum() const /*{{{*/
return hs != NULL ? hs->toStr() : "";
}
/*}}}*/
+void pkgAcquire::Item::FetchAfter(time_t FetchAfter) /*{{{*/
+{
+ d->FetchAfter = FetchAfter;
+}
+time_t pkgAcquire::Item::FetchAfter()
+{
+ return d->FetchAfter;
+}
+ /*}}}*/
bool pkgAcquire::Item::IsRedirectionLoop(std::string const &NewURI) /*{{{*/
{
// store can fail due to permission errors and the item will "loop" then
diff --git a/apt-pkg/acquire-item.h b/apt-pkg/acquire-item.h
index 3be8a9c62..33106226f 100644
--- a/apt-pkg/acquire-item.h
+++ b/apt-pkg/acquire-item.h
@@ -301,6 +301,9 @@ class APT_PUBLIC pkgAcquire::Item : public WeakPointable /*{{{*/
/** \brief The priority of the item, used for queuing */
int APT_HIDDEN Priority();
+ void APT_HIDDEN FetchAfter(time_t FetchAfter);
+ time_t APT_HIDDEN FetchAfter();
+
protected:
/** \brief The acquire object with which this item is associated. */
pkgAcquire * const Owner;
diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc
index 3fbc5c09f..5272ae0cb 100644
--- a/apt-pkg/acquire-worker.cc
+++ b/apt-pkg/acquire-worker.cc
@@ -632,6 +632,7 @@ void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &It
pkgAcquire::MethodConfig *const Config, pkgAcquireStatus *const Log,
std::string const &Message, bool const errTransient, bool const errAuthErr)
{
+ time_t currentTime = time(nullptr);
for (auto const Owner : ItmOwners)
{
std::string NewURI;
@@ -640,6 +641,16 @@ void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &It
--Owner->Retries;
Owner->FailMessage(Message);
auto SavedDesc = Owner->GetItemDesc();
+ if (_config->FindB("Acquire::Retries::Delay", true))
+ {
+ auto Iter = _config->FindI("Acquire::Retries", 3) - Owner->Retries - 1;
+ Owner->FetchAfter(currentTime + (1 << Iter));
+ }
+ else
+ {
+ Owner->FetchAfter(currentTime);
+ }
+
if (Log != nullptr)
Log->Fail(SavedDesc);
if (isDoomedItem(Owner) == false)
diff --git a/apt-pkg/acquire.cc b/apt-pkg/acquire.cc
index 8693de930..806f8e212 100644
--- a/apt-pkg/acquire.cc
+++ b/apt-pkg/acquire.cc
@@ -24,14 +24,15 @@
#include <algorithm>
#include <chrono>
+#include <cmath>
#include <iomanip>
#include <iostream>
#include <memory>
#include <numeric>
#include <sstream>
#include <string>
+#include <tuple>
#include <vector>
-#include <cmath>
#include <dirent.h>
#include <errno.h>
@@ -962,6 +963,7 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
};
QItem **OptimalI = &Items;
QItem **I = &Items;
+ auto insertLocation = std::make_tuple(Item.Owner->FetchAfter(), -Item.Owner->Priority());
// move to the end of the queue and check for duplicates here
for (; *I != 0; ) {
if (Item.URI == (*I)->URI && MetaKeysMatch(Item, *I))
@@ -974,10 +976,12 @@ bool pkgAcquire::Queue::Enqueue(ItemDesc &Item)
}
// Determine the optimal position to insert: before anything with a
// higher priority.
- int priority = (*I)->GetPriority();
+ auto queueLocation = std::make_tuple((*I)->GetFetchAfter(),
+ -(*I)->GetPriority());
I = &(*I)->Next;
- if (priority >= Item.Owner->Priority()) {
+ if (queueLocation <= insertLocation)
+ {
OptimalI = I;
}
}
@@ -1153,6 +1157,7 @@ bool pkgAcquire::Queue::Cycle()
// Look for a queable item
QItem *I = Items;
int ActivePriority = 0;
+ time_t currentTime = time(nullptr);
while (PipeDepth < static_cast<decltype(PipeDepth)>(MaxPipeDepth))
{
for (; I != 0; I = I->Next) {
@@ -1170,6 +1175,11 @@ bool pkgAcquire::Queue::Cycle()
// the queue is idle
if (I->GetPriority() < ActivePriority)
return true;
+
+ // Item is not ready yet, delay
+ if (I->GetFetchAfter() > currentTime)
+ return true;
+
I->Worker = Workers;
for (auto const &O: I->Owners)
O->Status = pkgAcquire::Item::StatFetching;
@@ -1237,6 +1247,15 @@ APT_PURE int pkgAcquire::Queue::QItem::GetPriority() const /*{{{*/
return Priority;
}
/*}}}*/
+APT_PURE time_t pkgAcquire::Queue::QItem::GetFetchAfter() const /*{{{*/
+{
+ time_t FetchAfter = 0;
+ for (auto const &O : Owners)
+ FetchAfter = std::max(FetchAfter, O->FetchAfter());
+
+ return FetchAfter;
+}
+ /*}}}*/
void pkgAcquire::Queue::QItem::SyncDestinationFiles() const /*{{{*/
{
/* ensure that the first owner has the best partial file of all and
diff --git a/apt-pkg/acquire.h b/apt-pkg/acquire.h
index a2c4fbc67..303554041 100644
--- a/apt-pkg/acquire.h
+++ b/apt-pkg/acquire.h
@@ -458,6 +458,8 @@ class APT_PUBLIC pkgAcquire::Queue
std::string Custom600Headers() const;
/** @return the maximum priority of this item */
int APT_HIDDEN GetPriority() const;
+ /** @return the maximum time to fetch this item at */
+ time_t APT_HIDDEN GetFetchAfter() const;
};
/** \brief The name of this queue. */
diff --git a/doc/examples/configure-index b/doc/examples/configure-index
index 312ee27d6..e74ebc3d7 100644
--- a/doc/examples/configure-index
+++ b/doc/examples/configure-index
@@ -230,7 +230,9 @@ APT
Acquire
{
Queue-Mode "<STRING>"; // host or access
- Retries "<INT>";
+ Retries "<INT>" {
+ Delay "<BOOL>"; // whether to backoff between retries using the delay: method
+ };
Source-Symlinks "<BOOL>";
ForceHash "<STRING>"; // hashmethod used for expected hash: sha256, sha1 or md5sum
Send-URI-Encoded "<BOOL>"; // false does the old encode/decode dance even if we could avoid it