From 7b9dea5a7c43fd91bcd23d1585d39a726d098285 Mon Sep 17 00:00:00 2001 From: Felix Moessbauer Date: Fri, 8 Nov 2024 15:04:17 +0100 Subject: http: use Retry-After HTTP header to optimize retries Some mirrors like snapshot.debian.org apply strict http rate limits. While apt already has an exponential backoff mechanism implemented, this is not sufficient due to the following reasons: 1. all retries happen roughly at the same time 2. the retry-after information from the server is not used We fix this by improving the algorithm: First, if present, the timestamp or duration of the Retry-After header is added to the exponential backoff, optimizing for success-on-second-try. Second, a random delay is added (delay += [0, delay]) to distribute the retries to not immediately run into the limit again. To avoid super-long delays, this is capped by the configured maximum delay. This logic only becomes active if the option Acquire::Retries::HandleRetryAfter is enabled. Signed-off-by: Felix Moessbauer --- apt-pkg/acquire-worker.cc | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) (limited to 'apt-pkg/acquire-worker.cc') diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 4840ea16a..f4326ab03 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -603,8 +603,8 @@ bool pkgAcquire::Worker::RunMessages() { std::string const failReason = LookupTag(Message, "FailReason"); { - auto const reasons = { "Timeout", "ConnectionRefused", - "ConnectionTimedOut", "ResolveFailure", "TmpResolveFailure" }; + auto const reasons = {"Timeout", "ConnectionRefused", + "ConnectionTimedOut", "ResolveFailure", "TmpResolveFailure", "TooManyRequests"}; errTransient = std::find(std::begin(reasons), std::end(reasons), failReason) != std::end(reasons); } if (errTransient == false) @@ -636,6 +636,7 @@ void pkgAcquire::Worker::HandleFailure(std::vector const &It std::string const &Message, bool const errTransient, bool const errAuthErr) { auto currentTime = clock::now(); + auto currentWall = std::chrono::system_clock::now(); for (auto const Owner : ItmOwners) { std::string NewURI; @@ -648,7 +649,24 @@ void pkgAcquire::Worker::HandleFailure(std::vector const &It { auto Iter = _config->FindI("Acquire::Retries", 3) - Owner->Retries - 1; auto const MaxDur = _config->FindI("Acquire::Retries::Delay::Maximum", 30); - auto Dur = std::chrono::seconds(std::min(1 << Iter, MaxDur)); + auto const handleRetryAfter = _config->FindB("Acquire::Retries::HandleRetryAfter", false); + auto Dur = std::chrono::seconds(1 << Iter); + auto const retryAfterStr = LookupTag(Message, "Retry-After"); + auto const failReason = LookupTag(Message, "FailReason"); + if (failReason == "TooManyRequests" && !retryAfterStr.empty() && handleRetryAfter) + { + // The webserver gave a retry time. Use it, but also add exponential + // backoff waiting as we might not be the only client. + const auto retryAfter = std::chrono::seconds(std::strtoul(retryAfterStr.c_str(), nullptr, 10)); + const auto epochCurrent = std::chrono::duration_cast(currentWall.time_since_epoch()); + // If the retryAfter is in the past, we can just continue. + if (retryAfter > epochCurrent) + Dur += retryAfter - epochCurrent; + // Usually, all requests run into the rate limit at the same time. + // Distribute the retries to avoid hitting the limit again. + Dur += std::chrono::seconds(std::rand() % Dur.count()); + } + Dur = std::min(Dur, std::chrono::seconds(MaxDur)); if (_config->FindB("Debug::Acquire::Retries")) std::clog << "Delaying " << SavedDesc.Description << " by " << Dur.count() << " seconds" << std::endl; Owner->FetchAfter(currentTime + Dur); -- cgit v1.2.3-70-g09d2