diff options
Diffstat (limited to 'apt-pkg')
| -rw-r--r-- | apt-pkg/acquire-method.cc | 9 | ||||
| -rw-r--r-- | apt-pkg/acquire-method.h | 1 | ||||
| -rw-r--r-- | apt-pkg/acquire-worker.cc | 24 |
3 files changed, 29 insertions, 5 deletions
diff --git a/apt-pkg/acquire-method.cc b/apt-pkg/acquire-method.cc index 7b1f4f1b5..13f3fa4dd 100644 --- a/apt-pkg/acquire-method.cc +++ b/apt-pkg/acquire-method.cc @@ -145,9 +145,15 @@ void pkgAcqMethod::Fail(bool Transient) Fail("", Transient); } + +void pkgAcqMethod::Fail(std::string Err, bool Tansient) +{ + std::unordered_map<std::string, std::string> fields; + FailWithContext(Err, Tansient, fields); +} /*}}}*/ // AcqMethod::Fail - A fetch has failed /*{{{*/ -void pkgAcqMethod::Fail(string Err, bool Transient) +void pkgAcqMethod::FailWithContext(std::string Err, bool Transient, std::unordered_map<std::string, std::string> &fields) { if (not _error->empty()) @@ -175,7 +181,6 @@ void pkgAcqMethod::Fail(string Err, bool Transient) if (IP.empty() == false && _config->FindB("Acquire::Failure::ShowIP", true) == true) Err.append(" ").append(IP); - std::unordered_map<std::string, std::string> fields; if (Queue != nullptr) try_emplace(fields, "URI", Queue->Uri); else diff --git a/apt-pkg/acquire-method.h b/apt-pkg/acquire-method.h index edee93996..387d0984d 100644 --- a/apt-pkg/acquire-method.h +++ b/apt-pkg/acquire-method.h @@ -92,6 +92,7 @@ class APT_PUBLIC pkgAcqMethod // Outgoing messages void Fail(bool Transient = false); + void FailWithContext(std::string Why, bool Transient, std::unordered_map<std::string, std::string> &fields); inline void Fail(const char *Why, bool Transient = false) {Fail(std::string(Why),Transient);}; virtual void Fail(std::string Why, bool Transient = false); virtual void URIStart(FetchResult &Res); diff --git a/apt-pkg/acquire-worker.cc b/apt-pkg/acquire-worker.cc index 4840ea16a..f4326ab03 100644 --- a/apt-pkg/acquire-worker.cc +++ b/apt-pkg/acquire-worker.cc @@ -603,8 +603,8 @@ bool pkgAcquire::Worker::RunMessages() { std::string const failReason = LookupTag(Message, "FailReason"); { - auto const reasons = { "Timeout", "ConnectionRefused", - "ConnectionTimedOut", "ResolveFailure", "TmpResolveFailure" }; + auto const reasons = {"Timeout", "ConnectionRefused", + "ConnectionTimedOut", "ResolveFailure", "TmpResolveFailure", "TooManyRequests"}; errTransient = std::find(std::begin(reasons), std::end(reasons), failReason) != std::end(reasons); } if (errTransient == false) @@ -636,6 +636,7 @@ void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &It std::string const &Message, bool const errTransient, bool const errAuthErr) { auto currentTime = clock::now(); + auto currentWall = std::chrono::system_clock::now(); for (auto const Owner : ItmOwners) { std::string NewURI; @@ -648,7 +649,24 @@ void pkgAcquire::Worker::HandleFailure(std::vector<pkgAcquire::Item *> const &It { auto Iter = _config->FindI("Acquire::Retries", 3) - Owner->Retries - 1; auto const MaxDur = _config->FindI("Acquire::Retries::Delay::Maximum", 30); - auto Dur = std::chrono::seconds(std::min(1 << Iter, MaxDur)); + auto const handleRetryAfter = _config->FindB("Acquire::Retries::HandleRetryAfter", false); + auto Dur = std::chrono::seconds(1 << Iter); + auto const retryAfterStr = LookupTag(Message, "Retry-After"); + auto const failReason = LookupTag(Message, "FailReason"); + if (failReason == "TooManyRequests" && !retryAfterStr.empty() && handleRetryAfter) + { + // The webserver gave a retry time. Use it, but also add exponential + // backoff waiting as we might not be the only client. + const auto retryAfter = std::chrono::seconds(std::strtoul(retryAfterStr.c_str(), nullptr, 10)); + const auto epochCurrent = std::chrono::duration_cast<std::chrono::seconds>(currentWall.time_since_epoch()); + // If the retryAfter is in the past, we can just continue. + if (retryAfter > epochCurrent) + Dur += retryAfter - epochCurrent; + // Usually, all requests run into the rate limit at the same time. + // Distribute the retries to avoid hitting the limit again. + Dur += std::chrono::seconds(std::rand() % Dur.count()); + } + Dur = std::min(Dur, std::chrono::seconds(MaxDur)); if (_config->FindB("Debug::Acquire::Retries")) std::clog << "Delaying " << SavedDesc.Description << " by " << Dur.count() << " seconds" << std::endl; Owner->FetchAfter(currentTime + Dur); |