feat(trello): support exponential backoff with jitter
Some checks failed
Test / build (map[name:stable]) (push) Successful in 1m56s
Test / build (map[name:nightly]) (push) Successful in 2m26s
Release Please / Release-plz (push) Failing after 17s

This commit is contained in:
Paul Campbell 2024-12-22 08:09:16 +00:00
parent 0e123898db
commit 4cdfdaec6f
4 changed files with 52 additions and 19 deletions

View file

@ -18,6 +18,7 @@ inquire = "0.7"
kameo = "0.13"
# kxio = {path = "../kxio/"}
kxio = "4.0"
rand = "0.8"
reqwest = { version = "0.12" , features = ["multipart", "stream"]}
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

24
src/macros/backoff.rs Normal file
View file

@ -0,0 +1,24 @@
#[macro_export]
macro_rules! with_exponential_backoff {
($ctx:expr, $operation:expr) => {{
let mut backoff_secs = 1;
loop {
match $operation {
Err(kxio::net::Error::Reqwest(e))
if e.status() == Some(kxio::net::StatusCode::TOO_MANY_REQUESTS) =>
{
backoff_secs *= 2;
let jitter = rand::random::<u64>() % 10;
let backoff_secs = 60.min(backoff_secs + jitter);
$crate::p!(
$ctx.prt,
">> Too many requests, backing off for {}s",
backoff_secs
);
tokio::time::sleep(tokio::time::Duration::from_secs(backoff_secs)).await;
}
result => break result,
}
}
}};
}

View file

@ -1,5 +1,6 @@
//
mod actor;
mod backoff;
mod newtype;
mod print;
mod send;

View file

@ -1,6 +1,5 @@
//
use std::collections::HashMap;
use std::path::PathBuf;
use std::{collections::HashMap, path::PathBuf};
use color_eyre::eyre::Context;
use kxio::net::{Net, ReqBuilder};
@ -11,15 +10,15 @@ use crate::{
trello::model::{
attachment::{TrelloAttachment, TrelloAttachmentId},
board::{TrelloBoard, TrelloBoardId},
card::TrelloCardId,
card::{TrelloLongCard, TrelloShortCard},
card::{TrelloCardId, TrelloLongCard, TrelloShortCard},
list::TrelloListId,
},
FullCtx,
with_exponential_backoff, FullCtx,
};
pub(crate) struct TrelloClient<'ctx> {
ctx: &'ctx FullCtx,
// 300 requests per 10 seconds for each API key and 100 requests per 10 second interval for each token
}
impl<'ctx> TrelloClient<'ctx> {
@ -55,11 +54,15 @@ impl<'ctx> TrelloClient<'ctx> {
url: impl Into<String>,
custom: fn(&Net, String) -> ReqBuilder,
) -> APIResult<T> {
let url = url.into();
APIResult::new(
custom(&self.ctx.net, self.url(url))
with_exponential_backoff!(
&self.ctx,
custom(&self.ctx.net, self.url(url.clone()))
.headers(self.common_headers())
.send()
.await,
.await
),
&self.ctx.prt,
)
.await
@ -77,16 +80,20 @@ impl<'ctx> TrelloClient<'ctx> {
.cloned()
.unwrap_or_else(|| PathBuf::from(attachment.file_name));
let file_name = self.ctx.fs.base().join(file_name);
let resp = self
.ctx
let resp = with_exponential_backoff!(
&self.ctx,
self.ctx
.net
.get(url)
.get(url.clone())
.headers(self.auth_headers())
.header("accept", "application/octet")
.send()
.await?
.bytes()
.await?;
.await
.map_err(Into::into)
)
.context("downloading attachment")?;
let file = self.ctx.fs.file(&file_name);
file.write(resp).context("writing to disk")?;
Ok(file_name)