feat(trello): support exponential backoff with jitter
This commit is contained in:
parent
0e123898db
commit
4cdfdaec6f
4 changed files with 52 additions and 19 deletions
|
@ -18,6 +18,7 @@ inquire = "0.7"
|
|||
kameo = "0.13"
|
||||
# kxio = {path = "../kxio/"}
|
||||
kxio = "4.0"
|
||||
rand = "0.8"
|
||||
reqwest = { version = "0.12" , features = ["multipart", "stream"]}
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
|
|
24
src/macros/backoff.rs
Normal file
24
src/macros/backoff.rs
Normal file
|
@ -0,0 +1,24 @@
|
|||
#[macro_export]
|
||||
macro_rules! with_exponential_backoff {
|
||||
($ctx:expr, $operation:expr) => {{
|
||||
let mut backoff_secs = 1;
|
||||
loop {
|
||||
match $operation {
|
||||
Err(kxio::net::Error::Reqwest(e))
|
||||
if e.status() == Some(kxio::net::StatusCode::TOO_MANY_REQUESTS) =>
|
||||
{
|
||||
backoff_secs *= 2;
|
||||
let jitter = rand::random::<u64>() % 10;
|
||||
let backoff_secs = 60.min(backoff_secs + jitter);
|
||||
$crate::p!(
|
||||
$ctx.prt,
|
||||
">> Too many requests, backing off for {}s",
|
||||
backoff_secs
|
||||
);
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(backoff_secs)).await;
|
||||
}
|
||||
result => break result,
|
||||
}
|
||||
}
|
||||
}};
|
||||
}
|
|
@ -1,5 +1,6 @@
|
|||
//
|
||||
mod actor;
|
||||
mod backoff;
|
||||
mod newtype;
|
||||
mod print;
|
||||
mod send;
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
//
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::{collections::HashMap, path::PathBuf};
|
||||
|
||||
use color_eyre::eyre::Context;
|
||||
use kxio::net::{Net, ReqBuilder};
|
||||
|
@ -11,15 +10,15 @@ use crate::{
|
|||
trello::model::{
|
||||
attachment::{TrelloAttachment, TrelloAttachmentId},
|
||||
board::{TrelloBoard, TrelloBoardId},
|
||||
card::TrelloCardId,
|
||||
card::{TrelloLongCard, TrelloShortCard},
|
||||
card::{TrelloCardId, TrelloLongCard, TrelloShortCard},
|
||||
list::TrelloListId,
|
||||
},
|
||||
FullCtx,
|
||||
with_exponential_backoff, FullCtx,
|
||||
};
|
||||
|
||||
pub(crate) struct TrelloClient<'ctx> {
|
||||
ctx: &'ctx FullCtx,
|
||||
// 300 requests per 10 seconds for each API key and 100 requests per 10 second interval for each token
|
||||
}
|
||||
|
||||
impl<'ctx> TrelloClient<'ctx> {
|
||||
|
@ -55,11 +54,15 @@ impl<'ctx> TrelloClient<'ctx> {
|
|||
url: impl Into<String>,
|
||||
custom: fn(&Net, String) -> ReqBuilder,
|
||||
) -> APIResult<T> {
|
||||
let url = url.into();
|
||||
APIResult::new(
|
||||
custom(&self.ctx.net, self.url(url))
|
||||
with_exponential_backoff!(
|
||||
&self.ctx,
|
||||
custom(&self.ctx.net, self.url(url.clone()))
|
||||
.headers(self.common_headers())
|
||||
.send()
|
||||
.await,
|
||||
.await
|
||||
),
|
||||
&self.ctx.prt,
|
||||
)
|
||||
.await
|
||||
|
@ -77,16 +80,20 @@ impl<'ctx> TrelloClient<'ctx> {
|
|||
.cloned()
|
||||
.unwrap_or_else(|| PathBuf::from(attachment.file_name));
|
||||
let file_name = self.ctx.fs.base().join(file_name);
|
||||
let resp = self
|
||||
.ctx
|
||||
let resp = with_exponential_backoff!(
|
||||
&self.ctx,
|
||||
self.ctx
|
||||
.net
|
||||
.get(url)
|
||||
.get(url.clone())
|
||||
.headers(self.auth_headers())
|
||||
.header("accept", "application/octet")
|
||||
.send()
|
||||
.await?
|
||||
.bytes()
|
||||
.await?;
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
)
|
||||
.context("downloading attachment")?;
|
||||
let file = self.ctx.fs.file(&file_name);
|
||||
file.write(resp).context("writing to disk")?;
|
||||
Ok(file_name)
|
||||
|
|
Loading…
Reference in a new issue