diff --git a/Cargo.toml b/Cargo.toml index 3a5c4e6..30aeb3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ inquire = "0.7" kameo = "0.13" # kxio = {path = "../kxio/"} kxio = "4.0" +rand = "0.8" reqwest = { version = "0.12" , features = ["multipart", "stream"]} serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/src/macros/backoff.rs b/src/macros/backoff.rs new file mode 100644 index 0000000..a9e886a --- /dev/null +++ b/src/macros/backoff.rs @@ -0,0 +1,24 @@ +#[macro_export] +macro_rules! with_exponential_backoff { + ($ctx:expr, $operation:expr) => {{ + let mut backoff_secs = 1; + loop { + match $operation { + Err(kxio::net::Error::Reqwest(e)) + if e.status() == Some(kxio::net::StatusCode::TOO_MANY_REQUESTS) => + { + backoff_secs *= 2; + let jitter = rand::random::() % 10; + let backoff_secs = 60.min(backoff_secs + jitter); + $crate::p!( + $ctx.prt, + ">> Too many requests, backing off for {}s", + backoff_secs + ); + tokio::time::sleep(tokio::time::Duration::from_secs(backoff_secs)).await; + } + result => break result, + } + } + }}; +} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 12606ff..350d14c 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,5 +1,6 @@ // mod actor; +mod backoff; mod newtype; mod print; mod send; diff --git a/src/trello/client.rs b/src/trello/client.rs index b100f9c..43f10ed 100644 --- a/src/trello/client.rs +++ b/src/trello/client.rs @@ -1,6 +1,5 @@ // -use std::collections::HashMap; -use std::path::PathBuf; +use std::{collections::HashMap, path::PathBuf}; use color_eyre::eyre::Context; use kxio::net::{Net, ReqBuilder}; @@ -11,15 +10,15 @@ use crate::{ trello::model::{ attachment::{TrelloAttachment, TrelloAttachmentId}, board::{TrelloBoard, TrelloBoardId}, - card::TrelloCardId, - card::{TrelloLongCard, TrelloShortCard}, + card::{TrelloCardId, TrelloLongCard, TrelloShortCard}, list::TrelloListId, }, - FullCtx, + with_exponential_backoff, FullCtx, }; pub(crate) struct TrelloClient<'ctx> { ctx: &'ctx FullCtx, + // 300 requests per 10 seconds for each API key and 100 requests per 10 second interval for each token } impl<'ctx> TrelloClient<'ctx> { @@ -55,11 +54,15 @@ impl<'ctx> TrelloClient<'ctx> { url: impl Into, custom: fn(&Net, String) -> ReqBuilder, ) -> APIResult { + let url = url.into(); APIResult::new( - custom(&self.ctx.net, self.url(url)) - .headers(self.common_headers()) - .send() - .await, + with_exponential_backoff!( + &self.ctx, + custom(&self.ctx.net, self.url(url.clone())) + .headers(self.common_headers()) + .send() + .await + ), &self.ctx.prt, ) .await @@ -77,16 +80,20 @@ impl<'ctx> TrelloClient<'ctx> { .cloned() .unwrap_or_else(|| PathBuf::from(attachment.file_name)); let file_name = self.ctx.fs.base().join(file_name); - let resp = self - .ctx - .net - .get(url) - .headers(self.auth_headers()) - .header("accept", "application/octet") - .send() - .await? - .bytes() - .await?; + let resp = with_exponential_backoff!( + &self.ctx, + self.ctx + .net + .get(url.clone()) + .headers(self.auth_headers()) + .header("accept", "application/octet") + .send() + .await? + .bytes() + .await + .map_err(Into::into) + ) + .context("downloading attachment")?; let file = self.ctx.fs.file(&file_name); file.write(resp).context("writing to disk")?; Ok(file_name)