diff --git a/src/extractor.rs b/src/extractor.rs index 7d04bdd..ea7066f 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -74,7 +74,11 @@ impl Extractor { let abs_url = get_absolute_url(&img_url, article_origin); async_download_tasks.push(task::spawn(async move { - let mut img_response = surf::get(&abs_url).await.expect("Unable to retrieve file"); + let mut img_response = surf::Client::new() + .with(surf::middleware::Redirect::default()) + .get(&abs_url) + .await + .expect("Unable to retrieve file"); let img_content: Vec = img_response.body_bytes().await.unwrap(); let img_mime = img_response .content_type() diff --git a/src/main.rs b/src/main.rs index e61bef0..d4f6c1b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,24 +23,26 @@ fn main() { type HTMLResource = (String, String); -async fn fetch_url(url: &str) -> HTMLResource { +async fn fetch_url(url: &str) -> Result> { let client = surf::Client::new(); println!("Fetching..."); - // TODO: Add middleware for following redirects - ( - url.to_string(), - client - .get(url) - .recv_string() - .await - .expect("Unable to fetch URL"), - ) + let mut res = client + .with(surf::middleware::Redirect::default()) + .get(url) + .send() + .await + .expect(&format!("Unable to fetch {}", url)); + if res.status() == 200 { + Ok((url.to_string(), res.body_string().await?)) + } else { + Err("Request failed to return HTTP 200".into()) + } } fn download(urls: Vec) { let mut async_url_tasks = Vec::with_capacity(urls.len()); for url in urls { - async_url_tasks.push(task::spawn(async move { fetch_url(&url).await })); + async_url_tasks.push(task::spawn(async move { fetch_url(&url).await.unwrap() })); } task::block_on(async { for url_task in async_url_tasks {