Add basic redirect provided by surf and early exit of the program if the response is not a 200

src/extractor.rs

@@ -74,7 +74,11 @@ impl Extractor {
             let abs_url = get_absolute_url(&img_url, article_origin);
 
             async_download_tasks.push(task::spawn(async move {
-                let mut img_response = surf::get(&abs_url).await.expect("Unable to retrieve file");
+                let mut img_response = surf::Client::new()
+                    .with(surf::middleware::Redirect::default())
+                    .get(&abs_url)
+                    .await
+                    .expect("Unable to retrieve file");
                 let img_content: Vec<u8> = img_response.body_bytes().await.unwrap();
                 let img_mime = img_response
                     .content_type()

src/main.rs

@@ -23,24 +23,26 @@ fn main() {
 
 type HTMLResource = (String, String);
 
-async fn fetch_url(url: &str) -> HTMLResource {
+async fn fetch_url(url: &str) -> Result<HTMLResource, Box<dyn std::error::Error>> {
     let client = surf::Client::new();
     println!("Fetching...");
-    // TODO: Add middleware for following redirects
+    let mut res = client
-    (
+        .with(surf::middleware::Redirect::default())
-        url.to_string(),
-        client
         .get(url)
-            .recv_string()
+        .send()
         .await
-            .expect("Unable to fetch URL"),
+        .expect(&format!("Unable to fetch {}", url));
-    )
+    if res.status() == 200 {
+        Ok((url.to_string(), res.body_string().await?))
+    } else {
+        Err("Request failed to return HTTP 200".into())
+    }
 }
 
 fn download(urls: Vec<String>) {
     let mut async_url_tasks = Vec::with_capacity(urls.len());
     for url in urls {
-        async_url_tasks.push(task::spawn(async move { fetch_url(&url).await }));
+        async_url_tasks.push(task::spawn(async move { fetch_url(&url).await.unwrap() }));
     }
     task::block_on(async {
         for url_task in async_url_tasks {