Merge pull request #5 from hipstermojo/dev

Merge 0.2.2-alpha-1

Cargo.lock

@@ -1242,7 +1242,7 @@ dependencies = [
 
 [[package]]
 name = "paperoni"
-version = "0.2.1-alpha1"
+version = "0.2.2-alpha1"
 dependencies = [
  "async-std",
  "clap",

Cargo.toml

@@ -3,7 +3,7 @@ description = "A web article downloader"
 homepage = "https://github.com/hipstermojo/paperoni"
 repository = "https://github.com/hipstermojo/paperoni"
 name = "paperoni"
-version = "0.2.1-alpha1"
+version = "0.2.2-alpha1"
 authors = ["Kenneth Gitere <gitere81@gmail.com>"]
 edition = "2018"
 license = "MIT"

src/cli.rs

@@ -6,7 +6,7 @@ pub fn cli_init() -> App<'static, 'static> {
             AppSettings::ArgRequiredElseHelp,
             AppSettings::UnifiedHelpMessage,
         ])
-        .version("0.2.1-alpha1")
+        .version("0.2.2-alpha1")
         .about(
             "
 Paperoni is an article downloader.

src/extractor.rs

@@ -68,7 +68,9 @@ impl Extractor {
     pub async fn download_images(&mut self, article_origin: &Url) -> async_std::io::Result<()> {
         let mut async_download_tasks = Vec::with_capacity(self.img_urls.len());
         self.extract_img_urls();
+        if self.img_urls.len() > 0 {
             println!("Downloading images...");
+        }
         for img_url in &self.img_urls {
             let img_url = img_url.0.clone();
             let abs_url = get_absolute_url(&img_url, article_origin);
@@ -129,6 +131,9 @@ impl Extractor {
                 .expect("Image node does not exist");
             let mut img_node = img_ref.attributes.borrow_mut();
             *img_node.get_mut("src").unwrap() = img_path.clone();
+            // srcset is removed because readers such as Foliate then fail to display
+            // the image already downloaded and stored in src
+            img_node.remove("srcset");
             self.img_urls.push((img_path, img_mime));
         }
         Ok(())

src/main.rs

@@ -23,30 +23,59 @@ fn main() {
 
 type HTMLResource = (String, String);
 
-async fn fetch_url(url: &str) -> Result<HTMLResource, Box<dyn std::error::Error>> {
+async fn fetch_url(url: &str) -> Result<HTMLResource, Box<dyn std::error::Error + Send + Sync>> {
     let client = surf::Client::new();
     println!("Fetching...");
-    let mut res = client
+
-        .with(surf::middleware::Redirect::default())
+    let mut redirect_count: u8 = 0;
-        .get(url)
+    let base_url = Url::parse(&url)?;
-        .send()
+    let mut url = base_url.clone();
-        .await
+    while redirect_count < 5 {
-        .expect(&format!("Unable to fetch {}", url));
+        redirect_count += 1;
-    if res.status() == 200 {
+        let req = surf::get(&url);
-        Ok((url.to_string(), res.body_string().await?))
+        let mut res = client.send(req).await?;
-    } else {
+        if res.status().is_redirection() {
-        Err("Request failed to return HTTP 200".into())
+            if let Some(location) = res.header(surf::http::headers::LOCATION) {
+                match Url::parse(location.last().as_str()) {
+                    Ok(valid_url) => url = valid_url,
+                    Err(e) => match e {
+                        url::ParseError::RelativeUrlWithoutBase => {
+                            url = base_url.join(location.last().as_str())?
                         }
+                        e => return Err(e.into()),
+                    },
+                };
+            }
+        } else if res.status().is_success() {
+            if let Some(mime) = res.content_type() {
+                if mime.essence() == "text/html" {
+                    return Ok((url.to_string(), res.body_string().await?));
+                } else {
+                    return Err(format!(
+                        "Invalid HTTP response. Received {} instead of text/html",
+                        mime.essence()
+                    )
+                    .into());
+                }
+            } else {
+                return Err("Unknown HTTP response".into());
+            }
+        } else {
+            return Err(format!("Request failed: HTTP {}", res.status()).into());
+        }
+    }
+    Err("Unable to fetch HTML".into())
 }
 
 fn download(urls: Vec<String>) {
     let mut async_url_tasks = Vec::with_capacity(urls.len());
     for url in urls {
-        async_url_tasks.push(task::spawn(async move { fetch_url(&url).await.unwrap() }));
+        async_url_tasks.push(task::spawn(async move { fetch_url(&url).await }));
     }
     task::block_on(async {
         for url_task in async_url_tasks {
-            let (url, html) = url_task.await;
+            match url_task.await {
+                Ok((url, html)) => {
                     println!("Extracting");
                     let mut extractor = Extractor::from_html(&html);
                     extractor.extract_content(&url);
@@ -82,12 +111,19 @@ fn download(urls: Vec<String>) {
                             file_path.push(&img.0);
 
                             let img_buf = File::open(&file_path).expect("Can't read file");
-                    epub.add_resource(file_path.file_name().unwrap(), img_buf, img.1.unwrap())
+                            epub.add_resource(
+                                file_path.file_name().unwrap(),
+                                img_buf,
+                                img.1.unwrap(),
+                            )
                             .unwrap();
                         }
                         epub.generate(&mut out_file).unwrap();
                         println!("Created {:?}", file_name);
                     }
                 }
+                Err(e) => println!("{}", e),
+            }
+        }
     })
 }

src/moz_readability/mod.rs

@@ -193,10 +193,7 @@ impl Readability {
                                 .borrow_mut()
                                 .insert(attr_name, prev_value.value.clone());
                         }
-                        // WARN: This assumes `next_element` returns an element node!!
+                        prev_elem.insert_after(new_img.as_node().clone());
-                        let inner_node_child =
-                            Self::next_element(inner_node_ref.first_child(), true);
-                        prev_elem.insert_after(inner_node_child.unwrap());
                         prev_elem.detach();
                     }
                 }