fix(deps): update rust crate regex to 1.11.0

This PR contains the following updates:

| Package | Type | Update | Change |
|---|---|---|---|
| [flexi_logger](https://crates.io/crates/flexi_logger) ([source](https://github.com/emabee/flexi_logger)) | dependencies | minor | `0.28.0` -> `0.29.0` |

---

### Release Notes

<details>
<summary>emabee/flexi_logger (flexi_logger)</summary>

### [`v0.29.0`](https://github.com/emabee/flexi_logger/blob/HEAD/CHANGELOG.md#0290---2024-08-25)

Revised `SyslogWriter` (-> version bump): introduced builder pattern,
added a configuration option for the message format
(resolves [issue #&#8203;168](https://github.com/emabee/flexi_logger/issues/168), kudos to [krims0n32](https://github.com/krims0n32)).

`LoggerHandle::existing_log_files` now also returns a meaningful result if file rotation is not
used. Kudos to [drdo](https://github.com/drdo) for
[discussion 170](https://github.com/emabee/flexi_logger/discussions/170).

### [`v0.28.5`](https://github.com/emabee/flexi_logger/blob/HEAD/CHANGELOG.md#0285---2024-06-21)

Remove unnecessary dependency to `is-terminal`.

Add impl `From<LevelFilter>` for `LogSpecification`.

Kudos to [Oakchris1955](https://github.com/Oakchris1955).

### [`v0.28.4`](https://github.com/emabee/flexi_logger/blob/HEAD/CHANGELOG.md#0284---2024-06-14)

Fix [issue #&#8203;162](https://github.com/emabee/flexi_logger/issues/162)
(FileLogWriter does not follow its max_level), kudos to [JoeWildfong](https://github.com/JoeWildfong).

### [`v0.28.3`](https://github.com/emabee/flexi_logger/blob/HEAD/CHANGELOG.md#0283---2024-06-10)

Add special handling for empty current infix to `Naming::TimestampsCustomFormat`
([issue #&#8203;161](https://github.com/emabee/flexi_logger/issues/161)).

### [`v0.28.2`](https://github.com/emabee/flexi_logger/blob/HEAD/CHANGELOG.md#0282---2024-06-09)

Add variant `Naming::TimestampsCustomFormat` ([issue #&#8203;158](https://github.com/emabee/flexi_logger/issues/158)),
kudos to [jb-alvarado](https://github.com/jb-alvarado).

### [`v0.28.1`](https://github.com/emabee/flexi_logger/blob/HEAD/CHANGELOG.md#0281---2024-06-01)

Introduce `flexi_logger::init()` as super-minimal entry usage.

Update dependencies.

</details>

---

### Configuration

📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - At any time (no schedule defined).

🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied.

♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox.

🔕 **Ignore**: Close this PR and you won't be reminded about this update again.

---

 - [ ] <!-- rebase-check -->If you want to rebase/retry this PR, check this box

---

This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate).
<!--renovate-debug:eyJjcmVhdGVkSW5WZXIiOiIzOC41Mi4zIiwidXBkYXRlZEluVmVyIjoiMzguNTIuMyIsInRhcmdldEJyYW5jaCI6Im1haW4iLCJsYWJlbHMiOltdfQ==-->

Co-authored-by: Renovate Bot <renovate@kemitix.net>
Reviewed-on: #43
Co-authored-by: renovate <renovate@noreply.kemitix.net>
Co-committed-by: renovate <renovate@noreply.kemitix.net>

.woodpecker.yml

@@ -0,0 +1,20 @@
+steps:
+  build:
+    when:
+      - event: cron
+      - event: push
+        branch: main
+      - event: pull_request
+    image: docker.io/rust:latest
+    commands:
+      - apt-get install libssl-dev
+      - cargo build
+  test:
+    when:
+      - event: cron
+      - event: push
+        branch: main
+      - event: pull_request
+    image: docker.io/rust:latest
+    commands:
+      - cargo test

Cargo.toml

@@ -13,26 +13,27 @@ readme = "README.md"
 
 [dependencies]
 # atty = "0.2.14"
-async-std = "1.10.0"
-base64 = "0.13.0"
-chrono = "0.4.19"
+async-std = "1.12.0"
+base64 = "0.22.0"
+chrono = "0.4.38"
 clap = { version = "2.34.0", features = ["yaml"] }
-colored = "2.0.0"
-comfy-table = "3.0.0"
-derive_builder = "0.10.2"
-directories = "3.0.2"
-epub-builder = "0.4.10"
-flexi_logger = "0.22.2"
-futures = "0.3.19"
+colored = "2.1.0"
+comfy-table = "7.1.1"
+derive_builder = "0.20.0"
+directories = "5.0.1"
+epub-builder = "0.7.4"
+eyre = "0.6.12"
+flexi_logger = "0.29.0"
+futures = "0.3.30"
 html5ever = "0.25.1"
-indicatif = "0.16.2"
-itertools = "0.10.3"
+indicatif = "0.17.8"
+itertools = "0.13.0"
 kuchiki = "0.8.1"
 lazy_static = "1.4.0"
-log = "0.4.14"
+log = "0.4.21"
 md5 = "0.7.0"
-regex = "1.5.4"
-serde = "1.0.136"
+openssl-sys = "0.9.102"
+regex = "1.11.0"
 surf = "2.3.2"
-thiserror = "1.0.30"
-url = "2.2.2"
+thiserror = "1.0.59"
+url = "2.5.0"

renovate.json

@@ -0,0 +1,10 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": ["config:recommended"],
+  "packageRules": [
+    {
+      "matchManagers": ["cargo"],
+      "rangeStrategy": "replace"
+    }
+  ]
+}

rust-toolchain

@@ -1 +0,0 @@
-1.57.0

src/epub.rs

@@ -30,7 +30,11 @@ pub fn generate_epubs(
         let enabled_bar = ProgressBar::new(articles.len() as u64);
         let style = ProgressStyle::default_bar().template(
             "{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} epub {pos}/{len:7} {msg:.green}",
-        );
+        ).map_err(|e| {
+            let mut paperoni_err: PaperoniError = e.into();
+            paperoni_err.set_article_source("progress bar");
+            vec![paperoni_err]
+        })?;
         enabled_bar.set_style(style);
         if !articles.is_empty() {
             enabled_bar.set_message("Generating epubs");
@@ -249,10 +253,7 @@ fn replace_escaped_characters(value: &str) -> String {
         .replace(">", ">")
 }
 
-fn add_stylesheets<T: epub_builder::Zip>(
-    epub: &mut EpubBuilder<T>,
-    app_config: &AppConfig,
-) -> Result<(), epub_builder::Error> {
+fn add_stylesheets(epub: &mut EpubBuilder<ZipLibrary>, app_config: &AppConfig) -> eyre::Result<()> {
     let body_stylesheet: &[u8] = include_bytes!("./assets/body.min.css");
     let header_stylesheet: &[u8] = include_bytes!("./assets/headers.min.css");
     match app_config.css_config {

src/errors.rs

@@ -15,6 +15,8 @@ pub enum ErrorKind {
     UTF8Error(String),
     #[error("[ReadabilityError]: {0}")]
     ReadabilityError(String),
+    #[error("[TemplateError]: {0}")]
+    TemplateError(String),
 }
 
 #[derive(Error, Debug)]
@@ -98,9 +100,9 @@ impl From<ErrorKind> for PaperoniError {
     }
 }
 
-impl From<epub_builder::Error> for PaperoniError {
-    fn from(err: epub_builder::Error) -> Self {
-        PaperoniError::with_kind(ErrorKind::EpubError(err.description().to_owned()))
+impl From<eyre::Error> for PaperoniError {
+    fn from(err: eyre::Error) -> Self {
+        PaperoniError::with_kind(ErrorKind::EpubError(err.to_string()))
     }
 }
 
@@ -128,6 +130,12 @@ impl From<std::str::Utf8Error> for PaperoniError {
     }
 }
 
+impl From<indicatif::style::TemplateError> for PaperoniError {
+    fn from(err: indicatif::style::TemplateError) -> Self {
+        PaperoniError::with_kind(ErrorKind::TemplateError(err.to_string()))
+    }
+}
+
 #[derive(Debug, Error)]
 pub enum LogError {
     #[error(transparent)]

src/extractor.rs

@@ -1,11 +1,8 @@
-use std::collections::BTreeMap;
-
-use html5ever::{LocalName, Namespace, QualName};
 use itertools::Itertools;
 use kuchiki::{traits::*, NodeRef};
 
 use crate::errors::PaperoniError;
-use crate::moz_readability::{MetaData, Readability, HTML_NS};
+use crate::moz_readability::{MetaData, Readability};
 
 /// A tuple of the url and an Option of the resource's MIME type
 pub type ResourceInfo = (String, Option<String>);
@@ -32,7 +29,6 @@ impl Article {
     /// the source of the content
     pub fn extract_content(&mut self) -> Result<(), PaperoniError> {
         self.readability.parse(&self.url)?;
-        self.reinsert_title_heading();
         if let Some(article_node_ref) = &self.readability.article_node {
             let template = r#"
             <!DOCTYPE html>
@@ -78,20 +74,6 @@ impl Article {
         )
     }
 
-    fn reinsert_title_heading(&mut self) {
-        if let Some(article_node_ref) = &self.readability.article_node {
-            if let Ok(article_root_ref) = article_node_ref.select_first("div#readability-page-1") {
-                let article_root_elem = article_root_ref.as_node();
-                let h1_elem = NodeRef::new_element(
-                    QualName::new(None, Namespace::from(HTML_NS), LocalName::from("h1")),
-                    BTreeMap::new(),
-                );
-                h1_elem.append(NodeRef::new_text(self.readability.metadata.title()));
-                article_root_elem.prepend(h1_elem);
-            };
-        }
-    }
-
     pub fn metadata(&self) -> &MetaData {
         &self.readability.metadata
     }

src/html.rs

@@ -4,7 +4,8 @@ use std::{
     path::Path,
 };
 
-use base64::encode;
+use base64::prelude::*;
+
 use comfy_table::{Attribute, Cell, CellAlignment, Color, ContentArrangement, Table};
 use html5ever::{LocalName, Namespace, QualName};
 use indicatif::{ProgressBar, ProgressStyle};
@@ -43,7 +44,11 @@ pub fn generate_html_exports(
         let enabled_bar = ProgressBar::new(articles.len() as u64);
         let style = ProgressStyle::default_bar().template(
             "{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} html {pos}/{len:7} {msg:.green}",
-        );
+        ).map_err(|e| {
+            let mut paperoni_err: PaperoniError = e.into();
+            paperoni_err.set_article_source("progress bar");
+            vec![paperoni_err]
+        })?;
         enabled_bar.set_style(style);
         if !articles.is_empty() {
             enabled_bar.set_message("Generating html files");
@@ -267,7 +272,7 @@ fn update_imgs_base64(article: &Article) -> Result<(), std::io::Error> {
         let img_base64_str = format!(
             "data:image:{};base64,{}",
             mime_type.as_deref().unwrap_or("image/*"),
-            encode(img_bytes)
+            BASE64_STANDARD.encode(img_bytes)
         );
 
         let img_elems = article

src/http.rs

@@ -5,7 +5,6 @@ use futures::StreamExt;
 use indicatif::ProgressBar;
 use log::warn;
 use log::{debug, info};
-use serde::{Deserialize, Serialize};
 use url::Url;
 
 use crate::cli::AppConfig;
@@ -23,162 +22,51 @@ pub fn download(
         let urls_iter = app_config.urls.iter().map(|url| fetch_html(url));
         let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn);
         let mut articles = Vec::new();
-        // Collect all urls that couldn't extract here
-        // let mut retry_with_paperteer: Vec<String> = Vec::new();
         while let Some(fetch_result) = responses.next().await {
             match fetch_result {
                 Ok((url, html)) => {
-                    match extract_and_download_imgs(
-                        &url,
-                        html,
-                        bar,
-                        partial_downloads,
-                        &mut articles,
-                    )
-                    .await
-                    {
-                        Ok(_) => bar.inc(1),
-
-                        // All errors are pushed into here since they're readability issues.
-                        Err(e) => errors.push(e),
+                    debug!("Extracting {}", &url);
+                    let mut extractor = Article::from_html(&html, &url);
+                    bar.set_message("Extracting...");
+                    match extractor.extract_content() {
+                        Ok(_) => {
+                            extractor.extract_img_urls();
+                            if let Err(img_errors) =
+                                download_images(&mut extractor, &Url::parse(&url).unwrap(), &bar)
+                                    .await
+                            {
+                                partial_downloads
+                                    .push(PartialDownload::new(&url, extractor.metadata().title()));
+                                warn!(
+                                    "{} image{} failed to download for {}",
+                                    img_errors.len(),
+                                    if img_errors.len() > 1 { "s" } else { "" },
+                                    url
+                                );
+                                for img_error in img_errors {
+                                    warn!(
+                                        "{}\n\t\tReason {}",
+                                        img_error.url().as_ref().unwrap(),
+                                        img_error
+                                    );
+                                }
+                            }
+                            articles.push(extractor);
+                        }
+                        Err(mut e) => {
+                            e.set_article_source(&url);
+                            errors.push(e);
+                        }
                     }
-
-                    // Outside the stream, make a new one to retry with paperteer
                 }
                 Err(e) => errors.push(e),
             }
+            bar.inc(1);
         }
-        // if !retry_with_paperteer.is_empty() {
-        //     fetch_html_from_paperteer(
-        //         retry_with_paperteer,
-        //         app_config,
-        //         bar,
-        //         partial_downloads,
-        //         errors,
-        //         &mut articles,
-        //     )
-        //     .await
-        //     .unwrap();
-        // }
         articles
     })
 }
 
-async fn extract_and_download_imgs<'a>(
-    url: &str,
-    html: String,
-    bar: &ProgressBar,
-    partial_downloads: &mut Vec<PartialDownload>,
-    articles: &mut Vec<Article>,
-) -> Result<(), PaperoniError> {
-    debug!("Extracting {}", &url);
-    let mut extractor = Article::from_html(&html, &url);
-    bar.set_message("Extracting...");
-    match extractor.extract_content() {
-        Ok(_) => {
-            extractor.extract_img_urls();
-            if let Err(img_errors) =
-                download_images(&mut extractor, &Url::parse(&url).unwrap(), &bar).await
-            {
-                partial_downloads.push(PartialDownload::new(&url, extractor.metadata().title()));
-                warn!(
-                    "{} image{} failed to download for {}",
-                    img_errors.len(),
-                    if img_errors.len() > 1 { "s" } else { "" },
-                    &url
-                );
-                for img_error in img_errors {
-                    warn!(
-                        "{}\n\t\tReason {}",
-                        img_error.url().as_ref().unwrap(),
-                        img_error
-                    );
-                }
-            }
-            articles.push(extractor);
-            Ok(())
-        }
-        Err(mut e) => {
-            e.set_article_source(&url);
-            Err(e)
-        }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-struct PaperteerBody {
-    urls: Vec<String>,
-}
-
-impl PaperteerBody {
-    fn new(urls: Vec<String>) -> Self {
-        PaperteerBody { urls }
-    }
-}
-
-#[derive(Serialize, Deserialize)]
-struct PaperteerItem {
-    url: String,
-    response: String,
-    html: String,
-}
-
-#[derive(Serialize, Deserialize)]
-struct PaperteerResponse {
-    data: Vec<PaperteerItem>,
-}
-
-// TODO: Change signature to simply take a vec of urls and return a vec of urls with either html or an error
-// This also means that extracting and downloading imgs should be handled externally
-async fn _fetch_html_from_paperteer(
-    urls: Vec<String>,
-    _app_config: &AppConfig,
-    bar: &ProgressBar,
-    partial_downloads: &mut Vec<PartialDownload>,
-    errors: &mut Vec<PaperoniError>,
-    articles: &mut Vec<Article>,
-) -> Result<(), ()> {
-    // Get the paperteer url
-    let render_endpoint = "/api/render";
-    let paperteer_url = url::Url::parse("http://localhost:3000")
-        .unwrap()
-        .join(render_endpoint)
-        .unwrap();
-
-    // Build request body with urls
-    let urls_str = urls.into_iter().map(|url| url.to_string()).collect();
-    let body = PaperteerBody::new(urls_str);
-
-    // Send to the paperteer url
-    let mut res = surf::post(paperteer_url)
-        .body(surf::Body::from_json(&body).unwrap())
-        .await
-        .unwrap();
-
-    // Receive the json response
-    // TODO: Check for body response
-    let PaperteerResponse { data } = res.body_json().await.unwrap();
-
-    // For each url, extract the article and images
-    for item in data {
-        let PaperteerItem {
-            html,
-            url,
-            response,
-        } = item;
-        if response == "ok" {
-            // Run the extract and download fn
-            match extract_and_download_imgs(&url, html, bar, partial_downloads, articles).await {
-                Ok(_) => bar.inc(1),
-                Err(e) => errors.push(e),
-            }
-        } else {
-            errors.push(crate::errors::ErrorKind::HTTPError("Paperteer failed".into()).into());
-        }
-    }
-    Ok(())
-}
-
 pub async fn fetch_html(url: &str) -> Result<HTMLResource, PaperoniError> {
     let client = surf::Client::new();
     debug!("Fetching {}", url);

src/logs.rs

@@ -2,7 +2,7 @@ use std::fs;
 
 use chrono::{DateTime, Local};
 use colored::*;
-use comfy_table::presets::UTF8_HORIZONTAL_BORDERS_ONLY;
+use comfy_table::presets::UTF8_HORIZONTAL_ONLY;
 use comfy_table::{Cell, CellAlignment, ContentArrangement, Table};
 use flexi_logger::{FileSpec, LevelFilter};
 use log::error;
@@ -38,7 +38,7 @@ pub fn display_summary(
         println!("\n{}", "Partially failed downloads".yellow().bold());
         let mut table_partial = Table::new();
         table_partial
-            .load_preset(UTF8_HORIZONTAL_BORDERS_ONLY)
+            .load_preset(UTF8_HORIZONTAL_ONLY)
             .set_header(vec![
                 Cell::new("Link").set_alignment(CellAlignment::Center),
                 Cell::new("Title").set_alignment(CellAlignment::Center),
@@ -55,7 +55,7 @@ pub fn display_summary(
         println!("\n{}", "Failed article downloads".bright_red().bold());
         let mut table_failed = Table::new();
         table_failed
-            .load_preset(UTF8_HORIZONTAL_BORDERS_ONLY)
+            .load_preset(UTF8_HORIZONTAL_ONLY)
             .set_header(vec![
                 Cell::new("Link").set_alignment(CellAlignment::Center),
                 Cell::new("Reason").set_alignment(CellAlignment::Center),

src/main.rs

@@ -2,9 +2,10 @@
 extern crate lazy_static;
 
 use std::process::exit;
+use std::time::Duration;
 
 use colored::Colorize;
-use comfy_table::presets::{UTF8_FULL, UTF8_HORIZONTAL_BORDERS_ONLY};
+use comfy_table::presets::{UTF8_FULL, UTF8_HORIZONTAL_ONLY};
 use comfy_table::{ContentArrangement, Table};
 use http::download;
 use indicatif::{ProgressBar, ProgressStyle};
@@ -25,6 +26,8 @@ use epub::generate_epubs;
 use html::generate_html_exports;
 use logs::display_summary;
 
+use crate::errors::PaperoniError;
+
 fn main() {
     let app_config = match cli::AppConfig::init_with_cli() {
         Ok(app_config) => app_config,
@@ -58,9 +61,15 @@ fn run(app_config: AppConfig) {
         let enabled_bar = ProgressBar::new(app_config.urls.len() as u64);
         let style = ProgressStyle::default_bar().template(
             "{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} link {pos}/{len:7} {msg:.yellow/white}",
-        );
-        enabled_bar.set_style(style);
-        enabled_bar.enable_steady_tick(500);
+        ).map_err(|e| {
+            let mut paperoni_err: PaperoniError = e.into();
+            paperoni_err.set_article_source("progress bar");
+            vec![paperoni_err]
+        });
+        if let Ok(style) = style {
+            enabled_bar.set_style(style);
+        }
+        enabled_bar.enable_steady_tick(Duration::from_millis(500));
         enabled_bar
     };
 
@@ -70,7 +79,7 @@ fn run(app_config: AppConfig) {
     let mut successful_articles_table = Table::new();
     successful_articles_table
         .load_preset(UTF8_FULL)
-        .load_preset(UTF8_HORIZONTAL_BORDERS_ONLY)
+        .load_preset(UTF8_HORIZONTAL_ONLY)
         .set_content_arrangement(ContentArrangement::Dynamic);
 
     match app_config.export_type {

src/moz_readability/mod.rs

@@ -17,7 +17,7 @@ const FLAG_STRIP_UNLIKELYS: u32 = 0x1;
 const FLAG_WEIGHT_CLASSES: u32 = 0x2;
 const FLAG_CLEAN_CONDITIONALLY: u32 = 0x4;
 const READABILITY_SCORE: &'static str = "readability-score";
-pub const HTML_NS: &'static str = "http://www.w3.org/1999/xhtml";
+const HTML_NS: &'static str = "http://www.w3.org/1999/xhtml";
 // TODO: Change to HashSet
 const PHRASING_ELEMS: [&str; 39] = [
     "abbr", "audio", "b", "bdo", "br", "button", "cite", "code", "data", "datalist", "dfn", "em",