Add progress indicators for the cli

This commit is contained in:
Kenneth Gitere 2021-04-17 17:27:38 +03:00
parent 217cd3e442
commit 04a1eed4e2
5 changed files with 84 additions and 7 deletions

50
Cargo.lock generated
View file

@ -403,6 +403,21 @@ dependencies = [
"cache-padded",
]
[[package]]
name = "console"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"regex",
"terminal_size",
"unicode-width",
"winapi",
]
[[package]]
name = "const_fn"
version = "0.4.3"
@ -577,6 +592,12 @@ dependencies = [
"dtoa",
]
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
version = "0.8.26"
@ -960,6 +981,18 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "indicatif"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
dependencies = [
"console",
"lazy_static",
"number_prefix",
"regex",
]
[[package]]
name = "infer"
version = "0.2.3"
@ -1232,6 +1265,12 @@ dependencies = [
"libc",
]
[[package]]
name = "number_prefix"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
[[package]]
name = "object"
version = "0.22.0"
@ -1278,6 +1317,7 @@ dependencies = [
"epub-builder",
"futures",
"html5ever",
"indicatif",
"kuchiki",
"lazy_static",
"md5",
@ -1944,6 +1984,16 @@ dependencies = [
"utf-8",
]
[[package]]
name = "terminal_size"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "textwrap"
version = "0.11.0"

View file

@ -17,6 +17,7 @@ clap = "2.33.3"
epub-builder = "0.4.8"
futures = "0.3.12"
html5ever = "0.25.1"
indicatif = "0.15.0"
kuchiki = "0.8.1"
lazy_static = "1.4.0"
md5 = "0.7.0"

View file

@ -1,6 +1,7 @@
use std::fs::File;
use epub_builder::{EpubBuilder, EpubContent, ZipLibrary};
use indicatif::{ProgressBar, ProgressStyle};
use crate::{
errors::PaperoniError,
@ -11,6 +12,12 @@ pub fn generate_epubs(
articles: Vec<Extractor>,
merged: Option<&String>,
) -> Result<(), PaperoniError> {
let bar = ProgressBar::new(articles.len() as u64);
let style = ProgressStyle::default_bar().template(
"{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} epub {pos}/{len:7} {msg:.green}",
);
bar.set_style(style);
bar.set_message("Generating epubs");
match merged {
Some(name) => {
let mut epub = EpubBuilder::new(ZipLibrary::new()?)?;
@ -33,6 +40,8 @@ pub fn generate_epubs(
.unwrap();
article.img_urls.iter().for_each(|img| {
// TODO: Add error handling
bar.inc(1);
let mut file_path = std::env::temp_dir();
file_path.push(&img.0);
@ -48,6 +57,7 @@ pub fn generate_epubs(
});
let mut out_file = File::create(&name).unwrap();
epub.generate(&mut out_file)?;
bar.finish_with_message("Generated epub\n");
println!("Created {:?}", name);
}
None => {
@ -79,8 +89,10 @@ pub fn generate_epubs(
epub.add_resource(file_path.file_name().unwrap(), img_buf, img.1.unwrap())?;
}
epub.generate(&mut out_file)?;
println!("Created {:?}", file_name);
bar.inc(1);
// println!("Created {:?}", file_name);
}
bar.finish_with_message("Generated epubs\n");
}
}
Ok(())

View file

@ -1,6 +1,7 @@
use async_std::io::prelude::*;
use async_std::{fs::File, stream};
use futures::StreamExt;
use indicatif::ProgressBar;
use url::Url;
use crate::{errors::ErrorKind, errors::PaperoniError, extractor::Extractor};
@ -9,7 +10,7 @@ type HTMLResource = (String, String);
pub async fn fetch_html(url: &str) -> Result<HTMLResource, PaperoniError> {
let client = surf::Client::new();
println!("Fetching...");
// println!("Fetching...");
let mut redirect_count: u8 = 0;
let base_url = Url::parse(&url)?;
@ -56,10 +57,12 @@ pub async fn fetch_html(url: &str) -> Result<HTMLResource, PaperoniError> {
pub async fn download_images(
extractor: &mut Extractor,
article_origin: &Url,
bar: &ProgressBar,
) -> Result<(), Vec<PaperoniError>> {
if extractor.img_urls.len() > 0 {
println!("Downloading images...");
// println!("Downloading images...");
}
let img_count = extractor.img_urls.len();
let imgs_req_iter = extractor
.img_urls
@ -70,7 +73,9 @@ pub async fn download_images(
surf::Client::new().get(get_absolute_url(&url, article_origin)),
)
})
.map(|(url, req)| async move {
.enumerate()
.map(|(img_idx, (url, req))| async move {
bar.set_message(format!("Downloading images [{}/{}]", img_idx + 1, img_count).as_str());
match req.await {
Ok(mut img_response) => {
// let mut img_response = req.await.expect("Unable to retrieve image");

View file

@ -4,6 +4,7 @@ extern crate lazy_static;
use async_std::stream;
use async_std::task;
use futures::stream::StreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use url::Url;
mod cli;
@ -29,6 +30,12 @@ fn main() {
}
fn download(app_config: AppConfig) {
let bar = ProgressBar::new(app_config.urls().len() as u64);
let style = ProgressStyle::default_bar().template(
"{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} link {pos}/{len:7} {msg:.yellow/white}",
);
bar.set_style(style);
bar.enable_steady_tick(500);
let articles = task::block_on(async {
let urls_iter = app_config.urls().iter().map(|url| fetch_html(url));
let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn());
@ -36,15 +43,15 @@ fn download(app_config: AppConfig) {
while let Some(fetch_result) = responses.next().await {
match fetch_result {
Ok((url, html)) => {
println!("Extracting");
// println!("Extracting");
let mut extractor = Extractor::from_html(&html);
bar.set_message("Extracting...");
extractor.extract_content(&url);
if extractor.article().is_some() {
extractor.extract_img_urls();
if let Err(img_errors) =
download_images(&mut extractor, &Url::parse(&url).unwrap()).await
download_images(&mut extractor, &Url::parse(&url).unwrap(), &bar).await
{
eprintln!(
"{} image{} failed to download for {}",
@ -58,9 +65,11 @@ fn download(app_config: AppConfig) {
}
Err(e) => eprintln!("{}", e),
}
bar.inc(1);
}
articles
});
bar.finish_with_message("Downloaded articles");
match generate_epubs(articles, app_config.merged()) {
Ok(_) => (),
Err(e) => eprintln!("{}", e),