Add progress indicators for the cli
This commit is contained in:
parent
217cd3e442
commit
04a1eed4e2
5 changed files with 84 additions and 7 deletions
50
Cargo.lock
generated
50
Cargo.lock
generated
|
@ -403,6 +403,21 @@ dependencies = [
|
|||
"cache-padded",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45"
|
||||
dependencies = [
|
||||
"encode_unicode",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"regex",
|
||||
"terminal_size",
|
||||
"unicode-width",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const_fn"
|
||||
version = "0.4.3"
|
||||
|
@ -577,6 +592,12 @@ dependencies = [
|
|||
"dtoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encode_unicode"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.26"
|
||||
|
@ -960,6 +981,18 @@ dependencies = [
|
|||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
|
||||
dependencies = [
|
||||
"console",
|
||||
"lazy_static",
|
||||
"number_prefix",
|
||||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "infer"
|
||||
version = "0.2.3"
|
||||
|
@ -1232,6 +1265,12 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "number_prefix"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.22.0"
|
||||
|
@ -1278,6 +1317,7 @@ dependencies = [
|
|||
"epub-builder",
|
||||
"futures",
|
||||
"html5ever",
|
||||
"indicatif",
|
||||
"kuchiki",
|
||||
"lazy_static",
|
||||
"md5",
|
||||
|
@ -1944,6 +1984,16 @@ dependencies = [
|
|||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "terminal_size"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "textwrap"
|
||||
version = "0.11.0"
|
||||
|
|
|
@ -17,6 +17,7 @@ clap = "2.33.3"
|
|||
epub-builder = "0.4.8"
|
||||
futures = "0.3.12"
|
||||
html5ever = "0.25.1"
|
||||
indicatif = "0.15.0"
|
||||
kuchiki = "0.8.1"
|
||||
lazy_static = "1.4.0"
|
||||
md5 = "0.7.0"
|
||||
|
|
14
src/epub.rs
14
src/epub.rs
|
@ -1,6 +1,7 @@
|
|||
use std::fs::File;
|
||||
|
||||
use epub_builder::{EpubBuilder, EpubContent, ZipLibrary};
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
|
||||
use crate::{
|
||||
errors::PaperoniError,
|
||||
|
@ -11,6 +12,12 @@ pub fn generate_epubs(
|
|||
articles: Vec<Extractor>,
|
||||
merged: Option<&String>,
|
||||
) -> Result<(), PaperoniError> {
|
||||
let bar = ProgressBar::new(articles.len() as u64);
|
||||
let style = ProgressStyle::default_bar().template(
|
||||
"{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} epub {pos}/{len:7} {msg:.green}",
|
||||
);
|
||||
bar.set_style(style);
|
||||
bar.set_message("Generating epubs");
|
||||
match merged {
|
||||
Some(name) => {
|
||||
let mut epub = EpubBuilder::new(ZipLibrary::new()?)?;
|
||||
|
@ -33,6 +40,8 @@ pub fn generate_epubs(
|
|||
.unwrap();
|
||||
|
||||
article.img_urls.iter().for_each(|img| {
|
||||
// TODO: Add error handling
|
||||
bar.inc(1);
|
||||
let mut file_path = std::env::temp_dir();
|
||||
file_path.push(&img.0);
|
||||
|
||||
|
@ -48,6 +57,7 @@ pub fn generate_epubs(
|
|||
});
|
||||
let mut out_file = File::create(&name).unwrap();
|
||||
epub.generate(&mut out_file)?;
|
||||
bar.finish_with_message("Generated epub\n");
|
||||
println!("Created {:?}", name);
|
||||
}
|
||||
None => {
|
||||
|
@ -79,8 +89,10 @@ pub fn generate_epubs(
|
|||
epub.add_resource(file_path.file_name().unwrap(), img_buf, img.1.unwrap())?;
|
||||
}
|
||||
epub.generate(&mut out_file)?;
|
||||
println!("Created {:?}", file_name);
|
||||
bar.inc(1);
|
||||
// println!("Created {:?}", file_name);
|
||||
}
|
||||
bar.finish_with_message("Generated epubs\n");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
|
11
src/http.rs
11
src/http.rs
|
@ -1,6 +1,7 @@
|
|||
use async_std::io::prelude::*;
|
||||
use async_std::{fs::File, stream};
|
||||
use futures::StreamExt;
|
||||
use indicatif::ProgressBar;
|
||||
use url::Url;
|
||||
|
||||
use crate::{errors::ErrorKind, errors::PaperoniError, extractor::Extractor};
|
||||
|
@ -9,7 +10,7 @@ type HTMLResource = (String, String);
|
|||
|
||||
pub async fn fetch_html(url: &str) -> Result<HTMLResource, PaperoniError> {
|
||||
let client = surf::Client::new();
|
||||
println!("Fetching...");
|
||||
// println!("Fetching...");
|
||||
|
||||
let mut redirect_count: u8 = 0;
|
||||
let base_url = Url::parse(&url)?;
|
||||
|
@ -56,10 +57,12 @@ pub async fn fetch_html(url: &str) -> Result<HTMLResource, PaperoniError> {
|
|||
pub async fn download_images(
|
||||
extractor: &mut Extractor,
|
||||
article_origin: &Url,
|
||||
bar: &ProgressBar,
|
||||
) -> Result<(), Vec<PaperoniError>> {
|
||||
if extractor.img_urls.len() > 0 {
|
||||
println!("Downloading images...");
|
||||
// println!("Downloading images...");
|
||||
}
|
||||
let img_count = extractor.img_urls.len();
|
||||
|
||||
let imgs_req_iter = extractor
|
||||
.img_urls
|
||||
|
@ -70,7 +73,9 @@ pub async fn download_images(
|
|||
surf::Client::new().get(get_absolute_url(&url, article_origin)),
|
||||
)
|
||||
})
|
||||
.map(|(url, req)| async move {
|
||||
.enumerate()
|
||||
.map(|(img_idx, (url, req))| async move {
|
||||
bar.set_message(format!("Downloading images [{}/{}]", img_idx + 1, img_count).as_str());
|
||||
match req.await {
|
||||
Ok(mut img_response) => {
|
||||
// let mut img_response = req.await.expect("Unable to retrieve image");
|
||||
|
|
15
src/main.rs
15
src/main.rs
|
@ -4,6 +4,7 @@ extern crate lazy_static;
|
|||
use async_std::stream;
|
||||
use async_std::task;
|
||||
use futures::stream::StreamExt;
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use url::Url;
|
||||
|
||||
mod cli;
|
||||
|
@ -29,6 +30,12 @@ fn main() {
|
|||
}
|
||||
|
||||
fn download(app_config: AppConfig) {
|
||||
let bar = ProgressBar::new(app_config.urls().len() as u64);
|
||||
let style = ProgressStyle::default_bar().template(
|
||||
"{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} link {pos}/{len:7} {msg:.yellow/white}",
|
||||
);
|
||||
bar.set_style(style);
|
||||
bar.enable_steady_tick(500);
|
||||
let articles = task::block_on(async {
|
||||
let urls_iter = app_config.urls().iter().map(|url| fetch_html(url));
|
||||
let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn());
|
||||
|
@ -36,15 +43,15 @@ fn download(app_config: AppConfig) {
|
|||
while let Some(fetch_result) = responses.next().await {
|
||||
match fetch_result {
|
||||
Ok((url, html)) => {
|
||||
println!("Extracting");
|
||||
// println!("Extracting");
|
||||
let mut extractor = Extractor::from_html(&html);
|
||||
bar.set_message("Extracting...");
|
||||
extractor.extract_content(&url);
|
||||
|
||||
if extractor.article().is_some() {
|
||||
extractor.extract_img_urls();
|
||||
|
||||
if let Err(img_errors) =
|
||||
download_images(&mut extractor, &Url::parse(&url).unwrap()).await
|
||||
download_images(&mut extractor, &Url::parse(&url).unwrap(), &bar).await
|
||||
{
|
||||
eprintln!(
|
||||
"{} image{} failed to download for {}",
|
||||
|
@ -58,9 +65,11 @@ fn download(app_config: AppConfig) {
|
|||
}
|
||||
Err(e) => eprintln!("{}", e),
|
||||
}
|
||||
bar.inc(1);
|
||||
}
|
||||
articles
|
||||
});
|
||||
bar.finish_with_message("Downloaded articles");
|
||||
match generate_epubs(articles, app_config.merged()) {
|
||||
Ok(_) => (),
|
||||
Err(e) => eprintln!("{}", e),
|
||||
|
|
Loading…
Reference in a new issue