From 13ad14e73d54f21d0b4ae114f9dead66b1edb163 Mon Sep 17 00:00:00 2001 From: Mikhail Gorbachev Date: Tue, 1 Jun 2021 12:23:22 +0300 Subject: [PATCH 1/3] Add `output_dir` to cli argument - Add `output_dir` to cli argument - This argument allows you to save output files in a special folder, not just current dir - Refactor 'cli.rs' - Add `Builder` for `AppConfig` - Add `Error` instead separated panics - Upgrade dependencies --- Cargo.lock | 159 +++++++++++++++++++------ Cargo.toml | 13 +- README.md | 23 ++-- rust-toolchain | 1 + src/cli.rs | 315 ++++++++++++++++++++++++++++--------------------- src/epub.rs | 13 +- src/http.rs | 10 +- src/logs.rs | 45 +------ src/main.rs | 26 ++-- 9 files changed, 354 insertions(+), 251 deletions(-) create mode 100644 rust-toolchain diff --git a/Cargo.lock b/Cargo.lock index d07b91c..c2bcea0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "addr2line" version = "0.14.1" @@ -71,9 +73,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.15" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] @@ -389,7 +391,7 @@ dependencies = [ "ansi_term", "atty", "bitflags", - "strsim", + "strsim 0.8.0", "textwrap", "unicode-width", "vec_map", @@ -435,9 +437,7 @@ dependencies = [ "encode_unicode", "lazy_static", "libc", - "regex", "terminal_size", - "unicode-width", "winapi", ] @@ -614,6 +614,41 @@ dependencies = [ "winapi", ] +[[package]] +name = "darling" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "dashmap" version = "4.0.2" @@ -630,6 +665,37 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +[[package]] +name = "derive_builder" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d13202debe11181040ae9063d739fa32cfcaaebe2275fe387703460ae2365b30" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e616858f6187ed828df7c64a6d71720d83767a7f19740b2d1b6fe6327b36e5" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "derive_more" version = "0.99.13" @@ -822,9 +888,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" +checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" dependencies = [ "futures-channel", "futures-core", @@ -837,9 +903,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" +checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" dependencies = [ "futures-core", "futures-sink", @@ -847,15 +913,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" +checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" [[package]] name = "futures-executor" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" +checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" dependencies = [ "futures-core", "futures-task", @@ -864,9 +930,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" +checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" [[package]] name = "futures-lite" @@ -885,10 +951,11 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" +checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" dependencies = [ + "autocfg", "proc-macro-hack", "proc-macro2", "quote", @@ -897,22 +964,23 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" +checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" [[package]] name = "futures-task" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" +checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" [[package]] name = "futures-util" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" +checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" dependencies = [ + "autocfg", "futures-channel", "futures-core", "futures-io", @@ -1112,6 +1180,12 @@ dependencies = [ "url", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -1125,9 +1199,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.15.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" +checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b" dependencies = [ "console", "lazy_static", @@ -1305,9 +1379,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "mime" @@ -1419,9 +1493,9 @@ dependencies = [ [[package]] name = "number_prefix" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" @@ -1469,6 +1543,7 @@ dependencies = [ "clap", "colored", "comfy-table", + "derive_builder", "directories", "epub-builder", "flexi_logger", @@ -1829,9 +1904,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.6" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick", "memchr", @@ -1840,9 +1915,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.23" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "remove_dir_all" @@ -2172,6 +2247,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strum" version = "0.20.0" @@ -2277,18 +2358,18 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ "proc-macro2", "quote", @@ -2465,9 +2546,9 @@ dependencies = [ [[package]] name = "url" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ "form_urlencoded", "idna", diff --git a/Cargo.toml b/Cargo.toml index 8b8b6e6..e3f6055 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,23 +12,24 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -async-std = "1.9.0" # atty = "0.2.14" +async-std = "1.9.0" chrono = "0.4.19" clap = "2.33.3" colored = "2.0.0" comfy-table = "2.1.0" +derive_builder = "0.10.2" directories = "3.0.2" epub-builder = "0.4.8" flexi_logger = "0.17.1" -futures = "0.3.14" +futures = "0.3.15" html5ever = "0.25.1" -indicatif = "0.15.0" +indicatif = "0.16.2" kuchiki = "0.8.1" lazy_static = "1.4.0" log = "0.4.14" md5 = "0.7.0" -regex = "1.4.5" +regex = "1.5.4" surf = "2.2.0" -thiserror = "1.0.24" -url = "2.2.1" +thiserror = "1.0.25" +url = "2.2.2" diff --git a/README.md b/README.md index 2cfba38..aa94d06 100644 --- a/README.md +++ b/README.md @@ -48,15 +48,20 @@ USAGE: paperoni [OPTIONS] [urls]... OPTIONS: - -f, --file Input file containing links - -h, --help Prints help information - --log-to-file Enables logging of events to a file located in .paperoni/logs with a default log level - of debug. Use -v to specify the logging level - --max_conn The maximum number of concurrent HTTP connections when downloading articles. Default is - 8 - --merge Merge multiple articles into a single epub - -V, --version Prints version information - -v Enables logging of events and set the verbosity level. Use -h to read on its usage + -f, --file Input file containing links + -h, --help Prints help information + --log-to-file + Enables logging of events to a file located in .paperoni/logs with a default log level of debug. Use -v to + specify the logging level + --max_conn + The maximum number of concurrent HTTP connections when downloading articles. Default is 8 + + -o, --output_directory Directory for store output epub documents + --merge Merge multiple articles into a single epub + -V, --version Prints version information + -v + Enables logging of events and set the verbosity level. Use --help to read on its usage + ARGS: ... Urls of web articles diff --git a/rust-toolchain b/rust-toolchain new file mode 100644 index 0000000..154cb93 --- /dev/null +++ b/rust-toolchain @@ -0,0 +1 @@ +1.52.1 diff --git a/src/cli.rs b/src/cli.rs index 19ce379..63c4d89 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,13 +1,56 @@ -use std::{fs::File, io::Read, path::Path}; +use std::{ + collections::HashSet, + num::{NonZeroUsize, ParseIntError}, + path::Path, +}; use chrono::{DateTime, Local}; -use clap::{App, AppSettings, Arg}; -use flexi_logger::LevelFilter as LogLevel; +use clap::{App, AppSettings, Arg, ArgMatches}; +use flexi_logger::{FlexiLoggerError, LevelFilter as LogLevel}; +use std::fs; +use thiserror::Error; -use crate::logs::init_logger; +const DEFAULT_MAX_CONN: usize = 8; -pub fn cli_init() -> AppConfig { - let app = App::new("paperoni") +#[derive(derive_builder::Builder)] +pub struct AppConfig { + /// Urls for store in epub + pub urls: Vec, + pub max_conn: usize, + /// Path to file of multiple articles into a single epub + pub merged: Option, + pub output_directory: Option, + pub log_level: LogLevel, + pub can_disable_progress_bar: bool, + pub start_time: DateTime, + pub is_logging_to_file: bool, +} + +#[derive(Debug, Error)] +pub enum Error { + #[error("Failed to open file with urls: {0}")] + UrlFileError(#[from] std::io::Error), + #[error("Failed to parse max connection value: {0}")] + InvalidMaxConnectionCount(#[from] ParseIntError), + #[error("No urls for parse")] + NoUrls, + #[error("No urls for parse")] + AppBuildError(#[from] AppConfigBuilderError), + #[error("Invalid output path name for merged epubs: {0}")] + InvalidOutputPath(String), + #[error("Log error: {0}")] + LogDirectoryError(String), + #[error(transparent)] + LogError(#[from] FlexiLoggerError), + #[error("Wrong output directory")] + WrongOutputDirectory, + #[error("Output directory not exists")] + OutputDirectoryNotExists, +} + +impl AppConfig { + pub fn init_with_cli() -> Result { + let app = App::new("paperoni") .settings(&[ AppSettings::ArgRequiredElseHelp, AppSettings::UnifiedHelpMessage, @@ -28,11 +71,21 @@ pub fn cli_init() -> AppConfig { .help("Input file containing links") .takes_value(true), ) + .arg( + Arg::with_name("output_directory") + .long("output_directory") + .short("o") + .help("Directory for store output epub documents") + .conflicts_with("output_name") + .long_help("Directory for saving epub documents") + .takes_value(true), + ) .arg( Arg::with_name("output_name") .long("merge") .help("Merge multiple articles into a single epub") .long_help("Merge multiple articles into a single epub that will be given the name provided") + .conflicts_with("output_directory") .takes_value(true), ).arg( Arg::with_name("max_conn") @@ -60,143 +113,135 @@ pub fn cli_init() -> AppConfig { .long("log-to-file") .help("Enables logging of events to a file located in .paperoni/logs with a default log level of debug. Use -v to specify the logging level") .takes_value(false)); - let arg_matches = app.get_matches(); - let mut urls: Vec = match arg_matches.value_of("file") { - Some(file_name) => { - if let Ok(mut file) = File::open(file_name) { - let mut content = String::new(); - match file.read_to_string(&mut content) { - Ok(_) => content - .lines() - .filter(|line| !line.is_empty()) - .map(|line| line.to_owned()) - .collect(), - Err(_) => vec![], + Self::try_from(app.get_matches()) + } + + fn init_merge_file(self) -> Result { + self.merged + .as_deref() + .map(fs::File::create) + .transpose() + .err() + .map(|err| Err(Error::InvalidOutputPath(err.to_string()))) + .unwrap_or(Ok(self)) + } + + fn init_logger(self) -> Result { + use directories::UserDirs; + use flexi_logger::LogSpecBuilder; + + match UserDirs::new() { + Some(user_dirs) => { + let home_dir = user_dirs.home_dir(); + let paperoni_dir = home_dir.join(".paperoni"); + let log_dir = paperoni_dir.join("logs"); + + let log_spec = LogSpecBuilder::new() + .module("paperoni", self.log_level) + .build(); + let formatted_timestamp = self.start_time.format("%Y-%m-%d_%H-%M-%S"); + let mut logger = flexi_logger::Logger::with(log_spec); + + if self.is_logging_to_file && (!paperoni_dir.is_dir() || !log_dir.is_dir()) { + if let Err(e) = fs::create_dir_all(&log_dir) { + return Err(Error::LogDirectoryError(format!("Unable to create paperoni directories on home directory for logging purposes\n{}",e))); + } } - } else { - println!("Unable to open file: {}", file_name); - vec![] + if self.is_logging_to_file { + logger = logger + .directory(log_dir) + .discriminant(formatted_timestamp.to_string()) + .suppress_timestamp() + .log_to_file(); + } + logger.start()?; + Ok(self) } + None => Err(Error::LogDirectoryError( + "Unable to get user directories for logging purposes".to_string(), + )), } - None => vec![], - }; - - if let Some(vals) = arg_matches.values_of("urls") { - urls.extend( - vals.filter(|val| !val.is_empty()) - .map(|val| val.to_string()), - ); } - - let max_conn = arg_matches - .value_of("max_conn") - .map(|conn_str| conn_str.parse::().ok()) - .flatten() - .map(|max| if max > 0 { max } else { 1 }) - .unwrap_or(8); - - let mut app_config = AppConfig::new(max_conn); - app_config.set_urls(urls); - - if let Some(name) = arg_matches.value_of("output_name") { - let file_path = Path::new(name); - if file_path.is_dir() { - eprintln!("{:?} is a directory", name); - std::process::exit(1); - } - - let file_name = if file_path.extension().is_some() { - name.to_owned() - } else { - name.to_owned() + ".epub" - }; - - match std::fs::File::create(&file_name) { - Ok(_) => (), - Err(e) => { - eprintln!("Unable to create file {:?}\n{}", file_path, e); - std::process::exit(1) - } - } - app_config.merged = Some(file_name); - } - - if arg_matches.is_present("verbosity") { - if !arg_matches.is_present("log-to-file") { - app_config.can_disable_progress_bar = true; - } - let log_levels: [LogLevel; 5] = [ - LogLevel::Off, - LogLevel::Error, - LogLevel::Warn, - LogLevel::Info, - LogLevel::Debug, - ]; - let level = arg_matches.occurrences_of("verbosity").clamp(0, 4) as usize; - app_config.log_level = log_levels[level]; - } - if arg_matches.is_present("log-to-file") { - app_config.log_level = LogLevel::Debug; - app_config.is_logging_to_file = true; - } - - init_logger(&app_config); - - app_config } -pub struct AppConfig { - urls: Vec, - max_conn: usize, - merged: Option, - log_level: LogLevel, - can_disable_progress_bar: bool, - start_time: DateTime, - is_logging_to_file: bool, +use std::convert::TryFrom; + +impl<'a> TryFrom> for AppConfig { + type Error = Error; + + fn try_from(arg_matches: ArgMatches<'a>) -> Result { + AppConfigBuilder::default() + .urls({ + let url_filter = |url: &str| { + let url = url.trim(); + if !url.is_empty() { + Some(url.to_owned()) + } else { + None + } + }; + match ( + arg_matches + .values_of("urls") + .and_then(|urls| urls.map(url_filter).collect::>>()), + arg_matches + .value_of("file") + .map(fs::read_to_string) + .transpose()? + .and_then(|content| { + content + .lines() + .map(url_filter) + .collect::>>() + }), + ) { + (Some(direct_urls), Some(file_urls)) => Ok(direct_urls + .union(&file_urls) + .map(ToOwned::to_owned) + .collect::>()), + (Some(urls), None) | (None, Some(urls)) => Ok(urls.into_iter().collect()), + (None, None) => Err(Error::NoUrls), + } + }?) + .max_conn(match arg_matches.value_of("max_conn") { + Some(max_conn) => max_conn.parse::()?.get(), + None => DEFAULT_MAX_CONN, + }) + .merged(arg_matches.value_of("output_name").map(ToOwned::to_owned)) + .can_disable_progress_bar( + arg_matches.is_present("verbosity") && !arg_matches.is_present("log-to-file"), + ) + .log_level(match arg_matches.occurrences_of("verbosity") { + 0 => LogLevel::Off, + 1 => LogLevel::Error, + 2 => LogLevel::Warn, + 3 => LogLevel::Info, + 4..=u64::MAX => LogLevel::Debug, + }) + .is_logging_to_file(arg_matches.is_present("log-to_file")) + .output_directory( + arg_matches + .value_of("output_directory") + .map(|output_directory| { + let path = Path::new(output_directory); + if !path.exists() { + Err(Error::OutputDirectoryNotExists) + } else if !path.is_dir() { + Err(Error::WrongOutputDirectory) + } else { + Ok(output_directory.to_owned()) + } + }) + .transpose()?, + ) + .start_time(Local::now()) + .try_init() + } } -impl AppConfig { - fn new(max_conn: usize) -> Self { - Self { - urls: vec![], - max_conn, - merged: None, - log_level: LogLevel::Off, - can_disable_progress_bar: false, - start_time: Local::now(), - is_logging_to_file: false, - } - } - - fn set_urls(&mut self, urls: Vec) { - self.urls.extend(urls); - } - - pub fn urls(&self) -> &Vec { - &self.urls - } - pub fn max_conn(&self) -> usize { - self.max_conn - } - - pub fn merged(&self) -> Option<&String> { - self.merged.as_ref() - } - - pub fn log_level(&self) -> LogLevel { - self.log_level - } - - pub fn can_disable_progress_bar(&self) -> bool { - self.can_disable_progress_bar - } - - pub fn start_time(&self) -> &DateTime { - &self.start_time - } - - pub fn is_logging_to_file(&self) -> bool { - self.is_logging_to_file +impl AppConfigBuilder { + pub fn try_init(&self) -> Result { + self.build()?.init_logger()?.init_merge_file() } } diff --git a/src/epub.rs b/src/epub.rs index 75f2b9e..06dfeff 100644 --- a/src/epub.rs +++ b/src/epub.rs @@ -16,7 +16,7 @@ pub fn generate_epubs( app_config: &AppConfig, successful_articles_table: &mut Table, ) -> Result<(), Vec> { - let bar = if app_config.can_disable_progress_bar() { + let bar = if app_config.can_disable_progress_bar { ProgressBar::hidden() } else { let enabled_bar = ProgressBar::new(articles.len() as u64); @@ -32,8 +32,8 @@ pub fn generate_epubs( let mut errors: Vec = Vec::new(); - match app_config.merged() { - Some(name) => { + match app_config.merged { + Some(ref name) => { successful_articles_table.set_header(vec![Cell::new("Table of Contents") .add_attribute(Attribute::Bold) .set_alignment(CellAlignment::Center) @@ -103,7 +103,7 @@ pub fn generate_epubs( .title(replace_metadata_value("Article Sources")), ) { let mut paperoni_err: PaperoniError = err.into(); - paperoni_err.set_article_source(name); + paperoni_err.set_article_source(&name); errors.push(paperoni_err); return Err(errors); } @@ -113,7 +113,7 @@ pub fn generate_epubs( Ok(_) => (), Err(err) => { let mut paperoni_err: PaperoniError = err.into(); - paperoni_err.set_article_source(name); + paperoni_err.set_article_source(&name); errors.push(paperoni_err); return Err(errors); } @@ -135,7 +135,8 @@ pub fn generate_epubs( let mut result = || -> Result<(), PaperoniError> { let mut epub = EpubBuilder::new(ZipLibrary::new()?)?; let file_name = format!( - "{}.epub", + "{}/{}.epub", + app_config.output_directory.as_deref().unwrap_or("."), article .metadata() .title() diff --git a/src/http.rs b/src/http.rs index efd64b8..148fab0 100644 --- a/src/http.rs +++ b/src/http.rs @@ -153,7 +153,11 @@ pub async fn download_images( }) .enumerate() .map(|(img_idx, (url, req))| async move { - bar.set_message(format!("Downloading images [{}/{}]", img_idx + 1, img_count).as_str()); + bar.set_message(format!( + "Downloading images [{}/{}]", + img_idx + 1, + img_count + )); match req.await { Ok(mut img_response) => { let process_response = @@ -234,9 +238,9 @@ fn get_absolute_url(url: &str, request_url: &Url) -> String { .unwrap() .join(url) .unwrap() - .into_string() + .into() } else { - request_url.join(url).unwrap().into_string() + request_url.join(url).unwrap().into() } } diff --git a/src/logs.rs b/src/logs.rs index 87b5d1b..0feb27f 100644 --- a/src/logs.rs +++ b/src/logs.rs @@ -1,11 +1,9 @@ use colored::*; use comfy_table::presets::UTF8_HORIZONTAL_BORDERS_ONLY; use comfy_table::{Cell, CellAlignment, ContentArrangement, Table}; -use directories::UserDirs; -use flexi_logger::LogSpecBuilder; use log::error; -use crate::{cli::AppConfig, errors::PaperoniError}; +use crate::errors::PaperoniError; pub fn display_summary( initial_article_count: usize, @@ -143,47 +141,6 @@ impl DownloadCount { } } } - -pub fn init_logger(app_config: &AppConfig) { - match UserDirs::new() { - Some(user_dirs) => { - let home_dir = user_dirs.home_dir(); - let paperoni_dir = home_dir.join(".paperoni"); - let log_dir = paperoni_dir.join("logs"); - - let log_spec = LogSpecBuilder::new() - .module("paperoni", app_config.log_level()) - .build(); - let formatted_timestamp = app_config.start_time().format("%Y-%m-%d_%H-%M-%S"); - let mut logger = flexi_logger::Logger::with(log_spec); - - if app_config.is_logging_to_file() && (!paperoni_dir.is_dir() || !log_dir.is_dir()) { - match std::fs::create_dir_all(&log_dir) { - Ok(_) => (), - Err(e) => { - eprintln!("Unable to create paperoni directories on home directory for logging purposes\n{}",e); - std::process::exit(1); - } - }; - } - - if app_config.is_logging_to_file() { - logger = logger - .directory(log_dir) - .discriminant(formatted_timestamp.to_string()) - .suppress_timestamp() - .log_to_file(); - } - - match logger.start() { - Ok(_) => (), - Err(e) => eprintln!("Unable to start logger!\n{}", e), - } - } - None => eprintln!("Unable to get user directories for logging purposes"), - }; -} - #[cfg(test)] mod tests { use super::{short_summary, DownloadCount}; diff --git a/src/main.rs b/src/main.rs index 0f8b34a..fcb0cd4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,8 @@ #[macro_use] extern crate lazy_static; +use std::process::exit; + use async_std::stream; use async_std::task; use comfy_table::presets::{UTF8_FULL, UTF8_HORIZONTAL_BORDERS_ONLY}; @@ -27,9 +29,15 @@ use http::{download_images, fetch_html}; use logs::display_summary; fn main() { - let app_config = cli::cli_init(); + let app_config = match cli::AppConfig::init_with_cli() { + Ok(app_config) => app_config, + Err(err) => { + eprintln!("{}", err); + exit(1); + } + }; - if !app_config.urls().is_empty() { + if !app_config.urls.is_empty() { download(app_config); } } @@ -37,10 +45,10 @@ fn main() { fn download(app_config: AppConfig) { let mut errors = Vec::new(); let mut partial_download_count: usize = 0; - let bar = if app_config.can_disable_progress_bar() { + let bar = if app_config.can_disable_progress_bar { ProgressBar::hidden() } else { - let enabled_bar = ProgressBar::new(app_config.urls().len() as u64); + let enabled_bar = ProgressBar::new(app_config.urls.len() as u64); let style = ProgressStyle::default_bar().template( "{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} link {pos}/{len:7} {msg:.yellow/white}", ); @@ -49,8 +57,8 @@ fn download(app_config: AppConfig) { enabled_bar }; let articles = task::block_on(async { - let urls_iter = app_config.urls().iter().map(|url| fetch_html(url)); - let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn()); + let urls_iter = app_config.urls.iter().map(|url| fetch_html(url)); + let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn); let mut articles = Vec::new(); while let Some(fetch_result) = responses.next().await { match fetch_result { @@ -109,15 +117,15 @@ fn download(app_config: AppConfig) { }; let has_errors = !errors.is_empty(); display_summary( - app_config.urls().len(), + app_config.urls.len(), succesful_articles_table, partial_download_count, errors, ); - if app_config.is_logging_to_file() { + if app_config.is_logging_to_file { println!( "Log written to paperoni_{}.log\n", - app_config.start_time().format("%Y-%m-%d_%H-%M-%S") + app_config.start_time.format("%Y-%m-%d_%H-%M-%S") ); } if has_errors { From aa9258e12205f5d8a99a83a6d7b9d5b7eafbcf18 Mon Sep 17 00:00:00 2001 From: Mikhail Gorbachev Date: Sun, 6 Jun 2021 13:20:08 +0300 Subject: [PATCH 2/3] Fix from PR#15 - refactor comments - move `cli::Error` to `errors::ErrorCli` - removed mixing of order of input urls - move pure functionality if `init_logger` to clear function --- README.md | 36 ++++++++++++---- src/cli.rs | 113 +++++++++++++------------------------------------- src/errors.rs | 31 ++++++++++++++ src/logs.rs | 46 ++++++++++++++++++++ 4 files changed, 133 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index aa94d06..873f95a 100644 --- a/README.md +++ b/README.md @@ -48,23 +48,41 @@ USAGE: paperoni [OPTIONS] [urls]... OPTIONS: - -f, --file Input file containing links - -h, --help Prints help information + -f, --file + Input file containing links + + -h, --help + Prints help information + --log-to-file Enables logging of events to a file located in .paperoni/logs with a default log level of debug. Use -v to specify the logging level --max_conn - The maximum number of concurrent HTTP connections when downloading articles. Default is 8 + The maximum number of concurrent HTTP connections when downloading articles. Default is 8. + NOTE: It is advised to use as few connections as needed i.e between 1 and 50. Using more connections can end + up overloading your network card with too many concurrent requests. + -o, --output_directory + Directory for saving epub documents + + --merge + Merge multiple articles into a single epub that will be given the name provided + + -V, --version + Prints version information - -o, --output_directory Directory for store output epub documents - --merge Merge multiple articles into a single epub - -V, --version Prints version information -v - Enables logging of events and set the verbosity level. Use --help to read on its usage - + This takes upto 4 levels of verbosity in the following order. + - Error (-v) + - Warn (-vv) + - Info (-vvv) + - Debug (-vvvv) + When this flag is passed, it disables the progress bars and logs to stderr. + If you would like to send the logs to a file (and enable progress bars), pass the log-to-file flag. ARGS: - ... Urls of web articles + ... + Urls of web articles + ``` To download a single article pass in its URL diff --git a/src/cli.rs b/src/cli.rs index 63c4d89..5827f56 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,14 +1,10 @@ -use std::{ - collections::HashSet, - num::{NonZeroUsize, ParseIntError}, - path::Path, -}; +use std::{collections::BTreeSet, fs, num::NonZeroUsize, path::Path}; use chrono::{DateTime, Local}; use clap::{App, AppSettings, Arg, ArgMatches}; -use flexi_logger::{FlexiLoggerError, LevelFilter as LogLevel}; -use std::fs; -use thiserror::Error; +use flexi_logger::LevelFilter as LogLevel; + +type Error = crate::errors::CliError; const DEFAULT_MAX_CONN: usize = 8; @@ -26,28 +22,6 @@ pub struct AppConfig { pub is_logging_to_file: bool, } -#[derive(Debug, Error)] -pub enum Error { - #[error("Failed to open file with urls: {0}")] - UrlFileError(#[from] std::io::Error), - #[error("Failed to parse max connection value: {0}")] - InvalidMaxConnectionCount(#[from] ParseIntError), - #[error("No urls for parse")] - NoUrls, - #[error("No urls for parse")] - AppBuildError(#[from] AppConfigBuilderError), - #[error("Invalid output path name for merged epubs: {0}")] - InvalidOutputPath(String), - #[error("Log error: {0}")] - LogDirectoryError(String), - #[error(transparent)] - LogError(#[from] FlexiLoggerError), - #[error("Wrong output directory")] - WrongOutputDirectory, - #[error("Output directory not exists")] - OutputDirectoryNotExists, -} - impl AppConfig { pub fn init_with_cli() -> Result { let app = App::new("paperoni") @@ -73,11 +47,10 @@ impl AppConfig { ) .arg( Arg::with_name("output_directory") - .long("output_directory") + .long("output-directory") .short("o") - .help("Directory for store output epub documents") + .help("Directory to store output epub documents") .conflicts_with("output_name") - .long_help("Directory for saving epub documents") .takes_value(true), ) .arg( @@ -128,40 +101,10 @@ impl AppConfig { } fn init_logger(self) -> Result { - use directories::UserDirs; - use flexi_logger::LogSpecBuilder; - - match UserDirs::new() { - Some(user_dirs) => { - let home_dir = user_dirs.home_dir(); - let paperoni_dir = home_dir.join(".paperoni"); - let log_dir = paperoni_dir.join("logs"); - - let log_spec = LogSpecBuilder::new() - .module("paperoni", self.log_level) - .build(); - let formatted_timestamp = self.start_time.format("%Y-%m-%d_%H-%M-%S"); - let mut logger = flexi_logger::Logger::with(log_spec); - - if self.is_logging_to_file && (!paperoni_dir.is_dir() || !log_dir.is_dir()) { - if let Err(e) = fs::create_dir_all(&log_dir) { - return Err(Error::LogDirectoryError(format!("Unable to create paperoni directories on home directory for logging purposes\n{}",e))); - } - } - if self.is_logging_to_file { - logger = logger - .directory(log_dir) - .discriminant(formatted_timestamp.to_string()) - .suppress_timestamp() - .log_to_file(); - } - logger.start()?; - Ok(self) - } - None => Err(Error::LogDirectoryError( - "Unable to get user directories for logging purposes".to_string(), - )), - } + use crate::logs; + logs::init_logger(self.log_level, &self.start_time, self.is_logging_to_file) + .map(|_| self) + .map_err(Error::LogError) } } @@ -181,21 +124,20 @@ impl<'a> TryFrom> for AppConfig { None } }; - match ( - arg_matches - .values_of("urls") - .and_then(|urls| urls.map(url_filter).collect::>>()), - arg_matches - .value_of("file") - .map(fs::read_to_string) - .transpose()? - .and_then(|content| { - content - .lines() - .map(url_filter) - .collect::>>() - }), - ) { + let direct_urls = arg_matches + .values_of("urls") + .and_then(|urls| urls.map(url_filter).collect::>>()); + let file_urls = arg_matches + .value_of("file") + .map(fs::read_to_string) + .transpose()? + .and_then(|content| { + content + .lines() + .map(url_filter) + .collect::>>() + }); + match (direct_urls, file_urls) { (Some(direct_urls), Some(file_urls)) => Ok(direct_urls .union(&file_urls) .map(ToOwned::to_owned) @@ -219,7 +161,7 @@ impl<'a> TryFrom> for AppConfig { 3 => LogLevel::Info, 4..=u64::MAX => LogLevel::Debug, }) - .is_logging_to_file(arg_matches.is_present("log-to_file")) + .is_logging_to_file(arg_matches.is_present("log-to-file")) .output_directory( arg_matches .value_of("output_directory") @@ -242,6 +184,9 @@ impl<'a> TryFrom> for AppConfig { impl AppConfigBuilder { pub fn try_init(&self) -> Result { - self.build()?.init_logger()?.init_merge_file() + self.build() + .map_err(Error::AppBuildError)? + .init_logger()? + .init_merge_file() } } diff --git a/src/errors.rs b/src/errors.rs index 84d1535..eb8cbe1 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,3 +1,6 @@ +use std::fmt::{Debug, Display}; + +use flexi_logger::FlexiLoggerError; use thiserror::Error; #[derive(Error, Debug)] @@ -124,3 +127,31 @@ impl From for PaperoniError { PaperoniError::with_kind(ErrorKind::UTF8Error(err.to_string())) } } + +#[derive(Debug, Error)] +pub enum LogError { + #[error(transparent)] + FlexiError(#[from] FlexiLoggerError), + #[error("Wrong log directory: {0}")] + LogDirectoryError(String), +} + +#[derive(Debug, Error)] +pub enum CliError { + #[error("Failed to open file with urls: {0}")] + UrlFileError(#[from] std::io::Error), + #[error("Failed to parse max connection value: {0}")] + InvalidMaxConnectionCount(#[from] std::num::ParseIntError), + #[error("No urls were provided")] + NoUrls, + #[error("Failed to build cli application: {0}")] + AppBuildError(BuilderError), + #[error("Invalid output path name for merged epubs: {0}")] + InvalidOutputPath(String), + #[error("Wrong output directory")] + WrongOutputDirectory, + #[error("Output directory not exists")] + OutputDirectoryNotExists, + #[error("Unable to start logger!\n{0}")] + LogError(#[from] LogError), +} diff --git a/src/logs.rs b/src/logs.rs index 0feb27f..e8f89de 100644 --- a/src/logs.rs +++ b/src/logs.rs @@ -1,6 +1,10 @@ +use std::fs; + +use chrono::{DateTime, Local}; use colored::*; use comfy_table::presets::UTF8_HORIZONTAL_BORDERS_ONLY; use comfy_table::{Cell, CellAlignment, ContentArrangement, Table}; +use flexi_logger::LevelFilter; use log::error; use crate::errors::PaperoniError; @@ -141,6 +145,48 @@ impl DownloadCount { } } } + +use crate::errors::LogError as Error; + +pub fn init_logger( + log_level: LevelFilter, + start_time: &DateTime, + is_logging_to_file: bool, +) -> Result<(), Error> { + use directories::UserDirs; + use flexi_logger::LogSpecBuilder; + + match UserDirs::new() { + Some(user_dirs) => { + let home_dir = user_dirs.home_dir(); + let paperoni_dir = home_dir.join(".paperoni"); + let log_dir = paperoni_dir.join("logs"); + + let log_spec = LogSpecBuilder::new().module("paperoni", log_level).build(); + let formatted_timestamp = start_time.format("%Y-%m-%d_%H-%M-%S"); + let mut logger = flexi_logger::Logger::with(log_spec); + + if is_logging_to_file && (!paperoni_dir.is_dir() || !log_dir.is_dir()) { + if let Err(e) = fs::create_dir_all(&log_dir) { + return Err(Error::LogDirectoryError(format!("Unable to create paperoni directories on home directory for logging purposes\n{}",e))); + } + } + if is_logging_to_file { + logger = logger + .directory(log_dir) + .discriminant(formatted_timestamp.to_string()) + .suppress_timestamp() + .log_to_file(); + } + logger.start()?; + Ok(()) + } + None => Err(Error::LogDirectoryError( + "Unable to get user directories for logging purposes".to_string(), + )), + } +} + #[cfg(test)] mod tests { use super::{short_summary, DownloadCount}; From 67e86e4d74a745904a422f20a52c53312086f5a1 Mon Sep 17 00:00:00 2001 From: Mikhail Gorbachev Date: Sun, 6 Jun 2021 15:52:30 +0300 Subject: [PATCH 3/3] Refactor `LogError` --- src/errors.rs | 6 ++++-- src/logs.rs | 12 ++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index eb8cbe1..a479268 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -132,8 +132,10 @@ impl From for PaperoniError { pub enum LogError { #[error(transparent)] FlexiError(#[from] FlexiLoggerError), - #[error("Wrong log directory: {0}")] - LogDirectoryError(String), + #[error("Unable to get user directories for logging purposes")] + UserDirectoriesError, + #[error("Can't create log directory: {0}")] + CreateLogDirectoryError(#[from] std::io::Error), } #[derive(Debug, Error)] diff --git a/src/logs.rs b/src/logs.rs index e8f89de..526921d 100644 --- a/src/logs.rs +++ b/src/logs.rs @@ -166,12 +166,10 @@ pub fn init_logger( let formatted_timestamp = start_time.format("%Y-%m-%d_%H-%M-%S"); let mut logger = flexi_logger::Logger::with(log_spec); - if is_logging_to_file && (!paperoni_dir.is_dir() || !log_dir.is_dir()) { - if let Err(e) = fs::create_dir_all(&log_dir) { - return Err(Error::LogDirectoryError(format!("Unable to create paperoni directories on home directory for logging purposes\n{}",e))); - } - } if is_logging_to_file { + if !paperoni_dir.is_dir() || !log_dir.is_dir() { + fs::create_dir_all(&log_dir)?; + } logger = logger .directory(log_dir) .discriminant(formatted_timestamp.to_string()) @@ -181,9 +179,7 @@ pub fn init_logger( logger.start()?; Ok(()) } - None => Err(Error::LogDirectoryError( - "Unable to get user directories for logging purposes".to_string(), - )), + None => Err(Error::UserDirectoriesError), } }