Add output_dir to cli argument

- Add `output_dir` to cli argument
    - This argument allows you to save output files in a special folder, not just current dir
- Refactor 'cli.rs'
    - Add `Builder` for `AppConfig`
    - Add `Error` instead separated panics
- Upgrade dependencies
This commit is contained in:
Mikhail Gorbachev 2021-06-01 12:23:22 +03:00
parent 1cbbc7527f
commit 13ad14e73d
9 changed files with 354 additions and 251 deletions

159
Cargo.lock generated
View file

@ -1,5 +1,7 @@
# This file is automatically @generated by Cargo. # This file is automatically @generated by Cargo.
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3
[[package]] [[package]]
name = "addr2line" name = "addr2line"
version = "0.14.1" version = "0.14.1"
@ -71,9 +73,9 @@ dependencies = [
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "0.7.15" version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
dependencies = [ dependencies = [
"memchr", "memchr",
] ]
@ -389,7 +391,7 @@ dependencies = [
"ansi_term", "ansi_term",
"atty", "atty",
"bitflags", "bitflags",
"strsim", "strsim 0.8.0",
"textwrap", "textwrap",
"unicode-width", "unicode-width",
"vec_map", "vec_map",
@ -435,9 +437,7 @@ dependencies = [
"encode_unicode", "encode_unicode",
"lazy_static", "lazy_static",
"libc", "libc",
"regex",
"terminal_size", "terminal_size",
"unicode-width",
"winapi", "winapi",
] ]
@ -614,6 +614,41 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "darling"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim 0.10.0",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]] [[package]]
name = "dashmap" name = "dashmap"
version = "4.0.2" version = "4.0.2"
@ -630,6 +665,37 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
[[package]]
name = "derive_builder"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d13202debe11181040ae9063d739fa32cfcaaebe2275fe387703460ae2365b30"
dependencies = [
"derive_builder_macro",
]
[[package]]
name = "derive_builder_core"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66e616858f6187ed828df7c64a6d71720d83767a7f19740b2d1b6fe6327b36e5"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "derive_builder_macro"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73"
dependencies = [
"derive_builder_core",
"syn",
]
[[package]] [[package]]
name = "derive_more" name = "derive_more"
version = "0.99.13" version = "0.99.13"
@ -822,9 +888,9 @@ dependencies = [
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253" checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27"
dependencies = [ dependencies = [
"futures-channel", "futures-channel",
"futures-core", "futures-core",
@ -837,9 +903,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-channel" name = "futures-channel"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25" checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -847,15 +913,15 @@ dependencies = [
[[package]] [[package]]
name = "futures-core" name = "futures-core"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815" checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
[[package]] [[package]]
name = "futures-executor" name = "futures-executor"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d" checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-task", "futures-task",
@ -864,9 +930,9 @@ dependencies = [
[[package]] [[package]]
name = "futures-io" name = "futures-io"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04" checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
[[package]] [[package]]
name = "futures-lite" name = "futures-lite"
@ -885,10 +951,11 @@ dependencies = [
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b" checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121"
dependencies = [ dependencies = [
"autocfg",
"proc-macro-hack", "proc-macro-hack",
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -897,22 +964,23 @@ dependencies = [
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23" checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282"
[[package]] [[package]]
name = "futures-task" name = "futures-task"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc" checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
[[package]] [[package]]
name = "futures-util" name = "futures-util"
version = "0.3.14" version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025" checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
dependencies = [ dependencies = [
"autocfg",
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"futures-io", "futures-io",
@ -1112,6 +1180,12 @@ dependencies = [
"url", "url",
] ]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.2.3" version = "0.2.3"
@ -1125,9 +1199,9 @@ dependencies = [
[[package]] [[package]]
name = "indicatif" name = "indicatif"
version = "0.15.0" version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4" checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b"
dependencies = [ dependencies = [
"console", "console",
"lazy_static", "lazy_static",
@ -1305,9 +1379,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.3.4" version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
[[package]] [[package]]
name = "mime" name = "mime"
@ -1419,9 +1493,9 @@ dependencies = [
[[package]] [[package]]
name = "number_prefix" name = "number_prefix"
version = "0.3.0" version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]] [[package]]
name = "object" name = "object"
@ -1469,6 +1543,7 @@ dependencies = [
"clap", "clap",
"colored", "colored",
"comfy-table", "comfy-table",
"derive_builder",
"directories", "directories",
"epub-builder", "epub-builder",
"flexi_logger", "flexi_logger",
@ -1829,9 +1904,9 @@ dependencies = [
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.4.6" version = "1.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759" checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
dependencies = [ dependencies = [
"aho-corasick", "aho-corasick",
"memchr", "memchr",
@ -1840,9 +1915,9 @@ dependencies = [
[[package]] [[package]]
name = "regex-syntax" name = "regex-syntax"
version = "0.6.23" version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]] [[package]]
name = "remove_dir_all" name = "remove_dir_all"
@ -2172,6 +2247,12 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]] [[package]]
name = "strum" name = "strum"
version = "0.20.0" version = "0.20.0"
@ -2277,18 +2358,18 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.24" version = "1.0.25"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6"
dependencies = [ dependencies = [
"thiserror-impl", "thiserror-impl",
] ]
[[package]] [[package]]
name = "thiserror-impl" name = "thiserror-impl"
version = "1.0.24" version = "1.0.25"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -2465,9 +2546,9 @@ dependencies = [
[[package]] [[package]]
name = "url" name = "url"
version = "2.2.1" version = "2.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
dependencies = [ dependencies = [
"form_urlencoded", "form_urlencoded",
"idna", "idna",

View file

@ -12,23 +12,24 @@ readme = "README.md"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
async-std = "1.9.0"
# atty = "0.2.14" # atty = "0.2.14"
async-std = "1.9.0"
chrono = "0.4.19" chrono = "0.4.19"
clap = "2.33.3" clap = "2.33.3"
colored = "2.0.0" colored = "2.0.0"
comfy-table = "2.1.0" comfy-table = "2.1.0"
derive_builder = "0.10.2"
directories = "3.0.2" directories = "3.0.2"
epub-builder = "0.4.8" epub-builder = "0.4.8"
flexi_logger = "0.17.1" flexi_logger = "0.17.1"
futures = "0.3.14" futures = "0.3.15"
html5ever = "0.25.1" html5ever = "0.25.1"
indicatif = "0.15.0" indicatif = "0.16.2"
kuchiki = "0.8.1" kuchiki = "0.8.1"
lazy_static = "1.4.0" lazy_static = "1.4.0"
log = "0.4.14" log = "0.4.14"
md5 = "0.7.0" md5 = "0.7.0"
regex = "1.4.5" regex = "1.5.4"
surf = "2.2.0" surf = "2.2.0"
thiserror = "1.0.24" thiserror = "1.0.25"
url = "2.2.1" url = "2.2.2"

View file

@ -48,15 +48,20 @@ USAGE:
paperoni [OPTIONS] [urls]... paperoni [OPTIONS] [urls]...
OPTIONS: OPTIONS:
-f, --file <file> Input file containing links -f, --file <file> Input file containing links
-h, --help Prints help information -h, --help Prints help information
--log-to-file Enables logging of events to a file located in .paperoni/logs with a default log level --log-to-file
of debug. Use -v to specify the logging level Enables logging of events to a file located in .paperoni/logs with a default log level of debug. Use -v to
--max_conn <max_conn> The maximum number of concurrent HTTP connections when downloading articles. Default is specify the logging level
8 --max_conn <max_conn>
--merge <output_name> Merge multiple articles into a single epub The maximum number of concurrent HTTP connections when downloading articles. Default is 8
-V, --version Prints version information
-v Enables logging of events and set the verbosity level. Use -h to read on its usage -o, --output_directory <output_directory> Directory for store output epub documents
--merge <output_name> Merge multiple articles into a single epub
-V, --version Prints version information
-v
Enables logging of events and set the verbosity level. Use --help to read on its usage
ARGS: ARGS:
<urls>... Urls of web articles <urls>... Urls of web articles

1
rust-toolchain Normal file
View file

@ -0,0 +1 @@
1.52.1

View file

@ -1,13 +1,56 @@
use std::{fs::File, io::Read, path::Path}; use std::{
collections::HashSet,
num::{NonZeroUsize, ParseIntError},
path::Path,
};
use chrono::{DateTime, Local}; use chrono::{DateTime, Local};
use clap::{App, AppSettings, Arg}; use clap::{App, AppSettings, Arg, ArgMatches};
use flexi_logger::LevelFilter as LogLevel; use flexi_logger::{FlexiLoggerError, LevelFilter as LogLevel};
use std::fs;
use thiserror::Error;
use crate::logs::init_logger; const DEFAULT_MAX_CONN: usize = 8;
pub fn cli_init() -> AppConfig { #[derive(derive_builder::Builder)]
let app = App::new("paperoni") pub struct AppConfig {
/// Urls for store in epub
pub urls: Vec<String>,
pub max_conn: usize,
/// Path to file of multiple articles into a single epub
pub merged: Option<String>,
pub output_directory: Option<String>,
pub log_level: LogLevel,
pub can_disable_progress_bar: bool,
pub start_time: DateTime<Local>,
pub is_logging_to_file: bool,
}
#[derive(Debug, Error)]
pub enum Error {
#[error("Failed to open file with urls: {0}")]
UrlFileError(#[from] std::io::Error),
#[error("Failed to parse max connection value: {0}")]
InvalidMaxConnectionCount(#[from] ParseIntError),
#[error("No urls for parse")]
NoUrls,
#[error("No urls for parse")]
AppBuildError(#[from] AppConfigBuilderError),
#[error("Invalid output path name for merged epubs: {0}")]
InvalidOutputPath(String),
#[error("Log error: {0}")]
LogDirectoryError(String),
#[error(transparent)]
LogError(#[from] FlexiLoggerError),
#[error("Wrong output directory")]
WrongOutputDirectory,
#[error("Output directory not exists")]
OutputDirectoryNotExists,
}
impl AppConfig {
pub fn init_with_cli() -> Result<AppConfig, Error> {
let app = App::new("paperoni")
.settings(&[ .settings(&[
AppSettings::ArgRequiredElseHelp, AppSettings::ArgRequiredElseHelp,
AppSettings::UnifiedHelpMessage, AppSettings::UnifiedHelpMessage,
@ -28,11 +71,21 @@ pub fn cli_init() -> AppConfig {
.help("Input file containing links") .help("Input file containing links")
.takes_value(true), .takes_value(true),
) )
.arg(
Arg::with_name("output_directory")
.long("output_directory")
.short("o")
.help("Directory for store output epub documents")
.conflicts_with("output_name")
.long_help("Directory for saving epub documents")
.takes_value(true),
)
.arg( .arg(
Arg::with_name("output_name") Arg::with_name("output_name")
.long("merge") .long("merge")
.help("Merge multiple articles into a single epub") .help("Merge multiple articles into a single epub")
.long_help("Merge multiple articles into a single epub that will be given the name provided") .long_help("Merge multiple articles into a single epub that will be given the name provided")
.conflicts_with("output_directory")
.takes_value(true), .takes_value(true),
).arg( ).arg(
Arg::with_name("max_conn") Arg::with_name("max_conn")
@ -60,143 +113,135 @@ pub fn cli_init() -> AppConfig {
.long("log-to-file") .long("log-to-file")
.help("Enables logging of events to a file located in .paperoni/logs with a default log level of debug. Use -v to specify the logging level") .help("Enables logging of events to a file located in .paperoni/logs with a default log level of debug. Use -v to specify the logging level")
.takes_value(false)); .takes_value(false));
let arg_matches = app.get_matches();
let mut urls: Vec<String> = match arg_matches.value_of("file") { Self::try_from(app.get_matches())
Some(file_name) => { }
if let Ok(mut file) = File::open(file_name) {
let mut content = String::new(); fn init_merge_file(self) -> Result<Self, Error> {
match file.read_to_string(&mut content) { self.merged
Ok(_) => content .as_deref()
.lines() .map(fs::File::create)
.filter(|line| !line.is_empty()) .transpose()
.map(|line| line.to_owned()) .err()
.collect(), .map(|err| Err(Error::InvalidOutputPath(err.to_string())))
Err(_) => vec![], .unwrap_or(Ok(self))
}
fn init_logger(self) -> Result<Self, Error> {
use directories::UserDirs;
use flexi_logger::LogSpecBuilder;
match UserDirs::new() {
Some(user_dirs) => {
let home_dir = user_dirs.home_dir();
let paperoni_dir = home_dir.join(".paperoni");
let log_dir = paperoni_dir.join("logs");
let log_spec = LogSpecBuilder::new()
.module("paperoni", self.log_level)
.build();
let formatted_timestamp = self.start_time.format("%Y-%m-%d_%H-%M-%S");
let mut logger = flexi_logger::Logger::with(log_spec);
if self.is_logging_to_file && (!paperoni_dir.is_dir() || !log_dir.is_dir()) {
if let Err(e) = fs::create_dir_all(&log_dir) {
return Err(Error::LogDirectoryError(format!("Unable to create paperoni directories on home directory for logging purposes\n{}",e)));
}
} }
} else { if self.is_logging_to_file {
println!("Unable to open file: {}", file_name); logger = logger
vec![] .directory(log_dir)
.discriminant(formatted_timestamp.to_string())
.suppress_timestamp()
.log_to_file();
}
logger.start()?;
Ok(self)
} }
None => Err(Error::LogDirectoryError(
"Unable to get user directories for logging purposes".to_string(),
)),
} }
None => vec![],
};
if let Some(vals) = arg_matches.values_of("urls") {
urls.extend(
vals.filter(|val| !val.is_empty())
.map(|val| val.to_string()),
);
} }
let max_conn = arg_matches
.value_of("max_conn")
.map(|conn_str| conn_str.parse::<usize>().ok())
.flatten()
.map(|max| if max > 0 { max } else { 1 })
.unwrap_or(8);
let mut app_config = AppConfig::new(max_conn);
app_config.set_urls(urls);
if let Some(name) = arg_matches.value_of("output_name") {
let file_path = Path::new(name);
if file_path.is_dir() {
eprintln!("{:?} is a directory", name);
std::process::exit(1);
}
let file_name = if file_path.extension().is_some() {
name.to_owned()
} else {
name.to_owned() + ".epub"
};
match std::fs::File::create(&file_name) {
Ok(_) => (),
Err(e) => {
eprintln!("Unable to create file {:?}\n{}", file_path, e);
std::process::exit(1)
}
}
app_config.merged = Some(file_name);
}
if arg_matches.is_present("verbosity") {
if !arg_matches.is_present("log-to-file") {
app_config.can_disable_progress_bar = true;
}
let log_levels: [LogLevel; 5] = [
LogLevel::Off,
LogLevel::Error,
LogLevel::Warn,
LogLevel::Info,
LogLevel::Debug,
];
let level = arg_matches.occurrences_of("verbosity").clamp(0, 4) as usize;
app_config.log_level = log_levels[level];
}
if arg_matches.is_present("log-to-file") {
app_config.log_level = LogLevel::Debug;
app_config.is_logging_to_file = true;
}
init_logger(&app_config);
app_config
} }
pub struct AppConfig { use std::convert::TryFrom;
urls: Vec<String>,
max_conn: usize, impl<'a> TryFrom<ArgMatches<'a>> for AppConfig {
merged: Option<String>, type Error = Error;
log_level: LogLevel,
can_disable_progress_bar: bool, fn try_from(arg_matches: ArgMatches<'a>) -> Result<Self, Self::Error> {
start_time: DateTime<Local>, AppConfigBuilder::default()
is_logging_to_file: bool, .urls({
let url_filter = |url: &str| {
let url = url.trim();
if !url.is_empty() {
Some(url.to_owned())
} else {
None
}
};
match (
arg_matches
.values_of("urls")
.and_then(|urls| urls.map(url_filter).collect::<Option<HashSet<_>>>()),
arg_matches
.value_of("file")
.map(fs::read_to_string)
.transpose()?
.and_then(|content| {
content
.lines()
.map(url_filter)
.collect::<Option<HashSet<_>>>()
}),
) {
(Some(direct_urls), Some(file_urls)) => Ok(direct_urls
.union(&file_urls)
.map(ToOwned::to_owned)
.collect::<Vec<_>>()),
(Some(urls), None) | (None, Some(urls)) => Ok(urls.into_iter().collect()),
(None, None) => Err(Error::NoUrls),
}
}?)
.max_conn(match arg_matches.value_of("max_conn") {
Some(max_conn) => max_conn.parse::<NonZeroUsize>()?.get(),
None => DEFAULT_MAX_CONN,
})
.merged(arg_matches.value_of("output_name").map(ToOwned::to_owned))
.can_disable_progress_bar(
arg_matches.is_present("verbosity") && !arg_matches.is_present("log-to-file"),
)
.log_level(match arg_matches.occurrences_of("verbosity") {
0 => LogLevel::Off,
1 => LogLevel::Error,
2 => LogLevel::Warn,
3 => LogLevel::Info,
4..=u64::MAX => LogLevel::Debug,
})
.is_logging_to_file(arg_matches.is_present("log-to_file"))
.output_directory(
arg_matches
.value_of("output_directory")
.map(|output_directory| {
let path = Path::new(output_directory);
if !path.exists() {
Err(Error::OutputDirectoryNotExists)
} else if !path.is_dir() {
Err(Error::WrongOutputDirectory)
} else {
Ok(output_directory.to_owned())
}
})
.transpose()?,
)
.start_time(Local::now())
.try_init()
}
} }
impl AppConfig { impl AppConfigBuilder {
fn new(max_conn: usize) -> Self { pub fn try_init(&self) -> Result<AppConfig, Error> {
Self { self.build()?.init_logger()?.init_merge_file()
urls: vec![],
max_conn,
merged: None,
log_level: LogLevel::Off,
can_disable_progress_bar: false,
start_time: Local::now(),
is_logging_to_file: false,
}
}
fn set_urls(&mut self, urls: Vec<String>) {
self.urls.extend(urls);
}
pub fn urls(&self) -> &Vec<String> {
&self.urls
}
pub fn max_conn(&self) -> usize {
self.max_conn
}
pub fn merged(&self) -> Option<&String> {
self.merged.as_ref()
}
pub fn log_level(&self) -> LogLevel {
self.log_level
}
pub fn can_disable_progress_bar(&self) -> bool {
self.can_disable_progress_bar
}
pub fn start_time(&self) -> &DateTime<Local> {
&self.start_time
}
pub fn is_logging_to_file(&self) -> bool {
self.is_logging_to_file
} }
} }

View file

@ -16,7 +16,7 @@ pub fn generate_epubs(
app_config: &AppConfig, app_config: &AppConfig,
successful_articles_table: &mut Table, successful_articles_table: &mut Table,
) -> Result<(), Vec<PaperoniError>> { ) -> Result<(), Vec<PaperoniError>> {
let bar = if app_config.can_disable_progress_bar() { let bar = if app_config.can_disable_progress_bar {
ProgressBar::hidden() ProgressBar::hidden()
} else { } else {
let enabled_bar = ProgressBar::new(articles.len() as u64); let enabled_bar = ProgressBar::new(articles.len() as u64);
@ -32,8 +32,8 @@ pub fn generate_epubs(
let mut errors: Vec<PaperoniError> = Vec::new(); let mut errors: Vec<PaperoniError> = Vec::new();
match app_config.merged() { match app_config.merged {
Some(name) => { Some(ref name) => {
successful_articles_table.set_header(vec![Cell::new("Table of Contents") successful_articles_table.set_header(vec![Cell::new("Table of Contents")
.add_attribute(Attribute::Bold) .add_attribute(Attribute::Bold)
.set_alignment(CellAlignment::Center) .set_alignment(CellAlignment::Center)
@ -103,7 +103,7 @@ pub fn generate_epubs(
.title(replace_metadata_value("Article Sources")), .title(replace_metadata_value("Article Sources")),
) { ) {
let mut paperoni_err: PaperoniError = err.into(); let mut paperoni_err: PaperoniError = err.into();
paperoni_err.set_article_source(name); paperoni_err.set_article_source(&name);
errors.push(paperoni_err); errors.push(paperoni_err);
return Err(errors); return Err(errors);
} }
@ -113,7 +113,7 @@ pub fn generate_epubs(
Ok(_) => (), Ok(_) => (),
Err(err) => { Err(err) => {
let mut paperoni_err: PaperoniError = err.into(); let mut paperoni_err: PaperoniError = err.into();
paperoni_err.set_article_source(name); paperoni_err.set_article_source(&name);
errors.push(paperoni_err); errors.push(paperoni_err);
return Err(errors); return Err(errors);
} }
@ -135,7 +135,8 @@ pub fn generate_epubs(
let mut result = || -> Result<(), PaperoniError> { let mut result = || -> Result<(), PaperoniError> {
let mut epub = EpubBuilder::new(ZipLibrary::new()?)?; let mut epub = EpubBuilder::new(ZipLibrary::new()?)?;
let file_name = format!( let file_name = format!(
"{}.epub", "{}/{}.epub",
app_config.output_directory.as_deref().unwrap_or("."),
article article
.metadata() .metadata()
.title() .title()

View file

@ -153,7 +153,11 @@ pub async fn download_images(
}) })
.enumerate() .enumerate()
.map(|(img_idx, (url, req))| async move { .map(|(img_idx, (url, req))| async move {
bar.set_message(format!("Downloading images [{}/{}]", img_idx + 1, img_count).as_str()); bar.set_message(format!(
"Downloading images [{}/{}]",
img_idx + 1,
img_count
));
match req.await { match req.await {
Ok(mut img_response) => { Ok(mut img_response) => {
let process_response = let process_response =
@ -234,9 +238,9 @@ fn get_absolute_url(url: &str, request_url: &Url) -> String {
.unwrap() .unwrap()
.join(url) .join(url)
.unwrap() .unwrap()
.into_string() .into()
} else { } else {
request_url.join(url).unwrap().into_string() request_url.join(url).unwrap().into()
} }
} }

View file

@ -1,11 +1,9 @@
use colored::*; use colored::*;
use comfy_table::presets::UTF8_HORIZONTAL_BORDERS_ONLY; use comfy_table::presets::UTF8_HORIZONTAL_BORDERS_ONLY;
use comfy_table::{Cell, CellAlignment, ContentArrangement, Table}; use comfy_table::{Cell, CellAlignment, ContentArrangement, Table};
use directories::UserDirs;
use flexi_logger::LogSpecBuilder;
use log::error; use log::error;
use crate::{cli::AppConfig, errors::PaperoniError}; use crate::errors::PaperoniError;
pub fn display_summary( pub fn display_summary(
initial_article_count: usize, initial_article_count: usize,
@ -143,47 +141,6 @@ impl DownloadCount {
} }
} }
} }
pub fn init_logger(app_config: &AppConfig) {
match UserDirs::new() {
Some(user_dirs) => {
let home_dir = user_dirs.home_dir();
let paperoni_dir = home_dir.join(".paperoni");
let log_dir = paperoni_dir.join("logs");
let log_spec = LogSpecBuilder::new()
.module("paperoni", app_config.log_level())
.build();
let formatted_timestamp = app_config.start_time().format("%Y-%m-%d_%H-%M-%S");
let mut logger = flexi_logger::Logger::with(log_spec);
if app_config.is_logging_to_file() && (!paperoni_dir.is_dir() || !log_dir.is_dir()) {
match std::fs::create_dir_all(&log_dir) {
Ok(_) => (),
Err(e) => {
eprintln!("Unable to create paperoni directories on home directory for logging purposes\n{}",e);
std::process::exit(1);
}
};
}
if app_config.is_logging_to_file() {
logger = logger
.directory(log_dir)
.discriminant(formatted_timestamp.to_string())
.suppress_timestamp()
.log_to_file();
}
match logger.start() {
Ok(_) => (),
Err(e) => eprintln!("Unable to start logger!\n{}", e),
}
}
None => eprintln!("Unable to get user directories for logging purposes"),
};
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{short_summary, DownloadCount}; use super::{short_summary, DownloadCount};

View file

@ -1,6 +1,8 @@
#[macro_use] #[macro_use]
extern crate lazy_static; extern crate lazy_static;
use std::process::exit;
use async_std::stream; use async_std::stream;
use async_std::task; use async_std::task;
use comfy_table::presets::{UTF8_FULL, UTF8_HORIZONTAL_BORDERS_ONLY}; use comfy_table::presets::{UTF8_FULL, UTF8_HORIZONTAL_BORDERS_ONLY};
@ -27,9 +29,15 @@ use http::{download_images, fetch_html};
use logs::display_summary; use logs::display_summary;
fn main() { fn main() {
let app_config = cli::cli_init(); let app_config = match cli::AppConfig::init_with_cli() {
Ok(app_config) => app_config,
Err(err) => {
eprintln!("{}", err);
exit(1);
}
};
if !app_config.urls().is_empty() { if !app_config.urls.is_empty() {
download(app_config); download(app_config);
} }
} }
@ -37,10 +45,10 @@ fn main() {
fn download(app_config: AppConfig) { fn download(app_config: AppConfig) {
let mut errors = Vec::new(); let mut errors = Vec::new();
let mut partial_download_count: usize = 0; let mut partial_download_count: usize = 0;
let bar = if app_config.can_disable_progress_bar() { let bar = if app_config.can_disable_progress_bar {
ProgressBar::hidden() ProgressBar::hidden()
} else { } else {
let enabled_bar = ProgressBar::new(app_config.urls().len() as u64); let enabled_bar = ProgressBar::new(app_config.urls.len() as u64);
let style = ProgressStyle::default_bar().template( let style = ProgressStyle::default_bar().template(
"{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} link {pos}/{len:7} {msg:.yellow/white}", "{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} link {pos}/{len:7} {msg:.yellow/white}",
); );
@ -49,8 +57,8 @@ fn download(app_config: AppConfig) {
enabled_bar enabled_bar
}; };
let articles = task::block_on(async { let articles = task::block_on(async {
let urls_iter = app_config.urls().iter().map(|url| fetch_html(url)); let urls_iter = app_config.urls.iter().map(|url| fetch_html(url));
let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn()); let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn);
let mut articles = Vec::new(); let mut articles = Vec::new();
while let Some(fetch_result) = responses.next().await { while let Some(fetch_result) = responses.next().await {
match fetch_result { match fetch_result {
@ -109,15 +117,15 @@ fn download(app_config: AppConfig) {
}; };
let has_errors = !errors.is_empty(); let has_errors = !errors.is_empty();
display_summary( display_summary(
app_config.urls().len(), app_config.urls.len(),
succesful_articles_table, succesful_articles_table,
partial_download_count, partial_download_count,
errors, errors,
); );
if app_config.is_logging_to_file() { if app_config.is_logging_to_file {
println!( println!(
"Log written to paperoni_{}.log\n", "Log written to paperoni_{}.log\n",
app_config.start_time().format("%Y-%m-%d_%H-%M-%S") app_config.start_time.format("%Y-%m-%d_%H-%M-%S")
); );
} }
if has_errors { if has_errors {