Merge branch 'dev' of github.com:hipstermojo/paperoni into dev
This commit is contained in:
Kenneth Gitere
commit
95bd22f339
10 changed files with 381 additions and 240 deletions
159
Cargo.lock
generated
159
Cargo.lock
generated
|
|
@ -1,5 +1,7 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.14.1"
|
||||
|
|
@ -71,9 +73,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.15"
|
||||
version = "0.7.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
|
||||
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
|
@ -389,7 +391,7 @@ dependencies = [
|
|||
"ansi_term",
|
||||
"atty",
|
||||
"bitflags",
|
||||
"strsim",
|
||||
"strsim 0.8.0",
|
||||
"textwrap",
|
||||
"unicode-width",
|
||||
"vec_map",
|
||||
|
|
@ -435,9 +437,7 @@ dependencies = [
|
|||
"encode_unicode",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"regex",
|
||||
"terminal_size",
|
||||
"unicode-width",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
|
|
@ -614,6 +614,41 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim 0.10.0",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "4.0.2"
|
||||
|
|
@ -630,6 +665,37 @@ version = "2.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d13202debe11181040ae9063d739fa32cfcaaebe2275fe387703460ae2365b30"
|
||||
dependencies = [
|
||||
"derive_builder_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_core"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66e616858f6187ed828df7c64a6d71720d83767a7f19740b2d1b6fe6327b36e5"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_macro"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58a94ace95092c5acb1e97a7e846b310cfbd499652f72297da7493f618a98d73"
|
||||
dependencies = [
|
||||
"derive_builder_core",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.13"
|
||||
|
|
@ -822,9 +888,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9d5813545e459ad3ca1bff9915e9ad7f1a47dc6a91b627ce321d5863b7dd253"
|
||||
checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
|
|
@ -837,9 +903,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce79c6a52a299137a6013061e0cf0e688fce5d7f1bc60125f520912fdb29ec25"
|
||||
checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
|
|
@ -847,15 +913,15 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-core"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "098cd1c6dda6ca01650f1a37a794245eb73181d0d4d4e955e2f3c37db7af1815"
|
||||
checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10f6cb7042eda00f0049b1d2080aa4b93442997ee507eb3828e8bd7577f94c9d"
|
||||
checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
|
|
@ -864,9 +930,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "365a1a1fb30ea1c03a830fdb2158f5236833ac81fa0ad12fe35b29cddc35cb04"
|
||||
checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
|
||||
|
||||
[[package]]
|
||||
name = "futures-lite"
|
||||
|
|
@ -885,10 +951,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "668c6733a182cd7deb4f1de7ba3bf2120823835b3bcfbeacf7d2c4a773c1bb8b"
|
||||
checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"proc-macro-hack",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -897,22 +964,23 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c5629433c555de3d82861a7a4e3794a4c40040390907cfbfd7143a92a426c23"
|
||||
checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282"
|
||||
|
||||
[[package]]
|
||||
name = "futures-task"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba7aa51095076f3ba6d9a1f702f74bd05ec65f555d70d2033d55ba8d69f581bc"
|
||||
checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.14"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c144ad54d60f23927f0a6b6d816e4271278b64f005ad65e4e35291d2de9c025"
|
||||
checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
|
|
@ -1112,6 +1180,12 @@ dependencies = [
|
|||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.2.3"
|
||||
|
|
@ -1125,9 +1199,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "indicatif"
|
||||
version = "0.15.0"
|
||||
version = "0.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
|
||||
checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b"
|
||||
dependencies = [
|
||||
"console",
|
||||
"lazy_static",
|
||||
|
|
@ -1305,9 +1379,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
|
|||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.4"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
|
||||
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
|
|
@ -1419,9 +1493,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "number_prefix"
|
||||
version = "0.3.0"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
|
||||
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
|
|
@ -1469,6 +1543,7 @@ dependencies = [
|
|||
"clap",
|
||||
"colored",
|
||||
"comfy-table",
|
||||
"derive_builder",
|
||||
"directories",
|
||||
"epub-builder",
|
||||
"flexi_logger",
|
||||
|
|
@ -1829,9 +1904,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.4.6"
|
||||
version = "1.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759"
|
||||
checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
|
|
@ -1840,9 +1915,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.23"
|
||||
version = "0.6.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"
|
||||
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
|
|
@ -2172,6 +2247,12 @@ version = "0.8.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.20.0"
|
||||
|
|
@ -2277,18 +2358,18 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
|
|||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.24"
|
||||
version = "1.0.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
|
||||
checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.24"
|
||||
version = "1.0.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
|
||||
checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
|
@ -2465,9 +2546,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.2.1"
|
||||
version = "2.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b"
|
||||
checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
|
|
|
|||
13
Cargo.toml
13
Cargo.toml
|
|
@ -12,23 +12,24 @@ readme = "README.md"
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
async-std = "1.9.0"
|
||||
# atty = "0.2.14"
|
||||
async-std = "1.9.0"
|
||||
chrono = "0.4.19"
|
||||
clap = "2.33.3"
|
||||
colored = "2.0.0"
|
||||
comfy-table = "2.1.0"
|
||||
derive_builder = "0.10.2"
|
||||
directories = "3.0.2"
|
||||
epub-builder = "0.4.8"
|
||||
flexi_logger = "0.17.1"
|
||||
futures = "0.3.14"
|
||||
futures = "0.3.15"
|
||||
html5ever = "0.25.1"
|
||||
indicatif = "0.15.0"
|
||||
indicatif = "0.16.2"
|
||||
kuchiki = "0.8.1"
|
||||
lazy_static = "1.4.0"
|
||||
log = "0.4.14"
|
||||
md5 = "0.7.0"
|
||||
regex = "1.4.5"
|
||||
regex = "1.5.4"
|
||||
surf = "2.2.0"
|
||||
thiserror = "1.0.24"
|
||||
url = "2.2.1"
|
||||
thiserror = "1.0.25"
|
||||
url = "2.2.2"
|
||||
|
|
|
|||
43
README.md
43
README.md
|
|
@ -48,18 +48,41 @@ USAGE:
|
|||
paperoni [OPTIONS] [urls]...
|
||||
|
||||
OPTIONS:
|
||||
-f, --file <file> Input file containing links
|
||||
-h, --help Prints help information
|
||||
--log-to-file Enables logging of events to a file located in .paperoni/logs with a default log level
|
||||
of debug. Use -v to specify the logging level
|
||||
--max_conn <max_conn> The maximum number of concurrent HTTP connections when downloading articles. Default is
|
||||
8
|
||||
--merge <output_name> Merge multiple articles into a single epub
|
||||
-V, --version Prints version information
|
||||
-v Enables logging of events and set the verbosity level. Use -h to read on its usage
|
||||
-f, --file <file>
|
||||
Input file containing links
|
||||
|
||||
-h, --help
|
||||
Prints help information
|
||||
|
||||
--log-to-file
|
||||
Enables logging of events to a file located in .paperoni/logs with a default log level of debug. Use -v to
|
||||
specify the logging level
|
||||
--max_conn <max_conn>
|
||||
The maximum number of concurrent HTTP connections when downloading articles. Default is 8.
|
||||
NOTE: It is advised to use as few connections as needed i.e between 1 and 50. Using more connections can end
|
||||
up overloading your network card with too many concurrent requests.
|
||||
-o, --output_directory <output_directory>
|
||||
Directory for saving epub documents
|
||||
|
||||
--merge <output_name>
|
||||
Merge multiple articles into a single epub that will be given the name provided
|
||||
|
||||
-V, --version
|
||||
Prints version information
|
||||
|
||||
-v
|
||||
This takes upto 4 levels of verbosity in the following order.
|
||||
- Error (-v)
|
||||
- Warn (-vv)
|
||||
- Info (-vvv)
|
||||
- Debug (-vvvv)
|
||||
When this flag is passed, it disables the progress bars and logs to stderr.
|
||||
If you would like to send the logs to a file (and enable progress bars), pass the log-to-file flag.
|
||||
|
||||
ARGS:
|
||||
<urls>... Urls of web articles
|
||||
<urls>...
|
||||
Urls of web articles
|
||||
|
||||
```
|
||||
|
||||
To download a single article pass in its URL
|
||||
|
|
|
|||
1
rust-toolchain
Normal file
1
rust-toolchain
Normal file
|
|
@ -0,0 +1 @@
|
|||
1.52.1
|
||||
272
src/cli.rs
272
src/cli.rs
|
|
@ -1,13 +1,30 @@
|
|||
use std::{fs::File, io::Read, path::Path};
|
||||
use std::{collections::BTreeSet, fs, num::NonZeroUsize, path::Path};
|
||||
|
||||
use chrono::{DateTime, Local};
|
||||
use clap::{App, AppSettings, Arg};
|
||||
use clap::{App, AppSettings, Arg, ArgMatches};
|
||||
use flexi_logger::LevelFilter as LogLevel;
|
||||
|
||||
use crate::logs::init_logger;
|
||||
type Error = crate::errors::CliError<AppConfigBuilderError>;
|
||||
|
||||
pub fn cli_init() -> AppConfig {
|
||||
let app = App::new("paperoni")
|
||||
const DEFAULT_MAX_CONN: usize = 8;
|
||||
|
||||
#[derive(derive_builder::Builder)]
|
||||
pub struct AppConfig {
|
||||
/// Urls for store in epub
|
||||
pub urls: Vec<String>,
|
||||
pub max_conn: usize,
|
||||
/// Path to file of multiple articles into a single epub
|
||||
pub merged: Option<String>,
|
||||
pub output_directory: Option<String>,
|
||||
pub log_level: LogLevel,
|
||||
pub can_disable_progress_bar: bool,
|
||||
pub start_time: DateTime<Local>,
|
||||
pub is_logging_to_file: bool,
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
pub fn init_with_cli() -> Result<AppConfig, Error> {
|
||||
let app = App::new("paperoni")
|
||||
.settings(&[
|
||||
AppSettings::ArgRequiredElseHelp,
|
||||
AppSettings::UnifiedHelpMessage,
|
||||
|
|
@ -28,11 +45,20 @@ pub fn cli_init() -> AppConfig {
|
|||
.help("Input file containing links")
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("output_directory")
|
||||
.long("output-directory")
|
||||
.short("o")
|
||||
.help("Directory to store output epub documents")
|
||||
.conflicts_with("output_name")
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("output_name")
|
||||
.long("merge")
|
||||
.help("Merge multiple articles into a single epub")
|
||||
.long_help("Merge multiple articles into a single epub that will be given the name provided")
|
||||
.conflicts_with("output_directory")
|
||||
.takes_value(true),
|
||||
).arg(
|
||||
Arg::with_name("max_conn")
|
||||
|
|
@ -60,143 +86,107 @@ pub fn cli_init() -> AppConfig {
|
|||
.long("log-to-file")
|
||||
.help("Enables logging of events to a file located in .paperoni/logs with a default log level of debug. Use -v to specify the logging level")
|
||||
.takes_value(false));
|
||||
let arg_matches = app.get_matches();
|
||||
|
||||
let mut urls: Vec<String> = match arg_matches.value_of("file") {
|
||||
Some(file_name) => {
|
||||
if let Ok(mut file) = File::open(file_name) {
|
||||
let mut content = String::new();
|
||||
match file.read_to_string(&mut content) {
|
||||
Ok(_) => content
|
||||
.lines()
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(|line| line.to_owned())
|
||||
.collect(),
|
||||
Err(_) => vec![],
|
||||
Self::try_from(app.get_matches())
|
||||
}
|
||||
|
||||
fn init_merge_file(self) -> Result<Self, Error> {
|
||||
self.merged
|
||||
.as_deref()
|
||||
.map(fs::File::create)
|
||||
.transpose()
|
||||
.err()
|
||||
.map(|err| Err(Error::InvalidOutputPath(err.to_string())))
|
||||
.unwrap_or(Ok(self))
|
||||
}
|
||||
|
||||
fn init_logger(self) -> Result<Self, Error> {
|
||||
use crate::logs;
|
||||
logs::init_logger(self.log_level, &self.start_time, self.is_logging_to_file)
|
||||
.map(|_| self)
|
||||
.map_err(Error::LogError)
|
||||
}
|
||||
}
|
||||
|
||||
use std::convert::TryFrom;
|
||||
|
||||
impl<'a> TryFrom<ArgMatches<'a>> for AppConfig {
|
||||
type Error = Error;
|
||||
|
||||
fn try_from(arg_matches: ArgMatches<'a>) -> Result<Self, Self::Error> {
|
||||
AppConfigBuilder::default()
|
||||
.urls({
|
||||
let url_filter = |url: &str| {
|
||||
let url = url.trim();
|
||||
if !url.is_empty() {
|
||||
Some(url.to_owned())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let direct_urls = arg_matches
|
||||
.values_of("urls")
|
||||
.and_then(|urls| urls.map(url_filter).collect::<Option<BTreeSet<_>>>());
|
||||
let file_urls = arg_matches
|
||||
.value_of("file")
|
||||
.map(fs::read_to_string)
|
||||
.transpose()?
|
||||
.and_then(|content| {
|
||||
content
|
||||
.lines()
|
||||
.map(url_filter)
|
||||
.collect::<Option<BTreeSet<_>>>()
|
||||
});
|
||||
match (direct_urls, file_urls) {
|
||||
(Some(direct_urls), Some(file_urls)) => Ok(direct_urls
|
||||
.union(&file_urls)
|
||||
.map(ToOwned::to_owned)
|
||||
.collect::<Vec<_>>()),
|
||||
(Some(urls), None) | (None, Some(urls)) => Ok(urls.into_iter().collect()),
|
||||
(None, None) => Err(Error::NoUrls),
|
||||
}
|
||||
} else {
|
||||
println!("Unable to open file: {}", file_name);
|
||||
vec![]
|
||||
}
|
||||
}
|
||||
None => vec![],
|
||||
};
|
||||
|
||||
if let Some(vals) = arg_matches.values_of("urls") {
|
||||
urls.extend(
|
||||
vals.filter(|val| !val.is_empty())
|
||||
.map(|val| val.to_string()),
|
||||
);
|
||||
}
|
||||
|
||||
let max_conn = arg_matches
|
||||
.value_of("max_conn")
|
||||
.map(|conn_str| conn_str.parse::<usize>().ok())
|
||||
.flatten()
|
||||
.map(|max| if max > 0 { max } else { 1 })
|
||||
.unwrap_or(8);
|
||||
|
||||
let mut app_config = AppConfig::new(max_conn);
|
||||
app_config.set_urls(urls);
|
||||
|
||||
if let Some(name) = arg_matches.value_of("output_name") {
|
||||
let file_path = Path::new(name);
|
||||
if file_path.is_dir() {
|
||||
eprintln!("{:?} is a directory", name);
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let file_name = if file_path.extension().is_some() {
|
||||
name.to_owned()
|
||||
} else {
|
||||
name.to_owned() + ".epub"
|
||||
};
|
||||
|
||||
match std::fs::File::create(&file_name) {
|
||||
Ok(_) => (),
|
||||
Err(e) => {
|
||||
eprintln!("Unable to create file {:?}\n{}", file_path, e);
|
||||
std::process::exit(1)
|
||||
}
|
||||
}
|
||||
app_config.merged = Some(file_name);
|
||||
}
|
||||
|
||||
if arg_matches.is_present("verbosity") {
|
||||
if !arg_matches.is_present("log-to-file") {
|
||||
app_config.can_disable_progress_bar = true;
|
||||
}
|
||||
let log_levels: [LogLevel; 5] = [
|
||||
LogLevel::Off,
|
||||
LogLevel::Error,
|
||||
LogLevel::Warn,
|
||||
LogLevel::Info,
|
||||
LogLevel::Debug,
|
||||
];
|
||||
let level = arg_matches.occurrences_of("verbosity").clamp(0, 4) as usize;
|
||||
app_config.log_level = log_levels[level];
|
||||
}
|
||||
if arg_matches.is_present("log-to-file") {
|
||||
app_config.log_level = LogLevel::Debug;
|
||||
app_config.is_logging_to_file = true;
|
||||
}
|
||||
|
||||
init_logger(&app_config);
|
||||
|
||||
app_config
|
||||
}
|
||||
|
||||
pub struct AppConfig {
|
||||
urls: Vec<String>,
|
||||
max_conn: usize,
|
||||
merged: Option<String>,
|
||||
log_level: LogLevel,
|
||||
can_disable_progress_bar: bool,
|
||||
start_time: DateTime<Local>,
|
||||
is_logging_to_file: bool,
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
fn new(max_conn: usize) -> Self {
|
||||
Self {
|
||||
urls: vec![],
|
||||
max_conn,
|
||||
merged: None,
|
||||
log_level: LogLevel::Off,
|
||||
can_disable_progress_bar: false,
|
||||
start_time: Local::now(),
|
||||
is_logging_to_file: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn set_urls(&mut self, urls: Vec<String>) {
|
||||
self.urls.extend(urls);
|
||||
}
|
||||
|
||||
pub fn urls(&self) -> &Vec<String> {
|
||||
&self.urls
|
||||
}
|
||||
pub fn max_conn(&self) -> usize {
|
||||
self.max_conn
|
||||
}
|
||||
|
||||
pub fn merged(&self) -> Option<&String> {
|
||||
self.merged.as_ref()
|
||||
}
|
||||
|
||||
pub fn log_level(&self) -> LogLevel {
|
||||
self.log_level
|
||||
}
|
||||
|
||||
pub fn can_disable_progress_bar(&self) -> bool {
|
||||
self.can_disable_progress_bar
|
||||
}
|
||||
|
||||
pub fn start_time(&self) -> &DateTime<Local> {
|
||||
&self.start_time
|
||||
}
|
||||
|
||||
pub fn is_logging_to_file(&self) -> bool {
|
||||
self.is_logging_to_file
|
||||
}?)
|
||||
.max_conn(match arg_matches.value_of("max_conn") {
|
||||
Some(max_conn) => max_conn.parse::<NonZeroUsize>()?.get(),
|
||||
None => DEFAULT_MAX_CONN,
|
||||
})
|
||||
.merged(arg_matches.value_of("output_name").map(ToOwned::to_owned))
|
||||
.can_disable_progress_bar(
|
||||
arg_matches.is_present("verbosity") && !arg_matches.is_present("log-to-file"),
|
||||
)
|
||||
.log_level(match arg_matches.occurrences_of("verbosity") {
|
||||
0 => LogLevel::Off,
|
||||
1 => LogLevel::Error,
|
||||
2 => LogLevel::Warn,
|
||||
3 => LogLevel::Info,
|
||||
4..=u64::MAX => LogLevel::Debug,
|
||||
})
|
||||
.is_logging_to_file(arg_matches.is_present("log-to-file"))
|
||||
.output_directory(
|
||||
arg_matches
|
||||
.value_of("output_directory")
|
||||
.map(|output_directory| {
|
||||
let path = Path::new(output_directory);
|
||||
if !path.exists() {
|
||||
Err(Error::OutputDirectoryNotExists)
|
||||
} else if !path.is_dir() {
|
||||
Err(Error::WrongOutputDirectory)
|
||||
} else {
|
||||
Ok(output_directory.to_owned())
|
||||
}
|
||||
})
|
||||
.transpose()?,
|
||||
)
|
||||
.start_time(Local::now())
|
||||
.try_init()
|
||||
}
|
||||
}
|
||||
|
||||
impl AppConfigBuilder {
|
||||
pub fn try_init(&self) -> Result<AppConfig, Error> {
|
||||
self.build()
|
||||
.map_err(Error::AppBuildError)?
|
||||
.init_logger()?
|
||||
.init_merge_file()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
13
src/epub.rs
13
src/epub.rs
|
|
@ -18,7 +18,7 @@ pub fn generate_epubs(
|
|||
app_config: &AppConfig,
|
||||
successful_articles_table: &mut Table,
|
||||
) -> Result<(), Vec<PaperoniError>> {
|
||||
let bar = if app_config.can_disable_progress_bar() {
|
||||
let bar = if app_config.can_disable_progress_bar {
|
||||
ProgressBar::hidden()
|
||||
} else {
|
||||
let enabled_bar = ProgressBar::new(articles.len() as u64);
|
||||
|
|
@ -34,8 +34,8 @@ pub fn generate_epubs(
|
|||
|
||||
let mut errors: Vec<PaperoniError> = Vec::new();
|
||||
|
||||
match app_config.merged() {
|
||||
Some(name) => {
|
||||
match app_config.merged {
|
||||
Some(ref name) => {
|
||||
successful_articles_table.set_header(vec![Cell::new("Table of Contents")
|
||||
.add_attribute(Attribute::Bold)
|
||||
.set_alignment(CellAlignment::Center)
|
||||
|
|
@ -112,7 +112,7 @@ pub fn generate_epubs(
|
|||
.title(replace_escaped_characters("Article Sources")),
|
||||
) {
|
||||
let mut paperoni_err: PaperoniError = err.into();
|
||||
paperoni_err.set_article_source(name);
|
||||
paperoni_err.set_article_source(&name);
|
||||
errors.push(paperoni_err);
|
||||
return Err(errors);
|
||||
}
|
||||
|
|
@ -122,7 +122,7 @@ pub fn generate_epubs(
|
|||
Ok(_) => (),
|
||||
Err(err) => {
|
||||
let mut paperoni_err: PaperoniError = err.into();
|
||||
paperoni_err.set_article_source(name);
|
||||
paperoni_err.set_article_source(&name);
|
||||
errors.push(paperoni_err);
|
||||
return Err(errors);
|
||||
}
|
||||
|
|
@ -144,7 +144,8 @@ pub fn generate_epubs(
|
|||
let mut result = || -> Result<(), PaperoniError> {
|
||||
let mut epub = EpubBuilder::new(ZipLibrary::new()?)?;
|
||||
let file_name = format!(
|
||||
"{}.epub",
|
||||
"{}/{}.epub",
|
||||
app_config.output_directory.as_deref().unwrap_or("."),
|
||||
article
|
||||
.metadata()
|
||||
.title()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
use std::fmt::{Debug, Display};
|
||||
|
||||
use flexi_logger::FlexiLoggerError;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
|
|
@ -124,3 +127,33 @@ impl From<std::str::Utf8Error> for PaperoniError {
|
|||
PaperoniError::with_kind(ErrorKind::UTF8Error(err.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LogError {
|
||||
#[error(transparent)]
|
||||
FlexiError(#[from] FlexiLoggerError),
|
||||
#[error("Unable to get user directories for logging purposes")]
|
||||
UserDirectoriesError,
|
||||
#[error("Can't create log directory: {0}")]
|
||||
CreateLogDirectoryError(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CliError<BuilderError: Debug + Display> {
|
||||
#[error("Failed to open file with urls: {0}")]
|
||||
UrlFileError(#[from] std::io::Error),
|
||||
#[error("Failed to parse max connection value: {0}")]
|
||||
InvalidMaxConnectionCount(#[from] std::num::ParseIntError),
|
||||
#[error("No urls were provided")]
|
||||
NoUrls,
|
||||
#[error("Failed to build cli application: {0}")]
|
||||
AppBuildError(BuilderError),
|
||||
#[error("Invalid output path name for merged epubs: {0}")]
|
||||
InvalidOutputPath(String),
|
||||
#[error("Wrong output directory")]
|
||||
WrongOutputDirectory,
|
||||
#[error("Output directory not exists")]
|
||||
OutputDirectoryNotExists,
|
||||
#[error("Unable to start logger!\n{0}")]
|
||||
LogError(#[from] LogError),
|
||||
}
|
||||
|
|
|
|||
10
src/http.rs
10
src/http.rs
|
|
@ -153,7 +153,11 @@ pub async fn download_images(
|
|||
})
|
||||
.enumerate()
|
||||
.map(|(img_idx, (url, req))| async move {
|
||||
bar.set_message(format!("Downloading images [{}/{}]", img_idx + 1, img_count).as_str());
|
||||
bar.set_message(format!(
|
||||
"Downloading images [{}/{}]",
|
||||
img_idx + 1,
|
||||
img_count
|
||||
));
|
||||
match req.await {
|
||||
Ok(mut img_response) => {
|
||||
let process_response =
|
||||
|
|
@ -234,9 +238,9 @@ fn get_absolute_url(url: &str, request_url: &Url) -> String {
|
|||
.unwrap()
|
||||
.join(url)
|
||||
.unwrap()
|
||||
.into_string()
|
||||
.into()
|
||||
} else {
|
||||
request_url.join(url).unwrap().into_string()
|
||||
request_url.join(url).unwrap().into()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
51
src/logs.rs
51
src/logs.rs
|
|
@ -1,11 +1,13 @@
|
|||
use std::fs;
|
||||
|
||||
use chrono::{DateTime, Local};
|
||||
use colored::*;
|
||||
use comfy_table::presets::UTF8_HORIZONTAL_BORDERS_ONLY;
|
||||
use comfy_table::{Cell, CellAlignment, ContentArrangement, Table};
|
||||
use directories::UserDirs;
|
||||
use flexi_logger::LogSpecBuilder;
|
||||
use flexi_logger::LevelFilter;
|
||||
use log::error;
|
||||
|
||||
use crate::{cli::AppConfig, errors::PaperoniError};
|
||||
use crate::errors::PaperoniError;
|
||||
|
||||
pub fn display_summary(
|
||||
initial_article_count: usize,
|
||||
|
|
@ -123,44 +125,41 @@ impl DownloadCount {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn init_logger(app_config: &AppConfig) {
|
||||
use crate::errors::LogError as Error;
|
||||
|
||||
pub fn init_logger(
|
||||
log_level: LevelFilter,
|
||||
start_time: &DateTime<Local>,
|
||||
is_logging_to_file: bool,
|
||||
) -> Result<(), Error> {
|
||||
use directories::UserDirs;
|
||||
use flexi_logger::LogSpecBuilder;
|
||||
|
||||
match UserDirs::new() {
|
||||
Some(user_dirs) => {
|
||||
let home_dir = user_dirs.home_dir();
|
||||
let paperoni_dir = home_dir.join(".paperoni");
|
||||
let log_dir = paperoni_dir.join("logs");
|
||||
|
||||
let log_spec = LogSpecBuilder::new()
|
||||
.module("paperoni", app_config.log_level())
|
||||
.build();
|
||||
let formatted_timestamp = app_config.start_time().format("%Y-%m-%d_%H-%M-%S");
|
||||
let log_spec = LogSpecBuilder::new().module("paperoni", log_level).build();
|
||||
let formatted_timestamp = start_time.format("%Y-%m-%d_%H-%M-%S");
|
||||
let mut logger = flexi_logger::Logger::with(log_spec);
|
||||
|
||||
if app_config.is_logging_to_file() && (!paperoni_dir.is_dir() || !log_dir.is_dir()) {
|
||||
match std::fs::create_dir_all(&log_dir) {
|
||||
Ok(_) => (),
|
||||
Err(e) => {
|
||||
eprintln!("Unable to create paperoni directories on home directory for logging purposes\n{}",e);
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
if app_config.is_logging_to_file() {
|
||||
if is_logging_to_file {
|
||||
if !paperoni_dir.is_dir() || !log_dir.is_dir() {
|
||||
fs::create_dir_all(&log_dir)?;
|
||||
}
|
||||
logger = logger
|
||||
.directory(log_dir)
|
||||
.discriminant(formatted_timestamp.to_string())
|
||||
.suppress_timestamp()
|
||||
.log_to_file();
|
||||
}
|
||||
|
||||
match logger.start() {
|
||||
Ok(_) => (),
|
||||
Err(e) => eprintln!("Unable to start logger!\n{}", e),
|
||||
}
|
||||
logger.start()?;
|
||||
Ok(())
|
||||
}
|
||||
None => eprintln!("Unable to get user directories for logging purposes"),
|
||||
};
|
||||
None => Err(Error::UserDirectoriesError),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
26
src/main.rs
26
src/main.rs
|
|
@ -1,6 +1,8 @@
|
|||
#[macro_use]
|
||||
extern crate lazy_static;
|
||||
|
||||
use std::process::exit;
|
||||
|
||||
use async_std::stream;
|
||||
use async_std::task;
|
||||
use comfy_table::presets::{UTF8_FULL, UTF8_HORIZONTAL_BORDERS_ONLY};
|
||||
|
|
@ -27,9 +29,15 @@ use http::{download_images, fetch_html};
|
|||
use logs::display_summary;
|
||||
|
||||
fn main() {
|
||||
let app_config = cli::cli_init();
|
||||
let app_config = match cli::AppConfig::init_with_cli() {
|
||||
Ok(app_config) => app_config,
|
||||
Err(err) => {
|
||||
eprintln!("{}", err);
|
||||
exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
if !app_config.urls().is_empty() {
|
||||
if !app_config.urls.is_empty() {
|
||||
download(app_config);
|
||||
}
|
||||
}
|
||||
|
|
@ -37,10 +45,10 @@ fn main() {
|
|||
fn download(app_config: AppConfig) {
|
||||
let mut errors = Vec::new();
|
||||
let mut partial_download_count: usize = 0;
|
||||
let bar = if app_config.can_disable_progress_bar() {
|
||||
let bar = if app_config.can_disable_progress_bar {
|
||||
ProgressBar::hidden()
|
||||
} else {
|
||||
let enabled_bar = ProgressBar::new(app_config.urls().len() as u64);
|
||||
let enabled_bar = ProgressBar::new(app_config.urls.len() as u64);
|
||||
let style = ProgressStyle::default_bar().template(
|
||||
"{spinner:.cyan} [{elapsed_precise}] {bar:40.white} {:>8} link {pos}/{len:7} {msg:.yellow/white}",
|
||||
);
|
||||
|
|
@ -49,8 +57,8 @@ fn download(app_config: AppConfig) {
|
|||
enabled_bar
|
||||
};
|
||||
let articles = task::block_on(async {
|
||||
let urls_iter = app_config.urls().iter().map(|url| fetch_html(url));
|
||||
let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn());
|
||||
let urls_iter = app_config.urls.iter().map(|url| fetch_html(url));
|
||||
let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn);
|
||||
let mut articles = Vec::new();
|
||||
while let Some(fetch_result) = responses.next().await {
|
||||
match fetch_result {
|
||||
|
|
@ -109,15 +117,15 @@ fn download(app_config: AppConfig) {
|
|||
};
|
||||
let has_errors = !errors.is_empty();
|
||||
display_summary(
|
||||
app_config.urls().len(),
|
||||
app_config.urls.len(),
|
||||
succesful_articles_table,
|
||||
partial_download_count,
|
||||
errors,
|
||||
);
|
||||
if app_config.is_logging_to_file() {
|
||||
if app_config.is_logging_to_file {
|
||||
println!(
|
||||
"Log written to paperoni_{}.log\n",
|
||||
app_config.start_time().format("%Y-%m-%d_%H-%M-%S")
|
||||
app_config.start_time.format("%Y-%m-%d_%H-%M-%S")
|
||||
);
|
||||
}
|
||||
if has_errors {
|
||||
|
|
|
|||
Loading…
Reference in a new issue