Add flag for configuring maximum concurrent requests

Change printing macro for error messages to go out to stderr
This commit is contained in:
Kenneth Gitere 2021-02-21 12:40:17 +03:00
parent b0c4c47413
commit 912bc9d915
2 changed files with 17 additions and 5 deletions

View file

@ -33,7 +33,12 @@ It takes a url and downloads the article content from it and saves it to an epub
.help("Merge multiple articles into a single epub") .help("Merge multiple articles into a single epub")
.long_help("Merge multiple articles into a single epub that will be given the name provided") .long_help("Merge multiple articles into a single epub that will be given the name provided")
.takes_value(true), .takes_value(true),
); ).arg(
Arg::with_name("max_conn")
.long("max_conn")
.help("The maximum number of concurrent HTTP connections when downloading articles. Default is 8")
.long_help("The maximum number of concurrent HTTP connections when downloading articles. Default is 8.\nNOTE: It is advised to use as few connections as needed i.e between 1 and 50. Using more connections can end up overloading your network card with too many concurrent requests.")
.takes_value(true));
let arg_matches = app.get_matches(); let arg_matches = app.get_matches();
let mut urls: Vec<String> = match arg_matches.value_of("file") { let mut urls: Vec<String> = match arg_matches.value_of("file") {
Some(file_name) => { Some(file_name) => {
@ -62,7 +67,14 @@ It takes a url and downloads the article content from it and saves it to an epub
); );
} }
let mut app_config = AppConfig::new(); let max_conn = arg_matches
.value_of("max_conn")
.map(|conn_str| conn_str.parse::<usize>().ok())
.flatten()
.map(|max| if max > 0 { max } else { 1 })
.unwrap_or(8);
let mut app_config = AppConfig::new(max_conn);
app_config.set_urls(urls); app_config.set_urls(urls);
if let Some(name) = arg_matches.value_of("output_name") { if let Some(name) = arg_matches.value_of("output_name") {
let file_name = if name.ends_with(".epub") && name.len() > 5 { let file_name = if name.ends_with(".epub") && name.len() > 5 {
@ -82,10 +94,10 @@ pub struct AppConfig {
} }
impl AppConfig { impl AppConfig {
fn new() -> Self { fn new(max_conn: usize) -> Self {
Self { Self {
urls: vec![], urls: vec![],
max_conn: 8, max_conn,
merged: None, merged: None,
} }
} }

View file

@ -47,7 +47,7 @@ fn download(app_config: AppConfig) {
articles.push(extractor); articles.push(extractor);
} }
} }
Err(e) => println!("{}", e), Err(e) => eprintln!("{}", e),
} }
} }
articles articles