2020-10-12 19:33:01 +01:00
|
|
|
#[macro_use]
|
|
|
|
extern crate lazy_static;
|
|
|
|
|
2021-02-06 14:03:02 +00:00
|
|
|
use async_std::stream;
|
2020-11-23 06:39:56 +00:00
|
|
|
use async_std::task;
|
2021-02-06 14:03:02 +00:00
|
|
|
use futures::stream::StreamExt;
|
2020-05-02 16:33:45 +01:00
|
|
|
use url::Url;
|
2020-04-30 09:05:53 +01:00
|
|
|
|
2020-05-16 08:09:44 +01:00
|
|
|
mod cli;
|
2021-02-06 09:59:03 +00:00
|
|
|
mod epub;
|
2021-04-17 10:04:06 +01:00
|
|
|
mod errors;
|
2020-05-01 14:17:59 +01:00
|
|
|
mod extractor;
|
2021-02-06 09:59:03 +00:00
|
|
|
/// This module is responsible for async HTTP calls for downloading
|
|
|
|
/// the HTML content and images
|
|
|
|
mod http;
|
2020-08-31 17:30:09 +01:00
|
|
|
mod moz_readability;
|
2020-05-01 14:17:59 +01:00
|
|
|
|
2021-02-06 14:03:02 +00:00
|
|
|
use cli::AppConfig;
|
2021-02-11 10:51:21 +00:00
|
|
|
use epub::generate_epubs;
|
2021-02-06 14:03:02 +00:00
|
|
|
use extractor::Extractor;
|
2021-04-17 10:08:24 +01:00
|
|
|
use http::{download_images, fetch_html};
|
2021-02-06 09:59:03 +00:00
|
|
|
|
2020-04-30 09:05:53 +01:00
|
|
|
fn main() {
|
2021-02-06 09:59:03 +00:00
|
|
|
let app_config = cli::cli_init();
|
2021-02-01 08:28:07 +00:00
|
|
|
|
2021-02-06 09:59:03 +00:00
|
|
|
if !app_config.urls().is_empty() {
|
2021-02-06 14:03:02 +00:00
|
|
|
download(app_config);
|
2020-05-16 08:09:44 +01:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-02-06 14:03:02 +00:00
|
|
|
fn download(app_config: AppConfig) {
|
2021-02-11 10:51:21 +00:00
|
|
|
let articles = task::block_on(async {
|
2021-04-17 10:08:24 +01:00
|
|
|
let urls_iter = app_config.urls().iter().map(|url| fetch_html(url));
|
2021-02-06 14:03:02 +00:00
|
|
|
let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn());
|
2021-02-11 10:51:21 +00:00
|
|
|
let mut articles = Vec::new();
|
2021-02-06 14:03:02 +00:00
|
|
|
while let Some(fetch_result) = responses.next().await {
|
|
|
|
match fetch_result {
|
2021-01-24 14:49:42 +00:00
|
|
|
Ok((url, html)) => {
|
|
|
|
println!("Extracting");
|
|
|
|
let mut extractor = Extractor::from_html(&html);
|
|
|
|
extractor.extract_content(&url);
|
2021-02-06 09:59:03 +00:00
|
|
|
|
2021-01-24 14:49:42 +00:00
|
|
|
if extractor.article().is_some() {
|
2021-02-06 14:03:02 +00:00
|
|
|
extractor.extract_img_urls();
|
2021-04-17 10:04:06 +01:00
|
|
|
|
|
|
|
if let Err(img_errors) =
|
|
|
|
download_images(&mut extractor, &Url::parse(&url).unwrap()).await
|
|
|
|
{
|
|
|
|
eprintln!(
|
|
|
|
"{} image{} failed to download for {}",
|
|
|
|
img_errors.len(),
|
|
|
|
if img_errors.len() > 1 { "s" } else { "" },
|
|
|
|
url
|
|
|
|
);
|
|
|
|
}
|
2021-02-11 10:51:21 +00:00
|
|
|
articles.push(extractor);
|
2021-01-24 14:49:42 +00:00
|
|
|
}
|
2020-10-22 13:22:56 +01:00
|
|
|
}
|
2021-02-21 09:40:17 +00:00
|
|
|
Err(e) => eprintln!("{}", e),
|
2020-10-22 10:12:30 +01:00
|
|
|
}
|
2020-05-05 10:24:11 +01:00
|
|
|
}
|
2021-02-11 10:51:21 +00:00
|
|
|
articles
|
|
|
|
});
|
2021-04-17 10:04:06 +01:00
|
|
|
match generate_epubs(articles, app_config.merged()) {
|
|
|
|
Ok(_) => (),
|
|
|
|
Err(e) => eprintln!("{}", e),
|
|
|
|
};
|
2020-04-30 09:05:53 +01:00
|
|
|
}
|