kemitix/paperoni
1
0
Fork 0
Code Issues 4 Pull requests 3 Projects Releases Packages Wiki Activity

dev: update packages and disable paperteer features

Browse source
This commit is contained in:
Kenneth Gitere 2022-02-01 20:16:29 +03:00
parent e777426c1b
commit abaa7d37df
4 changed files with 380 additions and 371 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

701
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

18
Cargo.toml
View file

@ -13,26 +13,26 @@ readme = "README.md"
[dependencies] [dependencies]
# atty = "0.2.14" # atty = "0.2.14"
async-std = "1.9.0" async-std = "1.10.0"
base64 = "0.13.0" base64 = "0.13.0"
chrono = "0.4.19" chrono = "0.4.19"
clap = { version = "2.33.3", features = ["yaml"] } clap = { version = "2.34.0", features = ["yaml"] }
colored = "2.0.0" colored = "2.0.0"
comfy-table = "3.0.0" comfy-table = "3.0.0"
derive_builder = "0.10.2" derive_builder = "0.10.2"
directories = "3.0.2" directories = "3.0.2"
epub-builder = "0.4.8" epub-builder = "0.4.10"
flexi_logger = "0.18.0" flexi_logger = "0.22.2"
futures = "0.3.15" futures = "0.3.19"
html5ever = "0.25.1" html5ever = "0.25.1"
indicatif = "0.16.2" indicatif = "0.16.2"
itertools = "0.10.1" itertools = "0.10.3"
kuchiki = "0.8.1" kuchiki = "0.8.1"
lazy_static = "1.4.0" lazy_static = "1.4.0"
log = "0.4.14" log = "0.4.14"
md5 = "0.7.0" md5 = "0.7.0"
regex = "1.5.4" regex = "1.5.4"
serde = "1.0.130" serde = "1.0.136"
surf = "2.2.0" surf = "2.3.2"
thiserror = "1.0.25" thiserror = "1.0.30"
url = "2.2.2" url = "2.2.2"

2
rust-toolchain
View file

@ -1 +1 @@
1.52.1 1.57.0

30
src/http.rs
View file

@ -24,7 +24,7 @@ pub fn download(
let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn); let mut responses = stream::from_iter(urls_iter).buffered(app_config.max_conn);
let mut articles = Vec::new(); let mut articles = Vec::new();
// Collect all urls that couldn't extract here // Collect all urls that couldn't extract here
let mut retry_with_paperteer: Vec<String> = Vec::new(); // let mut retry_with_paperteer: Vec<String> = Vec::new();
while let Some(fetch_result) = responses.next().await { while let Some(fetch_result) = responses.next().await {
match fetch_result { match fetch_result {
Ok((url, html)) => { Ok((url, html)) => {
@ -40,7 +40,7 @@ pub fn download(
Ok(_) => bar.inc(1), Ok(_) => bar.inc(1),
// All errors are pushed into here since they're readability issues. // All errors are pushed into here since they're readability issues.
Err(_) => retry_with_paperteer.push(url), Err(e) => errors.push(e),
} }
// Outside the stream, make a new one to retry with paperteer // Outside the stream, make a new one to retry with paperteer
@ -48,18 +48,18 @@ pub fn download(
Err(e) => errors.push(e), Err(e) => errors.push(e),
} }
} }
if !retry_with_paperteer.is_empty() { // if !retry_with_paperteer.is_empty() {
fetch_html_from_paperteer( // fetch_html_from_paperteer(
retry_with_paperteer, // retry_with_paperteer,
app_config, // app_config,
bar, // bar,
partial_downloads, // partial_downloads,
errors, // errors,
&mut articles, // &mut articles,
) // )
.await // .await
.unwrap(); // .unwrap();
} // }
articles articles
}) })
} }
@ -130,7 +130,7 @@ struct PaperteerResponse {
// TODO: Change signature to simply take a vec of urls and return a vec of urls with either html or an error // TODO: Change signature to simply take a vec of urls and return a vec of urls with either html or an error
// This also means that extracting and downloading imgs should be handled externally // This also means that extracting and downloading imgs should be handled externally
async fn fetch_html_from_paperteer( async fn _fetch_html_from_paperteer(
urls: Vec<String>, urls: Vec<String>,
_app_config: &AppConfig, _app_config: &AppConfig,
bar: &ProgressBar, bar: &ProgressBar,