kemitix/paperoni
Archived
1
0
Fork 0
Code Issues 4 Pull requests 3 Projects Releases Packages Wiki Activity

Add ImgError struct for representing errors in downloading article images

Browse source
This commit is contained in:
Kenneth Gitere 2021-04-24 13:57:06 +03:00
parent 910c45abf7
commit a3de3fb6ff
3 changed files with 110 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
src/errors.rs
View file

@ -14,6 +14,53 @@ pub enum ErrorKind {
ReadabilityError(String),
}
#[derive(Error, Debug)]
#[error("{kind}")]
/// Used to represent errors from downloading images. Errors from here are used solely for debugging
/// as they are considered recoverable.
pub struct ImgError {
kind: ErrorKind,
url: Option<String>,
}
impl ImgError {
pub fn with_kind(kind: ErrorKind) -> Self {
ImgError { url: None, kind }
}
pub fn set_url(&mut self, url: &str) {
self.url = Some(url.to_string());
}
pub fn url(&self) -> &Option<String> {
&self.url
}
}
impl From<ErrorKind> for ImgError {
fn from(kind: ErrorKind) -> Self {
ImgError::with_kind(kind)
}
}
impl From<surf::Error> for ImgError {
fn from(err: surf::Error) -> Self {
ImgError::with_kind(ErrorKind::HTTPError(err.to_string()))
}
}
impl From<url::ParseError> for ImgError {
fn from(err: url::ParseError) -> Self {
ImgError::with_kind(ErrorKind::HTTPError(err.to_string()))
}
}
impl From<std::io::Error> for ImgError {
fn from(err: std::io::Error) -> Self {
ImgError::with_kind(ErrorKind::IOError(err.to_string()))
}
}
#[derive(Error, Debug)]
#[error("{kind}")]
pub struct PaperoniError {

103
src/http.rs
View file

@ -5,8 +5,8 @@ use indicatif::ProgressBar;
use log::{debug, info};
use url::Url;
use crate::{errors::ErrorKind, errors::PaperoniError, extractor::Extractor};
use crate::errors::{ErrorKind, ImgError, PaperoniError};
use crate::extractor::Extractor;
type HTMLResource = (String, String);
pub async fn fetch_html(url: &str) -> Result<HTMLResource, PaperoniError> {
@ -76,7 +76,7 @@ pub async fn download_images(
extractor: &mut Extractor,
article_origin: &Url,
bar: &ProgressBar,
) -> Result<(), Vec<PaperoniError>> {
) -> Result<(), Vec<ImgError>> {
if extractor.img_urls.len() > 0 {
debug!(
"Downloading {} images for {}",
@ -102,53 +102,62 @@ pub async fn download_images(
bar.set_message(format!("Downloading images [{}/{}]", img_idx + 1, img_count).as_str());
match req.await {
Ok(mut img_response) => {
// let mut img_response = req.await.expect("Unable to retrieve image");
let img_content: Vec<u8> = match img_response.body_bytes().await {
Ok(bytes) => bytes,
Err(e) => return Err(e.into()),
};
let img_mime = img_response
.content_type()
.map(|mime| mime.essence().to_string());
let img_ext = match img_response
.content_type()
.map(|mime| map_mime_subtype_to_ext(mime.subtype()).to_string())
{
Some(mime_str) => mime_str,
None => {
return Err(ErrorKind::HTTPError(
"Image has no Content-Type".to_owned(),
)
.into())
let process_response = async {
let img_content: Vec<u8> = match img_response.body_bytes().await {
Ok(bytes) => bytes,
Err(e) => return Err(e.into()),
};
let img_mime = img_response
.content_type()
.map(|mime| mime.essence().to_string());
let img_ext = match img_response
.content_type()
.map(|mime| map_mime_subtype_to_ext(mime.subtype()).to_string())
{
Some(mime_str) => mime_str,
None => {
return Err(ErrorKind::HTTPError(
"Image has no Content-Type".to_owned(),
)
.into())
}
};
let mut img_path = std::env::temp_dir();
img_path.push(format!("{}.{}", hash_url(&url), &img_ext));
let mut img_file = match File::create(&img_path).await {
Ok(file) => file,
Err(e) => return Err(e.into()),
};
match img_file.write_all(&img_content).await {
Ok(_) => (),
Err(e) => return Err(e.into()),
}
};
let mut img_path = std::env::temp_dir();
img_path.push(format!("{}.{}", hash_url(&url), &img_ext));
let mut img_file = match File::create(&img_path).await {
Ok(file) => file,
Err(e) => return Err(e.into()),
Ok((
url,
img_path
.file_name()
.map(|os_str_name| {
os_str_name
.to_str()
.expect("Unable to get image file name")
.to_string()
})
.unwrap(),
img_mime,
))
};
match img_file.write_all(&img_content).await {
Ok(_) => (),
Err(e) => return Err(e.into()),
}
Ok((
url,
img_path
.file_name()
.map(|os_str_name| {
os_str_name
.to_str()
.expect("Unable to get image file name")
.to_string()
})
.unwrap(),
img_mime,
))
process_response.await.map_err(|mut e: ImgError| {
e.set_url(url);
e
})
}
Err(e) => {
let mut img_err: ImgError = e.into();
img_err.set_url(url);
Err(img_err)
}
Err(e) => Err(e.into()),
}
});
@ -170,7 +179,7 @@ pub async fn download_images(
let imgs_req_iter = stream::from_iter(imgs_req_iter)
.buffered(10)
.collect::<Vec<Result<_, PaperoniError>>>()
.collect::<Vec<Result<_, ImgError>>>()
.await;
let mut errors = Vec::new();
let mut replaced_imgs = Vec::new();

7
src/main.rs
View file

@ -70,6 +70,13 @@ fn download(app_config: AppConfig) {
if img_errors.len() > 1 { "s" } else { "" },
url
);
for img_error in img_errors {
warn!(
"{}\n\t\tReason {}",
img_error.url().as_ref().unwrap(),
img_error
);
}
}
articles.push(extractor);
}