kemitix/paperoni
1
0
Fork 0
Code Issues 4 Pull requests 3 Projects Releases Packages Wiki Activity

Refactor to use temp directory and update surf

Browse source 
Change from using res directory for image downloads to using temp directories.
Update surf to v2 which required changing the way Content-Type headers are
read from.
This commit is contained in:
Kenneth Gitere 2020-11-23 09:39:56 +03:00
parent ab800d0174
commit ef3efdba81
4 changed files with 1186 additions and 838 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1933
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

2
Cargo.toml
View file

@ -19,6 +19,6 @@ kuchiki = "0.8.1"
lazy_static = "1.3.9"
md5 = "0.7.0"
regex = "1.3.9"
surf = "1.0.3"
surf = "2.1.0"
structopt = { version = "0.3" }
url = "2.1.1"

63
src/extractor.rs
View file

@ -62,22 +62,23 @@ impl Extractor {
pub async fn download_images(&mut self, article_origin: &Url) -> async_std::io::Result<()> {
let mut async_download_tasks = Vec::with_capacity(self.img_urls.len());
self.extract_img_urls();
println!("Downloading images to res/");
println!("Downloading images...");
for img_url in &self.img_urls {
let img_url = img_url.0.clone();
let abs_url = get_absolute_url(&img_url, article_origin);
async_download_tasks.push(task::spawn(async move {
let mut img_response = surf::get(&abs_url).await.expect("Unable to retrieve file");
let img_content: Vec<u8> = img_response.body_bytes().await.unwrap();
let img_mime = img_response
.header("Content-Type")
.map(|header| header.to_string());
.content_type()
.map(|mime| mime.essence().to_string());
let img_ext = img_response
.header("Content-Type")
.and_then(map_mime_type_to_ext)
.content_type()
.map(|mime| map_mime_subtype_to_ext(mime.subtype()).to_string())
.unwrap();
let img_path = format!("res/{}{}", hash_url(&abs_url), &img_ext);
let mut img_path = std::env::temp_dir();
img_path.push(format!("{}.{}", hash_url(&abs_url), &img_ext));
let mut img_file = File::create(&img_path)
.await
.expect("Unable to create file");
@ -86,7 +87,19 @@ impl Extractor {
.await
.expect("Unable to save to file");
(img_url, img_path, img_mime)
(
img_url,
img_path
.file_name()
.map(|os_str_name| {
os_str_name
.to_str()
.expect("Unable to get image file name")
.to_string()
})
.unwrap(),
img_mime,
)
}));
}
@ -123,21 +136,15 @@ fn hash_url(url: &str) -> String {
format!("{:x}", md5::compute(url.as_bytes()))
}
/// Handles getting the extension from a given MIME type. The extension starts with a dot
fn map_mime_type_to_ext(mime_type: &str) -> Option<String> {
mime_type
.split("/")
.last()
.map(|format| {
if format == ("svg+xml") {
/// Handles getting the extension from a given MIME subtype.
fn map_mime_subtype_to_ext(subtype: &str) -> &str {
if subtype == ("svg+xml") {
return "svg";
} else if format == "x-icon" {
} else if subtype == "x-icon" {
"ico"
} else {
format
subtype
}
})
.map(|format| String::from(".") + format)
}
fn get_absolute_url(url: &str, request_url: &Url) -> String {
@ -204,23 +211,15 @@ mod test {
#[test]
fn test_map_mime_type_to_ext() {
let mime_types = vec![
"image/apng",
"image/bmp",
"image/gif",
"image/x-icon",
"image/jpeg",
"image/png",
"image/svg+xml",
"image/tiff",
"image/webp",
let mime_subtypes = vec![
"apng", "bmp", "gif", "x-icon", "jpeg", "png", "svg+xml", "tiff", "webp",
];
let exts = mime_types
let exts = mime_subtypes
.into_iter()
.map(|mime_type| map_mime_type_to_ext(mime_type).unwrap())
.map(|mime_type| map_mime_subtype_to_ext(mime_type))
.collect::<Vec<_>>();
assert_eq!(
vec![".apng", ".bmp", ".gif", ".ico", ".jpeg", ".png", ".svg", ".tiff", ".webp"],
vec!["apng", "bmp", "gif", "ico", "jpeg", "png", "svg", "tiff", "webp"],
exts
);
}

18
src/main.rs
View file

@ -2,9 +2,8 @@
extern crate lazy_static;
use std::fs::File;
use std::path::Path;
use async_std::{fs::create_dir, fs::remove_dir_all, task};
use async_std::task;
use epub_builder::{EpubBuilder, EpubContent, ZipLibrary};
use structopt::StructOpt;
use url::Url;
@ -50,11 +49,6 @@ fn download(urls: Vec<String>) {
let mut extractor = Extractor::from_html(&html);
extractor.extract_content(&url);
if extractor.article().is_some() {
if !Path::new("res/").exists() {
create_dir("res/")
.await
.expect("Unable to create res/ output folder");
}
extractor
.download_images(&Url::parse(&url).unwrap())
.await
@ -79,14 +73,14 @@ fn download(urls: Vec<String>) {
epub.add_content(EpubContent::new("code.xhtml", html_buf.as_bytes()))
.unwrap();
for img in extractor.img_urls {
let file_path = format!("{}", &img.0);
let mut file_path = std::env::temp_dir();
file_path.push(&img.0);
let img_buf = File::open(file_path).expect("Can't read file");
epub.add_resource(img.0, img_buf, img.1.unwrap()).unwrap();
let img_buf = File::open(&file_path).expect("Can't read file");
epub.add_resource(file_path.file_name().unwrap(), img_buf, img.1.unwrap())
.unwrap();
}
epub.generate(&mut out_file).unwrap();
println!("Cleaning up");
remove_dir_all("res/").await.unwrap();
println!("Created {:?}", file_name);
}
}