kemitix/paperoni
Archived
1
0
Fork 0
Code Issues 4 Pull requests 3 Projects Releases Packages Wiki Activity

Update img tags with new src values to point to the local files

Browse source
This commit is contained in:
Kenneth Gitere 2020-05-02 19:06:03 +03:00
parent 78ba40f57a
commit e5a318282d
2 changed files with 21 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/extractor.rs
View file

@ -5,7 +5,7 @@ use url::Url;
pub struct Extractor {
pub root_node: NodeRef,
content: Option<NodeDataRef<ElementData>>,
pub content: Option<NodeDataRef<ElementData>>,
img_urls: Vec<String>,
}
@ -96,7 +96,6 @@ impl Extractor {
pub async fn download_images(&mut self, article_origin: &Url) -> async_std::io::Result<()> {
self.extract_img_urls();
for img_url in &self.img_urls {
dbg!(&self.img_urls);
let mut img_url = img_url.clone();
get_absolute_url(&mut img_url, article_origin);
@ -108,13 +107,21 @@ impl Extractor {
.header("Content-Type")
.and_then(map_mime_type_to_ext)
.unwrap();
let img_path = format!("{}{}", hash_url(&img_url), &img_ext);
let mut img_file = File::create(format!("{}{}", hash_url(&img_url), &img_ext)).await?;
let mut img_file = File::create(&img_path).await?;
img_file.write_all(&img_content).await?;
println!("Image file downloaded successfully");
// Update img URLs
// self.content.as_ref().map(|content_ref| {});
let img_ref = self
.content
.as_mut()
.expect("Unable to get mutable ref")
.as_node()
.select_first(&format!("img[src='{}']", img_url))
.expect("Image node does not exist");
let mut img_node = img_ref.attributes.borrow_mut();
*img_node.get_mut("src").unwrap() = img_path;
}
Ok(())
}

11
src/main.rs
View file

@ -16,14 +16,21 @@ fn main() {
"https://medium.com/typeforms-engineering-blog/the-beginners-guide-to-oauth-dancing-4b8f3666de10",
"https://dev.to/steelwolf180/full-stack-development-in-django-3768"
];
let html = fetch_url(urls[3]).await;
let html = fetch_url(urls[5]).await;
let mut extractor = Extractor::from_html(&html);
println!("Extracting");
extractor.extract_content();
extractor
.download_images(&Url::parse(urls[3]).unwrap())
.download_images(&Url::parse(urls[5]).unwrap())
.await
.expect("Unable to download images");
let mut out_file = File::create("out.html").unwrap();
extractor
.content
.unwrap()
.as_node()
.serialize(&mut out_file)
.expect("Unable to serialize");
});
}