2021-06-10 18:16:31 +01:00
|
|
|
use itertools::Itertools;
|
2020-10-22 11:55:02 +01:00
|
|
|
use kuchiki::{traits::*, NodeRef};
|
2020-05-01 14:17:59 +01:00
|
|
|
|
2021-04-17 10:04:06 +01:00
|
|
|
use crate::errors::PaperoniError;
|
2020-10-22 11:55:02 +01:00
|
|
|
use crate::moz_readability::{MetaData, Readability};
|
2020-10-22 10:12:30 +01:00
|
|
|
|
2021-07-28 07:10:22 +01:00
|
|
|
/// A tuple of the url and an Option of the resource's MIME type
|
2020-05-05 10:24:11 +01:00
|
|
|
pub type ResourceInfo = (String, Option<String>);
|
|
|
|
|
|
2021-07-24 10:43:40 +01:00
|
|
|
pub struct Article {
|
|
|
|
|
node_ref_opt: Option<NodeRef>,
|
2020-05-05 10:24:11 +01:00
|
|
|
pub img_urls: Vec<ResourceInfo>,
|
2020-10-22 10:12:30 +01:00
|
|
|
readability: Readability,
|
2021-04-20 19:06:54 +01:00
|
|
|
pub url: String,
|
2020-05-01 14:17:59 +01:00
|
|
|
}
|
|
|
|
|
|
2021-07-24 10:43:40 +01:00
|
|
|
impl Article {
|
2020-05-01 14:17:59 +01:00
|
|
|
/// Create a new instance of an HTML extractor given an HTML string
|
2021-04-20 19:06:54 +01:00
|
|
|
pub fn from_html(html_str: &str, url: &str) -> Self {
|
2021-07-24 10:43:40 +01:00
|
|
|
Self {
|
|
|
|
|
node_ref_opt: None,
|
2020-05-02 16:33:45 +01:00
|
|
|
img_urls: Vec::new(),
|
2020-10-22 10:12:30 +01:00
|
|
|
readability: Readability::new(html_str),
|
2021-04-20 19:06:54 +01:00
|
|
|
url: url.to_string(),
|
2020-05-01 14:17:59 +01:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-02 12:51:53 +01:00
|
|
|
/// Locates and extracts the HTML in a document which is determined to be
|
|
|
|
|
/// the source of the content
|
2021-04-21 17:07:08 +01:00
|
|
|
pub fn extract_content(&mut self) -> Result<(), PaperoniError> {
|
|
|
|
|
self.readability.parse(&self.url)?;
|
2020-10-22 11:55:02 +01:00
|
|
|
if let Some(article_node_ref) = &self.readability.article_node {
|
|
|
|
|
let template = r#"
|
2021-07-24 10:03:36 +01:00
|
|
|
<!DOCTYPE html>
|
|
|
|
|
<html>
|
2020-10-22 11:55:02 +01:00
|
|
|
<head>
|
2021-06-09 06:04:50 +01:00
|
|
|
<link rel="stylesheet" href="stylesheet.css" type="text/css"></link>
|
2020-10-22 11:55:02 +01:00
|
|
|
</head>
|
|
|
|
|
<body>
|
|
|
|
|
</body>
|
|
|
|
|
</html>
|
|
|
|
|
"#;
|
|
|
|
|
let doc = kuchiki::parse_html().one(template);
|
|
|
|
|
let body = doc.select_first("body").unwrap();
|
|
|
|
|
body.as_node().append(article_node_ref.clone());
|
2021-07-24 10:43:40 +01:00
|
|
|
self.node_ref_opt = Some(doc);
|
2020-10-22 11:55:02 +01:00
|
|
|
}
|
2021-04-21 17:07:08 +01:00
|
|
|
Ok(())
|
2020-05-02 12:51:53 +01:00
|
|
|
}
|
2020-05-01 14:17:59 +01:00
|
|
|
|
2020-05-02 16:33:45 +01:00
|
|
|
/// Traverses the DOM tree of the content and retrieves the IMG URLs
|
2021-02-06 09:59:03 +00:00
|
|
|
pub fn extract_img_urls(&mut self) {
|
2021-07-24 10:43:40 +01:00
|
|
|
if let Some(content_ref) = &self.node_ref_opt {
|
2021-06-10 18:16:31 +01:00
|
|
|
self.img_urls = content_ref
|
|
|
|
|
.select("img")
|
|
|
|
|
.unwrap()
|
|
|
|
|
.filter_map(|img_ref| {
|
|
|
|
|
let attrs = img_ref.attributes.borrow();
|
|
|
|
|
attrs
|
|
|
|
|
.get("src")
|
|
|
|
|
.filter(|val| !(val.is_empty() || val.starts_with("data:image")))
|
|
|
|
|
.map(ToString::to_string)
|
|
|
|
|
})
|
|
|
|
|
.unique()
|
|
|
|
|
.map(|val| (val, None))
|
|
|
|
|
.collect();
|
2020-05-01 14:17:59 +01:00
|
|
|
}
|
|
|
|
|
}
|
2020-05-02 16:33:45 +01:00
|
|
|
|
2021-04-21 17:07:08 +01:00
|
|
|
/// Returns the extracted article [NodeRef]. It should only be called *AFTER* calling parse
|
2021-07-24 10:43:40 +01:00
|
|
|
pub fn node_ref(&self) -> &NodeRef {
|
|
|
|
|
self.node_ref_opt.as_ref().expect(
|
2021-04-21 17:07:08 +01:00
|
|
|
"Article node doesn't exist. This may be because the document has not been parsed",
|
|
|
|
|
)
|
2020-10-22 11:55:02 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pub fn metadata(&self) -> &MetaData {
|
|
|
|
|
&self.readability.metadata
|
2020-10-22 10:12:30 +01:00
|
|
|
}
|
2020-05-01 14:17:59 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
mod test {
|
|
|
|
|
use super::*;
|
|
|
|
|
const TEST_HTML: &'static str = r#"
|
|
|
|
|
<!doctype html>
|
|
|
|
|
<html lang="en">
|
|
|
|
|
<head>
|
|
|
|
|
<meta charset="utf-8">
|
|
|
|
|
<meta name="description" content="A sample document">
|
|
|
|
|
<meta name="keywords" content="test,Rust">
|
|
|
|
|
<meta name="author" content="Paperoni">
|
|
|
|
|
<title>Testing Paperoni</title>
|
|
|
|
|
</head>
|
|
|
|
|
<body>
|
|
|
|
|
<header>
|
|
|
|
|
<!-- Unimportant information -->
|
|
|
|
|
<h1>Testing Paperoni</h1>
|
|
|
|
|
</header>
|
|
|
|
|
<article>
|
|
|
|
|
<h1>Starting out</h1>
|
|
|
|
|
<p>Some Lorem Ipsum text here</p>
|
|
|
|
|
<p>Observe this picture</p>
|
2020-05-05 10:29:08 +01:00
|
|
|
<img src="./img.jpg" alt="Random image">
|
2020-12-24 09:16:30 +00:00
|
|
|
<img src="data:image/png;base64,lJGWEIUQOIQWIDYVIVEDYFOUYQFWD">
|
2020-05-01 14:17:59 +01:00
|
|
|
</article>
|
|
|
|
|
<footer>
|
|
|
|
|
<p>Made in HTML</p>
|
|
|
|
|
</footer>
|
|
|
|
|
</body>
|
|
|
|
|
</html>
|
|
|
|
|
"#;
|
|
|
|
|
|
2020-05-02 16:33:45 +01:00
|
|
|
#[test]
|
|
|
|
|
fn test_extract_img_urls() {
|
2021-07-24 10:43:40 +01:00
|
|
|
let mut article = Article::from_html(TEST_HTML, "http://example.com/");
|
|
|
|
|
article
|
2021-04-21 17:07:08 +01:00
|
|
|
.extract_content()
|
|
|
|
|
.expect("Article extraction failed unexpectedly");
|
2021-07-24 10:43:40 +01:00
|
|
|
article.extract_img_urls();
|
2020-05-02 16:33:45 +01:00
|
|
|
|
2021-07-24 10:43:40 +01:00
|
|
|
assert!(article.img_urls.len() > 0);
|
2020-10-22 10:12:30 +01:00
|
|
|
assert_eq!(
|
|
|
|
|
vec![("http://example.com/img.jpg".to_string(), None)],
|
2021-07-24 10:43:40 +01:00
|
|
|
article.img_urls
|
2020-10-22 10:12:30 +01:00
|
|
|
);
|
2020-05-02 16:33:45 +01:00
|
|
|
}
|
2020-05-01 14:17:59 +01:00
|
|
|
}
|
