Initial extraction code to get meta information on a blog

.gitignore

@@ -0,0 +1 @@
+/target

Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "paperoni"
+version = "0.1.0"
+authors = ["Kenneth Gitere <gitere81@gmail.com>"]
+edition = "2018"
+license = "MIT"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-std = "1.5.0"
+kuchiki = "0.8.0"
+surf = "1.0.3"

src/main.rs

@@ -0,0 +1,112 @@
+use async_std::task;
+use kuchiki::traits::*;
+
+fn main() {
+    task::block_on(async {
+        let urls = vec![
+            "https://saveandrun.com/posts/2020-01-24-generating-mazes-with-haskell-part-1.html",
+            "https://saveandrun.com/posts/2020-04-05-querying-pacman-with-datalog.html",
+            "https://saveandrun.com/posts/2020-01-08-working-with-git.html",
+            "https://blog.hipstermojo.xyz/posts/redis-orm-preface/",
+            "https://vuejsdevelopers.com/2020/03/31/vue-js-form-composition-api/?utm_campaign=xl5&utm_medium=article&utm_source=vuejsnews#adding-validators",
+            "https://medium.com/typeforms-engineering-blog/the-beginners-guide-to-oauth-dancing-4b8f3666de10"
+        ];
+        let html = fetch_url(urls[0]).await;
+        extract_content(html);
+    });
+}
+
+async fn fetch_url(url: &str) -> String {
+    let client = surf::Client::new();
+    println!("Fetching...");
+    client
+        .get(url)
+        .recv_string()
+        .await
+        .expect("Unable to fetch URL")
+}
+
+fn extract_content(html_str: String) {
+    let document = kuchiki::parse_html().one(html_str);
+    let author: Option<String> =
+        document
+            .select_first("meta[name='author']")
+            .ok()
+            .and_then(|data| {
+                data.attributes
+                    .borrow()
+                    .get("content")
+                    .map(|name| name.to_string())
+            });
+    let description = document
+        .select_first("meta[name='description']")
+        .ok()
+        .and_then(|data| {
+            data.attributes
+                .borrow()
+                .get("content")
+                .map(|description| description.to_string())
+        });
+    let tags = document
+        .select_first("meta[name='keywords']")
+        .ok()
+        .and_then(|data| {
+            data.attributes.borrow().get("content").map(|tags| {
+                tags.split(",")
+                    .map(|tag_str| tag_str.trim().to_string())
+                    .collect::<Vec<String>>()
+            })
+        });
+    let title = if let Some(title_node) = document.select_first("title").ok() {
+        title_node
+            .as_node()
+            .first_child()
+            .and_then(|text_node| {
+                text_node
+                    .as_text()
+                    .map(|text_ref| text_ref.borrow().to_string())
+            })
+            .unwrap_or("".to_string())
+    } else {
+        "".to_string()
+    };
+    let lang = document
+        .select_first("html")
+        .ok()
+        .and_then(|data| {
+            data.attributes
+                .borrow()
+                .get("lang")
+                .map(|val| val.to_string())
+        })
+        .unwrap_or("en".to_string());
+    let meta_attrs = MetaAttr::new(author, description, lang, tags, title);
+    dbg!(meta_attrs);
+}
+
+#[derive(Debug)]
+struct MetaAttr {
+    author: Option<String>,
+    description: Option<String>,
+    language: String,
+    tags: Option<Vec<String>>,
+    title: String,
+}
+
+impl MetaAttr {
+    pub fn new(
+        author: Option<String>,
+        description: Option<String>,
+        language: String,
+        tags: Option<Vec<String>>,
+        title: String,
+    ) -> Self {
+        MetaAttr {
+            author,
+            description,
+            language,
+            tags,
+            title,
+        }
+    }
+}