.gitignore

@@ -1,4 +1,5 @@
 target
 *.mp3
 *.webm
-*.txt
+/subscriptions.txt
+/downloaded.txt

.woodpecker.yml

@@ -0,0 +1,9 @@
+pipeline:
+  build:
+    image: rust
+    commands:
+      - rustup component add rustfmt clippy
+      - cargo --version
+      - cargo fmt --check
+      - cargo clippy --fix -- -Dwarnings -W clippy::nursery -W clippy::unwrap_used -W clippy::expect_used
+      - cargo test

Cargo.lock

@@ -35,6 +35,55 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
 
+[[package]]
+name = "anstream"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is-terminal",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
+
+[[package]]
+name = "anstyle-parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
+dependencies = [
+ "anstyle",
+ "windows-sys",
+]
+
 [[package]]
 name = "atom_syndication"
 version = "0.12.1"
@@ -127,6 +176,39 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "clap"
+version = "4.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d"
+dependencies = [
+ "clap_builder",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
+
 [[package]]
 name = "core-foundation"
 version = "0.9.3"
@@ -580,6 +662,17 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
 
+[[package]]
+name = "is-terminal"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
+dependencies = [
+ "hermit-abi",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.9"
@@ -921,8 +1014,10 @@ name = "podal"
 version = "0.1.0"
 dependencies = [
  "atom_syndication",
+ "clap",
  "reqwest",
  "scraper",
+ "tempfile",
 ]
 
 [[package]]
@@ -1425,6 +1520,12 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
 
+[[package]]
+name = "utf8parse"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"

Cargo.toml

@@ -9,3 +9,7 @@ edition = "2021"
 atom_syndication = "0.12.1"
 reqwest = { version = "0.11.18", features = ["json", "blocking"] }
 scraper = "0.17.1"
+clap = "4.3.19"
+
+[dev-dependencies]
+tempfile = "*"

src/errors.rs

@@ -0,0 +1,51 @@
+use std::{fmt::Display, str::Utf8Error, string::FromUtf8Error};
+
+#[derive(Debug)]
+pub struct Error {
+    details: String,
+}
+impl Display for Error {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.details.to_string().as_str())
+    }
+}
+impl From<Utf8Error> for Error {
+    fn from(value: Utf8Error) -> Self {
+        Self {
+            details: value.to_string(),
+        }
+    }
+}
+impl From<FromUtf8Error> for Error {
+    fn from(value: FromUtf8Error) -> Self {
+        Self {
+            details: value.to_string(),
+        }
+    }
+}
+impl From<String> for Error {
+    fn from(details: String) -> Self {
+        Self { details }
+    }
+}
+impl From<std::io::Error> for Error {
+    fn from(value: std::io::Error) -> Self {
+        Self {
+            details: value.to_string(),
+        }
+    }
+}
+impl From<atom_syndication::Error> for Error {
+    fn from(value: atom_syndication::Error) -> Self {
+        Self {
+            details: value.to_string(),
+        }
+    }
+}
+impl From<reqwest::Error> for Error {
+    fn from(value: reqwest::Error) -> Self {
+        Self {
+            details: value.to_string(),
+        }
+    }
+}

src/feed/find.rs

@@ -0,0 +1,20 @@
+use crate::prelude::*;
+
+pub fn find(site: &str, channel_name: &str) -> Result<String> {
+    if let Some(channel_prefix) = channel_name.chars().next() {
+        if channel_prefix != '@' {
+            return Err(format!("Channel Name must begin with an '@': {}", channel_name).into());
+        }
+    }
+    let channel_url = format!("{}{}", site, channel_name);
+    let response = reqwest::blocking::get(channel_url)?;
+    let rss_url = scraper::Html::parse_document(&response.text()?)
+        .select(&scraper::Selector::parse("link[title='RSS']").unwrap())
+        .next()
+        .unwrap()
+        .value()
+        .attr("href")
+        .unwrap()
+        .to_string();
+    Ok(rss_url)
+}

src/feed/get.rs

@@ -0,0 +1,9 @@
+use atom_syndication::Feed;
+
+use crate::prelude::*;
+
+pub fn get(url: &str) -> Result<Feed> {
+    let content = reqwest::blocking::get(url)?.bytes()?;
+    let channel = Feed::read_from(&content[..])?;
+    Ok(channel)
+}

src/feed/mod.rs

@@ -0,0 +1,12 @@
+use crate::prelude::*;
+
+mod find;
+mod get;
+
+pub use find::find;
+pub use get::get;
+
+type Feed = atom_syndication::Feed;
+
+pub type FeedFind = fn(&str, &str) -> Result<String>;
+pub type FeedGet = fn(&str) -> Result<Feed>;

src/fetch.rs

@@ -0,0 +1,22 @@
+use crate::prelude::*;
+
+use atom_syndication::Link;
+use std::process::Command;
+
+pub type FetchDownload = fn(&Link) -> Result<()>;
+
+pub fn download(link: &Link) -> Result<()> {
+    let cmd = "yt-dlp";
+    // println!("{} --extract-audio --audio-format mp3 {}", cmd, &link.href);
+    let output = Command::new(cmd)
+        .arg("--extract-audio")
+        .arg("--audio-format")
+        .arg("mp3")
+        .arg(&link.href)
+        .output()?;
+    if !output.stderr.is_empty() {
+        eprintln!("Error: {}", String::from_utf8(output.stderr)?);
+        println!("{}", String::from_utf8(output.stdout)?);
+    }
+    Ok(())
+}

src/history/add.rs

@@ -0,0 +1,17 @@
+use crate::prelude::*;
+
+use atom_syndication::Link;
+use std::fs::OpenOptions;
+use std::io::prelude::*;
+
+pub fn add(link: &Link, file_name: &str) -> Result<()> {
+    let mut file = OpenOptions::new()
+        .write(true)
+        .append(true)
+        .create(true)
+        .open(file_name)
+        .unwrap();
+
+    writeln!(file, "{}", link.href)?;
+    Ok(())
+}

src/history/find.rs

@@ -0,0 +1,17 @@
+use crate::prelude::*;
+
+use atom_syndication::Link;
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+
+pub fn find(link: &Link, file_name: &str) -> Result<bool> {
+    if let Ok(file) = File::open(file_name) {
+        let reader = BufReader::new(file);
+        for line in reader.lines() {
+            if line? == link.href {
+                return Ok(true); // is already downloaded
+            }
+        }
+    }
+    Ok(false) // is not already downloaded
+}

src/history/mod.rs

@@ -0,0 +1,12 @@
+use crate::prelude::*;
+
+mod add;
+mod find;
+
+pub use add::add;
+pub use find::find;
+
+type Link = atom_syndication::Link;
+
+pub type HistoryFind = fn(&Link, &str) -> Result<bool>;
+pub type HistoryAdd = fn(&Link, &str) -> Result<()>;

src/lib.rs

@@ -0,0 +1,37 @@
+mod errors;
+pub mod feed;
+pub mod fetch;
+pub mod history;
+pub mod prelude;
+mod subscriptions;
+
+use feed::{FeedFind, FeedGet};
+use fetch::FetchDownload;
+use history::{HistoryAdd, HistoryFind};
+use prelude::*;
+
+pub fn run(
+    subscriptions: &str,
+    history: &str,
+    site: &str,
+    feed_find: FeedFind,
+    feed_get: FeedGet,
+    history_find: HistoryFind,
+    history_add: HistoryAdd,
+    fetch_download: FetchDownload,
+) -> Result<()> {
+    for channel_name in subscriptions::lines_from(subscriptions)? {
+        println!("Channel: {}", channel_name);
+        let feed_url = feed_find(site, &channel_name)?;
+        for entry in feed_get(&feed_url)?.entries() {
+            if let Some(link) = entry.links().get(0).cloned() {
+                if !history_find(&link, history)? {
+                    println!("Downloading {}: {}", &channel_name, entry.title().as_str());
+                    fetch_download(&link)?;
+                    history_add(&link, history)?;
+                }
+            }
+        }
+    }
+    Ok(())
+}

src/main.rs

@@ -1,170 +1,22 @@
-// https://www.phind.com/agent?cache=clke9xk39001cmj085upzho1t
+use podal::prelude::*;
 
-use std::{fmt::Display, fs::File, string::FromUtf8Error};
-
-use atom_syndication::{Entry, Feed, Link};
-
-//
-// ERRORS
-//
-#[derive(Debug)]
-struct Error {
-    details: String,
-}
-impl Display for Error {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str(self.details.to_string().as_str())
-    }
-}
-impl From<FromUtf8Error> for Error {
-    fn from(value: FromUtf8Error) -> Self {
-        Self {
-            details: value.to_string(),
-        }
-    }
-}
-impl From<String> for Error {
-    fn from(details: String) -> Self {
-        Self { details }
-    }
-}
-impl From<std::io::Error> for Error {
-    fn from(value: std::io::Error) -> Self {
-        Self {
-            details: value.to_string(),
-        }
-    }
-}
-impl From<atom_syndication::Error> for Error {
-    fn from(value: atom_syndication::Error) -> Self {
-        Self {
-            details: value.to_string(),
-        }
-    }
-}
-impl From<reqwest::Error> for Error {
-    fn from(value: reqwest::Error) -> Self {
-        Self {
-            details: value.to_string(),
-        }
-    }
-}
-
-//
-// RESULTS
-//
-type Result<T> = std::result::Result<T, Error>;
-
-//
-// MAIN
-//
 fn main() -> Result<()> {
     println!("Podal");
     let subscriptions = "subscriptions.txt";
     let history = "downloaded.txt";
     let site = "https://www.youtube.com/";
 
-    for channel_name in lines_from(subscriptions)? {
-        let channel_name = channel_name?;
-        println!("Channel: {}", channel_name);
-        let feed_url = get_feed_url(site, &channel_name)?;
-        for entry in get_feed(feed_url)?.entries() {
-            if let Some(link) = get_link(entry) {
-                if !is_already_downloaded(&link, history)? {
-                    println!("Downloading {}: {}", &channel_name, entry.title().as_str());
-                    download_audio(&link)?;
-                    mark_as_downloaded(&link, history)?;
-                }
-            }
-        }
-    }
+    podal::run(
+        subscriptions,
+        history,
+        site,
+        podal::feed::find,
+        podal::feed::get,
+        podal::history::find,
+        podal::history::add,
+        podal::fetch::download,
+    )?;
 
     println!("Done");
     Ok(())
 }
-
-fn get_feed_url(site: &str, channel_name: &str) -> Result<String> {
-    if let Some(channel_prefix) = channel_name.chars().next() {
-        if channel_prefix != '@' {
-            return Err(format!("Channel Name must begin with an '@': {}", channel_name).into());
-        }
-    }
-    let channel_url = format!("{}{}", site, channel_name);
-    let response = reqwest::blocking::get(channel_url)?;
-    let rss_url = scraper::Html::parse_document(&response.text()?)
-        .select(&scraper::Selector::parse("link[title='RSS']").unwrap())
-        .next()
-        .unwrap()
-        .value()
-        .attr("href")
-        .unwrap()
-        .to_string();
-    Ok(rss_url)
-}
-
-fn get_link(item: &Entry) -> Option<Link> {
-    item.links().get(0).cloned()
-}
-
-// read list of rss feed URLs from file 'feeds.txt'
-fn lines_from(file_name: &str) -> Result<std::io::Lines<std::io::BufReader<std::fs::File>>> {
-    use std::io::{BufRead, BufReader};
-
-    let file = File::open(file_name)?;
-    let reader = BufReader::new(file);
-    Ok(reader.lines())
-}
-
-// fetch the RSS feed
-fn get_feed(url: String) -> Result<Feed> {
-    let content = reqwest::blocking::get(url)?.bytes()?;
-    let channel = Feed::read_from(&content[..])?;
-    Ok(channel)
-}
-
-fn is_already_downloaded(link: &Link, file_name: &str) -> Result<bool> {
-    use std::io::{BufRead, BufReader};
-
-    if let Ok(file) = File::open(file_name) {
-        let reader = BufReader::new(file);
-        for line in reader.lines() {
-            if line? == link.href {
-                return Ok(true); // is already downloaded
-            }
-        }
-    }
-    Ok(false) // is not already downloaded
-}
-
-fn download_audio(link: &Link) -> Result<()> {
-    use std::process::Command;
-
-    let cmd = "yt-dlp";
-    // println!("{} --extract-audio --audio-format mp3 {}", cmd, &link.href);
-    let output = Command::new(cmd)
-        .arg("--extract-audio")
-        .arg("--audio-format")
-        .arg("mp3")
-        .arg(&link.href)
-        .output()?;
-    if !output.stderr.is_empty() {
-        eprintln!("Error: {}", String::from_utf8(output.stderr)?);
-        println!("{}", String::from_utf8(output.stdout)?);
-    }
-    Ok(())
-}
-
-fn mark_as_downloaded(link: &Link, file_name: &str) -> Result<()> {
-    use std::fs::OpenOptions;
-    use std::io::prelude::*;
-
-    let mut file = OpenOptions::new()
-        .write(true)
-        .append(true)
-        .create(true)
-        .open(file_name)
-        .unwrap();
-
-    writeln!(file, "{}", link.href)?;
-    Ok(())
-}

src/prelude.rs

@@ -0,0 +1,3 @@
+use crate::errors::Error;
+
+pub type Result<T> = std::result::Result<T, Error>;

src/subscriptions.rs

@@ -0,0 +1,87 @@
+use crate::prelude::*;
+
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+
+pub fn lines_from(file_name: &str) -> Result<Vec<String>> {
+    let file = File::open(file_name)?;
+    let reader = BufReader::new(file);
+    let mut lines = vec![];
+    for line in reader.lines() {
+        if let Ok(line) = line {
+            if line.starts_with('@') {
+                lines.push(line);
+            }
+        }
+    }
+    Ok(lines)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{fs::File, io::Write, str::from_utf8};
+
+    use tempfile::{tempdir, TempDir};
+
+    use super::*;
+
+    #[test]
+    fn can_load_file() -> Result<()> {
+        //given
+        let (dir, file_name) = create_text_file(
+            "subscriptions.txt",
+            include_bytes!("../test/data/subscriptions.txt"),
+        )?;
+
+        //when
+        let result = lines_from(&file_name)?;
+
+        //then
+        drop(dir);
+        assert_eq!(result, ["@sub1", "@sub2", "@sub3"]);
+        Ok(())
+    }
+
+    #[test]
+    fn ignores_blank_lines() -> Result<()> {
+        //given
+        let (dir, file_name) = create_text_file(
+            "subscriptions.txt",
+            include_bytes!("../test/data/subscriptions-blank-line.txt"),
+        )?;
+
+        //when
+        let result = lines_from(&file_name)?;
+
+        //then
+        drop(dir);
+        assert_eq!(result, ["@sub1", "@sub2", "@sub3"]);
+        Ok(())
+    }
+
+    #[test]
+    fn ignores_comments() -> Result<()> {
+        //given
+        let (dir, file_name) = create_text_file(
+            "subscriptions.txt",
+            include_bytes!("../test/data/subscriptions-comment.txt"),
+        )?;
+
+        //when
+        let result = lines_from(&file_name)?;
+
+        //then
+        drop(dir);
+        assert_eq!(result, ["@sub1", "@sub3"]);
+        Ok(())
+    }
+
+    fn create_text_file(name: &str, data: &[u8]) -> Result<(TempDir, String)> {
+        let data = from_utf8(data)?;
+        let dir = tempdir()?;
+        let filename = format!("{}", &dir.path().join(name).display());
+        let file = File::create(&filename)?;
+        write!(&file, "{data}")?;
+        Ok((dir, filename))
+    }
+}

test/data/subscriptions-blank-line.txt

@@ -0,0 +1,4 @@
+@sub1
+
+@sub2
+@sub3

test/data/subscriptions-comment.txt

@@ -0,0 +1,3 @@
+@sub1
+#@sub2
+@sub3

test/data/subscriptions.txt

@@ -0,0 +1,3 @@
+@sub1
+@sub2
+@sub3