diff --git a/.gitignore b/.gitignore index 2143bde..c233fd4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ target *.mp3 *.webm -*.txt +/subscriptions.txt +/downloaded.txt diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..98731e8 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,9 @@ +pipeline: + build: + image: rust + commands: + - rustup component add rustfmt clippy + - cargo --version + - cargo fmt --check + - cargo clippy --fix -- -Dwarnings -W clippy::nursery -W clippy::unwrap_used -W clippy::expect_used + - cargo test diff --git a/Cargo.lock b/Cargo.lock index 75d0f97..f20c98c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,6 +35,55 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "atom_syndication" version = "0.12.1" @@ -127,6 +176,39 @@ dependencies = [ "num-traits", ] +[[package]] +name = "clap" +version = "4.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "core-foundation" version = "0.9.3" @@ -580,6 +662,17 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi", + "rustix", + "windows-sys", +] + [[package]] name = "itoa" version = "1.0.9" @@ -921,8 +1014,10 @@ name = "podal" version = "0.1.0" dependencies = [ "atom_syndication", + "clap", "reqwest", "scraper", + "tempfile", ] [[package]] @@ -1425,6 +1520,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index 5e00eca..0c54643 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,7 @@ edition = "2021" atom_syndication = "0.12.1" reqwest = { version = "0.11.18", features = ["json", "blocking"] } scraper = "0.17.1" +clap = "4.3.19" + +[dev-dependencies] +tempfile = "*" diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..6769040 --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,51 @@ +use std::{fmt::Display, str::Utf8Error, string::FromUtf8Error}; + +#[derive(Debug)] +pub struct Error { + details: String, +} +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.details.to_string().as_str()) + } +} +impl From for Error { + fn from(value: Utf8Error) -> Self { + Self { + details: value.to_string(), + } + } +} +impl From for Error { + fn from(value: FromUtf8Error) -> Self { + Self { + details: value.to_string(), + } + } +} +impl From for Error { + fn from(details: String) -> Self { + Self { details } + } +} +impl From for Error { + fn from(value: std::io::Error) -> Self { + Self { + details: value.to_string(), + } + } +} +impl From for Error { + fn from(value: atom_syndication::Error) -> Self { + Self { + details: value.to_string(), + } + } +} +impl From for Error { + fn from(value: reqwest::Error) -> Self { + Self { + details: value.to_string(), + } + } +} diff --git a/src/feed/find.rs b/src/feed/find.rs new file mode 100644 index 0000000..b735e05 --- /dev/null +++ b/src/feed/find.rs @@ -0,0 +1,20 @@ +use crate::prelude::*; + +pub fn find(site: &str, channel_name: &str) -> Result { + if let Some(channel_prefix) = channel_name.chars().next() { + if channel_prefix != '@' { + return Err(format!("Channel Name must begin with an '@': {}", channel_name).into()); + } + } + let channel_url = format!("{}{}", site, channel_name); + let response = reqwest::blocking::get(channel_url)?; + let rss_url = scraper::Html::parse_document(&response.text()?) + .select(&scraper::Selector::parse("link[title='RSS']").unwrap()) + .next() + .unwrap() + .value() + .attr("href") + .unwrap() + .to_string(); + Ok(rss_url) +} diff --git a/src/feed/get.rs b/src/feed/get.rs new file mode 100644 index 0000000..dad28b0 --- /dev/null +++ b/src/feed/get.rs @@ -0,0 +1,9 @@ +use atom_syndication::Feed; + +use crate::prelude::*; + +pub fn get(url: &str) -> Result { + let content = reqwest::blocking::get(url)?.bytes()?; + let channel = Feed::read_from(&content[..])?; + Ok(channel) +} diff --git a/src/feed/mod.rs b/src/feed/mod.rs new file mode 100644 index 0000000..8b43067 --- /dev/null +++ b/src/feed/mod.rs @@ -0,0 +1,12 @@ +use crate::prelude::*; + +mod find; +mod get; + +pub use find::find; +pub use get::get; + +type Feed = atom_syndication::Feed; + +pub type FeedFind = fn(&str, &str) -> Result; +pub type FeedGet = fn(&str) -> Result; diff --git a/src/fetch.rs b/src/fetch.rs new file mode 100644 index 0000000..1f95f96 --- /dev/null +++ b/src/fetch.rs @@ -0,0 +1,22 @@ +use crate::prelude::*; + +use atom_syndication::Link; +use std::process::Command; + +pub type FetchDownload = fn(&Link) -> Result<()>; + +pub fn download(link: &Link) -> Result<()> { + let cmd = "yt-dlp"; + // println!("{} --extract-audio --audio-format mp3 {}", cmd, &link.href); + let output = Command::new(cmd) + .arg("--extract-audio") + .arg("--audio-format") + .arg("mp3") + .arg(&link.href) + .output()?; + if !output.stderr.is_empty() { + eprintln!("Error: {}", String::from_utf8(output.stderr)?); + println!("{}", String::from_utf8(output.stdout)?); + } + Ok(()) +} diff --git a/src/history/add.rs b/src/history/add.rs new file mode 100644 index 0000000..e904004 --- /dev/null +++ b/src/history/add.rs @@ -0,0 +1,17 @@ +use crate::prelude::*; + +use atom_syndication::Link; +use std::fs::OpenOptions; +use std::io::prelude::*; + +pub fn add(link: &Link, file_name: &str) -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .append(true) + .create(true) + .open(file_name) + .unwrap(); + + writeln!(file, "{}", link.href)?; + Ok(()) +} diff --git a/src/history/find.rs b/src/history/find.rs new file mode 100644 index 0000000..5b760b8 --- /dev/null +++ b/src/history/find.rs @@ -0,0 +1,17 @@ +use crate::prelude::*; + +use atom_syndication::Link; +use std::fs::File; +use std::io::{BufRead, BufReader}; + +pub fn find(link: &Link, file_name: &str) -> Result { + if let Ok(file) = File::open(file_name) { + let reader = BufReader::new(file); + for line in reader.lines() { + if line? == link.href { + return Ok(true); // is already downloaded + } + } + } + Ok(false) // is not already downloaded +} diff --git a/src/history/mod.rs b/src/history/mod.rs new file mode 100644 index 0000000..e36428d --- /dev/null +++ b/src/history/mod.rs @@ -0,0 +1,12 @@ +use crate::prelude::*; + +mod add; +mod find; + +pub use add::add; +pub use find::find; + +type Link = atom_syndication::Link; + +pub type HistoryFind = fn(&Link, &str) -> Result; +pub type HistoryAdd = fn(&Link, &str) -> Result<()>; diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..ee239ca --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,37 @@ +mod errors; +pub mod feed; +pub mod fetch; +pub mod history; +pub mod prelude; +mod subscriptions; + +use feed::{FeedFind, FeedGet}; +use fetch::FetchDownload; +use history::{HistoryAdd, HistoryFind}; +use prelude::*; + +pub fn run( + subscriptions: &str, + history: &str, + site: &str, + feed_find: FeedFind, + feed_get: FeedGet, + history_find: HistoryFind, + history_add: HistoryAdd, + fetch_download: FetchDownload, +) -> Result<()> { + for channel_name in subscriptions::lines_from(subscriptions)? { + println!("Channel: {}", channel_name); + let feed_url = feed_find(site, &channel_name)?; + for entry in feed_get(&feed_url)?.entries() { + if let Some(link) = entry.links().get(0).cloned() { + if !history_find(&link, history)? { + println!("Downloading {}: {}", &channel_name, entry.title().as_str()); + fetch_download(&link)?; + history_add(&link, history)?; + } + } + } + } + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 46cef06..3e4a741 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,170 +1,22 @@ -// https://www.phind.com/agent?cache=clke9xk39001cmj085upzho1t +use podal::prelude::*; -use std::{fmt::Display, fs::File, string::FromUtf8Error}; - -use atom_syndication::{Entry, Feed, Link}; - -// -// ERRORS -// -#[derive(Debug)] -struct Error { - details: String, -} -impl Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.details.to_string().as_str()) - } -} -impl From for Error { - fn from(value: FromUtf8Error) -> Self { - Self { - details: value.to_string(), - } - } -} -impl From for Error { - fn from(details: String) -> Self { - Self { details } - } -} -impl From for Error { - fn from(value: std::io::Error) -> Self { - Self { - details: value.to_string(), - } - } -} -impl From for Error { - fn from(value: atom_syndication::Error) -> Self { - Self { - details: value.to_string(), - } - } -} -impl From for Error { - fn from(value: reqwest::Error) -> Self { - Self { - details: value.to_string(), - } - } -} - -// -// RESULTS -// -type Result = std::result::Result; - -// -// MAIN -// fn main() -> Result<()> { println!("Podal"); let subscriptions = "subscriptions.txt"; let history = "downloaded.txt"; let site = "https://www.youtube.com/"; - for channel_name in lines_from(subscriptions)? { - let channel_name = channel_name?; - println!("Channel: {}", channel_name); - let feed_url = get_feed_url(site, &channel_name)?; - for entry in get_feed(feed_url)?.entries() { - if let Some(link) = get_link(entry) { - if !is_already_downloaded(&link, history)? { - println!("Downloading {}: {}", &channel_name, entry.title().as_str()); - download_audio(&link)?; - mark_as_downloaded(&link, history)?; - } - } - } - } + podal::run( + subscriptions, + history, + site, + podal::feed::find, + podal::feed::get, + podal::history::find, + podal::history::add, + podal::fetch::download, + )?; println!("Done"); Ok(()) } - -fn get_feed_url(site: &str, channel_name: &str) -> Result { - if let Some(channel_prefix) = channel_name.chars().next() { - if channel_prefix != '@' { - return Err(format!("Channel Name must begin with an '@': {}", channel_name).into()); - } - } - let channel_url = format!("{}{}", site, channel_name); - let response = reqwest::blocking::get(channel_url)?; - let rss_url = scraper::Html::parse_document(&response.text()?) - .select(&scraper::Selector::parse("link[title='RSS']").unwrap()) - .next() - .unwrap() - .value() - .attr("href") - .unwrap() - .to_string(); - Ok(rss_url) -} - -fn get_link(item: &Entry) -> Option { - item.links().get(0).cloned() -} - -// read list of rss feed URLs from file 'feeds.txt' -fn lines_from(file_name: &str) -> Result>> { - use std::io::{BufRead, BufReader}; - - let file = File::open(file_name)?; - let reader = BufReader::new(file); - Ok(reader.lines()) -} - -// fetch the RSS feed -fn get_feed(url: String) -> Result { - let content = reqwest::blocking::get(url)?.bytes()?; - let channel = Feed::read_from(&content[..])?; - Ok(channel) -} - -fn is_already_downloaded(link: &Link, file_name: &str) -> Result { - use std::io::{BufRead, BufReader}; - - if let Ok(file) = File::open(file_name) { - let reader = BufReader::new(file); - for line in reader.lines() { - if line? == link.href { - return Ok(true); // is already downloaded - } - } - } - Ok(false) // is not already downloaded -} - -fn download_audio(link: &Link) -> Result<()> { - use std::process::Command; - - let cmd = "yt-dlp"; - // println!("{} --extract-audio --audio-format mp3 {}", cmd, &link.href); - let output = Command::new(cmd) - .arg("--extract-audio") - .arg("--audio-format") - .arg("mp3") - .arg(&link.href) - .output()?; - if !output.stderr.is_empty() { - eprintln!("Error: {}", String::from_utf8(output.stderr)?); - println!("{}", String::from_utf8(output.stdout)?); - } - Ok(()) -} - -fn mark_as_downloaded(link: &Link, file_name: &str) -> Result<()> { - use std::fs::OpenOptions; - use std::io::prelude::*; - - let mut file = OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(file_name) - .unwrap(); - - writeln!(file, "{}", link.href)?; - Ok(()) -} diff --git a/src/prelude.rs b/src/prelude.rs new file mode 100644 index 0000000..476ec26 --- /dev/null +++ b/src/prelude.rs @@ -0,0 +1,3 @@ +use crate::errors::Error; + +pub type Result = std::result::Result; diff --git a/src/subscriptions.rs b/src/subscriptions.rs new file mode 100644 index 0000000..5e5a955 --- /dev/null +++ b/src/subscriptions.rs @@ -0,0 +1,87 @@ +use crate::prelude::*; + +use std::fs::File; +use std::io::{BufRead, BufReader}; + +pub fn lines_from(file_name: &str) -> Result> { + let file = File::open(file_name)?; + let reader = BufReader::new(file); + let mut lines = vec![]; + for line in reader.lines() { + if let Ok(line) = line { + if line.starts_with('@') { + lines.push(line); + } + } + } + Ok(lines) +} + +#[cfg(test)] +mod tests { + use std::{fs::File, io::Write, str::from_utf8}; + + use tempfile::{tempdir, TempDir}; + + use super::*; + + #[test] + fn can_load_file() -> Result<()> { + //given + let (dir, file_name) = create_text_file( + "subscriptions.txt", + include_bytes!("../test/data/subscriptions.txt"), + )?; + + //when + let result = lines_from(&file_name)?; + + //then + drop(dir); + assert_eq!(result, ["@sub1", "@sub2", "@sub3"]); + Ok(()) + } + + #[test] + fn ignores_blank_lines() -> Result<()> { + //given + let (dir, file_name) = create_text_file( + "subscriptions.txt", + include_bytes!("../test/data/subscriptions-blank-line.txt"), + )?; + + //when + let result = lines_from(&file_name)?; + + //then + drop(dir); + assert_eq!(result, ["@sub1", "@sub2", "@sub3"]); + Ok(()) + } + + #[test] + fn ignores_comments() -> Result<()> { + //given + let (dir, file_name) = create_text_file( + "subscriptions.txt", + include_bytes!("../test/data/subscriptions-comment.txt"), + )?; + + //when + let result = lines_from(&file_name)?; + + //then + drop(dir); + assert_eq!(result, ["@sub1", "@sub3"]); + Ok(()) + } + + fn create_text_file(name: &str, data: &[u8]) -> Result<(TempDir, String)> { + let data = from_utf8(data)?; + let dir = tempdir()?; + let filename = format!("{}", &dir.path().join(name).display()); + let file = File::create(&filename)?; + write!(&file, "{data}")?; + Ok((dir, filename)) + } +} diff --git a/test/data/subscriptions-blank-line.txt b/test/data/subscriptions-blank-line.txt new file mode 100644 index 0000000..ef718e3 --- /dev/null +++ b/test/data/subscriptions-blank-line.txt @@ -0,0 +1,4 @@ +@sub1 + +@sub2 +@sub3 diff --git a/test/data/subscriptions-comment.txt b/test/data/subscriptions-comment.txt new file mode 100644 index 0000000..851ac8a --- /dev/null +++ b/test/data/subscriptions-comment.txt @@ -0,0 +1,3 @@ +@sub1 +#@sub2 +@sub3 diff --git a/test/data/subscriptions.txt b/test/data/subscriptions.txt new file mode 100644 index 0000000..a1714b6 --- /dev/null +++ b/test/data/subscriptions.txt @@ -0,0 +1,3 @@ +@sub1 +@sub2 +@sub3