From 29f62a279b9532a4638237b65a3de0d358f343a7 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 23 Jan 2024 08:06:28 +0000 Subject: [PATCH] f4-dir-per-channel (#21) Closes kemitix/podal#4 Reviewed-on: https://git.kemitix.net/kemitix/podal/pulls/21 Co-authored-by: Paul Campbell Co-committed-by: Paul Campbell --- .tool-versions | 1 + src/feed/find.rs | 40 ++++++++++++++++++++++++---------------- src/feed/mod.rs | 17 +++++++++++++++-- src/file/read.rs | 30 ++++++++++++++++++++++++------ src/lib.rs | 25 +++++++++++++++++-------- src/network/env.rs | 36 ++++++++++++++++++++++++++---------- src/network/mod.rs | 1 + src/test_utils.rs | 27 ++++++++++++++++----------- 8 files changed, 124 insertions(+), 53 deletions(-) create mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..b36d43c --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +yt-dlp latest diff --git a/src/feed/find.rs b/src/feed/find.rs index 5817abd..81d71fd 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -1,9 +1,11 @@ use crate::prelude::*; -use crate::network::NetworkEnv; +use crate::network::{NetUrl, NetworkEnv}; -pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { - if let Some(channel_prefix) = channel_name.chars().next() { +use super::ChannelName; + +pub fn find(site: &str, channel_name: &ChannelName, e: &NetworkEnv) -> Result { + if let Some(channel_prefix) = channel_name.0.chars().next() { if channel_prefix != '@' { return Err(anyhow!( "Channel Name must begin with an '@': {}", @@ -11,19 +13,21 @@ pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { )); } } - let channel_url = format!("{}{}", site, channel_name); + let channel_url = NetUrl(format!("{}{}", site, channel_name)); let response = (e.fetch_as_text)(&channel_url) .context(format!("Fetching channel to find RSS: {}", channel_url))?; let rss_selector = scraper::Selector::parse("link[title='RSS']") .map_err(|e| anyhow!("Invalid selector: {}", e))?; - let rss_url = scraper::Html::parse_document(&response) - .select(&rss_selector) - .next() - .context("No RSS link found")? - .value() - .attr("href") - .context("No href attribute found in RSS link")? - .to_string(); + let rss_url = NetUrl( + scraper::Html::parse_document(&response) + .select(&rss_selector) + .next() + .context("No RSS link found")? + .value() + .attr("href") + .context("No href attribute found in RSS link")? + .to_string(), + ); Ok(rss_url) } @@ -42,16 +46,16 @@ mod tests { //given let network_env = NetworkEnv { fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([( - "site@channel", + NetUrl::from("site@channel"), "the-rss-url", )])), fetch_as_bytes: stub_network_fetch_as_bytes(), download_as_mp3: stub_network_download_as_mp3(), }; //when - let result = find("site", "@channel", &network_env)?; + let result = find("site", &ChannelName::from("@channel"), &network_env)?; //then - assert_eq!(result, "the-rss-url"); + assert_eq!(result, NetUrl::from("the-rss-url")); Ok(()) } @@ -64,7 +68,11 @@ mod tests { download_as_mp3: stub_network_download_as_mp3(), }; //when - let result = find("site", "invalid-channel-name", &network_env); + let result = find( + "site", + &ChannelName::from("invalid-channel-name"), + &network_env, + ); //then assert!(result.is_err()); Ok(()) diff --git a/src/feed/mod.rs b/src/feed/mod.rs index 728ebc3..266c5d5 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -1,14 +1,27 @@ use crate::prelude::*; -use crate::network::NetworkEnv; +use crate::network::{NetUrl, NetworkEnv}; use atom_syndication::Feed; mod find; pub use find::find; -pub fn get(url: &str, e: &NetworkEnv) -> Result { +pub fn get(url: &NetUrl, e: &NetworkEnv) -> Result { let content = (e.fetch_as_bytes)(url).context(format!("Fetching feed: {}", url))?; let channel = Feed::read_from(&content[..]).context("Could not parse RSS feed")?; Ok(channel) } + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ChannelName(pub String); +impl ChannelName { + pub fn from(channel_name: &str) -> Self { + Self(channel_name.to_string()) + } +} +impl std::fmt::Display for ChannelName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.as_str()) + } +} diff --git a/src/file/read.rs b/src/file/read.rs index 0c1dea0..5d26a67 100644 --- a/src/file/read.rs +++ b/src/file/read.rs @@ -1,15 +1,15 @@ -use crate::prelude::*; +use crate::{feed::ChannelName, prelude::*}; use crate::file::FileEnv; use std::io::{BufRead, BufReader}; -pub fn lines_from(file_name: &str, e: &FileEnv) -> Result> { +pub fn lines_from(file_name: &str, e: &FileEnv) -> Result> { let file = (e.open)(file_name).context(format!("Opening file: {file_name}"))?; let reader = BufReader::new(file); let mut lines = vec![]; for line in reader.lines().flatten() { if line.starts_with('@') { - lines.push(line); + lines.push(ChannelName(line)); } } Ok(lines) @@ -41,7 +41,13 @@ mod tests { //then drop(dir); - assert_eq!(result, ["@sub1", "@sub2", "@sub3"]); + assert_eq!( + result, + ["@sub1", "@sub2", "@sub3"] + .into_iter() + .map(ChannelName::from) + .collect::>() + ); Ok(()) } @@ -64,7 +70,13 @@ mod tests { //then drop(dir); - assert_eq!(result, ["@sub1", "@sub2", "@sub3"]); + assert_eq!( + result, + ["@sub1", "@sub2", "@sub3"] + .into_iter() + .map(ChannelName::from) + .collect::>() + ); Ok(()) } @@ -87,7 +99,13 @@ mod tests { //then drop(dir); - assert_eq!(result, ["@sub1", "@sub3"]); + assert_eq!( + result, + ["@sub1", "@sub3"] + .into_iter() + .map(ChannelName::from) + .collect::>() + ); Ok(()) } } diff --git a/src/lib.rs b/src/lib.rs index 4fc0976..50ab77e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,8 @@ mod test_utils; use file::FileEnv; use network::NetworkEnv; +use crate::network::NetUrl; + pub struct Env { pub network: NetworkEnv, pub file: FileEnv, @@ -32,7 +34,8 @@ pub fn run(site: &str, a: &Args, e: Env) -> Result<()> { if let Some(link) = entry.links().first() { if !history::find(link, &a.history, &e.file).context("Finding history")? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); - (e.network.download_as_mp3)(&link.href).context("Downloading as MP3")?; + (e.network.download_as_mp3)(&NetUrl(link.href.clone()), &channel_name) + .context("Downloading as MP3")?; history::add(link, &a.history, &e.file).context("Adding to history")?; } } @@ -60,7 +63,7 @@ mod tests { //given let site = "http://example.com/"; - let (tx, rx) = mpsc::channel::(); // channel to recieve notice of downloaded urls + let (tx, rx) = mpsc::channel::(); // channel to recieve notice of downloaded urls // two channels in subscriptions.txt let subs_file_name = "subs"; @@ -84,16 +87,16 @@ mod tests { let env = Env { network: NetworkEnv { fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([ - ("http://example.com/@channel1", "rss-feed-1"), - ("http://example.com/@channel2", "rss-feed-2"), + (NetUrl::from("http://example.com/@channel1"), "rss-feed-1"), + (NetUrl::from("http://example.com/@channel2"), "rss-feed-2"), ])), fetch_as_bytes: mock_network_fetch_as_bytes_with_rss_entries(HashMap::from([ ( - "rss-feed-1".into(), + NetUrl::from("rss-feed-1"), feed_with_three_links("c1-f1", "c1-f2", "c1-f3").to_string(), ), ( - "rss-feed-2".into(), + NetUrl::from("rss-feed-2"), feed_with_three_links("c2-f1", "c2-f2", "c2-f3").to_string(), ), ])), @@ -114,12 +117,18 @@ mod tests { drop(subs_dir); drop(history_dir); - let mut downloads: Vec = vec![]; + let mut downloads: Vec = vec![]; for m in rx { downloads.push(m); } - assert_eq!(downloads, vec!["c1-f1", "c1-f3", "c2-f1", "c2-f2"]); + assert_eq!( + downloads, + ["c1-f1", "c1-f3", "c2-f1", "c2-f2"] + .into_iter() + .map(NetUrl::from) + .collect::>() + ); Ok(()) } diff --git a/src/network/env.rs b/src/network/env.rs index 3d68d88..14d30fb 100644 --- a/src/network/env.rs +++ b/src/network/env.rs @@ -1,12 +1,25 @@ use std::process::Command; -use crate::prelude::*; +use crate::{feed::ChannelName, prelude::*}; -pub type NetworkFetchAsTextFn = Box Result>; +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct NetUrl(pub String); +impl NetUrl { + pub fn from(url: &str) -> Self { + Self(url.to_string()) + } +} +impl std::fmt::Display for NetUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.as_str()) + } +} -pub type NetworkFetchAsBytesFn = Box Result>; +pub type NetworkFetchAsTextFn = Box Result>; -pub type NetworkDownloadAsMp3Fn = Box Result<()>>; +pub type NetworkFetchAsBytesFn = Box Result>; + +pub type NetworkDownloadAsMp3Fn = Box Result<()>>; pub struct NetworkEnv { pub fetch_as_text: NetworkFetchAsTextFn, @@ -17,29 +30,32 @@ impl Default for NetworkEnv { fn default() -> Self { Self { fetch_as_text: Box::new(|url| { - reqwest::blocking::get(url) + reqwest::blocking::get(&url.0) .context(format!("Fetching {}", url))? .text() .context(format!("Parsing text from body of response for {}", url)) }), fetch_as_bytes: Box::new(|url| { - reqwest::blocking::get(url) + reqwest::blocking::get(&url.0) .context(format!("Fetching {}", url))? .bytes() .context(format!("Parsing bytes from body of response for {}", url)) }), - download_as_mp3: Box::new(|url| { + download_as_mp3: Box::new(|url, channel_name| { + println!("Downloading: {}", url); let cmd = "yt-dlp"; let output = Command::new(cmd) .arg("--extract-audio") .arg("--audio-format") .arg("mp3") - .arg(url) + .arg("-P") + .arg(format!("~/Music/{}", channel_name)) + .arg(&url.0) .output() .with_context(|| { format!( - "Running: {} --extract-audio --audio-format mp3 {}", - cmd, url + "Running: {} --extract-audio --audio-format mp3 -p ~/Music/{} {}", + cmd, channel_name, url ) })?; if !output.stderr.is_empty() { diff --git a/src/network/mod.rs b/src/network/mod.rs index 2d626a3..4a2672f 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,5 +1,6 @@ mod env; +pub use env::NetUrl; pub use env::NetworkDownloadAsMp3Fn; pub use env::NetworkEnv; pub use env::NetworkFetchAsBytesFn; diff --git a/src/test_utils.rs b/src/test_utils.rs index 3ebae16..779da72 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -11,8 +11,9 @@ use anyhow::Context; use tempfile::{tempdir, TempDir}; use crate::{ + feed::ChannelName, file::{FileAppendLineFn, FileOpenFn}, - network::{NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn}, + network::{NetUrl, NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn}, prelude::*, }; @@ -35,10 +36,8 @@ pub fn read_text_file(path: &Path, file_name: &str) -> Result> { .map(String::from) .collect()) } -pub fn mock_fetch_as_text_with_rss_url( - map: HashMap<&'static str, &'static str>, -) -> NetworkFetchAsTextFn { - Box::new(move |url: &str| { +pub fn mock_fetch_as_text_with_rss_url(map: HashMap) -> NetworkFetchAsTextFn { + Box::new(move |url: &NetUrl| { map.get(url).map_or_else( || Err(anyhow!("Unexpected request for {}", url)), |url| Ok(format!(r#""#, url)), @@ -46,7 +45,7 @@ pub fn mock_fetch_as_text_with_rss_url( }) } pub fn mock_network_fetch_as_bytes_with_rss_entries( - feeds: HashMap, + feeds: HashMap, ) -> NetworkFetchAsBytesFn { Box::new(move |url| { feeds.get(url).cloned().map_or_else( @@ -71,9 +70,9 @@ pub fn mock_file_open(real_paths: HashMap) -> FileOpenFn { }) } -pub fn mock_network_download_as_mp3(tx: Sender) -> NetworkDownloadAsMp3Fn { - Box::new(move |url: &str| { - tx.send(url.into())?; +pub fn mock_network_download_as_mp3(tx: Sender) -> NetworkDownloadAsMp3Fn { + Box::new(move |url: &NetUrl, _channel_name: &ChannelName| { + tx.send(url.clone())?; Ok(()) }) } @@ -83,8 +82,14 @@ pub fn mock_file_append_line() -> FileAppendLineFn { } pub fn stub_network_fetch_as_bytes() -> NetworkFetchAsBytesFn { - Box::new(|url: &str| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url))) + Box::new(|url: &NetUrl| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url))) } pub fn stub_network_download_as_mp3() -> NetworkDownloadAsMp3Fn { - Box::new(|url: &str| Err(anyhow!("Not implemented: network_download_as_mp3: {}", url))) + Box::new(|url: &NetUrl, channel_name: &ChannelName| { + Err(anyhow!( + "Not implemented: network_download_as_mp3: ({}) {}", + channel_name, + url, + )) + }) }