From 9a7287c1d9bddc1adecf531105e82680a229e0ea Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 20 Jan 2024 20:44:01 +0000 Subject: [PATCH 1/4] Add type NetUrl --- src/feed/find.rs | 28 +++++++++++++++------------- src/feed/mod.rs | 4 ++-- src/lib.rs | 25 +++++++++++++++++-------- src/network/env.rs | 25 +++++++++++++++++++------ src/network/mod.rs | 1 + src/test_utils.rs | 20 +++++++++----------- 6 files changed, 63 insertions(+), 40 deletions(-) diff --git a/src/feed/find.rs b/src/feed/find.rs index 5817abd..bd910ed 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -1,8 +1,8 @@ use crate::prelude::*; -use crate::network::NetworkEnv; +use crate::network::{NetUrl, NetworkEnv}; -pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { +pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { if let Some(channel_prefix) = channel_name.chars().next() { if channel_prefix != '@' { return Err(anyhow!( @@ -11,19 +11,21 @@ pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { )); } } - let channel_url = format!("{}{}", site, channel_name); + let channel_url = NetUrl(format!("{}{}", site, channel_name)); let response = (e.fetch_as_text)(&channel_url) .context(format!("Fetching channel to find RSS: {}", channel_url))?; let rss_selector = scraper::Selector::parse("link[title='RSS']") .map_err(|e| anyhow!("Invalid selector: {}", e))?; - let rss_url = scraper::Html::parse_document(&response) - .select(&rss_selector) - .next() - .context("No RSS link found")? - .value() - .attr("href") - .context("No href attribute found in RSS link")? - .to_string(); + let rss_url = NetUrl( + scraper::Html::parse_document(&response) + .select(&rss_selector) + .next() + .context("No RSS link found")? + .value() + .attr("href") + .context("No href attribute found in RSS link")? + .to_string(), + ); Ok(rss_url) } @@ -42,7 +44,7 @@ mod tests { //given let network_env = NetworkEnv { fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([( - "site@channel", + NetUrl::from("site@channel"), "the-rss-url", )])), fetch_as_bytes: stub_network_fetch_as_bytes(), @@ -51,7 +53,7 @@ mod tests { //when let result = find("site", "@channel", &network_env)?; //then - assert_eq!(result, "the-rss-url"); + assert_eq!(result, NetUrl::from("the-rss-url")); Ok(()) } diff --git a/src/feed/mod.rs b/src/feed/mod.rs index 728ebc3..bb655c6 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -1,13 +1,13 @@ use crate::prelude::*; -use crate::network::NetworkEnv; +use crate::network::{NetUrl, NetworkEnv}; use atom_syndication::Feed; mod find; pub use find::find; -pub fn get(url: &str, e: &NetworkEnv) -> Result { +pub fn get(url: &NetUrl, e: &NetworkEnv) -> Result { let content = (e.fetch_as_bytes)(url).context(format!("Fetching feed: {}", url))?; let channel = Feed::read_from(&content[..]).context("Could not parse RSS feed")?; Ok(channel) diff --git a/src/lib.rs b/src/lib.rs index 4fc0976..6bd8f82 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,8 @@ mod test_utils; use file::FileEnv; use network::NetworkEnv; +use crate::network::NetUrl; + pub struct Env { pub network: NetworkEnv, pub file: FileEnv, @@ -32,7 +34,8 @@ pub fn run(site: &str, a: &Args, e: Env) -> Result<()> { if let Some(link) = entry.links().first() { if !history::find(link, &a.history, &e.file).context("Finding history")? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); - (e.network.download_as_mp3)(&link.href).context("Downloading as MP3")?; + (e.network.download_as_mp3)(&NetUrl(link.href.clone())) + .context("Downloading as MP3")?; history::add(link, &a.history, &e.file).context("Adding to history")?; } } @@ -60,7 +63,7 @@ mod tests { //given let site = "http://example.com/"; - let (tx, rx) = mpsc::channel::(); // channel to recieve notice of downloaded urls + let (tx, rx) = mpsc::channel::(); // channel to recieve notice of downloaded urls // two channels in subscriptions.txt let subs_file_name = "subs"; @@ -84,16 +87,16 @@ mod tests { let env = Env { network: NetworkEnv { fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([ - ("http://example.com/@channel1", "rss-feed-1"), - ("http://example.com/@channel2", "rss-feed-2"), + (NetUrl::from("http://example.com/@channel1"), "rss-feed-1"), + (NetUrl::from("http://example.com/@channel2"), "rss-feed-2"), ])), fetch_as_bytes: mock_network_fetch_as_bytes_with_rss_entries(HashMap::from([ ( - "rss-feed-1".into(), + NetUrl::from("rss-feed-1"), feed_with_three_links("c1-f1", "c1-f2", "c1-f3").to_string(), ), ( - "rss-feed-2".into(), + NetUrl::from("rss-feed-2"), feed_with_three_links("c2-f1", "c2-f2", "c2-f3").to_string(), ), ])), @@ -114,12 +117,18 @@ mod tests { drop(subs_dir); drop(history_dir); - let mut downloads: Vec = vec![]; + let mut downloads: Vec = vec![]; for m in rx { downloads.push(m); } - assert_eq!(downloads, vec!["c1-f1", "c1-f3", "c2-f1", "c2-f2"]); + assert_eq!( + downloads, + ["c1-f1", "c1-f3", "c2-f1", "c2-f2"] + .into_iter() + .map(NetUrl::from) + .collect::>() + ); Ok(()) } diff --git a/src/network/env.rs b/src/network/env.rs index 3d68d88..65663cc 100644 --- a/src/network/env.rs +++ b/src/network/env.rs @@ -2,11 +2,24 @@ use std::process::Command; use crate::prelude::*; -pub type NetworkFetchAsTextFn = Box Result>; +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct NetUrl(pub String); +impl NetUrl { + pub fn from(url: &str) -> Self { + Self(url.to_string()) + } +} +impl std::fmt::Display for NetUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.as_str()) + } +} -pub type NetworkFetchAsBytesFn = Box Result>; +pub type NetworkFetchAsTextFn = Box Result>; -pub type NetworkDownloadAsMp3Fn = Box Result<()>>; +pub type NetworkFetchAsBytesFn = Box Result>; + +pub type NetworkDownloadAsMp3Fn = Box Result<()>>; pub struct NetworkEnv { pub fetch_as_text: NetworkFetchAsTextFn, @@ -17,13 +30,13 @@ impl Default for NetworkEnv { fn default() -> Self { Self { fetch_as_text: Box::new(|url| { - reqwest::blocking::get(url) + reqwest::blocking::get(&url.0) .context(format!("Fetching {}", url))? .text() .context(format!("Parsing text from body of response for {}", url)) }), fetch_as_bytes: Box::new(|url| { - reqwest::blocking::get(url) + reqwest::blocking::get(&url.0) .context(format!("Fetching {}", url))? .bytes() .context(format!("Parsing bytes from body of response for {}", url)) @@ -34,7 +47,7 @@ impl Default for NetworkEnv { .arg("--extract-audio") .arg("--audio-format") .arg("mp3") - .arg(url) + .arg(&url.0) .output() .with_context(|| { format!( diff --git a/src/network/mod.rs b/src/network/mod.rs index 2d626a3..4a2672f 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -1,5 +1,6 @@ mod env; +pub use env::NetUrl; pub use env::NetworkDownloadAsMp3Fn; pub use env::NetworkEnv; pub use env::NetworkFetchAsBytesFn; diff --git a/src/test_utils.rs b/src/test_utils.rs index 3ebae16..60eff79 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -12,7 +12,7 @@ use tempfile::{tempdir, TempDir}; use crate::{ file::{FileAppendLineFn, FileOpenFn}, - network::{NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn}, + network::{NetUrl, NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn}, prelude::*, }; @@ -35,10 +35,8 @@ pub fn read_text_file(path: &Path, file_name: &str) -> Result> { .map(String::from) .collect()) } -pub fn mock_fetch_as_text_with_rss_url( - map: HashMap<&'static str, &'static str>, -) -> NetworkFetchAsTextFn { - Box::new(move |url: &str| { +pub fn mock_fetch_as_text_with_rss_url(map: HashMap) -> NetworkFetchAsTextFn { + Box::new(move |url: &NetUrl| { map.get(url).map_or_else( || Err(anyhow!("Unexpected request for {}", url)), |url| Ok(format!(r#""#, url)), @@ -46,7 +44,7 @@ pub fn mock_fetch_as_text_with_rss_url( }) } pub fn mock_network_fetch_as_bytes_with_rss_entries( - feeds: HashMap, + feeds: HashMap, ) -> NetworkFetchAsBytesFn { Box::new(move |url| { feeds.get(url).cloned().map_or_else( @@ -71,9 +69,9 @@ pub fn mock_file_open(real_paths: HashMap) -> FileOpenFn { }) } -pub fn mock_network_download_as_mp3(tx: Sender) -> NetworkDownloadAsMp3Fn { - Box::new(move |url: &str| { - tx.send(url.into())?; +pub fn mock_network_download_as_mp3(tx: Sender) -> NetworkDownloadAsMp3Fn { + Box::new(move |url: &NetUrl| { + tx.send(url.clone())?; Ok(()) }) } @@ -83,8 +81,8 @@ pub fn mock_file_append_line() -> FileAppendLineFn { } pub fn stub_network_fetch_as_bytes() -> NetworkFetchAsBytesFn { - Box::new(|url: &str| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url))) + Box::new(|url: &NetUrl| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url))) } pub fn stub_network_download_as_mp3() -> NetworkDownloadAsMp3Fn { - Box::new(|url: &str| Err(anyhow!("Not implemented: network_download_as_mp3: {}", url))) + Box::new(|url: &NetUrl| Err(anyhow!("Not implemented: network_download_as_mp3: {}", url))) } -- 2.45.3 From c2f4070a36b02d23987cf5f8023fb9eea0183d4c Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 23 Jan 2024 07:21:19 +0000 Subject: [PATCH 2/4] Add type ChannelName --- src/feed/find.rs | 14 ++++++++++---- src/feed/mod.rs | 13 +++++++++++++ src/file/read.rs | 30 ++++++++++++++++++++++++------ src/lib.rs | 2 +- src/network/env.rs | 8 +++++--- src/test_utils.rs | 11 +++++++++-- 6 files changed, 62 insertions(+), 16 deletions(-) diff --git a/src/feed/find.rs b/src/feed/find.rs index bd910ed..81d71fd 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -2,8 +2,10 @@ use crate::prelude::*; use crate::network::{NetUrl, NetworkEnv}; -pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { - if let Some(channel_prefix) = channel_name.chars().next() { +use super::ChannelName; + +pub fn find(site: &str, channel_name: &ChannelName, e: &NetworkEnv) -> Result { + if let Some(channel_prefix) = channel_name.0.chars().next() { if channel_prefix != '@' { return Err(anyhow!( "Channel Name must begin with an '@': {}", @@ -51,7 +53,7 @@ mod tests { download_as_mp3: stub_network_download_as_mp3(), }; //when - let result = find("site", "@channel", &network_env)?; + let result = find("site", &ChannelName::from("@channel"), &network_env)?; //then assert_eq!(result, NetUrl::from("the-rss-url")); Ok(()) @@ -66,7 +68,11 @@ mod tests { download_as_mp3: stub_network_download_as_mp3(), }; //when - let result = find("site", "invalid-channel-name", &network_env); + let result = find( + "site", + &ChannelName::from("invalid-channel-name"), + &network_env, + ); //then assert!(result.is_err()); Ok(()) diff --git a/src/feed/mod.rs b/src/feed/mod.rs index bb655c6..266c5d5 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -12,3 +12,16 @@ pub fn get(url: &NetUrl, e: &NetworkEnv) -> Result { let channel = Feed::read_from(&content[..]).context("Could not parse RSS feed")?; Ok(channel) } + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ChannelName(pub String); +impl ChannelName { + pub fn from(channel_name: &str) -> Self { + Self(channel_name.to_string()) + } +} +impl std::fmt::Display for ChannelName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.as_str()) + } +} diff --git a/src/file/read.rs b/src/file/read.rs index 0c1dea0..5d26a67 100644 --- a/src/file/read.rs +++ b/src/file/read.rs @@ -1,15 +1,15 @@ -use crate::prelude::*; +use crate::{feed::ChannelName, prelude::*}; use crate::file::FileEnv; use std::io::{BufRead, BufReader}; -pub fn lines_from(file_name: &str, e: &FileEnv) -> Result> { +pub fn lines_from(file_name: &str, e: &FileEnv) -> Result> { let file = (e.open)(file_name).context(format!("Opening file: {file_name}"))?; let reader = BufReader::new(file); let mut lines = vec![]; for line in reader.lines().flatten() { if line.starts_with('@') { - lines.push(line); + lines.push(ChannelName(line)); } } Ok(lines) @@ -41,7 +41,13 @@ mod tests { //then drop(dir); - assert_eq!(result, ["@sub1", "@sub2", "@sub3"]); + assert_eq!( + result, + ["@sub1", "@sub2", "@sub3"] + .into_iter() + .map(ChannelName::from) + .collect::>() + ); Ok(()) } @@ -64,7 +70,13 @@ mod tests { //then drop(dir); - assert_eq!(result, ["@sub1", "@sub2", "@sub3"]); + assert_eq!( + result, + ["@sub1", "@sub2", "@sub3"] + .into_iter() + .map(ChannelName::from) + .collect::>() + ); Ok(()) } @@ -87,7 +99,13 @@ mod tests { //then drop(dir); - assert_eq!(result, ["@sub1", "@sub3"]); + assert_eq!( + result, + ["@sub1", "@sub3"] + .into_iter() + .map(ChannelName::from) + .collect::>() + ); Ok(()) } } diff --git a/src/lib.rs b/src/lib.rs index 6bd8f82..50ab77e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,7 +34,7 @@ pub fn run(site: &str, a: &Args, e: Env) -> Result<()> { if let Some(link) = entry.links().first() { if !history::find(link, &a.history, &e.file).context("Finding history")? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); - (e.network.download_as_mp3)(&NetUrl(link.href.clone())) + (e.network.download_as_mp3)(&NetUrl(link.href.clone()), &channel_name) .context("Downloading as MP3")?; history::add(link, &a.history, &e.file).context("Adding to history")?; } diff --git a/src/network/env.rs b/src/network/env.rs index 65663cc..0aa4eb2 100644 --- a/src/network/env.rs +++ b/src/network/env.rs @@ -1,6 +1,6 @@ use std::process::Command; -use crate::prelude::*; +use crate::{feed::ChannelName, prelude::*}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct NetUrl(pub String); @@ -19,7 +19,7 @@ pub type NetworkFetchAsTextFn = Box Result>; pub type NetworkFetchAsBytesFn = Box Result>; -pub type NetworkDownloadAsMp3Fn = Box Result<()>>; +pub type NetworkDownloadAsMp3Fn = Box Result<()>>; pub struct NetworkEnv { pub fetch_as_text: NetworkFetchAsTextFn, @@ -41,12 +41,14 @@ impl Default for NetworkEnv { .bytes() .context(format!("Parsing bytes from body of response for {}", url)) }), - download_as_mp3: Box::new(|url| { + download_as_mp3: Box::new(|url, channel_name| { let cmd = "yt-dlp"; let output = Command::new(cmd) .arg("--extract-audio") .arg("--audio-format") .arg("mp3") + .arg("-p") + .arg(format!("~/Music/{}", channel_name)) .arg(&url.0) .output() .with_context(|| { diff --git a/src/test_utils.rs b/src/test_utils.rs index 60eff79..779da72 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -11,6 +11,7 @@ use anyhow::Context; use tempfile::{tempdir, TempDir}; use crate::{ + feed::ChannelName, file::{FileAppendLineFn, FileOpenFn}, network::{NetUrl, NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn}, prelude::*, @@ -70,7 +71,7 @@ pub fn mock_file_open(real_paths: HashMap) -> FileOpenFn { } pub fn mock_network_download_as_mp3(tx: Sender) -> NetworkDownloadAsMp3Fn { - Box::new(move |url: &NetUrl| { + Box::new(move |url: &NetUrl, _channel_name: &ChannelName| { tx.send(url.clone())?; Ok(()) }) @@ -84,5 +85,11 @@ pub fn stub_network_fetch_as_bytes() -> NetworkFetchAsBytesFn { Box::new(|url: &NetUrl| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url))) } pub fn stub_network_download_as_mp3() -> NetworkDownloadAsMp3Fn { - Box::new(|url: &NetUrl| Err(anyhow!("Not implemented: network_download_as_mp3: {}", url))) + Box::new(|url: &NetUrl, channel_name: &ChannelName| { + Err(anyhow!( + "Not implemented: network_download_as_mp3: ({}) {}", + channel_name, + url, + )) + }) } -- 2.45.3 From 038340e3914518a3fe9546da77d6adc4d4ca1329 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 23 Jan 2024 07:26:32 +0000 Subject: [PATCH 3/4] Add .tool-versions for latest yt-dlp --- .tool-versions | 1 + 1 file changed, 1 insertion(+) create mode 100644 .tool-versions diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..b36d43c --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +yt-dlp latest -- 2.45.3 From 3ef63c146f25d7ed6c7b1ef9ba1217a8feadd9ae Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 23 Jan 2024 08:00:08 +0000 Subject: [PATCH 4/4] pass correct parameter to specify output path --- src/network/env.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/network/env.rs b/src/network/env.rs index 0aa4eb2..14d30fb 100644 --- a/src/network/env.rs +++ b/src/network/env.rs @@ -42,19 +42,20 @@ impl Default for NetworkEnv { .context(format!("Parsing bytes from body of response for {}", url)) }), download_as_mp3: Box::new(|url, channel_name| { + println!("Downloading: {}", url); let cmd = "yt-dlp"; let output = Command::new(cmd) .arg("--extract-audio") .arg("--audio-format") .arg("mp3") - .arg("-p") + .arg("-P") .arg(format!("~/Music/{}", channel_name)) .arg(&url.0) .output() .with_context(|| { format!( - "Running: {} --extract-audio --audio-format mp3 {}", - cmd, url + "Running: {} --extract-audio --audio-format mp3 -p ~/Music/{} {}", + cmd, channel_name, url ) })?; if !output.stderr.is_empty() { -- 2.45.3