f4-dir-per-channel #21

Merged
kemitix merged 4 commits from f4-dir-per-channel into main 2024-01-23 08:06:28 +00:00
8 changed files with 124 additions and 53 deletions

1
.tool-versions Normal file
View file

@ -0,0 +1 @@
yt-dlp latest

View file

@ -1,9 +1,11 @@
use crate::prelude::*; use crate::prelude::*;
use crate::network::NetworkEnv; use crate::network::{NetUrl, NetworkEnv};
pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result<String> { use super::ChannelName;
if let Some(channel_prefix) = channel_name.chars().next() {
pub fn find(site: &str, channel_name: &ChannelName, e: &NetworkEnv) -> Result<NetUrl> {
if let Some(channel_prefix) = channel_name.0.chars().next() {
if channel_prefix != '@' { if channel_prefix != '@' {
return Err(anyhow!( return Err(anyhow!(
"Channel Name must begin with an '@': {}", "Channel Name must begin with an '@': {}",
@ -11,19 +13,21 @@ pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result<String> {
)); ));
} }
} }
let channel_url = format!("{}{}", site, channel_name); let channel_url = NetUrl(format!("{}{}", site, channel_name));
let response = (e.fetch_as_text)(&channel_url) let response = (e.fetch_as_text)(&channel_url)
.context(format!("Fetching channel to find RSS: {}", channel_url))?; .context(format!("Fetching channel to find RSS: {}", channel_url))?;
let rss_selector = scraper::Selector::parse("link[title='RSS']") let rss_selector = scraper::Selector::parse("link[title='RSS']")
.map_err(|e| anyhow!("Invalid selector: {}", e))?; .map_err(|e| anyhow!("Invalid selector: {}", e))?;
let rss_url = scraper::Html::parse_document(&response) let rss_url = NetUrl(
scraper::Html::parse_document(&response)
.select(&rss_selector) .select(&rss_selector)
.next() .next()
.context("No RSS link found")? .context("No RSS link found")?
.value() .value()
.attr("href") .attr("href")
.context("No href attribute found in RSS link")? .context("No href attribute found in RSS link")?
.to_string(); .to_string(),
);
Ok(rss_url) Ok(rss_url)
} }
@ -42,16 +46,16 @@ mod tests {
//given //given
let network_env = NetworkEnv { let network_env = NetworkEnv {
fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([( fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([(
"site@channel", NetUrl::from("site@channel"),
"the-rss-url", "the-rss-url",
)])), )])),
fetch_as_bytes: stub_network_fetch_as_bytes(), fetch_as_bytes: stub_network_fetch_as_bytes(),
download_as_mp3: stub_network_download_as_mp3(), download_as_mp3: stub_network_download_as_mp3(),
}; };
//when //when
let result = find("site", "@channel", &network_env)?; let result = find("site", &ChannelName::from("@channel"), &network_env)?;
//then //then
assert_eq!(result, "the-rss-url"); assert_eq!(result, NetUrl::from("the-rss-url"));
Ok(()) Ok(())
} }
@ -64,7 +68,11 @@ mod tests {
download_as_mp3: stub_network_download_as_mp3(), download_as_mp3: stub_network_download_as_mp3(),
}; };
//when //when
let result = find("site", "invalid-channel-name", &network_env); let result = find(
"site",
&ChannelName::from("invalid-channel-name"),
&network_env,
);
//then //then
assert!(result.is_err()); assert!(result.is_err());
Ok(()) Ok(())

View file

@ -1,14 +1,27 @@
use crate::prelude::*; use crate::prelude::*;
use crate::network::NetworkEnv; use crate::network::{NetUrl, NetworkEnv};
use atom_syndication::Feed; use atom_syndication::Feed;
mod find; mod find;
pub use find::find; pub use find::find;
pub fn get(url: &str, e: &NetworkEnv) -> Result<Feed> { pub fn get(url: &NetUrl, e: &NetworkEnv) -> Result<Feed> {
let content = (e.fetch_as_bytes)(url).context(format!("Fetching feed: {}", url))?; let content = (e.fetch_as_bytes)(url).context(format!("Fetching feed: {}", url))?;
let channel = Feed::read_from(&content[..]).context("Could not parse RSS feed")?; let channel = Feed::read_from(&content[..]).context("Could not parse RSS feed")?;
Ok(channel) Ok(channel)
} }
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ChannelName(pub String);
impl ChannelName {
pub fn from(channel_name: &str) -> Self {
Self(channel_name.to_string())
}
}
impl std::fmt::Display for ChannelName {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.as_str())
}
}

View file

@ -1,15 +1,15 @@
use crate::prelude::*; use crate::{feed::ChannelName, prelude::*};
use crate::file::FileEnv; use crate::file::FileEnv;
use std::io::{BufRead, BufReader}; use std::io::{BufRead, BufReader};
pub fn lines_from(file_name: &str, e: &FileEnv) -> Result<Vec<String>> { pub fn lines_from(file_name: &str, e: &FileEnv) -> Result<Vec<ChannelName>> {
let file = (e.open)(file_name).context(format!("Opening file: {file_name}"))?; let file = (e.open)(file_name).context(format!("Opening file: {file_name}"))?;
let reader = BufReader::new(file); let reader = BufReader::new(file);
let mut lines = vec![]; let mut lines = vec![];
for line in reader.lines().flatten() { for line in reader.lines().flatten() {
if line.starts_with('@') { if line.starts_with('@') {
lines.push(line); lines.push(ChannelName(line));
} }
} }
Ok(lines) Ok(lines)
@ -41,7 +41,13 @@ mod tests {
//then //then
drop(dir); drop(dir);
assert_eq!(result, ["@sub1", "@sub2", "@sub3"]); assert_eq!(
result,
["@sub1", "@sub2", "@sub3"]
.into_iter()
.map(ChannelName::from)
.collect::<Vec<ChannelName>>()
);
Ok(()) Ok(())
} }
@ -64,7 +70,13 @@ mod tests {
//then //then
drop(dir); drop(dir);
assert_eq!(result, ["@sub1", "@sub2", "@sub3"]); assert_eq!(
result,
["@sub1", "@sub2", "@sub3"]
.into_iter()
.map(ChannelName::from)
.collect::<Vec<ChannelName>>()
);
Ok(()) Ok(())
} }
@ -87,7 +99,13 @@ mod tests {
//then //then
drop(dir); drop(dir);
assert_eq!(result, ["@sub1", "@sub3"]); assert_eq!(
result,
["@sub1", "@sub3"]
.into_iter()
.map(ChannelName::from)
.collect::<Vec<ChannelName>>()
);
Ok(()) Ok(())
} }
} }

View file

@ -14,6 +14,8 @@ mod test_utils;
use file::FileEnv; use file::FileEnv;
use network::NetworkEnv; use network::NetworkEnv;
use crate::network::NetUrl;
pub struct Env { pub struct Env {
pub network: NetworkEnv, pub network: NetworkEnv,
pub file: FileEnv, pub file: FileEnv,
@ -32,7 +34,8 @@ pub fn run(site: &str, a: &Args, e: Env) -> Result<()> {
if let Some(link) = entry.links().first() { if let Some(link) = entry.links().first() {
if !history::find(link, &a.history, &e.file).context("Finding history")? { if !history::find(link, &a.history, &e.file).context("Finding history")? {
println!("Downloading {}: {}", &channel_name, entry.title().as_str()); println!("Downloading {}: {}", &channel_name, entry.title().as_str());
(e.network.download_as_mp3)(&link.href).context("Downloading as MP3")?; (e.network.download_as_mp3)(&NetUrl(link.href.clone()), &channel_name)
.context("Downloading as MP3")?;
history::add(link, &a.history, &e.file).context("Adding to history")?; history::add(link, &a.history, &e.file).context("Adding to history")?;
} }
} }
@ -60,7 +63,7 @@ mod tests {
//given //given
let site = "http://example.com/"; let site = "http://example.com/";
let (tx, rx) = mpsc::channel::<String>(); // channel to recieve notice of downloaded urls let (tx, rx) = mpsc::channel::<NetUrl>(); // channel to recieve notice of downloaded urls
// two channels in subscriptions.txt // two channels in subscriptions.txt
let subs_file_name = "subs"; let subs_file_name = "subs";
@ -84,16 +87,16 @@ mod tests {
let env = Env { let env = Env {
network: NetworkEnv { network: NetworkEnv {
fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([ fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([
("http://example.com/@channel1", "rss-feed-1"), (NetUrl::from("http://example.com/@channel1"), "rss-feed-1"),
("http://example.com/@channel2", "rss-feed-2"), (NetUrl::from("http://example.com/@channel2"), "rss-feed-2"),
])), ])),
fetch_as_bytes: mock_network_fetch_as_bytes_with_rss_entries(HashMap::from([ fetch_as_bytes: mock_network_fetch_as_bytes_with_rss_entries(HashMap::from([
( (
"rss-feed-1".into(), NetUrl::from("rss-feed-1"),
feed_with_three_links("c1-f1", "c1-f2", "c1-f3").to_string(), feed_with_three_links("c1-f1", "c1-f2", "c1-f3").to_string(),
), ),
( (
"rss-feed-2".into(), NetUrl::from("rss-feed-2"),
feed_with_three_links("c2-f1", "c2-f2", "c2-f3").to_string(), feed_with_three_links("c2-f1", "c2-f2", "c2-f3").to_string(),
), ),
])), ])),
@ -114,12 +117,18 @@ mod tests {
drop(subs_dir); drop(subs_dir);
drop(history_dir); drop(history_dir);
let mut downloads: Vec<String> = vec![]; let mut downloads: Vec<NetUrl> = vec![];
for m in rx { for m in rx {
downloads.push(m); downloads.push(m);
} }
assert_eq!(downloads, vec!["c1-f1", "c1-f3", "c2-f1", "c2-f2"]); assert_eq!(
downloads,
["c1-f1", "c1-f3", "c2-f1", "c2-f2"]
.into_iter()
.map(NetUrl::from)
.collect::<Vec<NetUrl>>()
);
Ok(()) Ok(())
} }

View file

@ -1,12 +1,25 @@
use std::process::Command; use std::process::Command;
use crate::prelude::*; use crate::{feed::ChannelName, prelude::*};
pub type NetworkFetchAsTextFn = Box<dyn Fn(&str) -> Result<String>>; #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct NetUrl(pub String);
impl NetUrl {
pub fn from(url: &str) -> Self {
Self(url.to_string())
}
}
impl std::fmt::Display for NetUrl {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.as_str())
}
}
pub type NetworkFetchAsBytesFn = Box<dyn Fn(&str) -> Result<bytes::Bytes>>; pub type NetworkFetchAsTextFn = Box<dyn Fn(&NetUrl) -> Result<String>>;
pub type NetworkDownloadAsMp3Fn = Box<dyn Fn(&str) -> Result<()>>; pub type NetworkFetchAsBytesFn = Box<dyn Fn(&NetUrl) -> Result<bytes::Bytes>>;
pub type NetworkDownloadAsMp3Fn = Box<dyn Fn(&NetUrl, &ChannelName) -> Result<()>>;
pub struct NetworkEnv { pub struct NetworkEnv {
pub fetch_as_text: NetworkFetchAsTextFn, pub fetch_as_text: NetworkFetchAsTextFn,
@ -17,29 +30,32 @@ impl Default for NetworkEnv {
fn default() -> Self { fn default() -> Self {
Self { Self {
fetch_as_text: Box::new(|url| { fetch_as_text: Box::new(|url| {
reqwest::blocking::get(url) reqwest::blocking::get(&url.0)
.context(format!("Fetching {}", url))? .context(format!("Fetching {}", url))?
.text() .text()
.context(format!("Parsing text from body of response for {}", url)) .context(format!("Parsing text from body of response for {}", url))
}), }),
fetch_as_bytes: Box::new(|url| { fetch_as_bytes: Box::new(|url| {
reqwest::blocking::get(url) reqwest::blocking::get(&url.0)
.context(format!("Fetching {}", url))? .context(format!("Fetching {}", url))?
.bytes() .bytes()
.context(format!("Parsing bytes from body of response for {}", url)) .context(format!("Parsing bytes from body of response for {}", url))
}), }),
download_as_mp3: Box::new(|url| { download_as_mp3: Box::new(|url, channel_name| {
println!("Downloading: {}", url);
let cmd = "yt-dlp"; let cmd = "yt-dlp";
let output = Command::new(cmd) let output = Command::new(cmd)
.arg("--extract-audio") .arg("--extract-audio")
.arg("--audio-format") .arg("--audio-format")
.arg("mp3") .arg("mp3")
.arg(url) .arg("-P")
.arg(format!("~/Music/{}", channel_name))
.arg(&url.0)
.output() .output()
.with_context(|| { .with_context(|| {
format!( format!(
"Running: {} --extract-audio --audio-format mp3 {}", "Running: {} --extract-audio --audio-format mp3 -p ~/Music/{} {}",
cmd, url cmd, channel_name, url
) )
})?; })?;
if !output.stderr.is_empty() { if !output.stderr.is_empty() {

View file

@ -1,5 +1,6 @@
mod env; mod env;
pub use env::NetUrl;
pub use env::NetworkDownloadAsMp3Fn; pub use env::NetworkDownloadAsMp3Fn;
pub use env::NetworkEnv; pub use env::NetworkEnv;
pub use env::NetworkFetchAsBytesFn; pub use env::NetworkFetchAsBytesFn;

View file

@ -11,8 +11,9 @@ use anyhow::Context;
use tempfile::{tempdir, TempDir}; use tempfile::{tempdir, TempDir};
use crate::{ use crate::{
feed::ChannelName,
file::{FileAppendLineFn, FileOpenFn}, file::{FileAppendLineFn, FileOpenFn},
network::{NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn}, network::{NetUrl, NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn},
prelude::*, prelude::*,
}; };
@ -35,10 +36,8 @@ pub fn read_text_file(path: &Path, file_name: &str) -> Result<Vec<String>> {
.map(String::from) .map(String::from)
.collect()) .collect())
} }
pub fn mock_fetch_as_text_with_rss_url( pub fn mock_fetch_as_text_with_rss_url(map: HashMap<NetUrl, &'static str>) -> NetworkFetchAsTextFn {
map: HashMap<&'static str, &'static str>, Box::new(move |url: &NetUrl| {
) -> NetworkFetchAsTextFn {
Box::new(move |url: &str| {
map.get(url).map_or_else( map.get(url).map_or_else(
|| Err(anyhow!("Unexpected request for {}", url)), || Err(anyhow!("Unexpected request for {}", url)),
|url| Ok(format!(r#"<html><link title="RSS" href="{}"></html>"#, url)), |url| Ok(format!(r#"<html><link title="RSS" href="{}"></html>"#, url)),
@ -46,7 +45,7 @@ pub fn mock_fetch_as_text_with_rss_url(
}) })
} }
pub fn mock_network_fetch_as_bytes_with_rss_entries( pub fn mock_network_fetch_as_bytes_with_rss_entries(
feeds: HashMap<String, String>, feeds: HashMap<NetUrl, String>,
) -> NetworkFetchAsBytesFn { ) -> NetworkFetchAsBytesFn {
Box::new(move |url| { Box::new(move |url| {
feeds.get(url).cloned().map_or_else( feeds.get(url).cloned().map_or_else(
@ -71,9 +70,9 @@ pub fn mock_file_open(real_paths: HashMap<String, String>) -> FileOpenFn {
}) })
} }
pub fn mock_network_download_as_mp3(tx: Sender<String>) -> NetworkDownloadAsMp3Fn { pub fn mock_network_download_as_mp3(tx: Sender<NetUrl>) -> NetworkDownloadAsMp3Fn {
Box::new(move |url: &str| { Box::new(move |url: &NetUrl, _channel_name: &ChannelName| {
tx.send(url.into())?; tx.send(url.clone())?;
Ok(()) Ok(())
}) })
} }
@ -83,8 +82,14 @@ pub fn mock_file_append_line() -> FileAppendLineFn {
} }
pub fn stub_network_fetch_as_bytes() -> NetworkFetchAsBytesFn { pub fn stub_network_fetch_as_bytes() -> NetworkFetchAsBytesFn {
Box::new(|url: &str| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url))) Box::new(|url: &NetUrl| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url)))
} }
pub fn stub_network_download_as_mp3() -> NetworkDownloadAsMp3Fn { pub fn stub_network_download_as_mp3() -> NetworkDownloadAsMp3Fn {
Box::new(|url: &str| Err(anyhow!("Not implemented: network_download_as_mp3: {}", url))) Box::new(|url: &NetUrl, channel_name: &ChannelName| {
Err(anyhow!(
"Not implemented: network_download_as_mp3: ({}) {}",
channel_name,
url,
))
})
} }