f4-dir-per-channel #21
6 changed files with 63 additions and 40 deletions
|
@ -1,8 +1,8 @@
|
|||
use crate::prelude::*;
|
||||
|
||||
use crate::network::NetworkEnv;
|
||||
use crate::network::{NetUrl, NetworkEnv};
|
||||
|
||||
pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result<String> {
|
||||
pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result<NetUrl> {
|
||||
if let Some(channel_prefix) = channel_name.chars().next() {
|
||||
if channel_prefix != '@' {
|
||||
return Err(anyhow!(
|
||||
|
@ -11,19 +11,21 @@ pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result<String> {
|
|||
));
|
||||
}
|
||||
}
|
||||
let channel_url = format!("{}{}", site, channel_name);
|
||||
let channel_url = NetUrl(format!("{}{}", site, channel_name));
|
||||
let response = (e.fetch_as_text)(&channel_url)
|
||||
.context(format!("Fetching channel to find RSS: {}", channel_url))?;
|
||||
let rss_selector = scraper::Selector::parse("link[title='RSS']")
|
||||
.map_err(|e| anyhow!("Invalid selector: {}", e))?;
|
||||
let rss_url = scraper::Html::parse_document(&response)
|
||||
.select(&rss_selector)
|
||||
.next()
|
||||
.context("No RSS link found")?
|
||||
.value()
|
||||
.attr("href")
|
||||
.context("No href attribute found in RSS link")?
|
||||
.to_string();
|
||||
let rss_url = NetUrl(
|
||||
scraper::Html::parse_document(&response)
|
||||
.select(&rss_selector)
|
||||
.next()
|
||||
.context("No RSS link found")?
|
||||
.value()
|
||||
.attr("href")
|
||||
.context("No href attribute found in RSS link")?
|
||||
.to_string(),
|
||||
);
|
||||
Ok(rss_url)
|
||||
}
|
||||
|
||||
|
@ -42,7 +44,7 @@ mod tests {
|
|||
//given
|
||||
let network_env = NetworkEnv {
|
||||
fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([(
|
||||
"site@channel",
|
||||
NetUrl::from("site@channel"),
|
||||
"the-rss-url",
|
||||
)])),
|
||||
fetch_as_bytes: stub_network_fetch_as_bytes(),
|
||||
|
@ -51,7 +53,7 @@ mod tests {
|
|||
//when
|
||||
let result = find("site", "@channel", &network_env)?;
|
||||
//then
|
||||
assert_eq!(result, "the-rss-url");
|
||||
assert_eq!(result, NetUrl::from("the-rss-url"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
use crate::prelude::*;
|
||||
|
||||
use crate::network::NetworkEnv;
|
||||
use crate::network::{NetUrl, NetworkEnv};
|
||||
use atom_syndication::Feed;
|
||||
|
||||
mod find;
|
||||
|
||||
pub use find::find;
|
||||
|
||||
pub fn get(url: &str, e: &NetworkEnv) -> Result<Feed> {
|
||||
pub fn get(url: &NetUrl, e: &NetworkEnv) -> Result<Feed> {
|
||||
let content = (e.fetch_as_bytes)(url).context(format!("Fetching feed: {}", url))?;
|
||||
let channel = Feed::read_from(&content[..]).context("Could not parse RSS feed")?;
|
||||
Ok(channel)
|
||||
|
|
25
src/lib.rs
25
src/lib.rs
|
@ -14,6 +14,8 @@ mod test_utils;
|
|||
use file::FileEnv;
|
||||
use network::NetworkEnv;
|
||||
|
||||
use crate::network::NetUrl;
|
||||
|
||||
pub struct Env {
|
||||
pub network: NetworkEnv,
|
||||
pub file: FileEnv,
|
||||
|
@ -32,7 +34,8 @@ pub fn run(site: &str, a: &Args, e: Env) -> Result<()> {
|
|||
if let Some(link) = entry.links().first() {
|
||||
if !history::find(link, &a.history, &e.file).context("Finding history")? {
|
||||
println!("Downloading {}: {}", &channel_name, entry.title().as_str());
|
||||
(e.network.download_as_mp3)(&link.href).context("Downloading as MP3")?;
|
||||
(e.network.download_as_mp3)(&NetUrl(link.href.clone()))
|
||||
.context("Downloading as MP3")?;
|
||||
history::add(link, &a.history, &e.file).context("Adding to history")?;
|
||||
}
|
||||
}
|
||||
|
@ -60,7 +63,7 @@ mod tests {
|
|||
//given
|
||||
|
||||
let site = "http://example.com/";
|
||||
let (tx, rx) = mpsc::channel::<String>(); // channel to recieve notice of downloaded urls
|
||||
let (tx, rx) = mpsc::channel::<NetUrl>(); // channel to recieve notice of downloaded urls
|
||||
|
||||
// two channels in subscriptions.txt
|
||||
let subs_file_name = "subs";
|
||||
|
@ -84,16 +87,16 @@ mod tests {
|
|||
let env = Env {
|
||||
network: NetworkEnv {
|
||||
fetch_as_text: mock_fetch_as_text_with_rss_url(HashMap::from([
|
||||
("http://example.com/@channel1", "rss-feed-1"),
|
||||
("http://example.com/@channel2", "rss-feed-2"),
|
||||
(NetUrl::from("http://example.com/@channel1"), "rss-feed-1"),
|
||||
(NetUrl::from("http://example.com/@channel2"), "rss-feed-2"),
|
||||
])),
|
||||
fetch_as_bytes: mock_network_fetch_as_bytes_with_rss_entries(HashMap::from([
|
||||
(
|
||||
"rss-feed-1".into(),
|
||||
NetUrl::from("rss-feed-1"),
|
||||
feed_with_three_links("c1-f1", "c1-f2", "c1-f3").to_string(),
|
||||
),
|
||||
(
|
||||
"rss-feed-2".into(),
|
||||
NetUrl::from("rss-feed-2"),
|
||||
feed_with_three_links("c2-f1", "c2-f2", "c2-f3").to_string(),
|
||||
),
|
||||
])),
|
||||
|
@ -114,12 +117,18 @@ mod tests {
|
|||
drop(subs_dir);
|
||||
drop(history_dir);
|
||||
|
||||
let mut downloads: Vec<String> = vec![];
|
||||
let mut downloads: Vec<NetUrl> = vec![];
|
||||
for m in rx {
|
||||
downloads.push(m);
|
||||
}
|
||||
|
||||
assert_eq!(downloads, vec!["c1-f1", "c1-f3", "c2-f1", "c2-f2"]);
|
||||
assert_eq!(
|
||||
downloads,
|
||||
["c1-f1", "c1-f3", "c2-f1", "c2-f2"]
|
||||
.into_iter()
|
||||
.map(NetUrl::from)
|
||||
.collect::<Vec<NetUrl>>()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
@ -2,11 +2,24 @@ use std::process::Command;
|
|||
|
||||
use crate::prelude::*;
|
||||
|
||||
pub type NetworkFetchAsTextFn = Box<dyn Fn(&str) -> Result<String>>;
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct NetUrl(pub String);
|
||||
impl NetUrl {
|
||||
pub fn from(url: &str) -> Self {
|
||||
Self(url.to_string())
|
||||
}
|
||||
}
|
||||
impl std::fmt::Display for NetUrl {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.0.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
pub type NetworkFetchAsBytesFn = Box<dyn Fn(&str) -> Result<bytes::Bytes>>;
|
||||
pub type NetworkFetchAsTextFn = Box<dyn Fn(&NetUrl) -> Result<String>>;
|
||||
|
||||
pub type NetworkDownloadAsMp3Fn = Box<dyn Fn(&str) -> Result<()>>;
|
||||
pub type NetworkFetchAsBytesFn = Box<dyn Fn(&NetUrl) -> Result<bytes::Bytes>>;
|
||||
|
||||
pub type NetworkDownloadAsMp3Fn = Box<dyn Fn(&NetUrl) -> Result<()>>;
|
||||
|
||||
pub struct NetworkEnv {
|
||||
pub fetch_as_text: NetworkFetchAsTextFn,
|
||||
|
@ -17,13 +30,13 @@ impl Default for NetworkEnv {
|
|||
fn default() -> Self {
|
||||
Self {
|
||||
fetch_as_text: Box::new(|url| {
|
||||
reqwest::blocking::get(url)
|
||||
reqwest::blocking::get(&url.0)
|
||||
.context(format!("Fetching {}", url))?
|
||||
.text()
|
||||
.context(format!("Parsing text from body of response for {}", url))
|
||||
}),
|
||||
fetch_as_bytes: Box::new(|url| {
|
||||
reqwest::blocking::get(url)
|
||||
reqwest::blocking::get(&url.0)
|
||||
.context(format!("Fetching {}", url))?
|
||||
.bytes()
|
||||
.context(format!("Parsing bytes from body of response for {}", url))
|
||||
|
@ -34,7 +47,7 @@ impl Default for NetworkEnv {
|
|||
.arg("--extract-audio")
|
||||
.arg("--audio-format")
|
||||
.arg("mp3")
|
||||
.arg(url)
|
||||
.arg(&url.0)
|
||||
.output()
|
||||
.with_context(|| {
|
||||
format!(
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
mod env;
|
||||
|
||||
pub use env::NetUrl;
|
||||
pub use env::NetworkDownloadAsMp3Fn;
|
||||
pub use env::NetworkEnv;
|
||||
pub use env::NetworkFetchAsBytesFn;
|
||||
|
|
|
@ -12,7 +12,7 @@ use tempfile::{tempdir, TempDir};
|
|||
|
||||
use crate::{
|
||||
file::{FileAppendLineFn, FileOpenFn},
|
||||
network::{NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn},
|
||||
network::{NetUrl, NetworkDownloadAsMp3Fn, NetworkFetchAsBytesFn, NetworkFetchAsTextFn},
|
||||
prelude::*,
|
||||
};
|
||||
|
||||
|
@ -35,10 +35,8 @@ pub fn read_text_file(path: &Path, file_name: &str) -> Result<Vec<String>> {
|
|||
.map(String::from)
|
||||
.collect())
|
||||
}
|
||||
pub fn mock_fetch_as_text_with_rss_url(
|
||||
map: HashMap<&'static str, &'static str>,
|
||||
) -> NetworkFetchAsTextFn {
|
||||
Box::new(move |url: &str| {
|
||||
pub fn mock_fetch_as_text_with_rss_url(map: HashMap<NetUrl, &'static str>) -> NetworkFetchAsTextFn {
|
||||
Box::new(move |url: &NetUrl| {
|
||||
map.get(url).map_or_else(
|
||||
|| Err(anyhow!("Unexpected request for {}", url)),
|
||||
|url| Ok(format!(r#"<html><link title="RSS" href="{}"></html>"#, url)),
|
||||
|
@ -46,7 +44,7 @@ pub fn mock_fetch_as_text_with_rss_url(
|
|||
})
|
||||
}
|
||||
pub fn mock_network_fetch_as_bytes_with_rss_entries(
|
||||
feeds: HashMap<String, String>,
|
||||
feeds: HashMap<NetUrl, String>,
|
||||
) -> NetworkFetchAsBytesFn {
|
||||
Box::new(move |url| {
|
||||
feeds.get(url).cloned().map_or_else(
|
||||
|
@ -71,9 +69,9 @@ pub fn mock_file_open(real_paths: HashMap<String, String>) -> FileOpenFn {
|
|||
})
|
||||
}
|
||||
|
||||
pub fn mock_network_download_as_mp3(tx: Sender<String>) -> NetworkDownloadAsMp3Fn {
|
||||
Box::new(move |url: &str| {
|
||||
tx.send(url.into())?;
|
||||
pub fn mock_network_download_as_mp3(tx: Sender<NetUrl>) -> NetworkDownloadAsMp3Fn {
|
||||
Box::new(move |url: &NetUrl| {
|
||||
tx.send(url.clone())?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
@ -83,8 +81,8 @@ pub fn mock_file_append_line() -> FileAppendLineFn {
|
|||
}
|
||||
|
||||
pub fn stub_network_fetch_as_bytes() -> NetworkFetchAsBytesFn {
|
||||
Box::new(|url: &str| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url)))
|
||||
Box::new(|url: &NetUrl| Err(anyhow!("Not implemented: network_fetch_as_bytes: {}", url)))
|
||||
}
|
||||
pub fn stub_network_download_as_mp3() -> NetworkDownloadAsMp3Fn {
|
||||
Box::new(|url: &str| Err(anyhow!("Not implemented: network_download_as_mp3: {}", url)))
|
||||
Box::new(|url: &NetUrl| Err(anyhow!("Not implemented: network_download_as_mp3: {}", url)))
|
||||
}
|
||||
|
|
Loading…
Add table
Reference in a new issue