From 80803522770cb33e943ef3700af1bda5e4874897 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 25 Jul 2023 17:02:31 +0100 Subject: [PATCH 01/26] add stub for testing feed::find --- src/feed/find.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/feed/find.rs b/src/feed/find.rs index b735e05..c60dbc9 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -18,3 +18,15 @@ pub fn find(site: &str, channel_name: &str) -> Result { .to_string(); Ok(rss_url) } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn finds_rss_url() -> Result<()> { + // TODO: need to inject wrapper for reqwest::blocking::get + // + // todo!(); + Ok(()) + } +} -- 2.45.3 From 1fd5df71518c59adba575dbab4817605ce934f7e Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 25 Jul 2023 19:24:15 +0100 Subject: [PATCH 02/26] Use Env to pass in functions --- src/feed/mod.rs | 5 +++++ src/fetch.rs | 4 ++++ src/history/mod.rs | 5 +++++ src/lib.rs | 33 +++++++++++++++------------------ src/main.rs | 21 ++++++++++++++++----- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/src/feed/mod.rs b/src/feed/mod.rs index 8b43067..f06df54 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -8,5 +8,10 @@ pub use get::get; type Feed = atom_syndication::Feed; +pub struct FeedEnv { + pub find: FeedFind, + pub get: FeedGet, +} + pub type FeedFind = fn(&str, &str) -> Result; pub type FeedGet = fn(&str) -> Result; diff --git a/src/fetch.rs b/src/fetch.rs index 1f95f96..d7f836e 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -3,6 +3,10 @@ use crate::prelude::*; use atom_syndication::Link; use std::process::Command; +pub struct FetchEnv { + pub download: FetchDownload, +} + pub type FetchDownload = fn(&Link) -> Result<()>; pub fn download(link: &Link) -> Result<()> { diff --git a/src/history/mod.rs b/src/history/mod.rs index e36428d..d6f7255 100644 --- a/src/history/mod.rs +++ b/src/history/mod.rs @@ -8,5 +8,10 @@ pub use find::find; type Link = atom_syndication::Link; +pub struct HistoryEnv { + pub find: HistoryFind, + pub add: HistoryAdd, +} + pub type HistoryFind = fn(&Link, &str) -> Result; pub type HistoryAdd = fn(&Link, &str) -> Result<()>; diff --git a/src/lib.rs b/src/lib.rs index ee239ca..ecaae86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,30 +5,27 @@ pub mod history; pub mod prelude; mod subscriptions; -use feed::{FeedFind, FeedGet}; -use fetch::FetchDownload; -use history::{HistoryAdd, HistoryFind}; +use feed::FeedEnv; +use fetch::FetchEnv; +use history::HistoryEnv; use prelude::*; -pub fn run( - subscriptions: &str, - history: &str, - site: &str, - feed_find: FeedFind, - feed_get: FeedGet, - history_find: HistoryFind, - history_add: HistoryAdd, - fetch_download: FetchDownload, -) -> Result<()> { +pub struct Env { + pub feed: FeedEnv, + pub history: HistoryEnv, + pub fetch: FetchEnv, +} + +pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> { for channel_name in subscriptions::lines_from(subscriptions)? { println!("Channel: {}", channel_name); - let feed_url = feed_find(site, &channel_name)?; - for entry in feed_get(&feed_url)?.entries() { + let feed_url = (e.feed.find)(site, &channel_name)?; + for entry in (e.feed.get)(&feed_url)?.entries() { if let Some(link) = entry.links().get(0).cloned() { - if !history_find(&link, history)? { + if !(e.history.find)(&link, history)? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); - fetch_download(&link)?; - history_add(&link, history)?; + (e.fetch.download)(&link)?; + (e.history.add)(&link, history)?; } } } diff --git a/src/main.rs b/src/main.rs index 3e4a741..41e7d98 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,7 @@ use podal::prelude::*; +use podal::{feed::FeedEnv, fetch::FetchEnv, history::HistoryEnv}; + fn main() -> Result<()> { println!("Podal"); let subscriptions = "subscriptions.txt"; @@ -10,11 +12,20 @@ fn main() -> Result<()> { subscriptions, history, site, - podal::feed::find, - podal::feed::get, - podal::history::find, - podal::history::add, - podal::fetch::download, + podal::Env { + feed: FeedEnv { + find: podal::feed::find, + get: podal::feed::get, + }, + + history: HistoryEnv { + find: podal::history::find, + add: podal::history::add, + }, + fetch: FetchEnv { + download: podal::fetch::download, + }, + }, )?; println!("Done"); -- 2.45.3 From b8a3770a1aebeac24e2cedb95db8db5cb918fd0a Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 25 Jul 2023 19:25:45 +0100 Subject: [PATCH 03/26] clippy fix --- src/subscriptions.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/subscriptions.rs b/src/subscriptions.rs index 5e5a955..17ebde7 100644 --- a/src/subscriptions.rs +++ b/src/subscriptions.rs @@ -7,11 +7,9 @@ pub fn lines_from(file_name: &str) -> Result> { let file = File::open(file_name)?; let reader = BufReader::new(file); let mut lines = vec![]; - for line in reader.lines() { - if let Ok(line) = line { - if line.starts_with('@') { - lines.push(line); - } + for line in reader.lines().flatten() { + if line.starts_with('@') { + lines.push(line); } } Ok(lines) -- 2.45.3 From 50da08851898035eab15f681a1d146d35153b89d Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 25 Jul 2023 19:36:56 +0100 Subject: [PATCH 04/26] feed::find uses fetch::get --- src/feed/find.rs | 6 ++++-- src/feed/mod.rs | 4 +++- src/fetch.rs | 8 ++++++++ src/lib.rs | 2 +- src/main.rs | 1 + 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/feed/find.rs b/src/feed/find.rs index 3c0203e..c6ba1ba 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -1,13 +1,15 @@ use crate::prelude::*; -pub fn find(site: &str, channel_name: &str) -> Result { +use crate::fetch::FetchEnv; + +pub fn find(site: &str, channel_name: &str, e: &FetchEnv) -> Result { if let Some(channel_prefix) = channel_name.chars().next() { if channel_prefix != '@' { return Err(format!("Channel Name must begin with an '@': {}", channel_name).into()); } } let channel_url = format!("{}{}", site, channel_name); - let response = reqwest::blocking::get(channel_url)?; + let response = (e.get)(&channel_url)?; let rss_url = scraper::Html::parse_document(&response.text()?) .select(&scraper::Selector::parse("link[title='RSS']").unwrap()) .next() diff --git a/src/feed/mod.rs b/src/feed/mod.rs index f06df54..24ee1c9 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -1,5 +1,7 @@ use crate::prelude::*; +use crate::FetchEnv; + mod find; mod get; @@ -13,5 +15,5 @@ pub struct FeedEnv { pub get: FeedGet, } -pub type FeedFind = fn(&str, &str) -> Result; +pub type FeedFind = fn(&str, &str, &FetchEnv) -> Result; pub type FeedGet = fn(&str) -> Result; diff --git a/src/fetch.rs b/src/fetch.rs index d7f836e..a1aaeac 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -5,9 +5,11 @@ use std::process::Command; pub struct FetchEnv { pub download: FetchDownload, + pub get: FetchGet, } pub type FetchDownload = fn(&Link) -> Result<()>; +pub type FetchGet = fn(&str) -> Result; pub fn download(link: &Link) -> Result<()> { let cmd = "yt-dlp"; @@ -24,3 +26,9 @@ pub fn download(link: &Link) -> Result<()> { } Ok(()) } + +pub type Response = reqwest::blocking::Response; + +pub fn get(url: &str) -> Result { + Ok(reqwest::blocking::get(url)?) +} diff --git a/src/lib.rs b/src/lib.rs index ecaae86..8718f1b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,7 +19,7 @@ pub struct Env { pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> { for channel_name in subscriptions::lines_from(subscriptions)? { println!("Channel: {}", channel_name); - let feed_url = (e.feed.find)(site, &channel_name)?; + let feed_url = (e.feed.find)(site, &channel_name, &e.fetch)?; for entry in (e.feed.get)(&feed_url)?.entries() { if let Some(link) = entry.links().get(0).cloned() { if !(e.history.find)(&link, history)? { diff --git a/src/main.rs b/src/main.rs index 41e7d98..c97df09 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,7 @@ fn main() -> Result<()> { }, fetch: FetchEnv { download: podal::fetch::download, + get: podal::fetch::get, }, }, )?; -- 2.45.3 From d8861d6f7e9c266d5d15dac0b43d690b0d8000f9 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Tue, 25 Jul 2023 19:57:31 +0100 Subject: [PATCH 05/26] Add tests for feed::find --- src/feed/find.rs | 37 ++++++++++++++++++++++++++++++++----- src/feed/mod.rs | 4 ++-- src/fetch.rs | 4 ++-- src/lib.rs | 2 +- 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/feed/find.rs b/src/feed/find.rs index c6ba1ba..537776f 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -1,16 +1,16 @@ use crate::prelude::*; -use crate::fetch::FetchEnv; +use crate::fetch::FetchGet; -pub fn find(site: &str, channel_name: &str, e: &FetchEnv) -> Result { +pub fn find(site: &str, channel_name: &str, e: &FetchGet) -> Result { if let Some(channel_prefix) = channel_name.chars().next() { if channel_prefix != '@' { return Err(format!("Channel Name must begin with an '@': {}", channel_name).into()); } } let channel_url = format!("{}{}", site, channel_name); - let response = (e.get)(&channel_url)?; - let rss_url = scraper::Html::parse_document(&response.text()?) + let response = (e)(&channel_url)?; + let rss_url = scraper::Html::parse_document(&response) .select(&scraper::Selector::parse("link[title='RSS']").unwrap()) .next() .unwrap() @@ -23,10 +23,37 @@ pub fn find(site: &str, channel_name: &str, e: &FetchEnv) -> Result { #[cfg(test)] mod tests { + use crate::fetch::Response; + use super::*; #[test] fn finds_rss_url() -> Result<()> { - // TODO: need to inject wrapper for reqwest::blocking::get + //given + let fetch_get = &(get as FetchGet); + //when + let result = find("site", "@channel", &fetch_get)?; + //then + assert_eq!(result, "the-rss-url"); Ok(()) } + + #[test] + fn error_if_channel_name_is_invalid() -> Result<()> { + //given + let fetch_get = &(get as FetchGet); + //when + let result = find("site", "invalid-channel-name", &fetch_get); + //then + assert!(result.is_err()); + Ok(()) + } + + fn get(_url: &str) -> Result { + Ok(r#" + + + + "# + .to_string()) + } } diff --git a/src/feed/mod.rs b/src/feed/mod.rs index 24ee1c9..8246422 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -1,6 +1,6 @@ use crate::prelude::*; -use crate::FetchEnv; +use crate::fetch::FetchGet; mod find; mod get; @@ -15,5 +15,5 @@ pub struct FeedEnv { pub get: FeedGet, } -pub type FeedFind = fn(&str, &str, &FetchEnv) -> Result; +pub type FeedFind = fn(&str, &str, &FetchGet) -> Result; pub type FeedGet = fn(&str) -> Result; diff --git a/src/fetch.rs b/src/fetch.rs index a1aaeac..b0d47fd 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -27,8 +27,8 @@ pub fn download(link: &Link) -> Result<()> { Ok(()) } -pub type Response = reqwest::blocking::Response; +pub type Response = String; pub fn get(url: &str) -> Result { - Ok(reqwest::blocking::get(url)?) + Ok(reqwest::blocking::get(url)?.text()?) } diff --git a/src/lib.rs b/src/lib.rs index 8718f1b..243cbd3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,7 +19,7 @@ pub struct Env { pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> { for channel_name in subscriptions::lines_from(subscriptions)? { println!("Channel: {}", channel_name); - let feed_url = (e.feed.find)(site, &channel_name, &e.fetch)?; + let feed_url = (e.feed.find)(site, &channel_name, &e.fetch.get)?; for entry in (e.feed.get)(&feed_url)?.entries() { if let Some(link) = entry.links().get(0).cloned() { if !(e.history.find)(&link, history)? { -- 2.45.3 From d7729cc2a3d6fe43dc9569143bcb55e49c12dce3 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Wed, 26 Jul 2023 21:25:14 +0100 Subject: [PATCH 06/26] rename get as reqwest_blocking_get --- src/feed/find.rs | 4 ++-- src/feed/get.rs | 6 +++--- src/feed/mod.rs | 5 ++--- src/main.rs | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/feed/find.rs b/src/feed/find.rs index 537776f..f37ce84 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -31,7 +31,7 @@ mod tests { //given let fetch_get = &(get as FetchGet); //when - let result = find("site", "@channel", &fetch_get)?; + let result = find("site", "@channel", fetch_get)?; //then assert_eq!(result, "the-rss-url"); Ok(()) @@ -42,7 +42,7 @@ mod tests { //given let fetch_get = &(get as FetchGet); //when - let result = find("site", "invalid-channel-name", &fetch_get); + let result = find("site", "invalid-channel-name", fetch_get); //then assert!(result.is_err()); Ok(()) diff --git a/src/feed/get.rs b/src/feed/get.rs index dad28b0..1a7ea40 100644 --- a/src/feed/get.rs +++ b/src/feed/get.rs @@ -1,8 +1,8 @@ -use atom_syndication::Feed; - use crate::prelude::*; -pub fn get(url: &str) -> Result { +use atom_syndication::Feed; + +pub fn reqwest_blocking_get(url: &str) -> Result { let content = reqwest::blocking::get(url)?.bytes()?; let channel = Feed::read_from(&content[..])?; Ok(channel) diff --git a/src/feed/mod.rs b/src/feed/mod.rs index 8246422..253bf26 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -5,10 +5,9 @@ use crate::fetch::FetchGet; mod find; mod get; +use atom_syndication::Feed; pub use find::find; -pub use get::get; - -type Feed = atom_syndication::Feed; +pub use get::reqwest_blocking_get; pub struct FeedEnv { pub find: FeedFind, diff --git a/src/main.rs b/src/main.rs index c97df09..8e7f6c5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,7 +15,7 @@ fn main() -> Result<()> { podal::Env { feed: FeedEnv { find: podal::feed::find, - get: podal::feed::get, + get: podal::feed::reqwest_blocking_get, }, history: HistoryEnv { -- 2.45.3 From defa57de258e75d50f47b83774e8482c7b7bde8d Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Thu, 27 Jul 2023 21:20:32 +0100 Subject: [PATCH 07/26] extract test_utils module --- src/lib.rs | 3 +++ src/subscriptions.rs | 12 +----------- src/test_utils.rs | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 src/test_utils.rs diff --git a/src/lib.rs b/src/lib.rs index 243cbd3..ce77f4d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,6 +5,9 @@ pub mod history; pub mod prelude; mod subscriptions; +#[cfg(test)] +mod test_utils; + use feed::FeedEnv; use fetch::FetchEnv; use history::HistoryEnv; diff --git a/src/subscriptions.rs b/src/subscriptions.rs index 17ebde7..5ecc379 100644 --- a/src/subscriptions.rs +++ b/src/subscriptions.rs @@ -17,9 +17,8 @@ pub fn lines_from(file_name: &str) -> Result> { #[cfg(test)] mod tests { - use std::{fs::File, io::Write, str::from_utf8}; - use tempfile::{tempdir, TempDir}; + use crate::test_utils::create_text_file; use super::*; @@ -73,13 +72,4 @@ mod tests { assert_eq!(result, ["@sub1", "@sub3"]); Ok(()) } - - fn create_text_file(name: &str, data: &[u8]) -> Result<(TempDir, String)> { - let data = from_utf8(data)?; - let dir = tempdir()?; - let filename = format!("{}", &dir.path().join(name).display()); - let file = File::create(&filename)?; - write!(&file, "{data}")?; - Ok((dir, filename)) - } } diff --git a/src/test_utils.rs b/src/test_utils.rs new file mode 100644 index 0000000..36e67ff --- /dev/null +++ b/src/test_utils.rs @@ -0,0 +1,14 @@ +use std::{fs::File, io::Write, str::from_utf8}; + +use tempfile::{tempdir, TempDir}; + +use crate::prelude::*; + +pub fn create_text_file(name: &str, data: &[u8]) -> Result<(TempDir, String)> { + let data = from_utf8(data)?; + let dir = tempdir()?; + let filename = format!("{}", &dir.path().join(name).display()); + let file = File::create(&filename)?; + write!(&file, "{data}")?; + Ok((dir, filename)) +} -- 2.45.3 From 20a4ef59d9dd45fdd028f897e3270fbf57c5ae40 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Thu, 27 Jul 2023 21:39:55 +0100 Subject: [PATCH 08/26] add test for history::add --- src/history/add.rs | 72 +++++++++++++++++++++++++++++++++++++++++ src/test_utils.rs | 13 +++++++- test/data/downloads.txt | 2 ++ test/data/empty.txt | 0 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 test/data/downloads.txt create mode 100644 test/data/empty.txt diff --git a/src/history/add.rs b/src/history/add.rs index e904004..756edf2 100644 --- a/src/history/add.rs +++ b/src/history/add.rs @@ -15,3 +15,75 @@ pub fn add(link: &Link, file_name: &str) -> Result<()> { writeln!(file, "{}", link.href)?; Ok(()) } + +#[cfg(test)] +mod tests { + + use crate::{ + history::Link, + test_utils::{create_text_file, read_text_file}, + }; + + use super::*; + + #[test] + fn creates_file_if_missing() -> Result<()> { + //given + let (dir, file_name) = + create_text_file("download.txt", include_bytes!("../../test/data/empty.txt"))?; + std::fs::remove_file(&file_name)?; + + let link = Link { + href: "foo".to_string(), + rel: "bar".to_string(), + hreflang: None, + mime_type: None, + title: None, + length: None, + }; + //when + add(&link, &file_name)?; + + //then + let content: Vec = read_text_file(&file_name)?; + drop(dir); + + let expected = vec!["foo".to_string()]; + assert_eq!(content, expected); + + Ok(()) + } + + #[test] + fn appends_to_exising_file() -> Result<()> { + // given + let (dir, file_name) = create_text_file( + "download.txt", + include_bytes!("../../test/data/downloads.txt"), + )?; + + let link = Link { + href: "foo".to_string(), + rel: "bar".to_string(), + hreflang: None, + mime_type: None, + title: None, + length: None, + }; + //when + add(&link, &file_name)?; + + //then + let content: Vec = read_text_file(&file_name)?; + drop(dir); + + let expected = vec![ + "line-1".to_string(), + "line-2".to_string(), + "foo".to_string(), + ]; + assert_eq!(content, expected); + + Ok(()) + } +} diff --git a/src/test_utils.rs b/src/test_utils.rs index 36e67ff..305ae44 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -1,4 +1,8 @@ -use std::{fs::File, io::Write, str::from_utf8}; +use std::{ + fs::{read_to_string, File}, + io::Write, + str::from_utf8, +}; use tempfile::{tempdir, TempDir}; @@ -12,3 +16,10 @@ pub fn create_text_file(name: &str, data: &[u8]) -> Result<(TempDir, String)> { write!(&file, "{data}")?; Ok((dir, filename)) } + +pub fn read_text_file(file_name: &str) -> Result> { + Ok(read_to_string(file_name)? + .lines() + .map(String::from) + .collect()) +} diff --git a/test/data/downloads.txt b/test/data/downloads.txt new file mode 100644 index 0000000..1f26067 --- /dev/null +++ b/test/data/downloads.txt @@ -0,0 +1,2 @@ +line-1 +line-2 diff --git a/test/data/empty.txt b/test/data/empty.txt new file mode 100644 index 0000000..e69de29 -- 2.45.3 From 1219eef08933667183647bfcdf9c16a4e7408278 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 28 Jul 2023 07:50:55 +0100 Subject: [PATCH 09/26] add tests for history::find --- src/history/find.rs | 83 ++++++++++++++++++++++++++++++ test/data/with-embedded-llamma.txt | 3 ++ test/data/with-llamma.txt | 3 ++ test/data/without-llamma.txt | 2 + 4 files changed, 91 insertions(+) create mode 100644 test/data/with-embedded-llamma.txt create mode 100644 test/data/with-llamma.txt create mode 100644 test/data/without-llamma.txt diff --git a/src/history/find.rs b/src/history/find.rs index 5b760b8..c1dd584 100644 --- a/src/history/find.rs +++ b/src/history/find.rs @@ -15,3 +15,86 @@ pub fn find(link: &Link, file_name: &str) -> Result { } Ok(false) // is not already downloaded } + +#[cfg(test)] +mod test { + use crate::{history::Link, test_utils::create_text_file}; + + use super::*; + #[test] + fn true_if_line_exists() -> Result<()> { + //given + let (dir, file_name) = + create_text_file("file", include_bytes!("../../test/data/with-llamma.txt"))?; + let link = Link { + href: "llamma".to_string(), + rel: "".to_string(), + hreflang: None, + mime_type: None, + title: None, + length: None, + }; + + //when + let result = find(&link, &file_name)?; + + //then + drop(dir); + + assert_eq!(result, true); + + Ok(()) + } + + #[test] + fn false_if_line_absent() -> Result<()> { + //given + let (dir, file_name) = + create_text_file("file", include_bytes!("../../test/data/without-llamma.txt"))?; + let link = Link { + href: "llamma".to_string(), + rel: "".to_string(), + hreflang: None, + mime_type: None, + title: None, + length: None, + }; + + //when + let result = find(&link, &file_name)?; + + //then + drop(dir); + + assert_eq!(result, false); + + Ok(()) + } + + #[test] + fn false_if_embedded_within_line() -> Result<()> { + //given + let (dir, file_name) = create_text_file( + "file", + include_bytes!("../../test/data/with-embedded-llamma.txt"), + )?; + let link = Link { + href: "llamma".to_string(), + rel: "".to_string(), + hreflang: None, + mime_type: None, + title: None, + length: None, + }; + + //when + let result = find(&link, &file_name)?; + + //then + drop(dir); + + assert_eq!(result, false); + + Ok(()) + } +} diff --git a/test/data/with-embedded-llamma.txt b/test/data/with-embedded-llamma.txt new file mode 100644 index 0000000..3574d2b --- /dev/null +++ b/test/data/with-embedded-llamma.txt @@ -0,0 +1,3 @@ +this is a text file +embedded llamma +it has three lines diff --git a/test/data/with-llamma.txt b/test/data/with-llamma.txt new file mode 100644 index 0000000..cad2d82 --- /dev/null +++ b/test/data/with-llamma.txt @@ -0,0 +1,3 @@ +this is a text file +llamma +it has three lines diff --git a/test/data/without-llamma.txt b/test/data/without-llamma.txt new file mode 100644 index 0000000..61a0ef5 --- /dev/null +++ b/test/data/without-llamma.txt @@ -0,0 +1,2 @@ +this is a text file +it has three lines -- 2.45.3 From a25d16efe2067aa828d4cadf81b8156079b82261 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 28 Jul 2023 19:32:45 +0100 Subject: [PATCH 10/26] only run ci on PR and push to main --- .woodpecker.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 98731e8..9d90676 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,6 +1,11 @@ pipeline: build: image: rust + when: + - event: pull_request + repo: kemitix/podal + - event: push + branch: main commands: - rustup component add rustfmt clippy - cargo --version -- 2.45.3 From 5e5613dd018400ad021f2385e9ab4ea808a53268 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 28 Jul 2023 19:36:08 +0100 Subject: [PATCH 11/26] woodpeckerci: top level is steps --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 9d90676..83d59e3 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,4 +1,4 @@ -pipeline: +steps: build: image: rust when: -- 2.45.3 From 040e37d2a9f96adc30fd7d483b8a0d56ed958187 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 28 Jul 2023 19:39:08 +0100 Subject: [PATCH 12/26] woodpeckerci: always use latest run image --- .woodpecker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 83d59e3..45249c4 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,6 +1,7 @@ steps: build: - image: rust + image: rust:latest + pull: true when: - event: pull_request repo: kemitix/podal -- 2.45.3 From d23fd7e59032ee36e7f178622d61092061c4624a Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 28 Jul 2023 19:42:26 +0100 Subject: [PATCH 13/26] woodpeckerci: promote when to root --- .woodpecker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 45249c4..5941af9 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,12 +1,12 @@ +when: + - event: pull_request + repo: kemitix/podal + - event: push + branch: main steps: build: image: rust:latest pull: true - when: - - event: pull_request - repo: kemitix/podal - - event: push - branch: main commands: - rustup component add rustfmt clippy - cargo --version -- 2.45.3 From fd24f084285166de40b5cc3d461999100e13fc32 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 28 Jul 2023 19:48:37 +0100 Subject: [PATCH 14/26] woodpeckerci: revert when to build step --- .woodpecker.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 5941af9..45249c4 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,12 +1,12 @@ -when: - - event: pull_request - repo: kemitix/podal - - event: push - branch: main steps: build: image: rust:latest pull: true + when: + - event: pull_request + repo: kemitix/podal + - event: push + branch: main commands: - rustup component add rustfmt clippy - cargo --version -- 2.45.3 From e318f782504fcdc06e2da552f04e89bded807ec5 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Fri, 28 Jul 2023 20:23:04 +0100 Subject: [PATCH 15/26] add justfile with coverage creation rules --- .gitignore | 6 ++++-- justfile | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 justfile diff --git a/.gitignore b/.gitignore index c233fd4..a3bd0cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ target *.mp3 *.webm -/subscriptions.txt -/downloaded.txt +subscriptions.txt +downloaded.txt +coverage +*.profraw diff --git a/justfile b/justfile new file mode 100644 index 0000000..933e5e8 --- /dev/null +++ b/justfile @@ -0,0 +1,23 @@ +coverage-init: + cargo install grcov + rustup component add llvm-tools + +coverage: + #!/usr/bin/env bash + set -e + rm -rf ./target + just clean + export RUSTFLAGS="-Zinstrument-coverage" LLVM_PROFILE_FILE="just-%p-%m.profraw" + cargo +nightly build + cargo +nightly test + just generate + just server + +generate: + grcov . --binary-path ./target/debug/ -s . -t html --branch --ignore-not-existing --ignore "./target/" -o ./coverage/ + +server: + cd ./coverage && python3 -m http.server 8001 + +clean: + rm -rf ./*.prof* ./coverage/ -- 2.45.3 From 92e369beab7be92dfb9da838680bdf0a051898bc Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 18:42:14 +0100 Subject: [PATCH 16/26] clippy fixes --- src/history/find.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/history/find.rs b/src/history/find.rs index c1dd584..4171eb2 100644 --- a/src/history/find.rs +++ b/src/history/find.rs @@ -41,7 +41,7 @@ mod test { //then drop(dir); - assert_eq!(result, true); + assert!(result); Ok(()) } @@ -66,7 +66,7 @@ mod test { //then drop(dir); - assert_eq!(result, false); + assert!(!result); Ok(()) } @@ -93,7 +93,7 @@ mod test { //then drop(dir); - assert_eq!(result, false); + assert!(!result); Ok(()) } -- 2.45.3 From 5931014636a6319a1faaa57b37ee1b98fc04ef7f Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 19:08:52 +0100 Subject: [PATCH 17/26] extracted File::open use into FileEnv --- src/file.rs | 13 +++++++++++++ src/history/find.rs | 14 ++++++-------- src/history/mod.rs | 4 ++-- src/lib.rs | 7 +++++-- src/main.rs | 2 ++ src/subscriptions.rs | 15 +++++++++------ 6 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 src/file.rs diff --git a/src/file.rs b/src/file.rs new file mode 100644 index 0000000..83c6006 --- /dev/null +++ b/src/file.rs @@ -0,0 +1,13 @@ +use std::fs::File; + +pub struct FileEnv { + pub open: FileOpen, +} +impl Default for FileEnv { + fn default() -> Self { + Self { + open: |path| File::open(path), + } + } +} +pub type FileOpen = fn(path: &str) -> std::io::Result; diff --git a/src/history/find.rs b/src/history/find.rs index 4171eb2..70162d2 100644 --- a/src/history/find.rs +++ b/src/history/find.rs @@ -1,11 +1,10 @@ -use crate::prelude::*; +use crate::{file::FileEnv, prelude::*}; use atom_syndication::Link; -use std::fs::File; use std::io::{BufRead, BufReader}; -pub fn find(link: &Link, file_name: &str) -> Result { - if let Ok(file) = File::open(file_name) { +pub fn find(link: &Link, file_name: &str, e: &FileEnv) -> Result { + if let Ok(file) = (e.open)(file_name) { let reader = BufReader::new(file); for line in reader.lines() { if line? == link.href { @@ -34,9 +33,8 @@ mod test { title: None, length: None, }; - //when - let result = find(&link, &file_name)?; + let result = find(&link, &file_name, &FileEnv::default())?; //then drop(dir); @@ -61,7 +59,7 @@ mod test { }; //when - let result = find(&link, &file_name)?; + let result = find(&link, &file_name, &FileEnv::default())?; //then drop(dir); @@ -88,7 +86,7 @@ mod test { }; //when - let result = find(&link, &file_name)?; + let result = find(&link, &file_name, &FileEnv::default())?; //then drop(dir); diff --git a/src/history/mod.rs b/src/history/mod.rs index d6f7255..47f7e36 100644 --- a/src/history/mod.rs +++ b/src/history/mod.rs @@ -1,4 +1,4 @@ -use crate::prelude::*; +use crate::{file::FileEnv, prelude::*}; mod add; mod find; @@ -13,5 +13,5 @@ pub struct HistoryEnv { pub add: HistoryAdd, } -pub type HistoryFind = fn(&Link, &str) -> Result; +pub type HistoryFind = fn(&Link, &str, &FileEnv) -> Result; pub type HistoryAdd = fn(&Link, &str) -> Result<()>; diff --git a/src/lib.rs b/src/lib.rs index ce77f4d..e51e116 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ mod errors; pub mod feed; pub mod fetch; +pub mod file; pub mod history; pub mod prelude; mod subscriptions; @@ -10,6 +11,7 @@ mod test_utils; use feed::FeedEnv; use fetch::FetchEnv; +use file::FileEnv; use history::HistoryEnv; use prelude::*; @@ -17,15 +19,16 @@ pub struct Env { pub feed: FeedEnv, pub history: HistoryEnv, pub fetch: FetchEnv, + pub file: FileEnv, } pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> { - for channel_name in subscriptions::lines_from(subscriptions)? { + for channel_name in subscriptions::lines_from(subscriptions, &e.file)? { println!("Channel: {}", channel_name); let feed_url = (e.feed.find)(site, &channel_name, &e.fetch.get)?; for entry in (e.feed.get)(&feed_url)?.entries() { if let Some(link) = entry.links().get(0).cloned() { - if !(e.history.find)(&link, history)? { + if !(e.history.find)(&link, history, &e.file)? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); (e.fetch.download)(&link)?; (e.history.add)(&link, history)?; diff --git a/src/main.rs b/src/main.rs index 8e7f6c5..5f11f1c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use podal::file::FileEnv; use podal::prelude::*; use podal::{feed::FeedEnv, fetch::FetchEnv, history::HistoryEnv}; @@ -26,6 +27,7 @@ fn main() -> Result<()> { download: podal::fetch::download, get: podal::fetch::get, }, + file: FileEnv::default(), }, )?; diff --git a/src/subscriptions.rs b/src/subscriptions.rs index 5ecc379..edaf777 100644 --- a/src/subscriptions.rs +++ b/src/subscriptions.rs @@ -1,10 +1,10 @@ use crate::prelude::*; -use std::fs::File; +use crate::file::FileEnv; use std::io::{BufRead, BufReader}; -pub fn lines_from(file_name: &str) -> Result> { - let file = File::open(file_name)?; +pub fn lines_from(file_name: &str, e: &FileEnv) -> Result> { + let file = (e.open)(file_name)?; let reader = BufReader::new(file); let mut lines = vec![]; for line in reader.lines().flatten() { @@ -29,9 +29,10 @@ mod tests { "subscriptions.txt", include_bytes!("../test/data/subscriptions.txt"), )?; + let file_env = FileEnv::default(); //when - let result = lines_from(&file_name)?; + let result = lines_from(&file_name, &file_env)?; //then drop(dir); @@ -46,9 +47,10 @@ mod tests { "subscriptions.txt", include_bytes!("../test/data/subscriptions-blank-line.txt"), )?; + let file_env = FileEnv::default(); //when - let result = lines_from(&file_name)?; + let result = lines_from(&file_name, &file_env)?; //then drop(dir); @@ -63,9 +65,10 @@ mod tests { "subscriptions.txt", include_bytes!("../test/data/subscriptions-comment.txt"), )?; + let file_env = FileEnv::default(); //when - let result = lines_from(&file_name)?; + let result = lines_from(&file_name, &file_env)?; //then drop(dir); -- 2.45.3 From 87a53f8426f3e22a81ac8318ba47eec42110b750 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 19:23:03 +0100 Subject: [PATCH 18/26] rename subscriptions/rs as file/read.rs --- src/{file.rs => file/env.rs} | 0 src/file/mod.rs | 4 ++++ src/{subscriptions.rs => file/read.rs} | 6 +++--- src/lib.rs | 3 +-- 4 files changed, 8 insertions(+), 5 deletions(-) rename src/{file.rs => file/env.rs} (100%) create mode 100644 src/file/mod.rs rename src/{subscriptions.rs => file/read.rs} (88%) diff --git a/src/file.rs b/src/file/env.rs similarity index 100% rename from src/file.rs rename to src/file/env.rs diff --git a/src/file/mod.rs b/src/file/mod.rs new file mode 100644 index 0000000..3a55a73 --- /dev/null +++ b/src/file/mod.rs @@ -0,0 +1,4 @@ +mod env; +pub mod read; + +pub use env::FileEnv; diff --git a/src/subscriptions.rs b/src/file/read.rs similarity index 88% rename from src/subscriptions.rs rename to src/file/read.rs index edaf777..2bb892f 100644 --- a/src/subscriptions.rs +++ b/src/file/read.rs @@ -27,7 +27,7 @@ mod tests { //given let (dir, file_name) = create_text_file( "subscriptions.txt", - include_bytes!("../test/data/subscriptions.txt"), + include_bytes!("../../test/data/subscriptions.txt"), )?; let file_env = FileEnv::default(); @@ -45,7 +45,7 @@ mod tests { //given let (dir, file_name) = create_text_file( "subscriptions.txt", - include_bytes!("../test/data/subscriptions-blank-line.txt"), + include_bytes!("../../test/data/subscriptions-blank-line.txt"), )?; let file_env = FileEnv::default(); @@ -63,7 +63,7 @@ mod tests { //given let (dir, file_name) = create_text_file( "subscriptions.txt", - include_bytes!("../test/data/subscriptions-comment.txt"), + include_bytes!("../../test/data/subscriptions-comment.txt"), )?; let file_env = FileEnv::default(); diff --git a/src/lib.rs b/src/lib.rs index e51e116..74787a2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,6 @@ pub mod fetch; pub mod file; pub mod history; pub mod prelude; -mod subscriptions; #[cfg(test)] mod test_utils; @@ -23,7 +22,7 @@ pub struct Env { } pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> { - for channel_name in subscriptions::lines_from(subscriptions, &e.file)? { + for channel_name in file::read::lines_from(subscriptions, &e.file)? { println!("Channel: {}", channel_name); let feed_url = (e.feed.find)(site, &channel_name, &e.fetch.get)?; for entry in (e.feed.get)(&feed_url)?.entries() { -- 2.45.3 From 73b9ad960bb9fde93b52cdf47a14b3e6a70434f0 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 19:38:59 +0100 Subject: [PATCH 19/26] add NetworkEnv --- src/feed/find.rs | 22 ++++++++++++---------- src/feed/mod.rs | 4 ++-- src/fetch.rs | 9 +-------- src/lib.rs | 5 ++++- src/main.rs | 3 ++- src/network/env.rs | 12 ++++++++++++ src/network/mod.rs | 3 +++ 7 files changed, 36 insertions(+), 22 deletions(-) create mode 100644 src/network/env.rs create mode 100644 src/network/mod.rs diff --git a/src/feed/find.rs b/src/feed/find.rs index f37ce84..c72e35b 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -1,15 +1,14 @@ +use crate::network::NetworkEnv; use crate::prelude::*; -use crate::fetch::FetchGet; - -pub fn find(site: &str, channel_name: &str, e: &FetchGet) -> Result { +pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { if let Some(channel_prefix) = channel_name.chars().next() { if channel_prefix != '@' { return Err(format!("Channel Name must begin with an '@': {}", channel_name).into()); } } let channel_url = format!("{}{}", site, channel_name); - let response = (e)(&channel_url)?; + let response = (e.fetch_as_text)(&channel_url)?; let rss_url = scraper::Html::parse_document(&response) .select(&scraper::Selector::parse("link[title='RSS']").unwrap()) .next() @@ -23,15 +22,16 @@ pub fn find(site: &str, channel_name: &str, e: &FetchGet) -> Result { #[cfg(test)] mod tests { - use crate::fetch::Response; use super::*; #[test] fn finds_rss_url() -> Result<()> { //given - let fetch_get = &(get as FetchGet); + let network_env = NetworkEnv { + fetch_as_text: dummy_fetch_as_text, + }; //when - let result = find("site", "@channel", fetch_get)?; + let result = find("site", "@channel", &network_env)?; //then assert_eq!(result, "the-rss-url"); Ok(()) @@ -40,15 +40,17 @@ mod tests { #[test] fn error_if_channel_name_is_invalid() -> Result<()> { //given - let fetch_get = &(get as FetchGet); + let network_env = NetworkEnv { + fetch_as_text: dummy_fetch_as_text, + }; //when - let result = find("site", "invalid-channel-name", fetch_get); + let result = find("site", "invalid-channel-name", &network_env); //then assert!(result.is_err()); Ok(()) } - fn get(_url: &str) -> Result { + fn dummy_fetch_as_text(_url: &str) -> Result { Ok(r#" diff --git a/src/feed/mod.rs b/src/feed/mod.rs index 253bf26..30de824 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -1,6 +1,6 @@ use crate::prelude::*; -use crate::fetch::FetchGet; +use crate::network::NetworkEnv; mod find; mod get; @@ -14,5 +14,5 @@ pub struct FeedEnv { pub get: FeedGet, } -pub type FeedFind = fn(&str, &str, &FetchGet) -> Result; +pub type FeedFind = fn(&str, &str, &NetworkEnv) -> Result; pub type FeedGet = fn(&str) -> Result; diff --git a/src/fetch.rs b/src/fetch.rs index b0d47fd..cad48b0 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -5,11 +5,10 @@ use std::process::Command; pub struct FetchEnv { pub download: FetchDownload, - pub get: FetchGet, } pub type FetchDownload = fn(&Link) -> Result<()>; -pub type FetchGet = fn(&str) -> Result; +pub type FetchGet = fn(&str) -> Result; pub fn download(link: &Link) -> Result<()> { let cmd = "yt-dlp"; @@ -26,9 +25,3 @@ pub fn download(link: &Link) -> Result<()> { } Ok(()) } - -pub type Response = String; - -pub fn get(url: &str) -> Result { - Ok(reqwest::blocking::get(url)?.text()?) -} diff --git a/src/lib.rs b/src/lib.rs index 74787a2..549819e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ pub mod feed; pub mod fetch; pub mod file; pub mod history; +pub mod network; pub mod prelude; #[cfg(test)] @@ -12,10 +13,12 @@ use feed::FeedEnv; use fetch::FetchEnv; use file::FileEnv; use history::HistoryEnv; +use network::NetworkEnv; use prelude::*; pub struct Env { pub feed: FeedEnv, + pub network: NetworkEnv, pub history: HistoryEnv, pub fetch: FetchEnv, pub file: FileEnv, @@ -24,7 +27,7 @@ pub struct Env { pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> { for channel_name in file::read::lines_from(subscriptions, &e.file)? { println!("Channel: {}", channel_name); - let feed_url = (e.feed.find)(site, &channel_name, &e.fetch.get)?; + let feed_url = (e.feed.find)(site, &channel_name, &e.network)?; for entry in (e.feed.get)(&feed_url)?.entries() { if let Some(link) = entry.links().get(0).cloned() { if !(e.history.find)(&link, history, &e.file)? { diff --git a/src/main.rs b/src/main.rs index 5f11f1c..a13f07a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use podal::file::FileEnv; +use podal::network::NetworkEnv; use podal::prelude::*; use podal::{feed::FeedEnv, fetch::FetchEnv, history::HistoryEnv}; @@ -25,8 +26,8 @@ fn main() -> Result<()> { }, fetch: FetchEnv { download: podal::fetch::download, - get: podal::fetch::get, }, + network: NetworkEnv::default(), file: FileEnv::default(), }, )?; diff --git a/src/network/env.rs b/src/network/env.rs new file mode 100644 index 0000000..55209f5 --- /dev/null +++ b/src/network/env.rs @@ -0,0 +1,12 @@ +use crate::prelude::*; + +pub struct NetworkEnv { + pub fetch_as_text: fn(url: &str) -> Result, +} +impl Default for NetworkEnv { + fn default() -> Self { + Self { + fetch_as_text: |url| Ok(reqwest::blocking::get(url)?.text()?), + } + } +} diff --git a/src/network/mod.rs b/src/network/mod.rs new file mode 100644 index 0000000..3af1de6 --- /dev/null +++ b/src/network/mod.rs @@ -0,0 +1,3 @@ +mod env; + +pub use env::NetworkEnv; -- 2.45.3 From f21182388488d578ffb8ab0bf9164da455df0de5 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 19:55:37 +0100 Subject: [PATCH 20/26] move feed::get from FetchEnv into a pure function --- Cargo.lock | 1 + Cargo.toml | 1 + src/errors.rs | 7 +++++++ src/feed/find.rs | 7 +++++++ src/feed/get.rs | 9 --------- src/feed/mod.rs | 12 +++++++----- src/lib.rs | 2 +- src/main.rs | 1 - src/network/env.rs | 2 ++ 9 files changed, 26 insertions(+), 16 deletions(-) delete mode 100644 src/feed/get.rs diff --git a/Cargo.lock b/Cargo.lock index f20c98c..aba38b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1014,6 +1014,7 @@ name = "podal" version = "0.1.0" dependencies = [ "atom_syndication", + "bytes", "clap", "reqwest", "scraper", diff --git a/Cargo.toml b/Cargo.toml index 0c54643..a01c713 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ atom_syndication = "0.12.1" reqwest = { version = "0.11.18", features = ["json", "blocking"] } scraper = "0.17.1" clap = "4.3.19" +bytes = "1.4.0" [dev-dependencies] tempfile = "*" diff --git a/src/errors.rs b/src/errors.rs index 6769040..84c2ac6 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -4,6 +4,13 @@ use std::{fmt::Display, str::Utf8Error, string::FromUtf8Error}; pub struct Error { details: String, } +impl Error { + pub fn message(details: &str) -> Self { + Self { + details: details.to_string(), + } + } +} impl Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.details.to_string().as_str()) diff --git a/src/feed/find.rs b/src/feed/find.rs index c72e35b..9703c94 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -23,12 +23,15 @@ pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { #[cfg(test)] mod tests { + use crate::errors::Error; + use super::*; #[test] fn finds_rss_url() -> Result<()> { //given let network_env = NetworkEnv { fetch_as_text: dummy_fetch_as_text, + fetch_as_bytes: dummy_fetch_as_bytes, }; //when let result = find("site", "@channel", &network_env)?; @@ -42,6 +45,7 @@ mod tests { //given let network_env = NetworkEnv { fetch_as_text: dummy_fetch_as_text, + fetch_as_bytes: dummy_fetch_as_bytes, }; //when let result = find("site", "invalid-channel-name", &network_env); @@ -58,4 +62,7 @@ mod tests { "# .to_string()) } + fn dummy_fetch_as_bytes(_url: &str) -> Result { + Err(Error::message("Not implemented")) + } } diff --git a/src/feed/get.rs b/src/feed/get.rs deleted file mode 100644 index 1a7ea40..0000000 --- a/src/feed/get.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::prelude::*; - -use atom_syndication::Feed; - -pub fn reqwest_blocking_get(url: &str) -> Result { - let content = reqwest::blocking::get(url)?.bytes()?; - let channel = Feed::read_from(&content[..])?; - Ok(channel) -} diff --git a/src/feed/mod.rs b/src/feed/mod.rs index 30de824..d8c68d0 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -1,18 +1,20 @@ use crate::prelude::*; use crate::network::NetworkEnv; +use atom_syndication::Feed; mod find; -mod get; -use atom_syndication::Feed; pub use find::find; -pub use get::reqwest_blocking_get; pub struct FeedEnv { pub find: FeedFind, - pub get: FeedGet, } pub type FeedFind = fn(&str, &str, &NetworkEnv) -> Result; -pub type FeedGet = fn(&str) -> Result; + +pub fn get(url: &str, e: &NetworkEnv) -> Result { + let content = (e.fetch_as_bytes)(url)?; + let channel = Feed::read_from(&content[..])?; + Ok(channel) +} diff --git a/src/lib.rs b/src/lib.rs index 549819e..9137f13 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,7 +28,7 @@ pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> for channel_name in file::read::lines_from(subscriptions, &e.file)? { println!("Channel: {}", channel_name); let feed_url = (e.feed.find)(site, &channel_name, &e.network)?; - for entry in (e.feed.get)(&feed_url)?.entries() { + for entry in feed::get(&feed_url, &e.network)?.entries() { if let Some(link) = entry.links().get(0).cloned() { if !(e.history.find)(&link, history, &e.file)? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); diff --git a/src/main.rs b/src/main.rs index a13f07a..395e811 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,6 @@ fn main() -> Result<()> { podal::Env { feed: FeedEnv { find: podal::feed::find, - get: podal::feed::reqwest_blocking_get, }, history: HistoryEnv { diff --git a/src/network/env.rs b/src/network/env.rs index 55209f5..8ea937f 100644 --- a/src/network/env.rs +++ b/src/network/env.rs @@ -2,11 +2,13 @@ use crate::prelude::*; pub struct NetworkEnv { pub fetch_as_text: fn(url: &str) -> Result, + pub fetch_as_bytes: fn(url: &str) -> Result, } impl Default for NetworkEnv { fn default() -> Self { Self { fetch_as_text: |url| Ok(reqwest::blocking::get(url)?.text()?), + fetch_as_bytes: |url| Ok(reqwest::blocking::get(url)?.bytes()?), } } } -- 2.45.3 From 95ed1735e77b8dfe01e48b5864496142fe2217b9 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 19:59:16 +0100 Subject: [PATCH 21/26] remove FeedEnv --- src/feed/mod.rs | 6 ------ src/fetch.rs | 3 +-- src/history/add.rs | 3 ++- src/history/find.rs | 3 ++- src/history/mod.rs | 12 +----------- src/lib.rs | 10 +++------- src/main.rs | 10 +--------- 7 files changed, 10 insertions(+), 37 deletions(-) diff --git a/src/feed/mod.rs b/src/feed/mod.rs index d8c68d0..c3486dc 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -7,12 +7,6 @@ mod find; pub use find::find; -pub struct FeedEnv { - pub find: FeedFind, -} - -pub type FeedFind = fn(&str, &str, &NetworkEnv) -> Result; - pub fn get(url: &str, e: &NetworkEnv) -> Result { let content = (e.fetch_as_bytes)(url)?; let channel = Feed::read_from(&content[..])?; diff --git a/src/fetch.rs b/src/fetch.rs index cad48b0..e34dea4 100644 --- a/src/fetch.rs +++ b/src/fetch.rs @@ -1,6 +1,5 @@ -use crate::prelude::*; +use crate::{history::Link, prelude::*}; -use atom_syndication::Link; use std::process::Command; pub struct FetchEnv { diff --git a/src/history/add.rs b/src/history/add.rs index 756edf2..580c219 100644 --- a/src/history/add.rs +++ b/src/history/add.rs @@ -1,9 +1,10 @@ use crate::prelude::*; -use atom_syndication::Link; use std::fs::OpenOptions; use std::io::prelude::*; +use super::Link; + pub fn add(link: &Link, file_name: &str) -> Result<()> { let mut file = OpenOptions::new() .write(true) diff --git a/src/history/find.rs b/src/history/find.rs index 70162d2..bb6cdc2 100644 --- a/src/history/find.rs +++ b/src/history/find.rs @@ -1,8 +1,9 @@ use crate::{file::FileEnv, prelude::*}; -use atom_syndication::Link; use std::io::{BufRead, BufReader}; +use super::Link; + pub fn find(link: &Link, file_name: &str, e: &FileEnv) -> Result { if let Ok(file) = (e.open)(file_name) { let reader = BufReader::new(file); diff --git a/src/history/mod.rs b/src/history/mod.rs index 47f7e36..a658069 100644 --- a/src/history/mod.rs +++ b/src/history/mod.rs @@ -1,17 +1,7 @@ -use crate::{file::FileEnv, prelude::*}; - mod add; mod find; pub use add::add; pub use find::find; -type Link = atom_syndication::Link; - -pub struct HistoryEnv { - pub find: HistoryFind, - pub add: HistoryAdd, -} - -pub type HistoryFind = fn(&Link, &str, &FileEnv) -> Result; -pub type HistoryAdd = fn(&Link, &str) -> Result<()>; +pub type Link = atom_syndication::Link; diff --git a/src/lib.rs b/src/lib.rs index 9137f13..57fab72 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,17 +9,13 @@ pub mod prelude; #[cfg(test)] mod test_utils; -use feed::FeedEnv; use fetch::FetchEnv; use file::FileEnv; -use history::HistoryEnv; use network::NetworkEnv; use prelude::*; pub struct Env { - pub feed: FeedEnv, pub network: NetworkEnv, - pub history: HistoryEnv, pub fetch: FetchEnv, pub file: FileEnv, } @@ -27,13 +23,13 @@ pub struct Env { pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> { for channel_name in file::read::lines_from(subscriptions, &e.file)? { println!("Channel: {}", channel_name); - let feed_url = (e.feed.find)(site, &channel_name, &e.network)?; + let feed_url = feed::find(site, &channel_name, &e.network)?; for entry in feed::get(&feed_url, &e.network)?.entries() { if let Some(link) = entry.links().get(0).cloned() { - if !(e.history.find)(&link, history, &e.file)? { + if !history::find(&link, history, &e.file)? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); (e.fetch.download)(&link)?; - (e.history.add)(&link, history)?; + history::add(&link, history)?; } } } diff --git a/src/main.rs b/src/main.rs index 395e811..83d2966 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,7 @@ use podal::file::FileEnv; use podal::network::NetworkEnv; use podal::prelude::*; -use podal::{feed::FeedEnv, fetch::FetchEnv, history::HistoryEnv}; +use podal::fetch::FetchEnv; fn main() -> Result<()> { println!("Podal"); @@ -15,14 +15,6 @@ fn main() -> Result<()> { history, site, podal::Env { - feed: FeedEnv { - find: podal::feed::find, - }, - - history: HistoryEnv { - find: podal::history::find, - add: podal::history::add, - }, fetch: FetchEnv { download: podal::fetch::download, }, -- 2.45.3 From f5974acd256750779d03faa4569c518a00a674bf Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 20:13:28 +0100 Subject: [PATCH 22/26] extracted append_line to FileEnv --- src/file/env.rs | 15 ++++++++++++++- src/history/add.rs | 19 +++++-------------- src/lib.rs | 2 +- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/file/env.rs b/src/file/env.rs index 83c6006..005ced2 100644 --- a/src/file/env.rs +++ b/src/file/env.rs @@ -1,13 +1,26 @@ -use std::fs::File; +use std::fs::{File, OpenOptions}; +use std::io::Write; pub struct FileEnv { pub open: FileOpen, + pub append_line: FileAppendLine, } impl Default for FileEnv { fn default() -> Self { Self { open: |path| File::open(path), + append_line: |file_name, line| { + let mut file = OpenOptions::new() + .write(true) + .append(true) + .create(true) + .open(file_name) + .unwrap(); + writeln!(file, "{}", line)?; + Ok(()) + }, } } } pub type FileOpen = fn(path: &str) -> std::io::Result; +pub type FileAppendLine = fn(paht: &str, line: &str) -> std::io::Result<()>; diff --git a/src/history/add.rs b/src/history/add.rs index 580c219..90e2628 100644 --- a/src/history/add.rs +++ b/src/history/add.rs @@ -1,19 +1,10 @@ +use crate::file::FileEnv; use crate::prelude::*; -use std::fs::OpenOptions; -use std::io::prelude::*; - use super::Link; -pub fn add(link: &Link, file_name: &str) -> Result<()> { - let mut file = OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(file_name) - .unwrap(); - - writeln!(file, "{}", link.href)?; +pub fn add(link: &Link, file_name: &str, e: &FileEnv) -> Result<()> { + (e.append_line)(file_name, &link.href)?; Ok(()) } @@ -43,7 +34,7 @@ mod tests { length: None, }; //when - add(&link, &file_name)?; + add(&link, &file_name, &FileEnv::default())?; //then let content: Vec = read_text_file(&file_name)?; @@ -72,7 +63,7 @@ mod tests { length: None, }; //when - add(&link, &file_name)?; + add(&link, &file_name, &FileEnv::default())?; //then let content: Vec = read_text_file(&file_name)?; diff --git a/src/lib.rs b/src/lib.rs index 57fab72..8f4d03a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,7 @@ pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> if !history::find(&link, history, &e.file)? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); (e.fetch.download)(&link)?; - history::add(&link, history)?; + history::add(&link, history, &e.file)?; } } } -- 2.45.3 From 7c752b7ba7f1cdbf4e5eed4266b274d50c09e947 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 20:22:47 +0100 Subject: [PATCH 23/26] fold FetchEnv into NetworkEnv --- src/feed/find.rs | 5 +++++ src/fetch.rs | 26 -------------------------- src/lib.rs | 8 +++----- src/main.rs | 5 ----- src/network/env.rs | 18 ++++++++++++++++++ 5 files changed, 26 insertions(+), 36 deletions(-) delete mode 100644 src/fetch.rs diff --git a/src/feed/find.rs b/src/feed/find.rs index 9703c94..27cef15 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -32,6 +32,7 @@ mod tests { let network_env = NetworkEnv { fetch_as_text: dummy_fetch_as_text, fetch_as_bytes: dummy_fetch_as_bytes, + download_as_mp3: dummy_download_as_mp3, }; //when let result = find("site", "@channel", &network_env)?; @@ -46,6 +47,7 @@ mod tests { let network_env = NetworkEnv { fetch_as_text: dummy_fetch_as_text, fetch_as_bytes: dummy_fetch_as_bytes, + download_as_mp3: dummy_download_as_mp3, }; //when let result = find("site", "invalid-channel-name", &network_env); @@ -65,4 +67,7 @@ mod tests { fn dummy_fetch_as_bytes(_url: &str) -> Result { Err(Error::message("Not implemented")) } + fn dummy_download_as_mp3(_url: &str) -> Result<()> { + Err(Error::message("Not implemented")) + } } diff --git a/src/fetch.rs b/src/fetch.rs deleted file mode 100644 index e34dea4..0000000 --- a/src/fetch.rs +++ /dev/null @@ -1,26 +0,0 @@ -use crate::{history::Link, prelude::*}; - -use std::process::Command; - -pub struct FetchEnv { - pub download: FetchDownload, -} - -pub type FetchDownload = fn(&Link) -> Result<()>; -pub type FetchGet = fn(&str) -> Result; - -pub fn download(link: &Link) -> Result<()> { - let cmd = "yt-dlp"; - // println!("{} --extract-audio --audio-format mp3 {}", cmd, &link.href); - let output = Command::new(cmd) - .arg("--extract-audio") - .arg("--audio-format") - .arg("mp3") - .arg(&link.href) - .output()?; - if !output.stderr.is_empty() { - eprintln!("Error: {}", String::from_utf8(output.stderr)?); - println!("{}", String::from_utf8(output.stdout)?); - } - Ok(()) -} diff --git a/src/lib.rs b/src/lib.rs index 8f4d03a..b7f1a9d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ +use prelude::*; + mod errors; pub mod feed; -pub mod fetch; pub mod file; pub mod history; pub mod network; @@ -9,14 +10,11 @@ pub mod prelude; #[cfg(test)] mod test_utils; -use fetch::FetchEnv; use file::FileEnv; use network::NetworkEnv; -use prelude::*; pub struct Env { pub network: NetworkEnv, - pub fetch: FetchEnv, pub file: FileEnv, } @@ -28,7 +26,7 @@ pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> if let Some(link) = entry.links().get(0).cloned() { if !history::find(&link, history, &e.file)? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); - (e.fetch.download)(&link)?; + (e.network.download_as_mp3)(&link.href)?; history::add(&link, history, &e.file)?; } } diff --git a/src/main.rs b/src/main.rs index 83d2966..c1c31ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,8 +2,6 @@ use podal::file::FileEnv; use podal::network::NetworkEnv; use podal::prelude::*; -use podal::fetch::FetchEnv; - fn main() -> Result<()> { println!("Podal"); let subscriptions = "subscriptions.txt"; @@ -15,9 +13,6 @@ fn main() -> Result<()> { history, site, podal::Env { - fetch: FetchEnv { - download: podal::fetch::download, - }, network: NetworkEnv::default(), file: FileEnv::default(), }, diff --git a/src/network/env.rs b/src/network/env.rs index 8ea937f..473fac3 100644 --- a/src/network/env.rs +++ b/src/network/env.rs @@ -1,14 +1,32 @@ +use std::process::Command; + use crate::prelude::*; pub struct NetworkEnv { pub fetch_as_text: fn(url: &str) -> Result, pub fetch_as_bytes: fn(url: &str) -> Result, + pub download_as_mp3: fn(url: &str) -> Result<()>, } impl Default for NetworkEnv { fn default() -> Self { Self { fetch_as_text: |url| Ok(reqwest::blocking::get(url)?.text()?), fetch_as_bytes: |url| Ok(reqwest::blocking::get(url)?.bytes()?), + download_as_mp3: |url| { + let cmd = "yt-dlp"; + // println!("{} --extract-audio --audio-format mp3 {}", cmd, &link.href); + let output = Command::new(cmd) + .arg("--extract-audio") + .arg("--audio-format") + .arg("mp3") + .arg(&url) + .output()?; + if !output.stderr.is_empty() { + eprintln!("Error: {}", String::from_utf8(output.stderr)?); + println!("{}", String::from_utf8(output.stdout)?); + } + Ok(()) + }, } } } -- 2.45.3 From 9cb02f67900d281661d0a3d5f5cc3b19acda2510 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 20:22:57 +0100 Subject: [PATCH 24/26] clean up Error --- src/errors.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 84c2ac6..70fed6e 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,8 +1,8 @@ -use std::{fmt::Display, str::Utf8Error, string::FromUtf8Error}; +use std::{str::Utf8Error, string::FromUtf8Error}; #[derive(Debug)] pub struct Error { - details: String, + pub details: String, } impl Error { pub fn message(details: &str) -> Self { @@ -11,11 +11,6 @@ impl Error { } } } -impl Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.details.to_string().as_str()) - } -} impl From for Error { fn from(value: Utf8Error) -> Self { Self { -- 2.45.3 From 95fce1e6d8a204b379c4e99e77d3028a0d8def05 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 20:23:38 +0100 Subject: [PATCH 25/26] clippy fix --- src/network/env.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/network/env.rs b/src/network/env.rs index 473fac3..7cbd626 100644 --- a/src/network/env.rs +++ b/src/network/env.rs @@ -19,7 +19,7 @@ impl Default for NetworkEnv { .arg("--extract-audio") .arg("--audio-format") .arg("mp3") - .arg(&url) + .arg(url) .output()?; if !output.stderr.is_empty() { eprintln!("Error: {}", String::from_utf8(output.stderr)?); -- 2.45.3 From b5a3f5ece2f692cdeebaf968dee24c7d48e4c597 Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 20:30:57 +0100 Subject: [PATCH 26/26] woodpeckerci: remove when clause It's preventing build from running when pushing to a PR branch --- .woodpecker.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 45249c4..0aca8df 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -2,11 +2,6 @@ steps: build: image: rust:latest pull: true - when: - - event: pull_request - repo: kemitix/podal - - event: push - branch: main commands: - rustup component add rustfmt clippy - cargo --version -- 2.45.3