From 83081a20510338d5e63bc4c3010f8fa311fa840a Mon Sep 17 00:00:00 2001 From: Paul Campbell Date: Sat, 29 Jul 2023 20:36:03 +0100 Subject: [PATCH] i5-add-tests (part 3) (#9) Reviewed-on: https://git.kemitix.net/kemitix/podal/pulls/9 Co-authored-by: Paul Campbell Co-committed-by: Paul Campbell --- .gitignore | 6 +++-- .woodpecker.yml | 5 ++-- Cargo.lock | 1 + Cargo.toml | 1 + justfile | 23 +++++++++++++++++ src/errors.rs | 12 +++++---- src/feed/find.rs | 34 ++++++++++++++++++-------- src/feed/get.rs | 9 ------- src/feed/mod.rs | 16 +++++------- src/fetch.rs | 34 -------------------------- src/file/env.rs | 26 ++++++++++++++++++++ src/file/mod.rs | 4 +++ src/{subscriptions.rs => file/read.rs} | 21 +++++++++------- src/history/add.rs | 20 +++++---------- src/history/find.rs | 23 +++++++++-------- src/history/mod.rs | 12 +-------- src/lib.rs | 29 +++++++++++----------- src/main.rs | 19 +++----------- src/network/env.rs | 32 ++++++++++++++++++++++++ src/network/mod.rs | 3 +++ 20 files changed, 182 insertions(+), 148 deletions(-) create mode 100644 justfile delete mode 100644 src/feed/get.rs delete mode 100644 src/fetch.rs create mode 100644 src/file/env.rs create mode 100644 src/file/mod.rs rename src/{subscriptions.rs => file/read.rs} (65%) create mode 100644 src/network/env.rs create mode 100644 src/network/mod.rs diff --git a/.gitignore b/.gitignore index c233fd4..a3bd0cd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ target *.mp3 *.webm -/subscriptions.txt -/downloaded.txt +subscriptions.txt +downloaded.txt +coverage +*.profraw diff --git a/.woodpecker.yml b/.woodpecker.yml index 98731e8..0aca8df 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,6 +1,7 @@ -pipeline: +steps: build: - image: rust + image: rust:latest + pull: true commands: - rustup component add rustfmt clippy - cargo --version diff --git a/Cargo.lock b/Cargo.lock index f20c98c..aba38b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1014,6 +1014,7 @@ name = "podal" version = "0.1.0" dependencies = [ "atom_syndication", + "bytes", "clap", "reqwest", "scraper", diff --git a/Cargo.toml b/Cargo.toml index 0c54643..a01c713 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ atom_syndication = "0.12.1" reqwest = { version = "0.11.18", features = ["json", "blocking"] } scraper = "0.17.1" clap = "4.3.19" +bytes = "1.4.0" [dev-dependencies] tempfile = "*" diff --git a/justfile b/justfile new file mode 100644 index 0000000..933e5e8 --- /dev/null +++ b/justfile @@ -0,0 +1,23 @@ +coverage-init: + cargo install grcov + rustup component add llvm-tools + +coverage: + #!/usr/bin/env bash + set -e + rm -rf ./target + just clean + export RUSTFLAGS="-Zinstrument-coverage" LLVM_PROFILE_FILE="just-%p-%m.profraw" + cargo +nightly build + cargo +nightly test + just generate + just server + +generate: + grcov . --binary-path ./target/debug/ -s . -t html --branch --ignore-not-existing --ignore "./target/" -o ./coverage/ + +server: + cd ./coverage && python3 -m http.server 8001 + +clean: + rm -rf ./*.prof* ./coverage/ diff --git a/src/errors.rs b/src/errors.rs index 6769040..70fed6e 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,12 +1,14 @@ -use std::{fmt::Display, str::Utf8Error, string::FromUtf8Error}; +use std::{str::Utf8Error, string::FromUtf8Error}; #[derive(Debug)] pub struct Error { - details: String, + pub details: String, } -impl Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.details.to_string().as_str()) +impl Error { + pub fn message(details: &str) -> Self { + Self { + details: details.to_string(), + } } } impl From for Error { diff --git a/src/feed/find.rs b/src/feed/find.rs index f37ce84..27cef15 100644 --- a/src/feed/find.rs +++ b/src/feed/find.rs @@ -1,15 +1,14 @@ +use crate::network::NetworkEnv; use crate::prelude::*; -use crate::fetch::FetchGet; - -pub fn find(site: &str, channel_name: &str, e: &FetchGet) -> Result { +pub fn find(site: &str, channel_name: &str, e: &NetworkEnv) -> Result { if let Some(channel_prefix) = channel_name.chars().next() { if channel_prefix != '@' { return Err(format!("Channel Name must begin with an '@': {}", channel_name).into()); } } let channel_url = format!("{}{}", site, channel_name); - let response = (e)(&channel_url)?; + let response = (e.fetch_as_text)(&channel_url)?; let rss_url = scraper::Html::parse_document(&response) .select(&scraper::Selector::parse("link[title='RSS']").unwrap()) .next() @@ -23,15 +22,20 @@ pub fn find(site: &str, channel_name: &str, e: &FetchGet) -> Result { #[cfg(test)] mod tests { - use crate::fetch::Response; + + use crate::errors::Error; use super::*; #[test] fn finds_rss_url() -> Result<()> { //given - let fetch_get = &(get as FetchGet); + let network_env = NetworkEnv { + fetch_as_text: dummy_fetch_as_text, + fetch_as_bytes: dummy_fetch_as_bytes, + download_as_mp3: dummy_download_as_mp3, + }; //when - let result = find("site", "@channel", fetch_get)?; + let result = find("site", "@channel", &network_env)?; //then assert_eq!(result, "the-rss-url"); Ok(()) @@ -40,15 +44,19 @@ mod tests { #[test] fn error_if_channel_name_is_invalid() -> Result<()> { //given - let fetch_get = &(get as FetchGet); + let network_env = NetworkEnv { + fetch_as_text: dummy_fetch_as_text, + fetch_as_bytes: dummy_fetch_as_bytes, + download_as_mp3: dummy_download_as_mp3, + }; //when - let result = find("site", "invalid-channel-name", fetch_get); + let result = find("site", "invalid-channel-name", &network_env); //then assert!(result.is_err()); Ok(()) } - fn get(_url: &str) -> Result { + fn dummy_fetch_as_text(_url: &str) -> Result { Ok(r#" @@ -56,4 +64,10 @@ mod tests { "# .to_string()) } + fn dummy_fetch_as_bytes(_url: &str) -> Result { + Err(Error::message("Not implemented")) + } + fn dummy_download_as_mp3(_url: &str) -> Result<()> { + Err(Error::message("Not implemented")) + } } diff --git a/src/feed/get.rs b/src/feed/get.rs deleted file mode 100644 index 1a7ea40..0000000 --- a/src/feed/get.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::prelude::*; - -use atom_syndication::Feed; - -pub fn reqwest_blocking_get(url: &str) -> Result { - let content = reqwest::blocking::get(url)?.bytes()?; - let channel = Feed::read_from(&content[..])?; - Ok(channel) -} diff --git a/src/feed/mod.rs b/src/feed/mod.rs index 253bf26..c3486dc 100644 --- a/src/feed/mod.rs +++ b/src/feed/mod.rs @@ -1,18 +1,14 @@ use crate::prelude::*; -use crate::fetch::FetchGet; +use crate::network::NetworkEnv; +use atom_syndication::Feed; mod find; -mod get; -use atom_syndication::Feed; pub use find::find; -pub use get::reqwest_blocking_get; -pub struct FeedEnv { - pub find: FeedFind, - pub get: FeedGet, +pub fn get(url: &str, e: &NetworkEnv) -> Result { + let content = (e.fetch_as_bytes)(url)?; + let channel = Feed::read_from(&content[..])?; + Ok(channel) } - -pub type FeedFind = fn(&str, &str, &FetchGet) -> Result; -pub type FeedGet = fn(&str) -> Result; diff --git a/src/fetch.rs b/src/fetch.rs deleted file mode 100644 index b0d47fd..0000000 --- a/src/fetch.rs +++ /dev/null @@ -1,34 +0,0 @@ -use crate::prelude::*; - -use atom_syndication::Link; -use std::process::Command; - -pub struct FetchEnv { - pub download: FetchDownload, - pub get: FetchGet, -} - -pub type FetchDownload = fn(&Link) -> Result<()>; -pub type FetchGet = fn(&str) -> Result; - -pub fn download(link: &Link) -> Result<()> { - let cmd = "yt-dlp"; - // println!("{} --extract-audio --audio-format mp3 {}", cmd, &link.href); - let output = Command::new(cmd) - .arg("--extract-audio") - .arg("--audio-format") - .arg("mp3") - .arg(&link.href) - .output()?; - if !output.stderr.is_empty() { - eprintln!("Error: {}", String::from_utf8(output.stderr)?); - println!("{}", String::from_utf8(output.stdout)?); - } - Ok(()) -} - -pub type Response = String; - -pub fn get(url: &str) -> Result { - Ok(reqwest::blocking::get(url)?.text()?) -} diff --git a/src/file/env.rs b/src/file/env.rs new file mode 100644 index 0000000..005ced2 --- /dev/null +++ b/src/file/env.rs @@ -0,0 +1,26 @@ +use std::fs::{File, OpenOptions}; +use std::io::Write; + +pub struct FileEnv { + pub open: FileOpen, + pub append_line: FileAppendLine, +} +impl Default for FileEnv { + fn default() -> Self { + Self { + open: |path| File::open(path), + append_line: |file_name, line| { + let mut file = OpenOptions::new() + .write(true) + .append(true) + .create(true) + .open(file_name) + .unwrap(); + writeln!(file, "{}", line)?; + Ok(()) + }, + } + } +} +pub type FileOpen = fn(path: &str) -> std::io::Result; +pub type FileAppendLine = fn(paht: &str, line: &str) -> std::io::Result<()>; diff --git a/src/file/mod.rs b/src/file/mod.rs new file mode 100644 index 0000000..3a55a73 --- /dev/null +++ b/src/file/mod.rs @@ -0,0 +1,4 @@ +mod env; +pub mod read; + +pub use env::FileEnv; diff --git a/src/subscriptions.rs b/src/file/read.rs similarity index 65% rename from src/subscriptions.rs rename to src/file/read.rs index 5ecc379..2bb892f 100644 --- a/src/subscriptions.rs +++ b/src/file/read.rs @@ -1,10 +1,10 @@ use crate::prelude::*; -use std::fs::File; +use crate::file::FileEnv; use std::io::{BufRead, BufReader}; -pub fn lines_from(file_name: &str) -> Result> { - let file = File::open(file_name)?; +pub fn lines_from(file_name: &str, e: &FileEnv) -> Result> { + let file = (e.open)(file_name)?; let reader = BufReader::new(file); let mut lines = vec![]; for line in reader.lines().flatten() { @@ -27,11 +27,12 @@ mod tests { //given let (dir, file_name) = create_text_file( "subscriptions.txt", - include_bytes!("../test/data/subscriptions.txt"), + include_bytes!("../../test/data/subscriptions.txt"), )?; + let file_env = FileEnv::default(); //when - let result = lines_from(&file_name)?; + let result = lines_from(&file_name, &file_env)?; //then drop(dir); @@ -44,11 +45,12 @@ mod tests { //given let (dir, file_name) = create_text_file( "subscriptions.txt", - include_bytes!("../test/data/subscriptions-blank-line.txt"), + include_bytes!("../../test/data/subscriptions-blank-line.txt"), )?; + let file_env = FileEnv::default(); //when - let result = lines_from(&file_name)?; + let result = lines_from(&file_name, &file_env)?; //then drop(dir); @@ -61,11 +63,12 @@ mod tests { //given let (dir, file_name) = create_text_file( "subscriptions.txt", - include_bytes!("../test/data/subscriptions-comment.txt"), + include_bytes!("../../test/data/subscriptions-comment.txt"), )?; + let file_env = FileEnv::default(); //when - let result = lines_from(&file_name)?; + let result = lines_from(&file_name, &file_env)?; //then drop(dir); diff --git a/src/history/add.rs b/src/history/add.rs index 756edf2..90e2628 100644 --- a/src/history/add.rs +++ b/src/history/add.rs @@ -1,18 +1,10 @@ +use crate::file::FileEnv; use crate::prelude::*; -use atom_syndication::Link; -use std::fs::OpenOptions; -use std::io::prelude::*; +use super::Link; -pub fn add(link: &Link, file_name: &str) -> Result<()> { - let mut file = OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(file_name) - .unwrap(); - - writeln!(file, "{}", link.href)?; +pub fn add(link: &Link, file_name: &str, e: &FileEnv) -> Result<()> { + (e.append_line)(file_name, &link.href)?; Ok(()) } @@ -42,7 +34,7 @@ mod tests { length: None, }; //when - add(&link, &file_name)?; + add(&link, &file_name, &FileEnv::default())?; //then let content: Vec = read_text_file(&file_name)?; @@ -71,7 +63,7 @@ mod tests { length: None, }; //when - add(&link, &file_name)?; + add(&link, &file_name, &FileEnv::default())?; //then let content: Vec = read_text_file(&file_name)?; diff --git a/src/history/find.rs b/src/history/find.rs index c1dd584..bb6cdc2 100644 --- a/src/history/find.rs +++ b/src/history/find.rs @@ -1,11 +1,11 @@ -use crate::prelude::*; +use crate::{file::FileEnv, prelude::*}; -use atom_syndication::Link; -use std::fs::File; use std::io::{BufRead, BufReader}; -pub fn find(link: &Link, file_name: &str) -> Result { - if let Ok(file) = File::open(file_name) { +use super::Link; + +pub fn find(link: &Link, file_name: &str, e: &FileEnv) -> Result { + if let Ok(file) = (e.open)(file_name) { let reader = BufReader::new(file); for line in reader.lines() { if line? == link.href { @@ -34,14 +34,13 @@ mod test { title: None, length: None, }; - //when - let result = find(&link, &file_name)?; + let result = find(&link, &file_name, &FileEnv::default())?; //then drop(dir); - assert_eq!(result, true); + assert!(result); Ok(()) } @@ -61,12 +60,12 @@ mod test { }; //when - let result = find(&link, &file_name)?; + let result = find(&link, &file_name, &FileEnv::default())?; //then drop(dir); - assert_eq!(result, false); + assert!(!result); Ok(()) } @@ -88,12 +87,12 @@ mod test { }; //when - let result = find(&link, &file_name)?; + let result = find(&link, &file_name, &FileEnv::default())?; //then drop(dir); - assert_eq!(result, false); + assert!(!result); Ok(()) } diff --git a/src/history/mod.rs b/src/history/mod.rs index d6f7255..a658069 100644 --- a/src/history/mod.rs +++ b/src/history/mod.rs @@ -1,17 +1,7 @@ -use crate::prelude::*; - mod add; mod find; pub use add::add; pub use find::find; -type Link = atom_syndication::Link; - -pub struct HistoryEnv { - pub find: HistoryFind, - pub add: HistoryAdd, -} - -pub type HistoryFind = fn(&Link, &str) -> Result; -pub type HistoryAdd = fn(&Link, &str) -> Result<()>; +pub type Link = atom_syndication::Link; diff --git a/src/lib.rs b/src/lib.rs index ce77f4d..b7f1a9d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,34 +1,33 @@ +use prelude::*; + mod errors; pub mod feed; -pub mod fetch; +pub mod file; pub mod history; +pub mod network; pub mod prelude; -mod subscriptions; #[cfg(test)] mod test_utils; -use feed::FeedEnv; -use fetch::FetchEnv; -use history::HistoryEnv; -use prelude::*; +use file::FileEnv; +use network::NetworkEnv; pub struct Env { - pub feed: FeedEnv, - pub history: HistoryEnv, - pub fetch: FetchEnv, + pub network: NetworkEnv, + pub file: FileEnv, } pub fn run(subscriptions: &str, history: &str, site: &str, e: Env) -> Result<()> { - for channel_name in subscriptions::lines_from(subscriptions)? { + for channel_name in file::read::lines_from(subscriptions, &e.file)? { println!("Channel: {}", channel_name); - let feed_url = (e.feed.find)(site, &channel_name, &e.fetch.get)?; - for entry in (e.feed.get)(&feed_url)?.entries() { + let feed_url = feed::find(site, &channel_name, &e.network)?; + for entry in feed::get(&feed_url, &e.network)?.entries() { if let Some(link) = entry.links().get(0).cloned() { - if !(e.history.find)(&link, history)? { + if !history::find(&link, history, &e.file)? { println!("Downloading {}: {}", &channel_name, entry.title().as_str()); - (e.fetch.download)(&link)?; - (e.history.add)(&link, history)?; + (e.network.download_as_mp3)(&link.href)?; + history::add(&link, history, &e.file)?; } } } diff --git a/src/main.rs b/src/main.rs index 8e7f6c5..c1c31ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ +use podal::file::FileEnv; +use podal::network::NetworkEnv; use podal::prelude::*; -use podal::{feed::FeedEnv, fetch::FetchEnv, history::HistoryEnv}; - fn main() -> Result<()> { println!("Podal"); let subscriptions = "subscriptions.txt"; @@ -13,19 +13,8 @@ fn main() -> Result<()> { history, site, podal::Env { - feed: FeedEnv { - find: podal::feed::find, - get: podal::feed::reqwest_blocking_get, - }, - - history: HistoryEnv { - find: podal::history::find, - add: podal::history::add, - }, - fetch: FetchEnv { - download: podal::fetch::download, - get: podal::fetch::get, - }, + network: NetworkEnv::default(), + file: FileEnv::default(), }, )?; diff --git a/src/network/env.rs b/src/network/env.rs new file mode 100644 index 0000000..7cbd626 --- /dev/null +++ b/src/network/env.rs @@ -0,0 +1,32 @@ +use std::process::Command; + +use crate::prelude::*; + +pub struct NetworkEnv { + pub fetch_as_text: fn(url: &str) -> Result, + pub fetch_as_bytes: fn(url: &str) -> Result, + pub download_as_mp3: fn(url: &str) -> Result<()>, +} +impl Default for NetworkEnv { + fn default() -> Self { + Self { + fetch_as_text: |url| Ok(reqwest::blocking::get(url)?.text()?), + fetch_as_bytes: |url| Ok(reqwest::blocking::get(url)?.bytes()?), + download_as_mp3: |url| { + let cmd = "yt-dlp"; + // println!("{} --extract-audio --audio-format mp3 {}", cmd, &link.href); + let output = Command::new(cmd) + .arg("--extract-audio") + .arg("--audio-format") + .arg("mp3") + .arg(url) + .output()?; + if !output.stderr.is_empty() { + eprintln!("Error: {}", String::from_utf8(output.stderr)?); + println!("{}", String::from_utf8(output.stdout)?); + } + Ok(()) + }, + } + } +} diff --git a/src/network/mod.rs b/src/network/mod.rs new file mode 100644 index 0000000..3af1de6 --- /dev/null +++ b/src/network/mod.rs @@ -0,0 +1,3 @@ +mod env; + +pub use env::NetworkEnv;