kemitix/paperoni
1
0
Fork 0
Code Issues 4 Pull requests 2 Projects Releases Packages Wiki Activity

Add basic redirect provided by surf and early exit of the program if the response is not a 200

Browse source
This commit is contained in:
Kenneth Gitere 2020-11-24 17:44:31 +03:00
parent 5f99bddc10
commit 725c73c83f
2 changed files with 18 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/extractor.rs
View file

@ -74,7 +74,11 @@ impl Extractor {
let abs_url = get_absolute_url(&img_url, article_origin);
async_download_tasks.push(task::spawn(async move {
let mut img_response = surf::get(&abs_url).await.expect("Unable to retrieve file");
let mut img_response = surf::Client::new()
.with(surf::middleware::Redirect::default())
.get(&abs_url)
.await
.expect("Unable to retrieve file");
let img_content: Vec<u8> = img_response.body_bytes().await.unwrap();
let img_mime = img_response
.content_type()

24
src/main.rs
View file

@ -23,24 +23,26 @@ fn main() {
type HTMLResource = (String, String);
async fn fetch_url(url: &str) -> HTMLResource {
async fn fetch_url(url: &str) -> Result<HTMLResource, Box<dyn std::error::Error>> {
let client = surf::Client::new();
println!("Fetching...");
// TODO: Add middleware for following redirects
(
url.to_string(),
client
.get(url)
.recv_string()
.await
.expect("Unable to fetch URL"),
)
let mut res = client
.with(surf::middleware::Redirect::default())
.get(url)
.send()
.await
.expect(&format!("Unable to fetch {}", url));
if res.status() == 200 {
Ok((url.to_string(), res.body_string().await?))
} else {
Err("Request failed to return HTTP 200".into())
}
}
fn download(urls: Vec<String>) {
let mut async_url_tasks = Vec::with_capacity(urls.len());
for url in urls {
async_url_tasks.push(task::spawn(async move { fetch_url(&url).await }));
async_url_tasks.push(task::spawn(async move { fetch_url(&url).await.unwrap() }));
}
task::block_on(async {
for url_task in async_url_tasks {