Add basic redirect provided by surf and early exit of the program if the response is not a 200
This commit is contained in:
parent
5f99bddc10
commit
725c73c83f
2 changed files with 18 additions and 12 deletions
|
@ -74,7 +74,11 @@ impl Extractor {
|
|||
let abs_url = get_absolute_url(&img_url, article_origin);
|
||||
|
||||
async_download_tasks.push(task::spawn(async move {
|
||||
let mut img_response = surf::get(&abs_url).await.expect("Unable to retrieve file");
|
||||
let mut img_response = surf::Client::new()
|
||||
.with(surf::middleware::Redirect::default())
|
||||
.get(&abs_url)
|
||||
.await
|
||||
.expect("Unable to retrieve file");
|
||||
let img_content: Vec<u8> = img_response.body_bytes().await.unwrap();
|
||||
let img_mime = img_response
|
||||
.content_type()
|
||||
|
|
20
src/main.rs
20
src/main.rs
|
@ -23,24 +23,26 @@ fn main() {
|
|||
|
||||
type HTMLResource = (String, String);
|
||||
|
||||
async fn fetch_url(url: &str) -> HTMLResource {
|
||||
async fn fetch_url(url: &str) -> Result<HTMLResource, Box<dyn std::error::Error>> {
|
||||
let client = surf::Client::new();
|
||||
println!("Fetching...");
|
||||
// TODO: Add middleware for following redirects
|
||||
(
|
||||
url.to_string(),
|
||||
client
|
||||
let mut res = client
|
||||
.with(surf::middleware::Redirect::default())
|
||||
.get(url)
|
||||
.recv_string()
|
||||
.send()
|
||||
.await
|
||||
.expect("Unable to fetch URL"),
|
||||
)
|
||||
.expect(&format!("Unable to fetch {}", url));
|
||||
if res.status() == 200 {
|
||||
Ok((url.to_string(), res.body_string().await?))
|
||||
} else {
|
||||
Err("Request failed to return HTTP 200".into())
|
||||
}
|
||||
}
|
||||
|
||||
fn download(urls: Vec<String>) {
|
||||
let mut async_url_tasks = Vec::with_capacity(urls.len());
|
||||
for url in urls {
|
||||
async_url_tasks.push(task::spawn(async move { fetch_url(&url).await }));
|
||||
async_url_tasks.push(task::spawn(async move { fetch_url(&url).await.unwrap() }));
|
||||
}
|
||||
task::block_on(async {
|
||||
for url_task in async_url_tasks {
|
||||
|
|
Loading…
Reference in a new issue