diff --git a/README.md b/README.md index 7a75107..0a0d269 100644 --- a/README.md +++ b/README.md @@ -784,7 +784,7 @@ See also [About Rust’s Machine Learning Community](https://medium.com/@autumn_ ### Bioinformatics -* [nonnominandus/pdbtbx](https://github.com/nonnominandus/pdbtbx) [PDBTBX](https://crates.io/crates/pdbtbx) ![Compile & Test](https://github.com/nonnominandus/rust-pdb/workflows/Compile%20&%20Test/badge.svg) - PDB/mmCIF libary in Rust. +* [nonnominandus/pdbtbx](https://github.com/nonnominandus/pdbtbx) [PDBTBX](https://crates.io/crates/pdbtbx) ![Compile & Test](https://github.com/nonnominandus/pdbtbx/workflows/Compile%20&%20Test/badge.svg?branch=master) - PDB/mmCIF libary in Rust. * [Rust-Bio](https://github.com/rust-bio) — bioinformatics libraries in Rust. ### Caching @@ -1137,7 +1137,7 @@ See also [About Rust’s Machine Learning Community](https://medium.com/@autumn_ * Postman Collection * [mandrean/postman-collection-rs](https://github.com/mandrean/postman-collection-rs) — A Postman Collection v1, v2 & v2.1 serialization & deserialization library [![Build Status](https://api.travis-ci.org/mandrean/postman-collection-rs.svg?branch=master)](https://travis-ci.org/mandrean/postman-collection-rs) * ProtocolBuffers - * [danburkert/prost](https://github.com/danburkert/prost) — [![build badge](https://api.travis-ci.org/danburkert/prost.svg?branch=master)](https://travis-ci.org/danburkert/prost) + * [tokio-rs/prost](https://github.com/tokio-rs/prost) — [![continuous integration](https://github.com/tokio-rs/prost/workflows/continuous%20integration/badge.svg?branch=master)](https://github.com/tokio-rs/prost/actions) * [stepancheg/rust-protobuf](https://github.com/stepancheg/rust-protobuf) — [![build badge](https://api.travis-ci.org/stepancheg/rust-protobuf.svg?branch=master)](https://travis-ci.org/stepancheg/rust-protobuf) * RON (Rusty Object Notation) * [https://github.com/ron-rs/ron](https://github.com/ron-rs/ron) — [![build badge](https://api.travis-ci.org/ron-rs/ron.svg?branch=master)](https://travis-ci.org/https://github.com/ron-rs/ron) diff --git a/src/main.rs b/src/main.rs index e0324ce..8861ca3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,9 @@ enum CheckerError { location: Option, }, + #[fail(display = "too many requests")] + TooManyRequests, + #[fail(display = "reqwest error: {}", error)] ReqwestError { error: String, @@ -191,6 +194,12 @@ fn get_url_core(url: String) -> BoxFuture<'static, (String, Result<(), CheckerEr return (url, res); } + if status == StatusCode::TOO_MANY_REQUESTS { + // We get a lot of these, and we should not retry as they'll just fail again + warn!("Error while getting {}: {}", url, status); + return (url, Err(CheckerError::TooManyRequests)); + } + warn!("Error while getting {}, retrying: {}", url, status); if status.is_redirection() { res = Err(CheckerError::HttpError {status: status.as_u16(), location: ok.headers().get(header::LOCATION).and_then(|h| h.to_str().ok()).map(|x| x.to_string())}); @@ -365,6 +374,13 @@ async fn main() -> Result<(), Error> { failed +=1; continue; } + CheckerError::TooManyRequests => { + // too many tries + if link.last_working.is_some() { + info!("Ignoring 429 failure on {} as we've seen success before", url); + continue; + } + } _ => {} }; if let Some(last_working) = link.last_working {