Archives
/
awesome-rust
mirror of https://github.com/rust-unofficial/awesome-rust
2
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #866 from palfrey/error-immediately-for-redirects

Browse Source 
For most errors, intermittent failures are ok, not so much for redirects
pull/867/head
Tom Parker-Shemilt 5 years ago committed by GitHub
parent 63b369c50f 85317379bb
commit e167865f93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 53 additions and 43 deletions
Show Stats Download Patch File Download Diff File

4
README.md
Unescape Escape View File

@ -1271,7 +1271,7 @@ See also [Are we game yet?](https://arewegameyet.com)
* [mattnenterprise/rust-pop3](https://github.com/mattnenterprise/rust-pop3) — A [POP3](https://en.wikipedia.org/wiki/Post_Office_Protocol) client for Rust [<img src="https://api.travis-ci.org/mattnenterprise/rust-pop3.svg?branch=master">](https://travis-ci.org/mattnenterprise/rust-pop3)
* SSH
* [alexcrichton/ssh2-rs](https://github.com/alexcrichton/ssh2-rs) — [libssh2](https://www.libssh2.org/) bindings [<img src="https://api.travis-ci.com/alexcrichton/ssh2-rs.svg?branch=master">](https://travis-ci.org/alexcrichton/ssh2-rs)
* [Thrussh](https://github.com/pijul-scm/thrussh/) — an SSH library written from scratch in Rust, backed by [libsodium](https://download.libsodium.org/doc/)
* [Thrussh](https://github.com/pijul-scm/thrussh/) — an SSH library written from scratch in Rust, backed by [libsodium](https://doc.libsodium.org/)
* Stomp
* [zslayton/stomp-rs](https://github.com/zslayton/stomp-rs) — A [STOMP 1.2](http://stomp.github.io/stomp-specification-1.2.html) client implementation in Rust [<img src="https://api.travis-ci.org/zslayton/stomp-rs.svg?branch=master">](https://travis-ci.org/zslayton/stomp-rs)
* uTP
@ -1485,7 +1485,7 @@ A registry allows you to publish your Rust libraries as crate packages, to share
* [Rust for professionals](https://overexact.com/rust-for-professionals/) — A quick introduction to Rust for experienced software developers.
* [Rust in Motion](https://www.manning.com/livevideo/rust-in-motion?a_aid=cnichols&a_bid=6a993c2e) — A video series by [Carol Nichols](https://github.com/carols10cents) and [Jake Goulding](https://github.com/shepmaster) (paid)
* [rust-learning](https://github.com/ctjhoa/rust-learning) — A collection of useful resources to learn Rust
* [Rustlings](https://github.com/rust-lang/rustlings) — small exercises to get you used to reading and writing Rust code
* [Rustlings](https://github.com/fmoko/rustlings) — small exercises to get you used to reading and writing Rust code
* [stdx](https://github.com/brson/stdx) — Learn these crates first as an extension to std
* [University of Pennsylvania's Comp Sci Rust Programming Course](http://cis198-2016s.github.io/schedule/)
* [Build a language VM](https://blog.subnetzero.io/post/building-language-vm-part-00/)

84
src/main.rs
Unescape Escape View File

@ -15,20 +15,20 @@ use scraper::{Html, Selector};
use failure::{Fail, Error, format_err};
use chrono::{Local, DateTime, Duration};
#[derive(Debug, Fail)]
#[derive(Debug, Fail, Serialize, Deserialize)]
enum CheckerError {
#[fail(display = "failed to try url")]
NotTried, // Generally shouldn't happen, but useful to have
#[fail(display = "http error: {}", status)]
HttpError {
status: StatusCode,
status: u16,
location: Option<String>,
},
#[fail(display = "reqwest error: {}", error)]
ReqwestError {
error: reqwest::Error,
error: String,
},
#[fail(display = "travis build is unknown")]
@ -41,6 +41,33 @@ enum CheckerError {
GithubActionNoBranch,
}
fn formatter(err: &CheckerError, url: &String) -> String {
match err {
CheckerError::HttpError {status, location} => {
match location {
Some(loc) => {
format!("[{}] {} -> {}", status, url, loc)
}
None => {
format!("[{}] {}", status, url)
}
}
}
CheckerError::TravisBuildUnknown => {
format!("[Unknown travis build] {}", url)
}
CheckerError::TravisBuildNoBranch => {
format!("[Travis build image with no branch specified] {}", url)
}
CheckerError::GithubActionNoBranch => {
format!("[Github action image with no branch specified] {}", url)
}
_ => {
format!("{:?}", err)
}
}
}
struct MaxHandles {
remaining: AtomicU32
}
@ -102,7 +129,7 @@ fn get_url(url: String) -> BoxFuture<'static, (String, Result<(), CheckerError>)
match resp {
Err(err) => {
warn!("Error while getting {}, retrying: {}", url, err);
res = Err(CheckerError::ReqwestError{error: err});
res = Err(CheckerError::ReqwestError{error: err.to_string()});
continue;
}
Ok(ok) => {
@ -139,10 +166,10 @@ fn get_url(url: String) -> BoxFuture<'static, (String, Result<(), CheckerError>)
warn!("Error while getting {}, retrying: {}", url, status);
if status.is_redirection() {
res = Err(CheckerError::HttpError {status: status, location: ok.headers().get(header::LOCATION).and_then(|h| h.to_str().ok()).map(|x| x.to_string())});
res = Err(CheckerError::HttpError {status: status.as_u16(), location: ok.headers().get(header::LOCATION).and_then(|h| h.to_str().ok()).map(|x| x.to_string())});
break;
} else {
res = Err(CheckerError::HttpError {status: status, location: None});
res = Err(CheckerError::HttpError {status: status.as_u16(), location: None});
continue;
}
}
@ -185,7 +212,7 @@ fn get_url(url: String) -> BoxFuture<'static, (String, Result<(), CheckerError>)
#[derive(Debug, Serialize, Deserialize)]
enum Working {
Yes,
No(String)
No(CheckerError)
}
#[derive(Debug, Serialize, Deserialize)]
@ -289,37 +316,13 @@ async fn main() -> Result<(), Error> {
},
Err(err) => {
print!("\u{2718} ");
let message = match err {
CheckerError::HttpError {status, location} => {
match location {
Some(loc) => {
format!("[{}] {} -> {}", status.as_u16(), url, loc)
}
None => {
format!("[{}] {}", status.as_u16(), url)
}
}
}
CheckerError::TravisBuildUnknown => {
format!("[Unknown travis build] {}", url)
}
CheckerError::TravisBuildNoBranch => {
format!("[Travis build image with no branch specified] {}", url)
}
CheckerError::GithubActionNoBranch => {
format!("[Github action image with no branch specified] {}", url)
}
_ => {
format!("{:?}", err)
}
};
if let Some(link) = results.get_mut(&url) {
link.updated_at = Local::now();
link.working = Working::No(message);
link.working = Working::No(err);
} else {
results.insert(url.clone(), Link {
updated_at: Local::now(),
working: Working::No(message),
working: Working::No(err),
last_working: None
});
}
@ -339,21 +342,28 @@ async fn main() -> Result<(), Error> {
println!("");
let mut failed: u32 = 0;
for (_url, link) in results.iter() {
if let Working::No(ref msg) = link.working {
if link.last_working.is_none() {
for (url, link) in results.iter() {
if let Working::No(ref err) = link.working {
match err {
CheckerError::HttpError {status, ..} if *status == 301 || *status == 302 => {
println!("{:?}", link);
failed +=1;
continue;
}
_ => {}
};
if let Some(last_working) = link.last_working {
let since = Local::now() - last_working;
if since > max_allowed_failed {
println!("{:?}", link);
failed +=1;
} else {
println!("Failure occurred but only {}, so we're not worrying yet: {}", chrono_humanize::HumanTime::from(-since), msg);
println!("Failure occurred but only {}, so we're not worrying yet: {}", chrono_humanize::HumanTime::from(-since), formatter(err, url));
}
} else {
println!("{:?}", link);
failed +=1;
continue;
}
}
}

Write Preview
Loading…
Cancel
Save