diff --git a/src/main.rs b/src/main.rs index 7e927f5..db5a60f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,7 @@ use regex::Regex; use scraper::{Html, Selector}; use failure::{Fail, Error, format_err}; use chrono::{Local, DateTime, Duration}; +use std::env; #[derive(Debug, Fail, Serialize, Deserialize)] enum CheckerError { @@ -121,11 +122,31 @@ fn get_url(url: String) -> BoxFuture<'static, (String, Result<(), CheckerError>) let mut res = Err(CheckerError::NotTried); for _ in 0..5u8 { debug!("Running {}", url); - let resp = CLIENT + lazy_static! { + static ref GITHUB_REPO_REGEX: Regex = Regex::new(r"^https://github.com/(?P[^/]+)/(?P[^/]+)$").unwrap(); + static ref GITHUB_API_REGEX: Regex = Regex::new(r"https://api.github.com/").unwrap(); + } + if GITHUB_REPO_REGEX.is_match(&url) { + let rewritten = GITHUB_REPO_REGEX.replace_all(&url, "https://api.github.com/repos/$org/$repo"); + info!("Replacing {} with {} to workaround rate limits on Github", url, rewritten); + let (_new_url, res) = get_url(rewritten.to_string()).await; + return (url, res); + } + let mut req = CLIENT .get(&url) - .header(header::ACCEPT, "image/svg+xml, text/html, */*;q=0.8") - .send() - .await; + .header(header::ACCEPT, "image/svg+xml, text/html, */*;q=0.8"); + + if GITHUB_API_REGEX.is_match(&url) { + if let Ok(username) = env::var("GITHUB_USERNAME") { + if let Ok(password) = env::var("GITHUB_TOKEN") { + // needs a token with at least public_repo scope + info!("Using basic auth for {}", url); + req = req.basic_auth(username, Some(password)); + } + } + } + + let resp = req.send().await; match resp { Err(err) => { warn!("Error while getting {}, retrying: {}", url, err);