|
|
@ -1,6 +1,6 @@
|
|
|
|
use regex::Regex;
|
|
|
|
use regex::Regex;
|
|
|
|
use reqwest::header::{CONTENT_TYPE, USER_AGENT};
|
|
|
|
use reqwest::header::{CONTENT_TYPE, USER_AGENT};
|
|
|
|
use reqwest::Client;
|
|
|
|
use reqwest::{Client, RedirectPolicy};
|
|
|
|
use std::time::Duration;
|
|
|
|
use std::time::Duration;
|
|
|
|
use url::{ParseError, Url};
|
|
|
|
use url::{ParseError, Url};
|
|
|
|
use utils::data_to_dataurl;
|
|
|
|
use utils::data_to_dataurl;
|
|
|
@ -9,6 +9,13 @@ lazy_static! {
|
|
|
|
static ref REGEX_URL: Regex = Regex::new(r"^https?://").unwrap();
|
|
|
|
static ref REGEX_URL: Regex = Regex::new(r"^https?://").unwrap();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pub fn is_data_url(url: &str) -> Result<bool, String> {
|
|
|
|
|
|
|
|
match Url::parse(url) {
|
|
|
|
|
|
|
|
Ok(parsed_url) => Ok(parsed_url.scheme() == "data"),
|
|
|
|
|
|
|
|
Err(err) => Err(format!("{}", err)),
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub fn is_valid_url(path: &str) -> bool {
|
|
|
|
pub fn is_valid_url(path: &str) -> bool {
|
|
|
|
REGEX_URL.is_match(path)
|
|
|
|
REGEX_URL.is_match(path)
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -18,63 +25,23 @@ pub fn resolve_url(from: &str, to: &str) -> Result<String, ParseError> {
|
|
|
|
// (anything, http://site.com/css/main.css)
|
|
|
|
// (anything, http://site.com/css/main.css)
|
|
|
|
to.to_string()
|
|
|
|
to.to_string()
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
let mut re = String::new();
|
|
|
|
Url::parse(from)?.join(to)?.to_string()
|
|
|
|
if is_valid_url(from) {
|
|
|
|
|
|
|
|
// It's a remote resource (HTTP)
|
|
|
|
|
|
|
|
if to.chars().nth(0) == Some('/') {
|
|
|
|
|
|
|
|
// (http://site.com/article/1, /...?)
|
|
|
|
|
|
|
|
let from_url = Url::parse(from)?;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if to.chars().nth(1) == Some('/') {
|
|
|
|
|
|
|
|
// (http://site.com/article/1, //images/1.png)
|
|
|
|
|
|
|
|
re.push_str(from_url.scheme());
|
|
|
|
|
|
|
|
re.push_str(":");
|
|
|
|
|
|
|
|
re.push_str(to);
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
// (http://site.com/article/1, /css/main.css)
|
|
|
|
|
|
|
|
re.push_str(from_url.scheme());
|
|
|
|
|
|
|
|
re.push_str("://");
|
|
|
|
|
|
|
|
re.push_str(from_url.host_str().unwrap());
|
|
|
|
|
|
|
|
re.push_str(to);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
// (http://site.com, css/main.css)
|
|
|
|
|
|
|
|
// TODO improve to ensure no // or /// ever happen
|
|
|
|
|
|
|
|
re.push_str(from);
|
|
|
|
|
|
|
|
re.push_str("/");
|
|
|
|
|
|
|
|
re.push_str(to);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
// It's a local resource (fs)
|
|
|
|
|
|
|
|
// TODO improve to ensure no // or /// ever happen
|
|
|
|
|
|
|
|
// TODO for fs use basepath instead of $from
|
|
|
|
|
|
|
|
re.push_str(from);
|
|
|
|
|
|
|
|
re.push_str("/");
|
|
|
|
|
|
|
|
re.push_str(to);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
re
|
|
|
|
|
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
Ok(result)
|
|
|
|
Ok(result)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub fn url_is_data(url: &str) -> Result<bool, String> {
|
|
|
|
|
|
|
|
match Url::parse(url) {
|
|
|
|
|
|
|
|
Ok(parsed_url) => Ok(parsed_url.scheme() == "data"),
|
|
|
|
|
|
|
|
Err(err) => Err(format!("{}", err)),
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pub fn retrieve_asset(
|
|
|
|
pub fn retrieve_asset(
|
|
|
|
url: &str,
|
|
|
|
url: &str,
|
|
|
|
as_dataurl: bool,
|
|
|
|
as_dataurl: bool,
|
|
|
|
as_mime: &str,
|
|
|
|
as_mime: &str,
|
|
|
|
opt_user_agent: &str,
|
|
|
|
opt_user_agent: &str,
|
|
|
|
) -> Result<String, reqwest::Error> {
|
|
|
|
) -> Result<String, reqwest::Error> {
|
|
|
|
if url_is_data(&url).unwrap() {
|
|
|
|
if is_data_url(&url).unwrap() {
|
|
|
|
Ok(url.to_string())
|
|
|
|
Ok(url.to_string())
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
let client = Client::builder()
|
|
|
|
let client = Client::builder()
|
|
|
|
|
|
|
|
.redirect(RedirectPolicy::limited(3))
|
|
|
|
.timeout(Duration::from_secs(10))
|
|
|
|
.timeout(Duration::from_secs(10))
|
|
|
|
.build()
|
|
|
|
.build()
|
|
|
|
.unwrap();
|
|
|
|
.unwrap();
|
|
|
@ -125,13 +92,19 @@ mod tests {
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[test]
|
|
|
|
fn test_resolve_url() -> Result<(), ParseError> {
|
|
|
|
fn test_resolve_url() -> Result<(), ParseError> {
|
|
|
|
let resolved_url = resolve_url("https://www.kernel.org", "../category/signatures.html")?;
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
|
|
|
|
"https://www.kernel.org",
|
|
|
|
|
|
|
|
"../category/signatures.html",
|
|
|
|
|
|
|
|
)?;
|
|
|
|
assert_eq!(
|
|
|
|
assert_eq!(
|
|
|
|
resolved_url.as_str(),
|
|
|
|
resolved_url.as_str(),
|
|
|
|
"https://www.kernel.org/../category/signatures.html"
|
|
|
|
"https://www.kernel.org/category/signatures.html"
|
|
|
|
);
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
let resolved_url = resolve_url("https://www.kernel.org", "category/signatures.html")?;
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
|
|
|
|
"https://www.kernel.org",
|
|
|
|
|
|
|
|
"category/signatures.html",
|
|
|
|
|
|
|
|
)?;
|
|
|
|
assert_eq!(
|
|
|
|
assert_eq!(
|
|
|
|
resolved_url.as_str(),
|
|
|
|
resolved_url.as_str(),
|
|
|
|
"https://www.kernel.org/category/signatures.html"
|
|
|
|
"https://www.kernel.org/category/signatures.html"
|
|
|
@ -155,6 +128,15 @@ mod tests {
|
|
|
|
"https://www.kernel.org/theme/images/logos/tux.png"
|
|
|
|
"https://www.kernel.org/theme/images/logos/tux.png"
|
|
|
|
);
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
|
|
|
|
"https://www.kernel.org",
|
|
|
|
|
|
|
|
"//another-host.org/theme/images/logos/tux.png",
|
|
|
|
|
|
|
|
)?;
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
|
|
resolved_url.as_str(),
|
|
|
|
|
|
|
|
"https://another-host.org/theme/images/logos/tux.png"
|
|
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
"https://www.kernel.org/category/signatures.html",
|
|
|
|
"https://www.kernel.org/category/signatures.html",
|
|
|
|
"/theme/images/logos/tux.png",
|
|
|
|
"/theme/images/logos/tux.png",
|
|
|
@ -164,16 +146,25 @@ mod tests {
|
|
|
|
"https://www.kernel.org/theme/images/logos/tux.png"
|
|
|
|
"https://www.kernel.org/theme/images/logos/tux.png"
|
|
|
|
);
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
|
|
|
|
"https://www.w3schools.com/html/html_iframe.asp",
|
|
|
|
|
|
|
|
"default.asp",
|
|
|
|
|
|
|
|
)?;
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
|
|
resolved_url.as_str(),
|
|
|
|
|
|
|
|
"https://www.w3schools.com/html/default.asp"
|
|
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[test]
|
|
|
|
fn test_url_is_data() {
|
|
|
|
fn test_is_data_url() {
|
|
|
|
assert!(
|
|
|
|
assert!(
|
|
|
|
url_is_data("data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h")
|
|
|
|
is_data_url("data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h")
|
|
|
|
.unwrap_or(false)
|
|
|
|
.unwrap_or(false)
|
|
|
|
);
|
|
|
|
);
|
|
|
|
assert!(!url_is_data("https://kernel.org").unwrap_or(false));
|
|
|
|
assert!(!is_data_url("https://kernel.org").unwrap_or(false));
|
|
|
|
assert!(!url_is_data("//kernel.org").unwrap_or(false));
|
|
|
|
assert!(!is_data_url("//kernel.org").unwrap_or(false));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|