|
|
@ -1,53 +1,59 @@
|
|
|
|
extern crate regex;
|
|
|
|
use regex::Regex;
|
|
|
|
extern crate reqwest;
|
|
|
|
use reqwest::header::CONTENT_TYPE;
|
|
|
|
extern crate url;
|
|
|
|
use reqwest::Client;
|
|
|
|
|
|
|
|
|
|
|
|
use self::regex::Regex;
|
|
|
|
|
|
|
|
use self::reqwest::Client;
|
|
|
|
|
|
|
|
use self::reqwest::header::CONTENT_TYPE;
|
|
|
|
|
|
|
|
use std::time::Duration;
|
|
|
|
use std::time::Duration;
|
|
|
|
use self::url::{Url, ParseError};
|
|
|
|
use url::{ParseError, Url};
|
|
|
|
use utils::data_to_dataurl;
|
|
|
|
use utils::data_to_dataurl;
|
|
|
|
|
|
|
|
|
|
|
|
pub fn is_url(path: &str) -> bool {
|
|
|
|
lazy_static! {
|
|
|
|
let re = Regex::new(r"^https?://").unwrap();
|
|
|
|
static ref REGEX_URL: Regex = Regex::new(r"^https?://").unwrap();
|
|
|
|
re.is_match(path)
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub fn resolve_url(from: &str, to: &str) -> Result<String, ParseError> {
|
|
|
|
pub fn is_valid_url(path: &str) -> bool {
|
|
|
|
let mut result = String::new();
|
|
|
|
REGEX_URL.is_match(path)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if is_url(to) { // (anything, http://site.com/css/main.css)
|
|
|
|
pub fn resolve_url(from: &str, to: &str) -> Result<String, ParseError> {
|
|
|
|
result = to.to_string();
|
|
|
|
let result = if is_valid_url(to) {
|
|
|
|
|
|
|
|
// (anything, http://site.com/css/main.css)
|
|
|
|
|
|
|
|
to.to_string()
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
if is_url(from) { // It's a remote resource (HTTP)
|
|
|
|
let mut re = String::new();
|
|
|
|
if to.chars().nth(0) == Some('/') { // (http://site.com/article/1, /...?)
|
|
|
|
if is_valid_url(from) {
|
|
|
|
|
|
|
|
// It's a remote resource (HTTP)
|
|
|
|
|
|
|
|
if to.chars().nth(0) == Some('/') {
|
|
|
|
|
|
|
|
// (http://site.com/article/1, /...?)
|
|
|
|
let from_url = Url::parse(from)?;
|
|
|
|
let from_url = Url::parse(from)?;
|
|
|
|
|
|
|
|
|
|
|
|
if to.chars().nth(1) == Some('/') { // (http://site.com/article/1, //images/1.png)
|
|
|
|
if to.chars().nth(1) == Some('/') {
|
|
|
|
result.push_str(from_url.scheme());
|
|
|
|
// (http://site.com/article/1, //images/1.png)
|
|
|
|
result.push_str(":");
|
|
|
|
re.push_str(from_url.scheme());
|
|
|
|
result.push_str(to);
|
|
|
|
re.push_str(":");
|
|
|
|
} else { // (http://site.com/article/1, /css/main.css)
|
|
|
|
re.push_str(to);
|
|
|
|
result.push_str(from_url.scheme());
|
|
|
|
} else {
|
|
|
|
result.push_str("://");
|
|
|
|
// (http://site.com/article/1, /css/main.css)
|
|
|
|
result.push_str(from_url.host_str().unwrap());
|
|
|
|
re.push_str(from_url.scheme());
|
|
|
|
result.push_str(to);
|
|
|
|
re.push_str("://");
|
|
|
|
|
|
|
|
re.push_str(from_url.host_str().unwrap());
|
|
|
|
|
|
|
|
re.push_str(to);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else { // (http://site.com, css/main.css)
|
|
|
|
} else {
|
|
|
|
|
|
|
|
// (http://site.com, css/main.css)
|
|
|
|
// TODO improve to ensure no // or /// ever happen
|
|
|
|
// TODO improve to ensure no // or /// ever happen
|
|
|
|
result.push_str(from);
|
|
|
|
re.push_str(from);
|
|
|
|
result.push_str("/");
|
|
|
|
re.push_str("/");
|
|
|
|
result.push_str(to);
|
|
|
|
re.push_str(to);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else { // It's a local resource (fs)
|
|
|
|
} else {
|
|
|
|
|
|
|
|
// It's a local resource (fs)
|
|
|
|
// TODO improve to ensure no // or /// ever happen
|
|
|
|
// TODO improve to ensure no // or /// ever happen
|
|
|
|
// TODO for fs use basepath instead of $from
|
|
|
|
// TODO for fs use basepath instead of $from
|
|
|
|
result.push_str(from);
|
|
|
|
re.push_str(from);
|
|
|
|
result.push_str("/");
|
|
|
|
re.push_str("/");
|
|
|
|
result.push_str(to);
|
|
|
|
re.push_str(to);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
re
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
Ok(result)
|
|
|
|
Ok(result)
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -55,11 +61,15 @@ pub fn resolve_url(from: &str, to: &str) -> Result<String, ParseError> {
|
|
|
|
pub fn url_is_data(url: &str) -> Result<bool, String> {
|
|
|
|
pub fn url_is_data(url: &str) -> Result<bool, String> {
|
|
|
|
match Url::parse(url) {
|
|
|
|
match Url::parse(url) {
|
|
|
|
Ok(parsed_url) => Ok(parsed_url.scheme() == "data"),
|
|
|
|
Ok(parsed_url) => Ok(parsed_url.scheme() == "data"),
|
|
|
|
Err(err) => return Err(format!("{}", err)),
|
|
|
|
Err(err) => Err(format!("{}", err)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub fn retrieve_asset(url: &str, as_dataurl: bool, as_mime: &str) -> Result<String, reqwest::Error> {
|
|
|
|
pub fn retrieve_asset(
|
|
|
|
|
|
|
|
url: &str,
|
|
|
|
|
|
|
|
as_dataurl: bool,
|
|
|
|
|
|
|
|
as_mime: &str,
|
|
|
|
|
|
|
|
) -> Result<String, reqwest::Error> {
|
|
|
|
if url_is_data(&url).unwrap() {
|
|
|
|
if url_is_data(&url).unwrap() {
|
|
|
|
Ok(url.to_string())
|
|
|
|
Ok(url.to_string())
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
@ -67,10 +77,7 @@ pub fn retrieve_asset(url: &str, as_dataurl: bool, as_mime: &str) -> Result<Stri
|
|
|
|
.timeout(Duration::from_secs(10))
|
|
|
|
.timeout(Duration::from_secs(10))
|
|
|
|
.build()
|
|
|
|
.build()
|
|
|
|
.unwrap();
|
|
|
|
.unwrap();
|
|
|
|
let mut response = client
|
|
|
|
let mut response = client.get(url).send().unwrap();
|
|
|
|
.get(url)
|
|
|
|
|
|
|
|
.send()
|
|
|
|
|
|
|
|
.unwrap();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if as_dataurl {
|
|
|
|
if as_dataurl {
|
|
|
|
// Convert response into a byte array
|
|
|
|
// Convert response into a byte array
|
|
|
@ -78,13 +85,15 @@ pub fn retrieve_asset(url: &str, as_dataurl: bool, as_mime: &str) -> Result<Stri
|
|
|
|
response.copy_to(&mut data)?;
|
|
|
|
response.copy_to(&mut data)?;
|
|
|
|
|
|
|
|
|
|
|
|
// Attempt to obtain MIME type by reading the Content-Type header
|
|
|
|
// Attempt to obtain MIME type by reading the Content-Type header
|
|
|
|
let mut mimetype = as_mime;
|
|
|
|
let mimetype = if as_mime == "" {
|
|
|
|
if as_mime == "" {
|
|
|
|
response
|
|
|
|
mimetype = response.headers()
|
|
|
|
.headers()
|
|
|
|
.get(CONTENT_TYPE)
|
|
|
|
.get(CONTENT_TYPE)
|
|
|
|
.and_then(|header| header.to_str().ok())
|
|
|
|
.and_then(|header| header.to_str().ok())
|
|
|
|
.unwrap_or(&as_mime);
|
|
|
|
.unwrap_or(&as_mime)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
|
|
|
|
as_mime
|
|
|
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
Ok(data_to_dataurl(&mimetype, &data))
|
|
|
|
Ok(data_to_dataurl(&mimetype, &data))
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
@ -99,37 +108,66 @@ mod tests {
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[test]
|
|
|
|
fn test_is_url() {
|
|
|
|
fn test_is_url() {
|
|
|
|
assert!(is_url("https://www.rust-lang.org/"));
|
|
|
|
assert!(is_valid_url("https://www.rust-lang.org/"));
|
|
|
|
assert!(is_url("http://kernel.org"));
|
|
|
|
assert!(is_valid_url("http://kernel.org"));
|
|
|
|
assert!(!is_url("./index.html"));
|
|
|
|
assert!(!is_valid_url("./index.html"));
|
|
|
|
assert!(!is_url("some-local-page.htm"));
|
|
|
|
assert!(!is_valid_url("some-local-page.htm"));
|
|
|
|
assert!(!is_url("ftp://1.2.3.4/www/index.html"));
|
|
|
|
assert!(!is_valid_url("ftp://1.2.3.4/www/index.html"));
|
|
|
|
assert!(!is_url("data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"));
|
|
|
|
assert!(!is_valid_url(
|
|
|
|
|
|
|
|
"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
|
|
|
|
|
|
|
|
));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[test]
|
|
|
|
fn test_resolve_url() -> Result<(), ParseError> {
|
|
|
|
fn test_resolve_url() -> Result<(), ParseError> {
|
|
|
|
let resolved_url = resolve_url("https://www.kernel.org", "../category/signatures.html")?;
|
|
|
|
let resolved_url = resolve_url("https://www.kernel.org", "../category/signatures.html")?;
|
|
|
|
assert_eq!(resolved_url.as_str(), "https://www.kernel.org/../category/signatures.html");
|
|
|
|
assert_eq!(
|
|
|
|
|
|
|
|
resolved_url.as_str(),
|
|
|
|
|
|
|
|
"https://www.kernel.org/../category/signatures.html"
|
|
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
let resolved_url = resolve_url("https://www.kernel.org", "category/signatures.html")?;
|
|
|
|
let resolved_url = resolve_url("https://www.kernel.org", "category/signatures.html")?;
|
|
|
|
assert_eq!(resolved_url.as_str(), "https://www.kernel.org/category/signatures.html");
|
|
|
|
assert_eq!(
|
|
|
|
|
|
|
|
resolved_url.as_str(),
|
|
|
|
let resolved_url = resolve_url("saved_page.htm", "https://www.kernel.org/category/signatures.html")?;
|
|
|
|
"https://www.kernel.org/category/signatures.html"
|
|
|
|
assert_eq!(resolved_url.as_str(), "https://www.kernel.org/category/signatures.html");
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
let resolved_url = resolve_url("https://www.kernel.org", "//www.kernel.org/theme/images/logos/tux.png")?;
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
assert_eq!(resolved_url.as_str(), "https://www.kernel.org/theme/images/logos/tux.png");
|
|
|
|
"saved_page.htm",
|
|
|
|
|
|
|
|
"https://www.kernel.org/category/signatures.html",
|
|
|
|
let resolved_url = resolve_url("https://www.kernel.org/category/signatures.html", "/theme/images/logos/tux.png")?;
|
|
|
|
)?;
|
|
|
|
assert_eq!(resolved_url.as_str(), "https://www.kernel.org/theme/images/logos/tux.png");
|
|
|
|
assert_eq!(
|
|
|
|
|
|
|
|
resolved_url.as_str(),
|
|
|
|
|
|
|
|
"https://www.kernel.org/category/signatures.html"
|
|
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
|
|
|
|
"https://www.kernel.org",
|
|
|
|
|
|
|
|
"//www.kernel.org/theme/images/logos/tux.png",
|
|
|
|
|
|
|
|
)?;
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
|
|
resolved_url.as_str(),
|
|
|
|
|
|
|
|
"https://www.kernel.org/theme/images/logos/tux.png"
|
|
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
let resolved_url = resolve_url(
|
|
|
|
|
|
|
|
"https://www.kernel.org/category/signatures.html",
|
|
|
|
|
|
|
|
"/theme/images/logos/tux.png",
|
|
|
|
|
|
|
|
)?;
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
|
|
|
|
resolved_url.as_str(),
|
|
|
|
|
|
|
|
"https://www.kernel.org/theme/images/logos/tux.png"
|
|
|
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[test]
|
|
|
|
fn test_url_is_data() {
|
|
|
|
fn test_url_is_data() {
|
|
|
|
assert!(url_is_data("data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h").unwrap_or(false));
|
|
|
|
assert!(
|
|
|
|
|
|
|
|
url_is_data("data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h")
|
|
|
|
|
|
|
|
.unwrap_or(false)
|
|
|
|
|
|
|
|
);
|
|
|
|
assert!(!url_is_data("https://kernel.org").unwrap_or(false));
|
|
|
|
assert!(!url_is_data("https://kernel.org").unwrap_or(false));
|
|
|
|
assert!(!url_is_data("//kernel.org").unwrap_or(false));
|
|
|
|
assert!(!url_is_data("//kernel.org").unwrap_or(false));
|
|
|
|
}
|
|
|
|
}
|
|
|
|