|
|
|
@ -1,5 +1,5 @@
|
|
|
|
|
import re
|
|
|
|
|
from typing import List, Union
|
|
|
|
|
from typing import List, Optional, Union
|
|
|
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
|
|
|
|
|
|
PREFIXES_TO_IGNORE = ("javascript:", "mailto:", "#")
|
|
|
|
@ -37,16 +37,18 @@ def find_all_links(
|
|
|
|
|
|
|
|
|
|
def extract_sub_links(
|
|
|
|
|
raw_html: str,
|
|
|
|
|
base_url: str,
|
|
|
|
|
url: str,
|
|
|
|
|
*,
|
|
|
|
|
base_url: Optional[str] = None,
|
|
|
|
|
pattern: Union[str, re.Pattern, None] = None,
|
|
|
|
|
prevent_outside: bool = True,
|
|
|
|
|
) -> List[str]:
|
|
|
|
|
"""Extract all links from a raw html string and convert into absolute paths.
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
raw_html: original html
|
|
|
|
|
base_url: the base url of the html
|
|
|
|
|
raw_html: original html.
|
|
|
|
|
url: the url of the html.
|
|
|
|
|
base_url: the base url to check for outside links against.
|
|
|
|
|
pattern: Regex to use for extracting links from raw html.
|
|
|
|
|
prevent_outside: If True, ignore external links which are not children
|
|
|
|
|
of the base url.
|
|
|
|
@ -54,6 +56,7 @@ def extract_sub_links(
|
|
|
|
|
Returns:
|
|
|
|
|
List[str]: sub links
|
|
|
|
|
"""
|
|
|
|
|
base_url = base_url if base_url is not None else url
|
|
|
|
|
all_links = find_all_links(raw_html, pattern=pattern)
|
|
|
|
|
absolute_paths = set()
|
|
|
|
|
for link in all_links:
|
|
|
|
@ -62,9 +65,9 @@ def extract_sub_links(
|
|
|
|
|
absolute_paths.add(link)
|
|
|
|
|
# Some may have omitted the protocol like //to/path
|
|
|
|
|
elif link.startswith("//"):
|
|
|
|
|
absolute_paths.add(f"{urlparse(base_url).scheme}:{link}")
|
|
|
|
|
absolute_paths.add(f"{urlparse(url).scheme}:{link}")
|
|
|
|
|
else:
|
|
|
|
|
absolute_paths.add(urljoin(base_url, link))
|
|
|
|
|
absolute_paths.add(urljoin(url, link))
|
|
|
|
|
if prevent_outside:
|
|
|
|
|
return [p for p in absolute_paths if p.startswith(base_url)]
|
|
|
|
|
return list(absolute_paths)
|
|
|
|
|