|
|
|
@ -1,5 +1,5 @@
|
|
|
|
|
import re
|
|
|
|
|
from typing import List, Optional, Union
|
|
|
|
|
from typing import List, Optional, Sequence, Union
|
|
|
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
|
|
|
|
|
|
PREFIXES_TO_IGNORE = ("javascript:", "mailto:", "#")
|
|
|
|
@ -42,6 +42,7 @@ def extract_sub_links(
|
|
|
|
|
base_url: Optional[str] = None,
|
|
|
|
|
pattern: Union[str, re.Pattern, None] = None,
|
|
|
|
|
prevent_outside: bool = True,
|
|
|
|
|
exclude_prefixes: Sequence[str] = (),
|
|
|
|
|
) -> List[str]:
|
|
|
|
|
"""Extract all links from a raw html string and convert into absolute paths.
|
|
|
|
|
|
|
|
|
@ -52,6 +53,7 @@ def extract_sub_links(
|
|
|
|
|
pattern: Regex to use for extracting links from raw html.
|
|
|
|
|
prevent_outside: If True, ignore external links which are not children
|
|
|
|
|
of the base url.
|
|
|
|
|
exclude_prefixes: Exclude any URLs that start with one of these prefixes.
|
|
|
|
|
|
|
|
|
|
Returns:
|
|
|
|
|
List[str]: sub links
|
|
|
|
@ -60,8 +62,10 @@ def extract_sub_links(
|
|
|
|
|
all_links = find_all_links(raw_html, pattern=pattern)
|
|
|
|
|
absolute_paths = set()
|
|
|
|
|
for link in all_links:
|
|
|
|
|
if any(link.startswith(exclude) for exclude in exclude_prefixes):
|
|
|
|
|
continue
|
|
|
|
|
# Some may be absolute links like https://to/path
|
|
|
|
|
if link.startswith("http"):
|
|
|
|
|
elif link.startswith("http"):
|
|
|
|
|
absolute_paths.add(link)
|
|
|
|
|
# Some may have omitted the protocol like //to/path
|
|
|
|
|
elif link.startswith("//"):
|
|
|
|
|