mirror of https://github.com/hwchase17/langchain
fix recursive loader (#10752)
maintain same base url throughout recursion, yield initial page, fixing recursion depth trackingpull/10842/head
parent
276125a33b
commit
96a9c27116
@ -0,0 +1,69 @@
|
||||
import re
|
||||
from typing import List, Union
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
PREFIXES_TO_IGNORE = ("javascript:", "mailto:", "#")
|
||||
SUFFIXES_TO_IGNORE = (
|
||||
".css",
|
||||
".js",
|
||||
".ico",
|
||||
".png",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".gif",
|
||||
".svg",
|
||||
".csv",
|
||||
".bz2",
|
||||
".zip",
|
||||
".epub",
|
||||
)
|
||||
SUFFIXES_TO_IGNORE_REGEX = (
|
||||
"(?!" + "|".join([re.escape(s) + "[\#'\"]" for s in SUFFIXES_TO_IGNORE]) + ")"
|
||||
)
|
||||
PREFIXES_TO_IGNORE_REGEX = (
|
||||
"(?!" + "|".join([re.escape(s) for s in PREFIXES_TO_IGNORE]) + ")"
|
||||
)
|
||||
DEFAULT_LINK_REGEX = (
|
||||
f"href=[\"']{PREFIXES_TO_IGNORE_REGEX}((?:{SUFFIXES_TO_IGNORE_REGEX}.)*?)[\#'\"]"
|
||||
)
|
||||
|
||||
|
||||
def find_all_links(
|
||||
raw_html: str, *, pattern: Union[str, re.Pattern, None] = None
|
||||
) -> List[str]:
|
||||
pattern = pattern or DEFAULT_LINK_REGEX
|
||||
return list(set(re.findall(pattern, raw_html)))
|
||||
|
||||
|
||||
def extract_sub_links(
|
||||
raw_html: str,
|
||||
base_url: str,
|
||||
*,
|
||||
pattern: Union[str, re.Pattern, None] = None,
|
||||
prevent_outside: bool = True,
|
||||
) -> List[str]:
|
||||
"""Extract all links from a raw html string and convert into absolute paths.
|
||||
|
||||
Args:
|
||||
raw_html: original html
|
||||
base_url: the base url of the html
|
||||
pattern: Regex to use for extracting links from raw html.
|
||||
prevent_outside: If True, ignore external links which are not children
|
||||
of the base url.
|
||||
|
||||
Returns:
|
||||
List[str]: sub links
|
||||
"""
|
||||
all_links = find_all_links(raw_html, pattern=pattern)
|
||||
absolute_paths = set()
|
||||
for link in all_links:
|
||||
# Some may be absolute links like https://to/path
|
||||
if link.startswith("http"):
|
||||
if not prevent_outside or link.startswith(base_url):
|
||||
absolute_paths.add(link)
|
||||
# Some may have omitted the protocol like //to/path
|
||||
elif link.startswith("//"):
|
||||
absolute_paths.add(f"{urlparse(base_url).scheme}:{link}")
|
||||
else:
|
||||
absolute_paths.add(urljoin(base_url, link))
|
||||
return list(absolute_paths)
|
@ -1,30 +1,61 @@
|
||||
import pytest as pytest
|
||||
|
||||
from langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
def test_async_recursive_url_loader() -> None:
|
||||
url = "https://docs.python.org/3.9/"
|
||||
loader = RecursiveUrlLoader(
|
||||
url=url, extractor=lambda _: "placeholder", use_async=True, max_depth=1
|
||||
url,
|
||||
extractor=lambda _: "placeholder",
|
||||
use_async=True,
|
||||
max_depth=3,
|
||||
timeout=None,
|
||||
)
|
||||
docs = loader.load()
|
||||
assert len(docs) == 24
|
||||
assert len(docs) == 1024
|
||||
assert docs[0].page_content == "placeholder"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
def test_async_recursive_url_loader_deterministic() -> None:
|
||||
url = "https://docs.python.org/3.9/"
|
||||
loader = RecursiveUrlLoader(
|
||||
url,
|
||||
use_async=True,
|
||||
max_depth=3,
|
||||
timeout=None,
|
||||
)
|
||||
docs = sorted(loader.load(), key=lambda d: d.metadata["source"])
|
||||
docs_2 = sorted(loader.load(), key=lambda d: d.metadata["source"])
|
||||
assert docs == docs_2
|
||||
|
||||
|
||||
def test_sync_recursive_url_loader() -> None:
|
||||
url = "https://docs.python.org/3.9/"
|
||||
loader = RecursiveUrlLoader(
|
||||
url=url, extractor=lambda _: "placeholder", use_async=False, max_depth=1
|
||||
url, extractor=lambda _: "placeholder", use_async=False, max_depth=2
|
||||
)
|
||||
docs = loader.load()
|
||||
assert len(docs) == 24
|
||||
assert len(docs) == 27
|
||||
assert docs[0].page_content == "placeholder"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
def test_sync_async_equivalent() -> None:
|
||||
url = "https://docs.python.org/3.9/"
|
||||
loader = RecursiveUrlLoader(url, use_async=False, max_depth=2)
|
||||
async_loader = RecursiveUrlLoader(url, use_async=False, max_depth=2)
|
||||
docs = sorted(loader.load(), key=lambda d: d.metadata["source"])
|
||||
async_docs = sorted(async_loader.load(), key=lambda d: d.metadata["source"])
|
||||
assert docs == async_docs
|
||||
|
||||
|
||||
def test_loading_invalid_url() -> None:
|
||||
url = "https://this.url.is.invalid/this/is/a/test"
|
||||
loader = RecursiveUrlLoader(
|
||||
url=url, max_depth=1, extractor=lambda _: "placeholder", use_async=False
|
||||
url, max_depth=1, extractor=lambda _: "placeholder", use_async=False
|
||||
)
|
||||
docs = loader.load()
|
||||
assert len(docs) == 0
|
||||
|
@ -0,0 +1,109 @@
|
||||
from langchain.utils.html import (
|
||||
PREFIXES_TO_IGNORE,
|
||||
SUFFIXES_TO_IGNORE,
|
||||
extract_sub_links,
|
||||
find_all_links,
|
||||
)
|
||||
|
||||
|
||||
def test_find_all_links_none() -> None:
|
||||
html = "<span>Hello world</span>"
|
||||
actual = find_all_links(html)
|
||||
assert actual == []
|
||||
|
||||
|
||||
def test_find_all_links_single() -> None:
|
||||
htmls = [
|
||||
"href='foobar.com'",
|
||||
'href="foobar.com"',
|
||||
'<div><a class="blah" href="foobar.com">hullo</a></div>',
|
||||
]
|
||||
actual = [find_all_links(html) for html in htmls]
|
||||
assert actual == [["foobar.com"]] * 3
|
||||
|
||||
|
||||
def test_find_all_links_multiple() -> None:
|
||||
html = (
|
||||
'<div><a class="blah" href="https://foobar.com">hullo</a></div>'
|
||||
'<div><a class="bleh" href="/baz/cool">buhbye</a></div>'
|
||||
)
|
||||
actual = find_all_links(html)
|
||||
assert sorted(actual) == [
|
||||
"/baz/cool",
|
||||
"https://foobar.com",
|
||||
]
|
||||
|
||||
|
||||
def test_find_all_links_ignore_suffix() -> None:
|
||||
html = 'href="foobar{suffix}"'
|
||||
for suffix in SUFFIXES_TO_IGNORE:
|
||||
actual = find_all_links(html.format(suffix=suffix))
|
||||
assert actual == []
|
||||
|
||||
# Don't ignore if pattern doesn't occur at end of link.
|
||||
html = 'href="foobar{suffix}more"'
|
||||
for suffix in SUFFIXES_TO_IGNORE:
|
||||
actual = find_all_links(html.format(suffix=suffix))
|
||||
assert actual == [f"foobar{suffix}more"]
|
||||
|
||||
|
||||
def test_find_all_links_ignore_prefix() -> None:
|
||||
html = 'href="{prefix}foobar"'
|
||||
for prefix in PREFIXES_TO_IGNORE:
|
||||
actual = find_all_links(html.format(prefix=prefix))
|
||||
assert actual == []
|
||||
|
||||
# Don't ignore if pattern doesn't occur at beginning of link.
|
||||
html = 'href="foobar{prefix}more"'
|
||||
for prefix in PREFIXES_TO_IGNORE:
|
||||
# Pound signs are split on when not prefixes.
|
||||
if prefix == "#":
|
||||
continue
|
||||
actual = find_all_links(html.format(prefix=prefix))
|
||||
assert actual == [f"foobar{prefix}more"]
|
||||
|
||||
|
||||
def test_find_all_links_drop_fragment() -> None:
|
||||
html = 'href="foobar.com/woah#section_one"'
|
||||
actual = find_all_links(html)
|
||||
assert actual == ["foobar.com/woah"]
|
||||
|
||||
|
||||
def test_extract_sub_links() -> None:
|
||||
html = (
|
||||
'<a href="https://foobar.com">one</a>'
|
||||
'<a href="http://baz.net">two</a>'
|
||||
'<a href="//foobar.com/hello">three</a>'
|
||||
'<a href="/how/are/you/doing">four</a>'
|
||||
)
|
||||
expected = sorted(
|
||||
[
|
||||
"https://foobar.com",
|
||||
"https://foobar.com/hello",
|
||||
"https://foobar.com/how/are/you/doing",
|
||||
]
|
||||
)
|
||||
actual = sorted(extract_sub_links(html, "https://foobar.com"))
|
||||
assert actual == expected
|
||||
|
||||
actual = sorted(extract_sub_links(html, "https://foobar.com/hello"))
|
||||
expected = sorted(
|
||||
[
|
||||
"https://foobar.com/hello",
|
||||
"https://foobar.com/how/are/you/doing",
|
||||
]
|
||||
)
|
||||
assert actual == expected
|
||||
|
||||
actual = sorted(
|
||||
extract_sub_links(html, "https://foobar.com/hello", prevent_outside=False)
|
||||
)
|
||||
expected = sorted(
|
||||
[
|
||||
"https://foobar.com",
|
||||
"http://baz.net",
|
||||
"https://foobar.com/hello",
|
||||
"https://foobar.com/how/are/you/doing",
|
||||
]
|
||||
)
|
||||
assert actual == expected
|
Loading…
Reference in New Issue