mirror of https://github.com/hwchase17/langchain
fix recursive loader (#10752)
maintain same base url throughout recursion, yield initial page, fixing recursion depth trackingpull/10842/head
parent
276125a33b
commit
96a9c27116
@ -0,0 +1,69 @@
|
|||||||
|
import re
|
||||||
|
from typing import List, Union
|
||||||
|
from urllib.parse import urljoin, urlparse
|
||||||
|
|
||||||
|
PREFIXES_TO_IGNORE = ("javascript:", "mailto:", "#")
|
||||||
|
SUFFIXES_TO_IGNORE = (
|
||||||
|
".css",
|
||||||
|
".js",
|
||||||
|
".ico",
|
||||||
|
".png",
|
||||||
|
".jpg",
|
||||||
|
".jpeg",
|
||||||
|
".gif",
|
||||||
|
".svg",
|
||||||
|
".csv",
|
||||||
|
".bz2",
|
||||||
|
".zip",
|
||||||
|
".epub",
|
||||||
|
)
|
||||||
|
SUFFIXES_TO_IGNORE_REGEX = (
|
||||||
|
"(?!" + "|".join([re.escape(s) + "[\#'\"]" for s in SUFFIXES_TO_IGNORE]) + ")"
|
||||||
|
)
|
||||||
|
PREFIXES_TO_IGNORE_REGEX = (
|
||||||
|
"(?!" + "|".join([re.escape(s) for s in PREFIXES_TO_IGNORE]) + ")"
|
||||||
|
)
|
||||||
|
DEFAULT_LINK_REGEX = (
|
||||||
|
f"href=[\"']{PREFIXES_TO_IGNORE_REGEX}((?:{SUFFIXES_TO_IGNORE_REGEX}.)*?)[\#'\"]"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def find_all_links(
|
||||||
|
raw_html: str, *, pattern: Union[str, re.Pattern, None] = None
|
||||||
|
) -> List[str]:
|
||||||
|
pattern = pattern or DEFAULT_LINK_REGEX
|
||||||
|
return list(set(re.findall(pattern, raw_html)))
|
||||||
|
|
||||||
|
|
||||||
|
def extract_sub_links(
|
||||||
|
raw_html: str,
|
||||||
|
base_url: str,
|
||||||
|
*,
|
||||||
|
pattern: Union[str, re.Pattern, None] = None,
|
||||||
|
prevent_outside: bool = True,
|
||||||
|
) -> List[str]:
|
||||||
|
"""Extract all links from a raw html string and convert into absolute paths.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
raw_html: original html
|
||||||
|
base_url: the base url of the html
|
||||||
|
pattern: Regex to use for extracting links from raw html.
|
||||||
|
prevent_outside: If True, ignore external links which are not children
|
||||||
|
of the base url.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List[str]: sub links
|
||||||
|
"""
|
||||||
|
all_links = find_all_links(raw_html, pattern=pattern)
|
||||||
|
absolute_paths = set()
|
||||||
|
for link in all_links:
|
||||||
|
# Some may be absolute links like https://to/path
|
||||||
|
if link.startswith("http"):
|
||||||
|
if not prevent_outside or link.startswith(base_url):
|
||||||
|
absolute_paths.add(link)
|
||||||
|
# Some may have omitted the protocol like //to/path
|
||||||
|
elif link.startswith("//"):
|
||||||
|
absolute_paths.add(f"{urlparse(base_url).scheme}:{link}")
|
||||||
|
else:
|
||||||
|
absolute_paths.add(urljoin(base_url, link))
|
||||||
|
return list(absolute_paths)
|
@ -1,30 +1,61 @@
|
|||||||
|
import pytest as pytest
|
||||||
|
|
||||||
from langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader
|
from langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
def test_async_recursive_url_loader() -> None:
|
def test_async_recursive_url_loader() -> None:
|
||||||
url = "https://docs.python.org/3.9/"
|
url = "https://docs.python.org/3.9/"
|
||||||
loader = RecursiveUrlLoader(
|
loader = RecursiveUrlLoader(
|
||||||
url=url, extractor=lambda _: "placeholder", use_async=True, max_depth=1
|
url,
|
||||||
|
extractor=lambda _: "placeholder",
|
||||||
|
use_async=True,
|
||||||
|
max_depth=3,
|
||||||
|
timeout=None,
|
||||||
)
|
)
|
||||||
docs = loader.load()
|
docs = loader.load()
|
||||||
assert len(docs) == 24
|
assert len(docs) == 1024
|
||||||
assert docs[0].page_content == "placeholder"
|
assert docs[0].page_content == "placeholder"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
def test_async_recursive_url_loader_deterministic() -> None:
|
||||||
|
url = "https://docs.python.org/3.9/"
|
||||||
|
loader = RecursiveUrlLoader(
|
||||||
|
url,
|
||||||
|
use_async=True,
|
||||||
|
max_depth=3,
|
||||||
|
timeout=None,
|
||||||
|
)
|
||||||
|
docs = sorted(loader.load(), key=lambda d: d.metadata["source"])
|
||||||
|
docs_2 = sorted(loader.load(), key=lambda d: d.metadata["source"])
|
||||||
|
assert docs == docs_2
|
||||||
|
|
||||||
|
|
||||||
def test_sync_recursive_url_loader() -> None:
|
def test_sync_recursive_url_loader() -> None:
|
||||||
url = "https://docs.python.org/3.9/"
|
url = "https://docs.python.org/3.9/"
|
||||||
loader = RecursiveUrlLoader(
|
loader = RecursiveUrlLoader(
|
||||||
url=url, extractor=lambda _: "placeholder", use_async=False, max_depth=1
|
url, extractor=lambda _: "placeholder", use_async=False, max_depth=2
|
||||||
)
|
)
|
||||||
docs = loader.load()
|
docs = loader.load()
|
||||||
assert len(docs) == 24
|
assert len(docs) == 27
|
||||||
assert docs[0].page_content == "placeholder"
|
assert docs[0].page_content == "placeholder"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
def test_sync_async_equivalent() -> None:
|
||||||
|
url = "https://docs.python.org/3.9/"
|
||||||
|
loader = RecursiveUrlLoader(url, use_async=False, max_depth=2)
|
||||||
|
async_loader = RecursiveUrlLoader(url, use_async=False, max_depth=2)
|
||||||
|
docs = sorted(loader.load(), key=lambda d: d.metadata["source"])
|
||||||
|
async_docs = sorted(async_loader.load(), key=lambda d: d.metadata["source"])
|
||||||
|
assert docs == async_docs
|
||||||
|
|
||||||
|
|
||||||
def test_loading_invalid_url() -> None:
|
def test_loading_invalid_url() -> None:
|
||||||
url = "https://this.url.is.invalid/this/is/a/test"
|
url = "https://this.url.is.invalid/this/is/a/test"
|
||||||
loader = RecursiveUrlLoader(
|
loader = RecursiveUrlLoader(
|
||||||
url=url, max_depth=1, extractor=lambda _: "placeholder", use_async=False
|
url, max_depth=1, extractor=lambda _: "placeholder", use_async=False
|
||||||
)
|
)
|
||||||
docs = loader.load()
|
docs = loader.load()
|
||||||
assert len(docs) == 0
|
assert len(docs) == 0
|
||||||
|
@ -0,0 +1,109 @@
|
|||||||
|
from langchain.utils.html import (
|
||||||
|
PREFIXES_TO_IGNORE,
|
||||||
|
SUFFIXES_TO_IGNORE,
|
||||||
|
extract_sub_links,
|
||||||
|
find_all_links,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_all_links_none() -> None:
|
||||||
|
html = "<span>Hello world</span>"
|
||||||
|
actual = find_all_links(html)
|
||||||
|
assert actual == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_all_links_single() -> None:
|
||||||
|
htmls = [
|
||||||
|
"href='foobar.com'",
|
||||||
|
'href="foobar.com"',
|
||||||
|
'<div><a class="blah" href="foobar.com">hullo</a></div>',
|
||||||
|
]
|
||||||
|
actual = [find_all_links(html) for html in htmls]
|
||||||
|
assert actual == [["foobar.com"]] * 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_all_links_multiple() -> None:
|
||||||
|
html = (
|
||||||
|
'<div><a class="blah" href="https://foobar.com">hullo</a></div>'
|
||||||
|
'<div><a class="bleh" href="/baz/cool">buhbye</a></div>'
|
||||||
|
)
|
||||||
|
actual = find_all_links(html)
|
||||||
|
assert sorted(actual) == [
|
||||||
|
"/baz/cool",
|
||||||
|
"https://foobar.com",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_all_links_ignore_suffix() -> None:
|
||||||
|
html = 'href="foobar{suffix}"'
|
||||||
|
for suffix in SUFFIXES_TO_IGNORE:
|
||||||
|
actual = find_all_links(html.format(suffix=suffix))
|
||||||
|
assert actual == []
|
||||||
|
|
||||||
|
# Don't ignore if pattern doesn't occur at end of link.
|
||||||
|
html = 'href="foobar{suffix}more"'
|
||||||
|
for suffix in SUFFIXES_TO_IGNORE:
|
||||||
|
actual = find_all_links(html.format(suffix=suffix))
|
||||||
|
assert actual == [f"foobar{suffix}more"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_all_links_ignore_prefix() -> None:
|
||||||
|
html = 'href="{prefix}foobar"'
|
||||||
|
for prefix in PREFIXES_TO_IGNORE:
|
||||||
|
actual = find_all_links(html.format(prefix=prefix))
|
||||||
|
assert actual == []
|
||||||
|
|
||||||
|
# Don't ignore if pattern doesn't occur at beginning of link.
|
||||||
|
html = 'href="foobar{prefix}more"'
|
||||||
|
for prefix in PREFIXES_TO_IGNORE:
|
||||||
|
# Pound signs are split on when not prefixes.
|
||||||
|
if prefix == "#":
|
||||||
|
continue
|
||||||
|
actual = find_all_links(html.format(prefix=prefix))
|
||||||
|
assert actual == [f"foobar{prefix}more"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_all_links_drop_fragment() -> None:
|
||||||
|
html = 'href="foobar.com/woah#section_one"'
|
||||||
|
actual = find_all_links(html)
|
||||||
|
assert actual == ["foobar.com/woah"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_sub_links() -> None:
|
||||||
|
html = (
|
||||||
|
'<a href="https://foobar.com">one</a>'
|
||||||
|
'<a href="http://baz.net">two</a>'
|
||||||
|
'<a href="//foobar.com/hello">three</a>'
|
||||||
|
'<a href="/how/are/you/doing">four</a>'
|
||||||
|
)
|
||||||
|
expected = sorted(
|
||||||
|
[
|
||||||
|
"https://foobar.com",
|
||||||
|
"https://foobar.com/hello",
|
||||||
|
"https://foobar.com/how/are/you/doing",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
actual = sorted(extract_sub_links(html, "https://foobar.com"))
|
||||||
|
assert actual == expected
|
||||||
|
|
||||||
|
actual = sorted(extract_sub_links(html, "https://foobar.com/hello"))
|
||||||
|
expected = sorted(
|
||||||
|
[
|
||||||
|
"https://foobar.com/hello",
|
||||||
|
"https://foobar.com/how/are/you/doing",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert actual == expected
|
||||||
|
|
||||||
|
actual = sorted(
|
||||||
|
extract_sub_links(html, "https://foobar.com/hello", prevent_outside=False)
|
||||||
|
)
|
||||||
|
expected = sorted(
|
||||||
|
[
|
||||||
|
"https://foobar.com",
|
||||||
|
"http://baz.net",
|
||||||
|
"https://foobar.com/hello",
|
||||||
|
"https://foobar.com/how/are/you/doing",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert actual == expected
|
Loading…
Reference in New Issue