Fix web-base loader (#11135)

Fix initialization

https://github.com/langchain-ai/langchain/issues/11095
pull/11205/head
Eugene Yurtsev 11 months ago committed by GitHub
parent 3bc44b01c0
commit 2c114fcb5e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -76,10 +76,15 @@ class WebBaseLoader(BaseLoader):
)
if web_paths:
self.web_paths = list(web_paths)
elif isinstance(web_path, str):
self.web_paths = [web_path]
elif isinstance(web_path, Sequence):
self.web_paths = list(web_path)
else:
self.web_paths = [web_path]
raise TypeError(
f"web_path must be str or Sequence[str] got ({type(web_path)}) or"
f" web_paths must be Sequence[str] got ({type(web_paths)})"
)
self.requests_per_second = requests_per_second
self.default_parser = default_parser
self.requests_kwargs = requests_kwargs or {}

@ -11,3 +11,11 @@ class TestWebBaseLoader:
url = "https://www.example.com"
loader = WebBaseLoader(url, header_template=header_template)
assert loader.session.headers["User-Agent"] == user_specified_user_agent
def test_web_path_parameter(self) -> None:
web_base_loader = WebBaseLoader(web_paths=["https://www.example.com"])
assert web_base_loader.web_paths == ["https://www.example.com"]
web_base_loader = WebBaseLoader(web_path=["https://www.example.com"])
assert web_base_loader.web_paths == ["https://www.example.com"]
web_base_loader = WebBaseLoader(web_path="https://www.example.com")
assert web_base_loader.web_paths == ["https://www.example.com"]

Loading…
Cancel
Save