mirror of
https://github.com/hwchase17/langchain
synced 2024-11-04 06:00:26 +00:00
add continue to fix 'continue_on_failure' parameter for URL doc loader (#2735)
Currently, the function still fails if `continue_on_failure` is set to True, because `elements` is not set. --------- Co-authored-by: leecjohnny <johnny-lee1255@users.noreply.github.com>
This commit is contained in:
parent
4bdcedab54
commit
0ab364404e
@ -63,6 +63,7 @@ class UnstructuredURLLoader(BaseLoader):
|
||||
except Exception as e:
|
||||
if self.continue_on_failure:
|
||||
logger.error(f"Error fetching or processing {url}, exeption: {e}")
|
||||
continue
|
||||
else:
|
||||
raise e
|
||||
text = "\n\n".join([str(el) for el in elements])
|
||||
|
16
tests/integration_tests/document_loaders/test_url.py
Normal file
16
tests/integration_tests/document_loaders/test_url.py
Normal file
@ -0,0 +1,16 @@
|
||||
import pytest
|
||||
|
||||
from langchain.document_loaders import UnstructuredURLLoader
|
||||
|
||||
|
||||
def test_continue_on_failure_true() -> None:
|
||||
"""Test exception is not raised when continue_on_failure=True."""
|
||||
loader = UnstructuredURLLoader(["badurl.foobar"])
|
||||
loader.load()
|
||||
|
||||
|
||||
def test_continue_on_failure_false() -> None:
|
||||
"""Test exception is raised when continue_on_failure=False."""
|
||||
loader = UnstructuredURLLoader(["badurl.foobar"], continue_on_failure=False)
|
||||
with pytest.raises(Exception):
|
||||
loader.load()
|
Loading…
Reference in New Issue
Block a user