diff --git a/langchain/document_loaders/url.py b/langchain/document_loaders/url.py index a94c85e5..c0dca346 100644 --- a/langchain/document_loaders/url.py +++ b/langchain/document_loaders/url.py @@ -63,6 +63,7 @@ class UnstructuredURLLoader(BaseLoader): except Exception as e: if self.continue_on_failure: logger.error(f"Error fetching or processing {url}, exeption: {e}") + continue else: raise e text = "\n\n".join([str(el) for el in elements]) diff --git a/tests/integration_tests/document_loaders/test_url.py b/tests/integration_tests/document_loaders/test_url.py new file mode 100644 index 00000000..f61a8114 --- /dev/null +++ b/tests/integration_tests/document_loaders/test_url.py @@ -0,0 +1,16 @@ +import pytest + +from langchain.document_loaders import UnstructuredURLLoader + + +def test_continue_on_failure_true() -> None: + """Test exception is not raised when continue_on_failure=True.""" + loader = UnstructuredURLLoader(["badurl.foobar"]) + loader.load() + + +def test_continue_on_failure_false() -> None: + """Test exception is raised when continue_on_failure=False.""" + loader = UnstructuredURLLoader(["badurl.foobar"], continue_on_failure=False) + with pytest.raises(Exception): + loader.load()