add continue to fix 'continue_on_failure' parameter for URL doc loader (#2735)

Currently, the function still fails if `continue_on_failure` is set to
True, because `elements` is not set.

---------

Co-authored-by: leecjohnny <johnny-lee1255@users.noreply.github.com>
This commit is contained in:
Johnny Lee 2023-04-12 00:12:39 -04:00 committed by GitHub
parent 4bdcedab54
commit 0ab364404e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 0 deletions

View File

@ -63,6 +63,7 @@ class UnstructuredURLLoader(BaseLoader):
except Exception as e:
if self.continue_on_failure:
logger.error(f"Error fetching or processing {url}, exeption: {e}")
continue
else:
raise e
text = "\n\n".join([str(el) for el in elements])

View File

@ -0,0 +1,16 @@
import pytest
from langchain.document_loaders import UnstructuredURLLoader
def test_continue_on_failure_true() -> None:
"""Test exception is not raised when continue_on_failure=True."""
loader = UnstructuredURLLoader(["badurl.foobar"])
loader.load()
def test_continue_on_failure_false() -> None:
"""Test exception is raised when continue_on_failure=False."""
loader = UnstructuredURLLoader(["badurl.foobar"], continue_on_failure=False)
with pytest.raises(Exception):
loader.load()