forked from Archives/langchain
move tests to unit tests (#4481)
parent
b5d3e99f9b
commit
21ec50009b
@ -1,25 +0,0 @@
|
|||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from langchain.document_loaders import DirectoryLoader, TextLoader
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.requires("chardet")
|
|
||||||
def test_text_loader() -> None:
|
|
||||||
"""Test text loader."""
|
|
||||||
path = Path(__file__).parent.parent / "examples"
|
|
||||||
files = path.glob("**/*.txt")
|
|
||||||
loader = DirectoryLoader(str(path), glob="**/*.txt", loader_cls=TextLoader)
|
|
||||||
loader_detect_encoding = DirectoryLoader(
|
|
||||||
str(path),
|
|
||||||
glob="**/*.txt",
|
|
||||||
loader_kwargs={"autodetect_encoding": True},
|
|
||||||
loader_cls=TextLoader,
|
|
||||||
)
|
|
||||||
|
|
||||||
with pytest.raises((UnicodeDecodeError, RuntimeError)):
|
|
||||||
loader.load()
|
|
||||||
|
|
||||||
docs = loader_detect_encoding.load()
|
|
||||||
assert len(docs) == len(list(files))
|
|
@ -0,0 +1,26 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from langchain.document_loaders import DirectoryLoader, TextLoader
|
||||||
|
|
||||||
|
class TestTextLoader:
|
||||||
|
|
||||||
|
@pytest.mark.requires("chardet")
|
||||||
|
def test_load_directory(self) -> None:
|
||||||
|
"""Test text loader."""
|
||||||
|
path = Path(__file__).parent.parent / "examples"
|
||||||
|
files = path.glob("**/*.txt")
|
||||||
|
loader = DirectoryLoader(str(path), glob="**/*.txt", loader_cls=TextLoader)
|
||||||
|
loader_detect_encoding = DirectoryLoader(
|
||||||
|
str(path),
|
||||||
|
glob="**/*.txt",
|
||||||
|
loader_kwargs={"autodetect_encoding": True},
|
||||||
|
loader_cls=TextLoader,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises((UnicodeDecodeError, RuntimeError)):
|
||||||
|
loader.load()
|
||||||
|
|
||||||
|
docs = loader_detect_encoding.load()
|
||||||
|
assert len(docs) == len(list(files))
|
Loading…
Reference in New Issue