move tests to unit tests (#4479)

pyproject.toml

@@ -174,7 +174,7 @@ azure = ["azure-identity", "azure-cosmos", "openai", "azure-core"]
 all = ["anthropic", "cohere", "openai", "nlpcloud", "huggingface_hub", "jina", "manifest-ml", "elasticsearch", "opensearch-py", "google-search-results", "faiss-cpu", "sentence-transformers", "transformers", "spacy", "nltk", "wikipedia", "beautifulsoup4", "tiktoken", "torch", "jinja2", "pinecone-client", "pinecone-text", "weaviate-client", "redis", "google-api-python-client", "wolframalpha", "qdrant-client", "tensorflow-text", "pypdf", "networkx", "nomic", "aleph-alpha-client", "deeplake", "pgvector", "psycopg2-binary", "boto3", "pyowm", "pytesseract", "html2text", "atlassian-python-api", "gptcache", "duckduckgo-search", "arxiv", "azure-identity", "clickhouse-connect", "azure-cosmos", "lancedb", "lark", "pexpect", "pyvespa", "O365", "jq", "docarray", "protobuf", "hnswlib", "chardet"]
 # An extra used to be able to add extended testing.
 extended_testing = [
-  "pypdf", "pdfminer.six", "tqdm"
+  "pypdf", "pdfminer.six", "tqdm", "chardet"
 ]
 
 [tool.ruff]

tests/integration_tests/document_loaders/test_dir_text_loader.py

@@ -1,25 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-from langchain.document_loaders import DirectoryLoader, TextLoader
-
-
-@pytest.mark.requires("chardet")
-def test_text_loader() -> None:
-    """Test text loader."""
-    path = Path(__file__).parent.parent / "examples"
-    files = path.glob("**/*.txt")
-    loader = DirectoryLoader(str(path), glob="**/*.txt", loader_cls=TextLoader)
-    loader_detect_encoding = DirectoryLoader(
-        str(path),
-        glob="**/*.txt",
-        loader_kwargs={"autodetect_encoding": True},
-        loader_cls=TextLoader,
-    )
-
-    with pytest.raises((UnicodeDecodeError, RuntimeError)):
-        loader.load()
-
-    docs = loader_detect_encoding.load()
-    assert len(docs) == len(list(files))

tests/unit_tests/document_loader/test_text_loader.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+import pytest
+
+from langchain.document_loaders import DirectoryLoader, TextLoader
+
+class TestTextLoader:
+
+    @pytest.mark.requires("chardet")
+    def test_load_directory(self) -> None:
+        """Test text loader."""
+        path = Path(__file__).parent.parent / "examples"
+        files = path.glob("**/*.txt")
+        loader = DirectoryLoader(str(path), glob="**/*.txt", loader_cls=TextLoader)
+        loader_detect_encoding = DirectoryLoader(
+            str(path),
+            glob="**/*.txt",
+            loader_kwargs={"autodetect_encoding": True},
+            loader_cls=TextLoader,
+        )
+
+        with pytest.raises((UnicodeDecodeError, RuntimeError)):
+            loader.load()
+
+        docs = loader_detect_encoding.load()
+        assert len(docs) == len(list(files))