langchain/libs/community/tests/unit_tests/document_loaders/test_generic_loader.py

"""Test generic loader."""
import os
import tempfile
from pathlib import Path
from typing import Any, Generator, Iterator

import pytest
from langchain_core.documents import Document

from langchain_community.document_loaders.base import BaseBlobParser
from langchain_community.document_loaders.blob_loaders import Blob, FileSystemBlobLoader
from langchain_community.document_loaders.generic import GenericLoader
from langchain_community.document_loaders.parsers.txt import TextParser


@pytest.fixture
def toy_dir() -> Generator[Path, None, None]:
    """Yield a pre-populated directory to test the blob loader."""
    with tempfile.TemporaryDirectory() as temp_dir:
        # Create test.txt
        with open(os.path.join(temp_dir, "test.txt"), "w") as test_txt:
            test_txt.write("This is a test.txt file.")

        # Create test.html
        with open(os.path.join(temp_dir, "test.html"), "w") as test_html:
            test_html.write(
                "<html><body><h1>This is a test.html file.</h1></body></html>"
            )

        # Create .hidden_file
        with open(os.path.join(temp_dir, ".hidden_file"), "w") as hidden_file:
            hidden_file.write("This is a hidden file.")

        # Create some_dir/nested_file.txt
        some_dir = os.path.join(temp_dir, "some_dir")
        os.makedirs(some_dir)
        with open(os.path.join(some_dir, "nested_file.txt"), "w") as nested_file:
            nested_file.write("This is a nested_file.txt file.")

        # Create some_dir/other_dir/more_nested.txt
        other_dir = os.path.join(some_dir, "other_dir")
        os.makedirs(other_dir)
        with open(os.path.join(other_dir, "more_nested.txt"), "w") as nested_file:
            nested_file.write("This is a more_nested.txt file.")

        yield Path(temp_dir)


class AsIsParser(BaseBlobParser):
    """Parser created for testing purposes."""

    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
        """Extract the first character of a blob."""
        yield Document(page_content=blob.as_string())


def test__init__(toy_dir: str) -> None:
    """Test initialization from init."""
    loader = GenericLoader(
        FileSystemBlobLoader(toy_dir, suffixes=[".txt"]),
        AsIsParser(),
    )
    docs = loader.load()
    assert len(docs) == 3
    # Glob order seems to be deterministic with recursion. If this test becomes flaky,
    # we can sort the docs by page content.
    assert docs[0].page_content == "This is a test.txt file."


def test_from_filesystem_classmethod(toy_dir: str) -> None:
    """Test generic loader."""
    loader = GenericLoader.from_filesystem(
        toy_dir, suffixes=[".txt"], parser=AsIsParser()
    )
    docs = loader.load()
    assert len(docs) == 3
    # Glob order seems to be deterministic with recursion. If this test becomes flaky,
    # we can sort the docs by page content.
    assert docs[0].page_content == "This is a test.txt file."


def test_from_filesystem_classmethod_with_path(toy_dir: str) -> None:
    loader = GenericLoader.from_filesystem(os.path.join(toy_dir, "test.txt"))
    docs = loader.load()
    assert len(docs) == 1
    assert docs[0].page_content == "This is a test.txt file."


def test_from_filesystem_classmethod_with_glob(toy_dir: str) -> None:
    """Test that glob parameter is taken into account."""
    loader = GenericLoader.from_filesystem(toy_dir, glob="*.txt", parser=AsIsParser())
    docs = loader.load()
    assert len(docs) == 1
    # Glob order seems to be deterministic with recursion. If this test becomes flaky,
    # we can sort the docs by page content.
    assert docs[0].page_content == "This is a test.txt file."


@pytest.mark.requires("tqdm")
def test_from_filesystem_classmethod_show_progress(toy_dir: str) -> None:
    """Test that glob parameter is taken into account."""
    loader = GenericLoader.from_filesystem(
        toy_dir, glob="*.txt", parser=AsIsParser(), show_progress=True
    )
    docs = loader.load()
    assert len(docs) == 1
    # Glob order seems to be deterministic with recursion. If this test becomes flaky,
    # we can sort the docs by page content.
    assert docs[0].page_content == "This is a test.txt file."


def test_from_filesystem_using_default_parser(toy_dir: str) -> None:
    """Use the default generic parser."""
    loader = GenericLoader.from_filesystem(
        toy_dir,
        suffixes=[".txt"],
    )
    docs = loader.load()
    assert len(docs) == 3
    # Glob order seems to be deterministic with recursion. If this test becomes flaky,
    # we can sort the docs by page content.
    assert docs[0].page_content == "This is a test.txt file."


def test_specifying_parser_via_class_attribute(toy_dir: str) -> None:
    class TextLoader(GenericLoader):
        """Parser created for testing purposes."""

        @staticmethod
        def get_parser(**kwargs: Any) -> BaseBlobParser:
            return TextParser()

    loader = TextLoader.from_filesystem(toy_dir, suffixes=[".txt"])
    docs = loader.load()
    assert len(docs) == 3
    # Glob order seems to be deterministic with recursion. If this test becomes flaky,
    # we can sort the docs by page content.
    assert docs[0].page_content == "This is a test.txt file."
Add a generic document loader (#4875) # Add generic document loader * This PR adds a generic document loader which can assemble a loader from a blob loader and a parser * Adds a registry for parsers * Populate registry with a default mimetype based parser ## Expected changes - Parsing involves loading content via IO so can be sped up via: * Threading in sync * Async - The actual parsing logic may be computatinoally involved: may need to figure out to add multi-processing support - May want to add suffix based parser since suffixes are easier to specify in comparison to mime types ## Before submitting No notebooks yet, we first need to get a few of the basic parsers up (prior to advertising the interface) 1 year ago			`"""Test generic loader."""`
			`import os`
			`import tempfile`
			`from pathlib import Path`
Improve file system blob loader and generic loader (#14004) * Add support for passing a specific file to the file system blob loader * Allow specifying a class parameter for the parser for the generic loader ```python class AudioLoader(GenericLoader): @staticmethod def get_parser(kwargs): return MyAudioParser(kwargs): ``` The intent of the GenericLoader is to provide on-ramps from different sources (e.g., web, s3, file system). An alternative is to use pipelining syntax or creating a Pipeline ``` FileSystemBlobLoader(...) \| MyAudioParser ``` --------- Co-authored-by: Bagatur <baskaryan@gmail.com> 10 months ago			`from typing import Any, Generator, Iterator`
Add a generic document loader (#4875) # Add generic document loader * This PR adds a generic document loader which can assemble a loader from a blob loader and a parser * Adds a registry for parsers * Populate registry with a default mimetype based parser ## Expected changes - Parsing involves loading content via IO so can be sped up via: * Threading in sync * Async - The actual parsing logic may be computatinoally involved: may need to figure out to add multi-processing support - May want to add suffix based parser since suffixes are easier to specify in comparison to mime types ## Before submitting No notebooks yet, we first need to get a few of the basic parsers up (prior to advertising the interface) 1 year ago
			`import pytest`
REFACTOR: Refactor langchain_core (#13627) Changes: - remove langchain_core/schema since no clear distinction b/n schema and non-schema modules - make every module that doesn't end in -y plural - where easy have 1-2 classes per file - no more than one level of nesting in directories - only import from top level core modules in langchain 10 months ago			`from langchain_core.documents import Document`
Add a generic document loader (#4875) # Add generic document loader * This PR adds a generic document loader which can assemble a loader from a blob loader and a parser * Adds a registry for parsers * Populate registry with a default mimetype based parser ## Expected changes - Parsing involves loading content via IO so can be sped up via: * Threading in sync * Async - The actual parsing logic may be computatinoally involved: may need to figure out to add multi-processing support - May want to add suffix based parser since suffixes are easier to specify in comparison to mime types ## Before submitting No notebooks yet, we first need to get a few of the basic parsers up (prior to advertising the interface) 1 year ago
community[major], core[patch], langchain[patch], experimental[patch]: Create langchain-community (#14463) Moved the following modules to new package langchain-community in a backwards compatible fashion: ``` mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community mv langchain/langchain/adapters community/langchain_community mv langchain/langchain/callbacks community/langchain_community/callbacks mv langchain/langchain/chat_loaders community/langchain_community mv langchain/langchain/chat_models community/langchain_community mv langchain/langchain/document_loaders community/langchain_community mv langchain/langchain/docstore community/langchain_community mv langchain/langchain/document_transformers community/langchain_community mv langchain/langchain/embeddings community/langchain_community mv langchain/langchain/graphs community/langchain_community mv langchain/langchain/llms community/langchain_community mv langchain/langchain/memory/chat_message_histories community/langchain_community mv langchain/langchain/retrievers community/langchain_community mv langchain/langchain/storage community/langchain_community mv langchain/langchain/tools community/langchain_community mv langchain/langchain/utilities community/langchain_community mv langchain/langchain/vectorstores community/langchain_community mv langchain/langchain/agents/agent_toolkits community/langchain_community mv langchain/langchain/cache.py community/langchain_community ``` Moved the following to core ``` mv langchain/langchain/utils/json_schema.py core/langchain_core/utils mv langchain/langchain/utils/html.py core/langchain_core/utils mv langchain/langchain/utils/strings.py core/langchain_core/utils cat langchain/langchain/utils/env.py >> core/langchain_core/utils/env.py rm langchain/langchain/utils/env.py ``` See .scripts/community_split/script_integrations.sh for all changes 9 months ago			`from langchain_community.document_loaders.base import BaseBlobParser`
			`from langchain_community.document_loaders.blob_loaders import Blob, FileSystemBlobLoader`
			`from langchain_community.document_loaders.generic import GenericLoader`
			`from langchain_community.document_loaders.parsers.txt import TextParser`
Add a generic document loader (#4875) # Add generic document loader * This PR adds a generic document loader which can assemble a loader from a blob loader and a parser * Adds a registry for parsers * Populate registry with a default mimetype based parser ## Expected changes - Parsing involves loading content via IO so can be sped up via: * Threading in sync * Async - The actual parsing logic may be computatinoally involved: may need to figure out to add multi-processing support - May want to add suffix based parser since suffixes are easier to specify in comparison to mime types ## Before submitting No notebooks yet, we first need to get a few of the basic parsers up (prior to advertising the interface) 1 year ago

			`@pytest.fixture`
			`def toy_dir() -> Generator[Path, None, None]:`
			`"""Yield a pre-populated directory to test the blob loader."""`
			`with tempfile.TemporaryDirectory() as temp_dir:`
			`# Create test.txt`
			`with open(os.path.join(temp_dir, "test.txt"), "w") as test_txt:`
			`test_txt.write("This is a test.txt file.")`

			`# Create test.html`
			`with open(os.path.join(temp_dir, "test.html"), "w") as test_html:`
			`test_html.write(`
			`"<html><body><h1>This is a test.html file.</h1></body></html>"`
			`)`

			`# Create .hidden_file`
			`with open(os.path.join(temp_dir, ".hidden_file"), "w") as hidden_file:`
			`hidden_file.write("This is a hidden file.")`

			`# Create some_dir/nested_file.txt`
			`some_dir = os.path.join(temp_dir, "some_dir")`
			`os.makedirs(some_dir)`
			`with open(os.path.join(some_dir, "nested_file.txt"), "w") as nested_file:`
			`nested_file.write("This is a nested_file.txt file.")`

			`# Create some_dir/other_dir/more_nested.txt`
			`other_dir = os.path.join(some_dir, "other_dir")`
			`os.makedirs(other_dir)`
			`with open(os.path.join(other_dir, "more_nested.txt"), "w") as nested_file:`
			`nested_file.write("This is a more_nested.txt file.")`

			`yield Path(temp_dir)`


			`class AsIsParser(BaseBlobParser):`
			`"""Parser created for testing purposes."""`

			`def lazy_parse(self, blob: Blob) -> Iterator[Document]:`
			`"""Extract the first character of a blob."""`
			`yield Document(page_content=blob.as_string())`


			`def test__init__(toy_dir: str) -> None:`
			`"""Test initialization from init."""`
			`loader = GenericLoader(`
			`FileSystemBlobLoader(toy_dir, suffixes=[".txt"]),`
			`AsIsParser(),`
			`)`
			`docs = loader.load()`
			`assert len(docs) == 3`
			`# Glob order seems to be deterministic with recursion. If this test becomes flaky,`
			`# we can sort the docs by page content.`
			`assert docs[0].page_content == "This is a test.txt file."`


			`def test_from_filesystem_classmethod(toy_dir: str) -> None:`
			`"""Test generic loader."""`
			`loader = GenericLoader.from_filesystem(`
			`toy_dir, suffixes=[".txt"], parser=AsIsParser()`
			`)`
			`docs = loader.load()`
			`assert len(docs) == 3`
			`# Glob order seems to be deterministic with recursion. If this test becomes flaky,`
			`# we can sort the docs by page content.`
			`assert docs[0].page_content == "This is a test.txt file."`


Improve file system blob loader and generic loader (#14004) * Add support for passing a specific file to the file system blob loader * Allow specifying a class parameter for the parser for the generic loader ```python class AudioLoader(GenericLoader): @staticmethod def get_parser(kwargs): return MyAudioParser(kwargs): ``` The intent of the GenericLoader is to provide on-ramps from different sources (e.g., web, s3, file system). An alternative is to use pipelining syntax or creating a Pipeline ``` FileSystemBlobLoader(...) \| MyAudioParser ``` --------- Co-authored-by: Bagatur <baskaryan@gmail.com> 10 months ago			`def test_from_filesystem_classmethod_with_path(toy_dir: str) -> None:`
			`loader = GenericLoader.from_filesystem(os.path.join(toy_dir, "test.txt"))`
			`docs = loader.load()`
			`assert len(docs) == 1`
			`assert docs[0].page_content == "This is a test.txt file."`


Add a generic document loader (#4875) # Add generic document loader * This PR adds a generic document loader which can assemble a loader from a blob loader and a parser * Adds a registry for parsers * Populate registry with a default mimetype based parser ## Expected changes - Parsing involves loading content via IO so can be sped up via: * Threading in sync * Async - The actual parsing logic may be computatinoally involved: may need to figure out to add multi-processing support - May want to add suffix based parser since suffixes are easier to specify in comparison to mime types ## Before submitting No notebooks yet, we first need to get a few of the basic parsers up (prior to advertising the interface) 1 year ago			`def test_from_filesystem_classmethod_with_glob(toy_dir: str) -> None:`
			`"""Test that glob parameter is taken into account."""`
			`loader = GenericLoader.from_filesystem(toy_dir, glob="*.txt", parser=AsIsParser())`
			`docs = loader.load()`
			`assert len(docs) == 1`
			`# Glob order seems to be deterministic with recursion. If this test becomes flaky,`
			`# we can sort the docs by page content.`
			`assert docs[0].page_content == "This is a test.txt file."`


			`@pytest.mark.requires("tqdm")`
			`def test_from_filesystem_classmethod_show_progress(toy_dir: str) -> None:`
			`"""Test that glob parameter is taken into account."""`
			`loader = GenericLoader.from_filesystem(`
			`toy_dir, glob="*.txt", parser=AsIsParser(), show_progress=True`
			`)`
			`docs = loader.load()`
			`assert len(docs) == 1`
			`# Glob order seems to be deterministic with recursion. If this test becomes flaky,`
			`# we can sort the docs by page content.`
			`assert docs[0].page_content == "This is a test.txt file."`


			`def test_from_filesystem_using_default_parser(toy_dir: str) -> None:`
			`"""Use the default generic parser."""`
			`loader = GenericLoader.from_filesystem(`
			`toy_dir,`
			`suffixes=[".txt"],`
			`)`
			`docs = loader.load()`
			`assert len(docs) == 3`
			`# Glob order seems to be deterministic with recursion. If this test becomes flaky,`
			`# we can sort the docs by page content.`
			`assert docs[0].page_content == "This is a test.txt file."`
Improve file system blob loader and generic loader (#14004) * Add support for passing a specific file to the file system blob loader * Allow specifying a class parameter for the parser for the generic loader ```python class AudioLoader(GenericLoader): @staticmethod def get_parser(kwargs): return MyAudioParser(kwargs): ``` The intent of the GenericLoader is to provide on-ramps from different sources (e.g., web, s3, file system). An alternative is to use pipelining syntax or creating a Pipeline ``` FileSystemBlobLoader(...) \| MyAudioParser ``` --------- Co-authored-by: Bagatur <baskaryan@gmail.com> 10 months ago

			`def test_specifying_parser_via_class_attribute(toy_dir: str) -> None:`
			`class TextLoader(GenericLoader):`
			`"""Parser created for testing purposes."""`

			`@staticmethod`
			`def get_parser(**kwargs: Any) -> BaseBlobParser:`
			`return TextParser()`

			`loader = TextLoader.from_filesystem(toy_dir, suffixes=[".txt"])`
			`docs = loader.load()`
			`assert len(docs) == 3`
			`# Glob order seems to be deterministic with recursion. If this test becomes flaky,`
			`# we can sort the docs by page content.`
			`assert docs[0].page_content == "This is a test.txt file."`