mirror of
https://github.com/hwchase17/langchain
synced 2024-11-10 01:10:59 +00:00
ba54f1577f
Thank you for contributing to LangChain! - [x] **PR title**: "community: added support for llmsherpa library" - [x] **Add tests and docs**: 1. Integration test: 'docs/docs/integrations/document_loaders/test_llmsherpa.py'. 2. an example notebook: `docs/docs/integrations/document_loaders/llmsherpa.ipynb`. - [x] **Lint and test**: Run `make format`, `make lint` and `make test` from the root of the package(s) you've modified. See contribution guidelines for more: https://python.langchain.com/docs/contributing/ Additional guidelines: - Make sure optional dependencies are imported within a function. - Please do not add dependencies to pyproject.toml files (even optional ones) unless they are required for unit tests. - Most PRs should not touch more than one package. - Changes should be backwards compatible. - If you are adding something to community, do not re-import it in langchain. If no one reviews your PR within a few days, please @-mention one of baskaryan, efriis, eyurtsev, hwchase17. --------- Co-authored-by: Bagatur <22008038+baskaryan@users.noreply.github.com> Co-authored-by: Bagatur <baskaryan@gmail.com>
47 lines
1.3 KiB
Python
47 lines
1.3 KiB
Python
from langchain_community.document_loaders.llmsherpa import LLMSherpaFileLoader
|
|
|
|
file_path = "https://arxiv.org/pdf/2402.14207.pdf"
|
|
|
|
|
|
def test_llmsherpa_file_loader_initialization() -> None:
|
|
loader = LLMSherpaFileLoader(
|
|
file_path=file_path,
|
|
)
|
|
docs = loader.load()
|
|
assert isinstance(loader, LLMSherpaFileLoader)
|
|
assert hasattr(docs, "__iter__")
|
|
assert loader.strategy == "chunks"
|
|
assert (
|
|
loader.url
|
|
== "https://readers.llmsherpa.com/api/document/developer/parseDocument?renderFormat=all&useNewIndentParser=true&applyOcr=yes"
|
|
)
|
|
assert len(docs) > 1
|
|
|
|
|
|
def test_apply_ocr() -> None:
|
|
loader = LLMSherpaFileLoader(
|
|
file_path=file_path,
|
|
apply_ocr=True,
|
|
new_indent_parser=False,
|
|
)
|
|
docs = loader.load()
|
|
assert (
|
|
loader.url
|
|
== "https://readers.llmsherpa.com/api/document/developer/parseDocument?renderFormat=all&applyOcr=yes"
|
|
)
|
|
assert len(docs) > 1
|
|
|
|
|
|
def test_new_indent_parser() -> None:
|
|
loader = LLMSherpaFileLoader(
|
|
file_path=file_path,
|
|
apply_ocr=False,
|
|
new_indent_parser=True,
|
|
)
|
|
docs = loader.load()
|
|
assert (
|
|
loader.url
|
|
== "https://readers.llmsherpa.com/api/document/developer/parseDocument?renderFormat=all&useNewIndentParser=true"
|
|
)
|
|
assert len(docs) > 1
|