forked from Archives/langchain
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
29 lines
844 B
Python
29 lines
844 B
Python
"""Tests for the HTML parsers."""
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from langchain.document_loaders.blob_loaders import Blob
|
|
from langchain.document_loaders.parsers.html import BS4HTMLParser
|
|
|
|
HERE = Path(__file__).parent
|
|
EXAMPLES = HERE.parent.parent.parent / "integration_tests" / "examples"
|
|
|
|
|
|
@pytest.mark.requires("bs4", "lxml")
|
|
def test_bs_html_loader() -> None:
|
|
"""Test unstructured loader."""
|
|
file_path = EXAMPLES / "example.html"
|
|
blob = Blob.from_path(file_path)
|
|
parser = BS4HTMLParser(get_text_separator="|")
|
|
docs = list(parser.lazy_parse(blob))
|
|
assert isinstance(docs, list)
|
|
assert len(docs) == 1
|
|
|
|
metadata = docs[0].metadata
|
|
content = docs[0].page_content
|
|
|
|
assert metadata["title"] == "Chew dad's slippers"
|
|
assert metadata["source"] == str(file_path)
|
|
assert content[:2] == "\n|"
|