mirror of https://github.com/hwchase17/langchain
Allow readthedoc loader to pass custom html tag (#5175)
## Description The html structure of readthedocs can differ. Currently, the html tag is hardcoded in the reader, and unable to fit into some cases. This pr includes the following changes: 1. Replace `find_all` with `find` because we just want one tag. 2. Provide `custom_html_tag` to the loader. 3. Add tests for readthedoc loader 4. Refactor code ## Issues See more in https://github.com/hwchase17/langchain/pull/2609. The problem was not completely fixed in that pr. --------- Signed-off-by: byhsu <byhsu@linkedin.com> Co-authored-by: byhsu <byhsu@linkedin.com> Co-authored-by: Dev 2049 <dev.dev2049@gmail.com>pull/5162/head^2
parent
d8eed6018f
commit
f0730c6489
@ -0,0 +1,5 @@
|
||||
<html>
|
||||
<article role="main">
|
||||
Hello World!
|
||||
</article>
|
||||
</html>
|
@ -0,0 +1,5 @@
|
||||
<html>
|
||||
<div role="main">
|
||||
Hello World!
|
||||
</div>
|
||||
</html>
|
@ -0,0 +1,5 @@
|
||||
<html>
|
||||
<main id="main-content">
|
||||
Hello World!
|
||||
</main>
|
||||
</html>
|
@ -0,0 +1,40 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from langchain.document_loaders.readthedocs import ReadTheDocsLoader
|
||||
|
||||
PARENT_DIR = Path(__file__).parent / "test_docs" / "readthedocs"
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_main_id_main_content() -> None:
|
||||
loader = ReadTheDocsLoader(PARENT_DIR / "main_id_main_content")
|
||||
documents = loader.load()
|
||||
assert len(documents[0].page_content) != 0
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_div_role_main() -> None:
|
||||
loader = ReadTheDocsLoader(PARENT_DIR / "div_role_main")
|
||||
documents = loader.load()
|
||||
assert len(documents[0].page_content) != 0
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_custom() -> None:
|
||||
loader = ReadTheDocsLoader(
|
||||
PARENT_DIR / "custom",
|
||||
custom_html_tag=("article", {"role": "main"}),
|
||||
)
|
||||
documents = loader.load()
|
||||
assert len(documents[0].page_content) != 0
|
||||
|
||||
|
||||
@pytest.mark.requires("bs4")
|
||||
def test_empty() -> None:
|
||||
loader = ReadTheDocsLoader(
|
||||
PARENT_DIR / "custom",
|
||||
)
|
||||
documents = loader.load()
|
||||
assert len(documents[0].page_content) == 0
|
Loading…
Reference in New Issue