diff --git a/langchain/document_loaders/html_bs.py b/langchain/document_loaders/html_bs.py index fc636367a3..4a73187ad3 100644 --- a/langchain/document_loaders/html_bs.py +++ b/langchain/document_loaders/html_bs.py @@ -17,6 +17,7 @@ class BSHTMLLoader(BaseLoader): file_path: str, open_encoding: Union[str, None] = None, bs_kwargs: Union[dict, None] = None, + get_text_separator: str = "", ) -> None: """Initialise with path, and optionally, file encoding to use, and any kwargs to pass to the BeautifulSoup object.""" @@ -33,6 +34,7 @@ class BSHTMLLoader(BaseLoader): if bs_kwargs is None: bs_kwargs = {"features": "lxml"} self.bs_kwargs = bs_kwargs + self.get_text_separator = get_text_separator def load(self) -> List[Document]: from bs4 import BeautifulSoup @@ -41,7 +43,7 @@ class BSHTMLLoader(BaseLoader): with open(self.file_path, "r", encoding=self.open_encoding) as f: soup = BeautifulSoup(f, **self.bs_kwargs) - text = soup.get_text() + text = soup.get_text(self.get_text_separator) if soup.title: title = str(soup.title.string) diff --git a/tests/integration_tests/document_loaders/test_bshtml.py b/tests/integration_tests/document_loaders/test_bshtml.py index 2510788e1d..038371fab5 100644 --- a/tests/integration_tests/document_loaders/test_bshtml.py +++ b/tests/integration_tests/document_loaders/test_bshtml.py @@ -9,15 +9,17 @@ from langchain.document_loaders.html_bs import BSHTMLLoader def test_bs_html_loader() -> None: """Test unstructured loader.""" file_path = Path(__file__).parent.parent / "examples/example.html" - loader = BSHTMLLoader(str(file_path)) + loader = BSHTMLLoader(str(file_path), get_text_separator="|") docs = loader.load() assert len(docs) == 1 metadata = docs[0].metadata + content = docs[0].page_content assert metadata["title"] == "Chew dad's slippers" assert metadata["source"] == str(file_path) + assert content[:2] == "\n|" @pytest.mark.skipif(