From 030ce9f50637dcbff44e6ee67a019374f7cc7adc Mon Sep 17 00:00:00 2001
From: Tim Asp <707699+timothyasp@users.noreply.github.com>
Date: Thu, 23 Mar 2023 21:56:13 -0700
Subject: [PATCH] fix import error of bs4 (#1952)
Ran into a broken build if bs4 wasn't installed in the project.
Minor tweak to follow the other doc loaders optional package-loading
conventions.
Also updated html docs to include reference to this new html loader.
side note: Should there be 2 different html-to-text document loaders?
This new one only handles local files, while the existing unstructured
html loader handles HTML from local and remote. So it seems like the
improvement was adding the title to the metadata, which is useful but
could also be added to `html.py`
---
.../examples/example_data/fake-content.html | 3 ++
.../document_loaders/examples/html.ipynb | 52 +++++++++++++++++--
langchain/document_loaders/html_bs.py | 11 +++-
3 files changed, 59 insertions(+), 7 deletions(-)
diff --git a/docs/modules/document_loaders/examples/example_data/fake-content.html b/docs/modules/document_loaders/examples/example_data/fake-content.html
index 9ad19d30..acba7602 100644
--- a/docs/modules/document_loaders/examples/example_data/fake-content.html
+++ b/docs/modules/document_loaders/examples/example_data/fake-content.html
@@ -1,5 +1,8 @@
+
+ Test Title
+
My First Heading
diff --git a/docs/modules/document_loaders/examples/html.ipynb b/docs/modules/document_loaders/examples/html.ipynb
index 2a498828..91ff32e0 100644
--- a/docs/modules/document_loaders/examples/html.ipynb
+++ b/docs/modules/document_loaders/examples/html.ipynb
@@ -48,9 +48,7 @@
"outputs": [
{
"data": {
- "text/plain": [
- "[Document(page_content='My First Heading\\n\\nMy first paragraph.', lookup_str='', metadata={'source': 'example_data/fake-content.html'}, lookup_index=0)]"
- ]
+ "text/plain": "[Document(page_content='My First Heading\\n\\nMy first paragraph.', lookup_str='', metadata={'source': 'example_data/fake-content.html'}, lookup_index=0)]"
},
"execution_count": 4,
"metadata": {},
@@ -61,13 +59,57 @@
"data"
]
},
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Loading HTML with BeautifulSoup4\n",
+ "\n",
+ "We can also use BeautifulSoup4 to load HTML documents using the `BSHTMLLoader`. This will extract the text from the html into `page_content`, and the page title as `title` into `metadata`."
+ ],
+ "metadata": {
+ "collapsed": false
+ }
+ },
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 16,
"id": "79b1bce4",
"metadata": {},
"outputs": [],
- "source": []
+ "source": [
+ "from langchain.document_loaders import BSHTMLLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "outputs": [
+ {
+ "data": {
+ "text/plain": "[Document(page_content='\\n\\nTest Title\\n\\n\\nMy First Heading\\nMy first paragraph.\\n\\n\\n', lookup_str='', metadata={'source': 'example_data/fake-content.html', 'title': 'Test Title'}, lookup_index=0)]"
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "loader = BSHTMLLoader(\"example_data/fake-content.html\")\n",
+ "data = loader.load()\n",
+ "data"
+ ],
+ "metadata": {
+ "collapsed": false
+ }
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "outputs": [],
+ "source": [],
+ "metadata": {
+ "collapsed": false
+ }
}
],
"metadata": {
diff --git a/langchain/document_loaders/html_bs.py b/langchain/document_loaders/html_bs.py
index 92802ccb..568d8054 100644
--- a/langchain/document_loaders/html_bs.py
+++ b/langchain/document_loaders/html_bs.py
@@ -3,8 +3,6 @@
import logging
from typing import Dict, List, Union
-from bs4 import BeautifulSoup
-
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
@@ -15,9 +13,18 @@ class BSHTMLLoader(BaseLoader):
"""Loader that uses beautiful soup to parse HTML files."""
def __init__(self, file_path: str) -> None:
+ try:
+ import bs4 # noqa:F401
+ except ImportError:
+ raise ValueError(
+ "bs4 package not found, please install it with " "`pip install bs4`"
+ )
+
self.file_path = file_path
def load(self) -> List[Document]:
+ from bs4 import BeautifulSoup
+
"""Load HTML document into document objects."""
with open(self.file_path, "r") as f:
soup = BeautifulSoup(f, features="lxml")