diff --git a/docs/extras/modules/data_connection/document_loaders/integrations/mhtml.ipynb b/docs/extras/modules/data_connection/document_loaders/integrations/mhtml.ipynb new file mode 100644 index 0000000000..12ebd2a3e1 --- /dev/null +++ b/docs/extras/modules/data_connection/document_loaders/integrations/mhtml.ipynb @@ -0,0 +1,71 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "87067cdf", + "metadata": {}, + "source": [ + "# mhtml\n", + "\n", + "MHTML is a is used both for emails but also for archived webpages. MHTML, sometimes referred as MHT, stands for MIME HTML is a single file in which entire webpage is archived. When one saves a webpage as MHTML format, this file extension will contain HTML code, images, audio files, flash animation etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d4c6174", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders import MHTMLLoader" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "12dcebc8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "page_content='LangChain\\nLANG CHAIN 🦜️🔗Official Home Page\\xa0\\n\\n\\n\\n\\n\\n\\n\\nIntegrations\\n\\n\\n\\nFeatures\\n\\n\\n\\n\\nBlog\\n\\n\\n\\nConceptual Guide\\n\\n\\n\\n\\nPython Repo\\n\\n\\nJavaScript Repo\\n\\n\\n\\nPython Documentation \\n\\n\\nJavaScript Documentation\\n\\n\\n\\n\\nPython ChatLangChain \\n\\n\\nJavaScript ChatLangChain\\n\\n\\n\\n\\nDiscord \\n\\n\\nTwitter\\n\\n\\n\\n\\nIf you have any comments about our WEB page, you can \\nwrite us at the address shown above. However, due to \\nthe limited number of personnel in our corporate office, we are unable to \\nprovide a direct response.\\n\\nCopyright © 2023-2023 LangChain Inc.\\n\\n\\n' metadata={'source': '../../../../../../tests/integration_tests/examples/example.mht', 'title': 'LangChain'}\n" + ] + } + ], + "source": [ + "# Create a new loader object for the MHTML file\n", + "loader = MHTMLLoader(file_path='../../../../../../tests/integration_tests/examples/example.mht')\n", + "\n", + "# Load the document from the file\n", + "documents = loader.load()\n", + "\n", + "# Print the documents to see the results\n", + "for doc in documents:\n", + " print(doc)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/langchain/document_loaders/__init__.py b/langchain/document_loaders/__init__.py index 1503734f77..57e450c884 100644 --- a/langchain/document_loaders/__init__.py +++ b/langchain/document_loaders/__init__.py @@ -68,6 +68,7 @@ from langchain.document_loaders.mastodon import MastodonTootsLoader from langchain.document_loaders.max_compute import MaxComputeLoader from langchain.document_loaders.mediawikidump import MWDumpLoader from langchain.document_loaders.merge import MergedDataLoader +from langchain.document_loaders.mhtml import MHTMLLoader from langchain.document_loaders.modern_treasury import ModernTreasuryLoader from langchain.document_loaders.notebook import NotebookLoader from langchain.document_loaders.notion import NotionDirectoryLoader @@ -205,6 +206,7 @@ __all__ = [ "MathpixPDFLoader", "MaxComputeLoader", "MergedDataLoader", + "MHTMLLoader", "ModernTreasuryLoader", "NotebookLoader", "NotionDBLoader", diff --git a/langchain/document_loaders/mhtml.py b/langchain/document_loaders/mhtml.py new file mode 100644 index 0000000000..27d3eceb12 --- /dev/null +++ b/langchain/document_loaders/mhtml.py @@ -0,0 +1,69 @@ +"""Loader to load MHTML files, enriching metadata with page title.""" + +import email +import logging +from typing import Dict, List, Union + +from langchain.docstore.document import Document +from langchain.document_loaders.base import BaseLoader + +logger = logging.getLogger(__name__) + + +class MHTMLLoader(BaseLoader): + """Loader that uses beautiful soup to parse HTML files.""" + + def __init__( + self, + file_path: str, + open_encoding: Union[str, None] = None, + bs_kwargs: Union[dict, None] = None, + get_text_separator: str = "", + ) -> None: + """Initialise with path, and optionally, file encoding to use, and any kwargs + to pass to the BeautifulSoup object.""" + try: + import bs4 # noqa:F401 + except ImportError: + raise ValueError( + "beautifulsoup4 package not found, please install it with " + "`pip install beautifulsoup4`" + ) + + self.file_path = file_path + self.open_encoding = open_encoding + if bs_kwargs is None: + bs_kwargs = {"features": "lxml"} + self.bs_kwargs = bs_kwargs + self.get_text_separator = get_text_separator + + def load(self) -> List[Document]: + from bs4 import BeautifulSoup + + """Load MHTML document into document objects.""" + + with open(self.file_path, "r", encoding=self.open_encoding) as f: + message = email.message_from_string(f.read()) + parts = message.get_payload() + + if type(parts) is not list: + parts = [message] + + for part in parts: + if part.get_content_type() == "text/html": + html = part.get_payload(decode=True).decode() + + soup = BeautifulSoup(html, **self.bs_kwargs) + text = soup.get_text(self.get_text_separator) + + if soup.title: + title = str(soup.title.string) + else: + title = "" + + metadata: Dict[str, Union[str, None]] = { + "source": self.file_path, + "title": title, + } + return [Document(page_content=text, metadata=metadata)] + return [] diff --git a/tests/integration_tests/examples/example.mht b/tests/integration_tests/examples/example.mht new file mode 100644 index 0000000000..44a45ea020 --- /dev/null +++ b/tests/integration_tests/examples/example.mht @@ -0,0 +1,108 @@ +From: +Snapshot-Content-Location: https://langchain.com/ +Subject: +Date: Fri, 16 Jun 2023 19:32:59 -0000 +MIME-Version: 1.0 +Content-Type: multipart/related; + type="text/html"; + boundary="----MultipartBoundary--dYaUgeoeP18TqraaeOwkeZyu1vI09OtkFwH2rcnJMt----" + + +------MultipartBoundary--dYaUgeoeP18TqraaeOwkeZyu1vI09OtkFwH2rcnJMt---- +Content-Type: text/html +Content-ID: +Content-Transfer-Encoding: quoted-printable +Content-Location: https://langchain.com/ + +LangChain

+ LANG C= +HAIN =F0=9F=A6=9C=EF=B8=8F= +=F0=9F=94=97
Official Home Page = +

+ +
+
+ + + + + + + + + + + + + =20 +=09 + + + + + + + + + + =09 + + +
+ +
+ +
+ +
+ +
+ +
+ +
+
+ +

If you have any comments about our WEB page, you can=20 +write us at the address shown above. However, due to=20 +the limited number of personnel in our corporate office, we are unable to= +=20 +provide a direct response.

 +
+

Copyright =C2=A9 2023-2023 LangChain = +Inc.=20 +

+ + +------MultipartBoundary--dYaUgeoeP18TqraaeOwkeZyu1vI09OtkFwH2rcnJMt------ diff --git a/tests/unit_tests/document_loaders/test_mhtml.py b/tests/unit_tests/document_loaders/test_mhtml.py new file mode 100644 index 0000000000..2ab36defbb --- /dev/null +++ b/tests/unit_tests/document_loaders/test_mhtml.py @@ -0,0 +1,25 @@ +from pathlib import Path + +import pytest + +from langchain.document_loaders.mhtml import MHTMLLoader + +HERE = Path(__file__).parent +EXAMPLES = HERE.parent.parent / "integration_tests" / "examples" + + +@pytest.mark.requires("bs4", "lxml") +def test_mhtml_loader() -> None: + """Test mhtml loader.""" + file_path = EXAMPLES / "example.mht" + loader = MHTMLLoader(str(file_path)) + docs = loader.load() + + assert len(docs) == 1 + + metadata = docs[0].metadata + content = docs[0].page_content + + assert metadata["title"] == "LangChain" + assert metadata["source"] == str(file_path) + assert "LANG CHAIN 🦜️🔗Official Home Page" in content