diff --git a/docs/modules/indexes/document_loaders/examples/url.ipynb b/docs/modules/indexes/document_loaders/examples/url.ipynb index 581f1a33..517f2f66 100644 --- a/docs/modules/indexes/document_loaders/examples/url.ipynb +++ b/docs/modules/indexes/document_loaders/examples/url.ipynb @@ -112,6 +112,79 @@ "source": [ "data = loader.load()" ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "a2c1c79f", + "metadata": {}, + "source": [ + "# Playwright URL Loader\n", + "\n", + "This covers how to load HTML documents from a list of URLs using the `PlaywrightURLLoader`.\n", + "\n", + "As in the Selenium case, Playwright allows us to load pages that need JavaScript to render.\n", + "\n", + "## Setup\n", + "\n", + "To use the `PlaywrightURLLoader`, you will need to install `playwright` and `unstructured`. Additionally, you will need to install the Playwright Chromium browser:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53158417", + "metadata": {}, + "outputs": [], + "source": [ + "# Install playwright\n", + "!pip install \"playwright\"\n", + "!pip install \"unstructured\"\n", + "!playwright install" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ab4e115", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders import PlaywrightURLLoader" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce5a9a0a", + "metadata": {}, + "outputs": [], + "source": [ + "urls = [\n", + " \"https://www.youtube.com/watch?v=dQw4w9WgXcQ\",\n", + " \"https://goo.gl/maps/NDSHwePEyaHMFGwh8\"\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2dc3e0bc", + "metadata": {}, + "outputs": [], + "source": [ + "loader = PlaywrightURLLoader(urls=urls, remove_selectors=[\"header\", \"footer\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10b79f80", + "metadata": {}, + "outputs": [], + "source": [ + "data = loader.load()" + ] } ], "metadata": { @@ -130,7 +203,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.10.6" } }, "nbformat": 4, diff --git a/langchain/document_loaders/__init__.py b/langchain/document_loaders/__init__.py index c2ea430a..17d2cde9 100644 --- a/langchain/document_loaders/__init__.py +++ b/langchain/document_loaders/__init__.py @@ -64,6 +64,7 @@ from langchain.document_loaders.unstructured import ( UnstructuredFileLoader, ) from langchain.document_loaders.url import UnstructuredURLLoader +from langchain.document_loaders.url_playwright import PlaywrightURLLoader from langchain.document_loaders.url_selenium import SeleniumURLLoader from langchain.document_loaders.web_base import WebBaseLoader from langchain.document_loaders.whatsapp_chat import WhatsAppChatLoader @@ -82,6 +83,7 @@ __all__ = [ "UnstructuredFileIOLoader", "UnstructuredURLLoader", "SeleniumURLLoader", + "PlaywrightURLLoader", "DirectoryLoader", "NotionDirectoryLoader", "NotionDBLoader", diff --git a/langchain/document_loaders/url_playwright.py b/langchain/document_loaders/url_playwright.py new file mode 100644 index 00000000..15a5dbd7 --- /dev/null +++ b/langchain/document_loaders/url_playwright.py @@ -0,0 +1,87 @@ +"""Loader that uses Playwright to load a page, then uses unstructured to load the html. +""" +import logging +from typing import List, Optional + +from langchain.docstore.document import Document +from langchain.document_loaders.base import BaseLoader + +logger = logging.getLogger(__file__) + + +class PlaywrightURLLoader(BaseLoader): + """Loader that uses Playwright and to load a page and unstructured to load the html. + This is useful for loading pages that require javascript to render. + + Attributes: + urls (List[str]): List of URLs to load. + continue_on_failure (bool): If True, continue loading other URLs on failure. + headless (bool): If True, the browser will run in headless mode. + """ + + def __init__( + self, + urls: List[str], + continue_on_failure: bool = True, + headless: bool = True, + remove_selectors: Optional[List[str]] = None, + ): + """Load a list of URLs using Playwright and unstructured.""" + try: + import playwright # noqa:F401 + except ImportError: + raise ValueError( + "playwright package not found, please install it with " + "`pip install playwright`" + ) + + try: + import unstructured # noqa:F401 + except ImportError: + raise ValueError( + "unstructured package not found, please install it with " + "`pip install unstructured`" + ) + + self.urls = urls + self.continue_on_failure = continue_on_failure + self.headless = headless + self.remove_selectors = remove_selectors + + def load(self) -> List[Document]: + """Load the specified URLs using Playwright and create Document instances. + + Returns: + List[Document]: A list of Document instances with loaded content. + """ + from playwright.sync_api import sync_playwright + from unstructured.partition.html import partition_html + + docs: List[Document] = list() + + with sync_playwright() as p: + browser = p.chromium.launch(headless=self.headless) + for url in self.urls: + try: + page = browser.new_page() + page.goto(url) + + for selector in self.remove_selectors or []: + element = page.locator(selector) + if element.is_visible(): + element.evaluate("element => element.remove()") + + page_source = page.content() + elements = partition_html(text=page_source) + text = "\n\n".join([str(el) for el in elements]) + metadata = {"source": url} + docs.append(Document(page_content=text, metadata=metadata)) + except Exception as e: + if self.continue_on_failure: + logger.error( + f"Error fetching or processing {url}, exception: {e}" + ) + else: + raise e + browser.close() + return docs