Harrison/playwright (#2871)

Co-authored-by: Manuel Saelices <msaelices@gmail.com>

docs/modules/indexes/document_loaders/examples/url.ipynb

@@ -112,6 +112,79 @@
    "source": [
     "data = loader.load()"
    ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "a2c1c79f",
+   "metadata": {},
+   "source": [
+    "# Playwright URL Loader\n",
+    "\n",
+    "This covers how to load HTML documents from a list of URLs using the `PlaywrightURLLoader`.\n",
+    "\n",
+    "As in the Selenium case, Playwright allows us to load pages that need JavaScript to render.\n",
+    "\n",
+    "## Setup\n",
+    "\n",
+    "To use the `PlaywrightURLLoader`, you will need to install `playwright` and `unstructured`. Additionally, you will need to install the Playwright Chromium browser:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "53158417",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install playwright\n",
+    "!pip install \"playwright\"\n",
+    "!pip install \"unstructured\"\n",
+    "!playwright install"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ab4e115",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import PlaywrightURLLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ce5a9a0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "urls = [\n",
+    "    \"https://www.youtube.com/watch?v=dQw4w9WgXcQ\",\n",
+    "    \"https://goo.gl/maps/NDSHwePEyaHMFGwh8\"\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2dc3e0bc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = PlaywrightURLLoader(urls=urls, remove_selectors=[\"header\", \"footer\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10b79f80",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = loader.load()"
+   ]
   }
  ],
  "metadata": {
@@ -130,7 +203,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.13"
+   "version": "3.10.6"
   }
  },
  "nbformat": 4,

langchain/document_loaders/__init__.py

@@ -64,6 +64,7 @@ from langchain.document_loaders.unstructured import (
     UnstructuredFileLoader,
 )
 from langchain.document_loaders.url import UnstructuredURLLoader
+from langchain.document_loaders.url_playwright import PlaywrightURLLoader
 from langchain.document_loaders.url_selenium import SeleniumURLLoader
 from langchain.document_loaders.web_base import WebBaseLoader
 from langchain.document_loaders.whatsapp_chat import WhatsAppChatLoader
@@ -82,6 +83,7 @@ __all__ = [
     "UnstructuredFileIOLoader",
     "UnstructuredURLLoader",
     "SeleniumURLLoader",
+    "PlaywrightURLLoader",
     "DirectoryLoader",
     "NotionDirectoryLoader",
     "NotionDBLoader",

langchain/document_loaders/url_playwright.py

@@ -0,0 +1,87 @@
+"""Loader that uses Playwright to load a page, then uses unstructured to load the html.
+"""
+import logging
+from typing import List, Optional
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+logger = logging.getLogger(__file__)
+
+
+class PlaywrightURLLoader(BaseLoader):
+    """Loader that uses Playwright and to load a page and unstructured to load the html.
+    This is useful for loading pages that require javascript to render.
+
+    Attributes:
+        urls (List[str]): List of URLs to load.
+        continue_on_failure (bool): If True, continue loading other URLs on failure.
+        headless (bool): If True, the browser will run in headless mode.
+    """
+
+    def __init__(
+        self,
+        urls: List[str],
+        continue_on_failure: bool = True,
+        headless: bool = True,
+        remove_selectors: Optional[List[str]] = None,
+    ):
+        """Load a list of URLs using Playwright and unstructured."""
+        try:
+            import playwright  # noqa:F401
+        except ImportError:
+            raise ValueError(
+                "playwright package not found, please install it with "
+                "`pip install playwright`"
+            )
+
+        try:
+            import unstructured  # noqa:F401
+        except ImportError:
+            raise ValueError(
+                "unstructured package not found, please install it with "
+                "`pip install unstructured`"
+            )
+
+        self.urls = urls
+        self.continue_on_failure = continue_on_failure
+        self.headless = headless
+        self.remove_selectors = remove_selectors
+
+    def load(self) -> List[Document]:
+        """Load the specified URLs using Playwright and create Document instances.
+
+        Returns:
+            List[Document]: A list of Document instances with loaded content.
+        """
+        from playwright.sync_api import sync_playwright
+        from unstructured.partition.html import partition_html
+
+        docs: List[Document] = list()
+
+        with sync_playwright() as p:
+            browser = p.chromium.launch(headless=self.headless)
+            for url in self.urls:
+                try:
+                    page = browser.new_page()
+                    page.goto(url)
+
+                    for selector in self.remove_selectors or []:
+                        element = page.locator(selector)
+                        if element.is_visible():
+                            element.evaluate("element => element.remove()")
+
+                    page_source = page.content()
+                    elements = partition_html(text=page_source)
+                    text = "\n\n".join([str(el) for el in elements])
+                    metadata = {"source": url}
+                    docs.append(Document(page_content=text, metadata=metadata))
+                except Exception as e:
+                    if self.continue_on_failure:
+                        logger.error(
+                            f"Error fetching or processing {url}, exception: {e}"
+                        )
+                    else:
+                        raise e
+            browser.close()
+        return docs