gutenberg books (#946)

Co-authored-by: zanderchase <zander@unfold.ag>
Co-authored-by: Harrison Chase <harrisonchase@Harrisons-MBP.attlocal.net>

docs/modules/document_loaders/examples/gutenberg.ipynb

@@ -0,0 +1,83 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bda1f3f5",
+   "metadata": {},
+   "source": [
+    "# Gutenberg\n",
+    "\n",
+    "This covers how to load links to Gutenberg e-books into a document format that we can use downstream."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "9bfd5e46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import GutenbergLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "700e4ef2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = GutenbergLoader('https://www.gutenberg.org/cache/epub/69972/pg69972.txt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "b6f28930",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7d436441",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3b74d755",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/modules/document_loaders/how_to_guides.rst

@@ -37,6 +37,7 @@ There are a lot of different document loaders that LangChain supports. Below are
 
 `GCS Directory <./examples/gcs_directory.html>`_: A walkthrough of how to load all files in a directory from Google Cloud Storage (GCS).
 
+`Gutenberg <./examples/gutenberg.html>`_: A walkthrough of how to load data from a Gutenberg ebook text.
 
 .. toctree::
    :maxdepth: 1

langchain/document_loaders/__init__.py

@@ -6,6 +6,7 @@ from langchain.document_loaders.email import UnstructuredEmailLoader
 from langchain.document_loaders.gcs_directory import GCSDirectoryLoader
 from langchain.document_loaders.gcs_file import GCSFileLoader
 from langchain.document_loaders.googledrive import GoogleDriveLoader
+from langchain.document_loaders.gutenberg import GutenbergLoader
 from langchain.document_loaders.html import UnstructuredHTMLLoader
 from langchain.document_loaders.notion import NotionDirectoryLoader
 from langchain.document_loaders.obsidian import ObsidianLoader
@@ -36,4 +37,5 @@ __all__ = [
     "S3DirectoryLoader",
     "GCSFileLoader",
     "GCSDirectoryLoader",
+    "GutenbergLoader",
 ]

langchain/document_loaders/gutenberg.py

@@ -0,0 +1,28 @@
+"""Loader that loads .txt web files."""
+from typing import List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+
+class GutenbergLoader(BaseLoader):
+    """Loader that uses urllib to load .txt web files."""
+
+    def __init__(self, file_path: str):
+        """Initialize with file path."""
+        if not file_path.startswith("https://www.gutenberg.org"):
+            raise ValueError("file path must start with 'https://www.gutenberg.org'")
+
+        if not file_path.endswith(".txt"):
+            raise ValueError("file path must end with '.txt'")
+
+        self.file_path = file_path
+
+    def load(self) -> List[Document]:
+        """Load file."""
+        from urllib.request import urlopen
+
+        elements = urlopen(self.file_path)
+        text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements])
+        metadata = {"source": self.file_path}
+        return [Document(page_content=text, metadata=metadata)]