From 3e1901e1aacadabcf20389a901fa67782c15be1c Mon Sep 17 00:00:00 2001
From: Harrison Chase <hw.chase.17@gmail.com>
Date: Wed, 8 Feb 2023 12:00:47 -0800
Subject: [PATCH] gutenberg books (#946)

Co-authored-by: zanderchase <zander@unfold.ag>
Co-authored-by: Harrison Chase <harrisonchase@Harrisons-MBP.attlocal.net>
---
 .../document_loaders/examples/gutenberg.ipynb | 83 +++++++++++++++++++
 .../document_loaders/how_to_guides.rst        |  1 +
 langchain/document_loaders/__init__.py        |  2 +
 langchain/document_loaders/gutenberg.py       | 28 +++++++
 4 files changed, 114 insertions(+)
 create mode 100644 docs/modules/document_loaders/examples/gutenberg.ipynb
 create mode 100644 langchain/document_loaders/gutenberg.py

diff --git a/docs/modules/document_loaders/examples/gutenberg.ipynb b/docs/modules/document_loaders/examples/gutenberg.ipynb
new file mode 100644
index 0000000000..e2d28c3daf
--- /dev/null
+++ b/docs/modules/document_loaders/examples/gutenberg.ipynb
@@ -0,0 +1,83 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bda1f3f5",
+   "metadata": {},
+   "source": [
+    "# Gutenberg\n",
+    "\n",
+    "This covers how to load links to Gutenberg e-books into a document format that we can use downstream."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "9bfd5e46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import GutenbergLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "700e4ef2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = GutenbergLoader('https://www.gutenberg.org/cache/epub/69972/pg69972.txt')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "b6f28930",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7d436441",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3b74d755",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/docs/modules/document_loaders/how_to_guides.rst b/docs/modules/document_loaders/how_to_guides.rst
index ee807d4db5..1d5b08858c 100644
--- a/docs/modules/document_loaders/how_to_guides.rst
+++ b/docs/modules/document_loaders/how_to_guides.rst
@@ -37,6 +37,7 @@ There are a lot of different document loaders that LangChain supports. Below are
 
 `GCS Directory <./examples/gcs_directory.html>`_: A walkthrough of how to load all files in a directory from Google Cloud Storage (GCS).
 
+`Gutenberg <./examples/gutenberg.html>`_: A walkthrough of how to load data from a Gutenberg ebook text.
 
 .. toctree::
    :maxdepth: 1
diff --git a/langchain/document_loaders/__init__.py b/langchain/document_loaders/__init__.py
index c95f2d072d..52697daf68 100644
--- a/langchain/document_loaders/__init__.py
+++ b/langchain/document_loaders/__init__.py
@@ -6,6 +6,7 @@ from langchain.document_loaders.email import UnstructuredEmailLoader
 from langchain.document_loaders.gcs_directory import GCSDirectoryLoader
 from langchain.document_loaders.gcs_file import GCSFileLoader
 from langchain.document_loaders.googledrive import GoogleDriveLoader
+from langchain.document_loaders.gutenberg import GutenbergLoader
 from langchain.document_loaders.html import UnstructuredHTMLLoader
 from langchain.document_loaders.notion import NotionDirectoryLoader
 from langchain.document_loaders.obsidian import ObsidianLoader
@@ -36,4 +37,5 @@ __all__ = [
     "S3DirectoryLoader",
     "GCSFileLoader",
     "GCSDirectoryLoader",
+    "GutenbergLoader",
 ]
diff --git a/langchain/document_loaders/gutenberg.py b/langchain/document_loaders/gutenberg.py
new file mode 100644
index 0000000000..41a0a5f55a
--- /dev/null
+++ b/langchain/document_loaders/gutenberg.py
@@ -0,0 +1,28 @@
+"""Loader that loads .txt web files."""
+from typing import List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+
+class GutenbergLoader(BaseLoader):
+    """Loader that uses urllib to load .txt web files."""
+
+    def __init__(self, file_path: str):
+        """Initialize with file path."""
+        if not file_path.startswith("https://www.gutenberg.org"):
+            raise ValueError("file path must start with 'https://www.gutenberg.org'")
+
+        if not file_path.endswith(".txt"):
+            raise ValueError("file path must end with '.txt'")
+
+        self.file_path = file_path
+
+    def load(self) -> List[Document]:
+        """Load file."""
+        from urllib.request import urlopen
+
+        elements = urlopen(self.file_path)
+        text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements])
+        metadata = {"source": self.file_path}
+        return [Document(page_content=text, metadata=metadata)]