diff --git a/docs/modules/document_loaders/examples/gutenberg.ipynb b/docs/modules/document_loaders/examples/gutenberg.ipynb new file mode 100644 index 0000000000..e2d28c3daf --- /dev/null +++ b/docs/modules/document_loaders/examples/gutenberg.ipynb @@ -0,0 +1,83 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "bda1f3f5", + "metadata": {}, + "source": [ + "# Gutenberg\n", + "\n", + "This covers how to load links to Gutenberg e-books into a document format that we can use downstream." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "9bfd5e46", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.document_loaders import GutenbergLoader" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "700e4ef2", + "metadata": {}, + "outputs": [], + "source": [ + "loader = GutenbergLoader('https://www.gutenberg.org/cache/epub/69972/pg69972.txt')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "b6f28930", + "metadata": {}, + "outputs": [], + "source": [ + "data = loader.load()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d436441", + "metadata": {}, + "outputs": [], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b74d755", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/modules/document_loaders/how_to_guides.rst b/docs/modules/document_loaders/how_to_guides.rst index ee807d4db5..1d5b08858c 100644 --- a/docs/modules/document_loaders/how_to_guides.rst +++ b/docs/modules/document_loaders/how_to_guides.rst @@ -37,6 +37,7 @@ There are a lot of different document loaders that LangChain supports. Below are `GCS Directory <./examples/gcs_directory.html>`_: A walkthrough of how to load all files in a directory from Google Cloud Storage (GCS). +`Gutenberg <./examples/gutenberg.html>`_: A walkthrough of how to load data from a Gutenberg ebook text. .. toctree:: :maxdepth: 1 diff --git a/langchain/document_loaders/__init__.py b/langchain/document_loaders/__init__.py index c95f2d072d..52697daf68 100644 --- a/langchain/document_loaders/__init__.py +++ b/langchain/document_loaders/__init__.py @@ -6,6 +6,7 @@ from langchain.document_loaders.email import UnstructuredEmailLoader from langchain.document_loaders.gcs_directory import GCSDirectoryLoader from langchain.document_loaders.gcs_file import GCSFileLoader from langchain.document_loaders.googledrive import GoogleDriveLoader +from langchain.document_loaders.gutenberg import GutenbergLoader from langchain.document_loaders.html import UnstructuredHTMLLoader from langchain.document_loaders.notion import NotionDirectoryLoader from langchain.document_loaders.obsidian import ObsidianLoader @@ -36,4 +37,5 @@ __all__ = [ "S3DirectoryLoader", "GCSFileLoader", "GCSDirectoryLoader", + "GutenbergLoader", ] diff --git a/langchain/document_loaders/gutenberg.py b/langchain/document_loaders/gutenberg.py new file mode 100644 index 0000000000..41a0a5f55a --- /dev/null +++ b/langchain/document_loaders/gutenberg.py @@ -0,0 +1,28 @@ +"""Loader that loads .txt web files.""" +from typing import List + +from langchain.docstore.document import Document +from langchain.document_loaders.base import BaseLoader + + +class GutenbergLoader(BaseLoader): + """Loader that uses urllib to load .txt web files.""" + + def __init__(self, file_path: str): + """Initialize with file path.""" + if not file_path.startswith("https://www.gutenberg.org"): + raise ValueError("file path must start with 'https://www.gutenberg.org'") + + if not file_path.endswith(".txt"): + raise ValueError("file path must end with '.txt'") + + self.file_path = file_path + + def load(self) -> List[Document]: + """Load file.""" + from urllib.request import urlopen + + elements = urlopen(self.file_path) + text = "\n\n".join([str(el.decode("utf-8-sig")) for el in elements]) + metadata = {"source": self.file_path} + return [Document(page_content=text, metadata=metadata)]