From 96ebe98dc252a288ea6808a17bc9bb0520d957d7 Mon Sep 17 00:00:00 2001
From: Harrison Chase <hw.chase.17@gmail.com>
Date: Fri, 17 Mar 2023 08:10:27 -0700
Subject: [PATCH] Harrison/latex splitter (#1738)

Co-authored-by: Aidan Holland <thehappydinoa@gmail.com>
Co-authored-by: Jan de Boer <44832123+Janldeboer@users.noreply.github.com>
---
 .../indexes/examples/textsplitter.ipynb       | 71 +++++++++++++++++++
 langchain/text_splitter.py                    | 31 ++++++++
 2 files changed, 102 insertions(+)

diff --git a/docs/modules/indexes/examples/textsplitter.ipynb b/docs/modules/indexes/examples/textsplitter.ipynb
index 85cd9eda28..2d7bcf99ec 100644
--- a/docs/modules/indexes/examples/textsplitter.ipynb
+++ b/docs/modules/indexes/examples/textsplitter.ipynb
@@ -176,6 +176,77 @@
     "docs"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "3a2f572e",
+   "metadata": {},
+   "source": [
+    "## Latex Text Splitter\n",
+    "\n",
+    "LatexTextSplitter splits text along Latex headings, headlines, enumerations and more. It's implemented as a simple subclass of RecursiveCharacterSplitter with Latex-specific separators. See the source code to see the Latex syntax expected by default.\n",
+    "\n",
+    "1. How the text is split: by list of latex specific tags\n",
+    "2. How the chunk size is measured: by length function passed in (defaults to number of characters)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c2503917",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import LatexTextSplitter"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e46b753b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "latex_text = \"\"\"\n",
+    "\\documentclass{article}\n",
+    "\n",
+    "\\begin{document}\n",
+    "\n",
+    "\\maketitle\n",
+    "\n",
+    "\\section{Introduction}\n",
+    "Large language models (LLMs) are a type of machine learning model that can be trained on vast amounts of text data to generate human-like language. In recent years, LLMs have made significant advances in a variety of natural language processing tasks, including language translation, text generation, and sentiment analysis.\n",
+    "\n",
+    "\\subsection{History of LLMs}\n",
+    "The earliest LLMs were developed in the 1980s and 1990s, but they were limited by the amount of data that could be processed and the computational power available at the time. In the past decade, however, advances in hardware and software have made it possible to train LLMs on massive datasets, leading to significant improvements in performance.\n",
+    "\n",
+    "\\subsection{Applications of LLMs}\n",
+    "LLMs have many applications in industry, including chatbots, content creation, and virtual assistants. They can also be used in academia for research in linguistics, psychology, and computational linguistics.\n",
+    "\n",
+    "\\end{document}\n",
+    "\"\"\"\n",
+    "latex_splitter = LatexTextSplitter(chunk_size=400, chunk_overlap=0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73b5bd33",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docs = latex_splitter.create_documents([latex_text])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e1c7fbd5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docs"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "c350765d",
diff --git a/langchain/text_splitter.py b/langchain/text_splitter.py
index 4e297e3b0e..9da309a317 100644
--- a/langchain/text_splitter.py
+++ b/langchain/text_splitter.py
@@ -355,6 +355,37 @@ class MarkdownTextSplitter(RecursiveCharacterTextSplitter):
         super().__init__(separators=separators, **kwargs)
 
 
+class LatexTextSplitter(RecursiveCharacterTextSplitter):
+    """Attempts to split the text along Latex-formatted layout elements."""
+
+    def __init__(self, **kwargs: Any):
+        """Initialize a LatexTextSplitter."""
+        separators = [
+            # First, try to split along Latex sections
+            "\n\\chapter{",
+            "\n\\section{",
+            "\n\\subsection{",
+            "\n\\subsubsection{",
+            # Now split by environments
+            "\n\\begin{enumerate}",
+            "\n\\begin{itemize}",
+            "\n\\begin{description}",
+            "\n\\begin{list}",
+            "\n\\begin{quote}",
+            "\n\\begin{quotation}",
+            "\n\\begin{verse}",
+            "\n\\begin{verbatim}",
+            ## Now split by math environments
+            "\n\\begin{align}",
+            "$$",
+            "$",
+            # Now split by the normal type of lines
+            " ",
+            "",
+        ]
+        super().__init__(separators=separators, **kwargs)
+
+
 class PythonCodeTextSplitter(RecursiveCharacterTextSplitter):
     """Attempts to split the text along Python syntax."""