diff --git a/docs/api_reference/api_reference.rst b/docs/api_reference/api_reference.rst
index 31d4ae1909..95043e3a1e 100644
--- a/docs/api_reference/api_reference.rst
+++ b/docs/api_reference/api_reference.rst
@@ -660,6 +660,7 @@ Classes
document_loaders.word_document.Docx2txtLoader
document_loaders.word_document.UnstructuredWordDocumentLoader
document_loaders.xml.UnstructuredXMLLoader
+ document_loaders.xorbits.XorbitsLoader
document_loaders.youtube.GoogleApiYoutubeLoader
document_loaders.youtube.YoutubeLoader
diff --git a/docs/extras/modules/data_connection/document_loaders/integrations/xorbits.ipynb b/docs/extras/modules/data_connection/document_loaders/integrations/xorbits.ipynb
new file mode 100644
index 0000000000..cf5f60f028
--- /dev/null
+++ b/docs/extras/modules/data_connection/document_loaders/integrations/xorbits.ipynb
@@ -0,0 +1,304 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Xorbits Pandas DataFrame\n",
+ "\n",
+ "This notebook goes over how to load data from a [xorbits.pandas](https://doc.xorbits.io/en/latest/reference/pandas/frame.html) DataFrame."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#!pip install xorbits"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import xorbits.pandas as pd"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df = pd.read_csv(\"example_data/mlb_teams_2012.csv\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "b0d1d84e23c04f1296f63b3ea3dd1e5b",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ " 0%| | 0.00/100 [00:00, ?it/s]"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Team | \n",
+ " \"Payroll (millions)\" | \n",
+ " \"Wins\" | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 | \n",
+ " Nationals | \n",
+ " 81.34 | \n",
+ " 98 | \n",
+ "
\n",
+ " \n",
+ " 1 | \n",
+ " Reds | \n",
+ " 82.20 | \n",
+ " 97 | \n",
+ "
\n",
+ " \n",
+ " 2 | \n",
+ " Yankees | \n",
+ " 197.96 | \n",
+ " 95 | \n",
+ "
\n",
+ " \n",
+ " 3 | \n",
+ " Giants | \n",
+ " 117.62 | \n",
+ " 94 | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " Braves | \n",
+ " 83.31 | \n",
+ " 94 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " Team \"Payroll (millions)\" \"Wins\"\n",
+ "0 Nationals 81.34 98\n",
+ "1 Reds 82.20 97\n",
+ "2 Yankees 197.96 95\n",
+ "3 Giants 117.62 94\n",
+ "4 Braves 83.31 94"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from langchain.document_loaders import XorbitsLoader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "loader = XorbitsLoader(df, page_content_column=\"Team\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "c8c8b67f1aae4a3c9de7734bb6cf738e",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ " 0%| | 0.00/100 [00:00, ?it/s]"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[Document(page_content='Nationals', metadata={' \"Payroll (millions)\"': 81.34, ' \"Wins\"': 98}),\n",
+ " Document(page_content='Reds', metadata={' \"Payroll (millions)\"': 82.2, ' \"Wins\"': 97}),\n",
+ " Document(page_content='Yankees', metadata={' \"Payroll (millions)\"': 197.96, ' \"Wins\"': 95}),\n",
+ " Document(page_content='Giants', metadata={' \"Payroll (millions)\"': 117.62, ' \"Wins\"': 94}),\n",
+ " Document(page_content='Braves', metadata={' \"Payroll (millions)\"': 83.31, ' \"Wins\"': 94}),\n",
+ " Document(page_content='Athletics', metadata={' \"Payroll (millions)\"': 55.37, ' \"Wins\"': 94}),\n",
+ " Document(page_content='Rangers', metadata={' \"Payroll (millions)\"': 120.51, ' \"Wins\"': 93}),\n",
+ " Document(page_content='Orioles', metadata={' \"Payroll (millions)\"': 81.43, ' \"Wins\"': 93}),\n",
+ " Document(page_content='Rays', metadata={' \"Payroll (millions)\"': 64.17, ' \"Wins\"': 90}),\n",
+ " Document(page_content='Angels', metadata={' \"Payroll (millions)\"': 154.49, ' \"Wins\"': 89}),\n",
+ " Document(page_content='Tigers', metadata={' \"Payroll (millions)\"': 132.3, ' \"Wins\"': 88}),\n",
+ " Document(page_content='Cardinals', metadata={' \"Payroll (millions)\"': 110.3, ' \"Wins\"': 88}),\n",
+ " Document(page_content='Dodgers', metadata={' \"Payroll (millions)\"': 95.14, ' \"Wins\"': 86}),\n",
+ " Document(page_content='White Sox', metadata={' \"Payroll (millions)\"': 96.92, ' \"Wins\"': 85}),\n",
+ " Document(page_content='Brewers', metadata={' \"Payroll (millions)\"': 97.65, ' \"Wins\"': 83}),\n",
+ " Document(page_content='Phillies', metadata={' \"Payroll (millions)\"': 174.54, ' \"Wins\"': 81}),\n",
+ " Document(page_content='Diamondbacks', metadata={' \"Payroll (millions)\"': 74.28, ' \"Wins\"': 81}),\n",
+ " Document(page_content='Pirates', metadata={' \"Payroll (millions)\"': 63.43, ' \"Wins\"': 79}),\n",
+ " Document(page_content='Padres', metadata={' \"Payroll (millions)\"': 55.24, ' \"Wins\"': 76}),\n",
+ " Document(page_content='Mariners', metadata={' \"Payroll (millions)\"': 81.97, ' \"Wins\"': 75}),\n",
+ " Document(page_content='Mets', metadata={' \"Payroll (millions)\"': 93.35, ' \"Wins\"': 74}),\n",
+ " Document(page_content='Blue Jays', metadata={' \"Payroll (millions)\"': 75.48, ' \"Wins\"': 73}),\n",
+ " Document(page_content='Royals', metadata={' \"Payroll (millions)\"': 60.91, ' \"Wins\"': 72}),\n",
+ " Document(page_content='Marlins', metadata={' \"Payroll (millions)\"': 118.07, ' \"Wins\"': 69}),\n",
+ " Document(page_content='Red Sox', metadata={' \"Payroll (millions)\"': 173.18, ' \"Wins\"': 69}),\n",
+ " Document(page_content='Indians', metadata={' \"Payroll (millions)\"': 78.43, ' \"Wins\"': 68}),\n",
+ " Document(page_content='Twins', metadata={' \"Payroll (millions)\"': 94.08, ' \"Wins\"': 66}),\n",
+ " Document(page_content='Rockies', metadata={' \"Payroll (millions)\"': 78.06, ' \"Wins\"': 64}),\n",
+ " Document(page_content='Cubs', metadata={' \"Payroll (millions)\"': 88.19, ' \"Wins\"': 61}),\n",
+ " Document(page_content='Astros', metadata={' \"Payroll (millions)\"': 60.65, ' \"Wins\"': 55})]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "loader.load()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "fc85c9f59b3644689d05853159fbd358",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ " 0%| | 0.00/100 [00:00, ?it/s]"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "page_content='Nationals' metadata={' \"Payroll (millions)\"': 81.34, ' \"Wins\"': 98}\n",
+ "page_content='Reds' metadata={' \"Payroll (millions)\"': 82.2, ' \"Wins\"': 97}\n",
+ "page_content='Yankees' metadata={' \"Payroll (millions)\"': 197.96, ' \"Wins\"': 95}\n",
+ "page_content='Giants' metadata={' \"Payroll (millions)\"': 117.62, ' \"Wins\"': 94}\n",
+ "page_content='Braves' metadata={' \"Payroll (millions)\"': 83.31, ' \"Wins\"': 94}\n",
+ "page_content='Athletics' metadata={' \"Payroll (millions)\"': 55.37, ' \"Wins\"': 94}\n",
+ "page_content='Rangers' metadata={' \"Payroll (millions)\"': 120.51, ' \"Wins\"': 93}\n",
+ "page_content='Orioles' metadata={' \"Payroll (millions)\"': 81.43, ' \"Wins\"': 93}\n",
+ "page_content='Rays' metadata={' \"Payroll (millions)\"': 64.17, ' \"Wins\"': 90}\n",
+ "page_content='Angels' metadata={' \"Payroll (millions)\"': 154.49, ' \"Wins\"': 89}\n",
+ "page_content='Tigers' metadata={' \"Payroll (millions)\"': 132.3, ' \"Wins\"': 88}\n",
+ "page_content='Cardinals' metadata={' \"Payroll (millions)\"': 110.3, ' \"Wins\"': 88}\n",
+ "page_content='Dodgers' metadata={' \"Payroll (millions)\"': 95.14, ' \"Wins\"': 86}\n",
+ "page_content='White Sox' metadata={' \"Payroll (millions)\"': 96.92, ' \"Wins\"': 85}\n",
+ "page_content='Brewers' metadata={' \"Payroll (millions)\"': 97.65, ' \"Wins\"': 83}\n",
+ "page_content='Phillies' metadata={' \"Payroll (millions)\"': 174.54, ' \"Wins\"': 81}\n",
+ "page_content='Diamondbacks' metadata={' \"Payroll (millions)\"': 74.28, ' \"Wins\"': 81}\n",
+ "page_content='Pirates' metadata={' \"Payroll (millions)\"': 63.43, ' \"Wins\"': 79}\n",
+ "page_content='Padres' metadata={' \"Payroll (millions)\"': 55.24, ' \"Wins\"': 76}\n",
+ "page_content='Mariners' metadata={' \"Payroll (millions)\"': 81.97, ' \"Wins\"': 75}\n",
+ "page_content='Mets' metadata={' \"Payroll (millions)\"': 93.35, ' \"Wins\"': 74}\n",
+ "page_content='Blue Jays' metadata={' \"Payroll (millions)\"': 75.48, ' \"Wins\"': 73}\n",
+ "page_content='Royals' metadata={' \"Payroll (millions)\"': 60.91, ' \"Wins\"': 72}\n",
+ "page_content='Marlins' metadata={' \"Payroll (millions)\"': 118.07, ' \"Wins\"': 69}\n",
+ "page_content='Red Sox' metadata={' \"Payroll (millions)\"': 173.18, ' \"Wins\"': 69}\n",
+ "page_content='Indians' metadata={' \"Payroll (millions)\"': 78.43, ' \"Wins\"': 68}\n",
+ "page_content='Twins' metadata={' \"Payroll (millions)\"': 94.08, ' \"Wins\"': 66}\n",
+ "page_content='Rockies' metadata={' \"Payroll (millions)\"': 78.06, ' \"Wins\"': 64}\n",
+ "page_content='Cubs' metadata={' \"Payroll (millions)\"': 88.19, ' \"Wins\"': 61}\n",
+ "page_content='Astros' metadata={' \"Payroll (millions)\"': 60.65, ' \"Wins\"': 55}\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Use lazy load for larger table, which won't read the full table into memory\n",
+ "for i in loader.lazy_load():\n",
+ " print(i)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "base",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.13"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/langchain/document_loaders/__init__.py b/langchain/document_loaders/__init__.py
index fd32632354..bae2151032 100644
--- a/langchain/document_loaders/__init__.py
+++ b/langchain/document_loaders/__init__.py
@@ -142,6 +142,7 @@ from langchain.document_loaders.word_document import (
UnstructuredWordDocumentLoader,
)
from langchain.document_loaders.xml import UnstructuredXMLLoader
+from langchain.document_loaders.xorbits import XorbitsLoader
from langchain.document_loaders.youtube import (
GoogleApiClient,
GoogleApiYoutubeLoader,
@@ -287,6 +288,7 @@ __all__ = [
"WebBaseLoader",
"WhatsAppChatLoader",
"WikipediaLoader",
+ "XorbitsLoader",
"YoutubeAudioLoader",
"YoutubeLoader",
]
diff --git a/langchain/document_loaders/xorbits.py b/langchain/document_loaders/xorbits.py
new file mode 100644
index 0000000000..e8259d8f0a
--- /dev/null
+++ b/langchain/document_loaders/xorbits.py
@@ -0,0 +1,46 @@
+from typing import Any, Iterator, List
+
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseLoader
+
+
+class XorbitsLoader(BaseLoader):
+ """Load Xorbits DataFrame."""
+
+ def __init__(self, data_frame: Any, page_content_column: str = "text"):
+ """Initialize with dataframe object.
+
+ Requirements:
+ Must have xorbits installed. You can install with `pip install xorbits`.
+
+ Args:
+ data_frame: Xorbits DataFrame object.
+ page_content_column: Name of the column containing the page content.
+ Defaults to "text".
+ """
+ try:
+ import xorbits.pandas as pd
+ except ImportError as e:
+ raise ImportError(
+ "Cannot import xorbits, please install with 'pip install xorbits'."
+ ) from e
+
+ if not isinstance(data_frame, pd.DataFrame):
+ raise ValueError(
+ f"Expected data_frame to be a xorbits.pandas.DataFrame, \
+ got {type(data_frame)}"
+ )
+ self.data_frame = data_frame
+ self.page_content_column = page_content_column
+
+ def lazy_load(self) -> Iterator[Document]:
+ """Lazy load records from dataframe."""
+ for _, row in self.data_frame.iterrows():
+ text = row[self.page_content_column]
+ metadata = row.to_dict()
+ metadata.pop(self.page_content_column)
+ yield Document(page_content=text, metadata=metadata)
+
+ def load(self) -> List[Document]:
+ """Load full dataframe."""
+ return list(self.lazy_load())
diff --git a/pyproject.toml b/pyproject.toml
index 94559a75f5..f3c043d74c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -349,6 +349,7 @@ extended_testing = [
"pyspark",
"openai",
"rapidfuzz"
+
]
[[tool.poetry.source]]
diff --git a/tests/integration_tests/document_loaders/test_xorbits.py b/tests/integration_tests/document_loaders/test_xorbits.py
new file mode 100644
index 0000000000..a83df60827
--- /dev/null
+++ b/tests/integration_tests/document_loaders/test_xorbits.py
@@ -0,0 +1,64 @@
+import pytest
+
+from langchain.document_loaders import XorbitsLoader
+from langchain.schema import Document
+
+try:
+ import xorbits # noqa: F401
+
+ xorbits_installed = True
+except ImportError:
+ xorbits_installed = False
+
+
+@pytest.mark.skipif(not xorbits_installed, reason="xorbits not installed")
+def test_load_returns_list_of_documents() -> None:
+ import xorbits.pandas as pd
+
+ data = {
+ "text": ["Hello", "World"],
+ "author": ["Alice", "Bob"],
+ "date": ["2022-01-01", "2022-01-02"],
+ }
+ loader = XorbitsLoader(pd.DataFrame(data))
+ docs = loader.load()
+ assert isinstance(docs, list)
+ assert all(isinstance(doc, Document) for doc in docs)
+ assert len(docs) == 2
+
+
+@pytest.mark.skipif(not xorbits_installed, reason="xorbits not installed")
+def test_load_converts_dataframe_columns_to_document_metadata() -> None:
+ import xorbits.pandas as pd
+
+ data = {
+ "text": ["Hello", "World"],
+ "author": ["Alice", "Bob"],
+ "date": ["2022-01-01", "2022-01-02"],
+ }
+ loader = XorbitsLoader(pd.DataFrame(data))
+ docs = loader.load()
+ expected = {
+ "author": ["Alice", "Bob"],
+ "date": ["2022-01-01", "2022-01-02"],
+ }
+ for i, doc in enumerate(docs):
+ assert doc.metadata["author"] == expected["author"][i]
+ assert doc.metadata["date"] == expected["date"][i]
+
+
+@pytest.mark.skipif(not xorbits_installed, reason="xorbits not installed")
+def test_load_uses_page_content_column_to_create_document_text() -> None:
+ import xorbits.pandas as pd
+
+ data = {
+ "text": ["Hello", "World"],
+ "author": ["Alice", "Bob"],
+ "date": ["2022-01-01", "2022-01-02"],
+ }
+ sample_data_frame = pd.DataFrame(data)
+ sample_data_frame = sample_data_frame.rename(columns={"text": "dummy_test_column"})
+ loader = XorbitsLoader(sample_data_frame, page_content_column="dummy_test_column")
+ docs = loader.load()
+ assert docs[0].page_content == "Hello"
+ assert docs[1].page_content == "World"