forked from Archives/langchain
`NotebookLoader.load()` loads the `.ipynb` notebook file into a `Document` object. **Parameters**: * `include_outputs` (bool): whether to include cell outputs in the resulting document (default is False). * `max_output_length` (int): the maximum number of characters to include from each cell output (default is 10). * `remove_newline` (bool): whether to remove newline characters from the cell sources and outputs (default is False). * `traceback` (bool): whether to include full traceback (default is False).docker-utility-pexpect
Klein Tahiraj
1 year ago
committed by
GitHub
parent
4b5d427421
commit
8a0751dadd
@ -0,0 +1,83 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Notebook\n",
|
||||
"\n",
|
||||
"This notebook covers how to load data from an .ipynb notebook into a format suitable by LangChain."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.document_loaders import NotebookLoader"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loader = NotebookLoader(\"example_data/notebook.ipynb\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"`NotebookLoader.load()` loads the `.ipynb` notebook file into a `Document` object.\n",
|
||||
"\n",
|
||||
"**Parameters**:\n",
|
||||
"\n",
|
||||
"* `include_outputs` (bool): whether to include cell outputs in the resulting document (default is False).\n",
|
||||
"* `max_output_length` (int): the maximum number of characters to include from each cell output (default is 10).\n",
|
||||
"* `remove_newline` (bool): whether to remove newline characters from the cell sources and outputs (default is False).\n",
|
||||
"* `traceback` (bool): whether to include full traceback (default is False)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loader.load(include_outputs=True, max_output_length=20, remove_newline=True)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.1"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "981b6680a42bdb5eb22187741e1607b3aae2cf73db800d1af1f268d1de6a1f70"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@ -0,0 +1,83 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Notebook\n",
|
||||
"\n",
|
||||
"This notebook covers how to load data from an .ipynb notebook into a format suitable by LangChain."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.document_loaders import NotebookLoader"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loader = NotebookLoader(\"example_data/notebook.ipynb\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"`NotebookLoader.load()` loads the `.ipynb` notebook file into a `Document` object.\n",
|
||||
"\n",
|
||||
"**Parameters**:\n",
|
||||
"\n",
|
||||
"* `include_outputs` (bool): whether to include cell outputs in the resulting document (default is False).\n",
|
||||
"* `max_output_length` (int): the maximum number of characters to include from each cell output (default is 10).\n",
|
||||
"* `remove_newline` (bool): whether to remove newline characters from the cell sources and outputs (default is False).\n",
|
||||
"* `traceback` (bool): whether to include full traceback (default is False)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loader.load(include_outputs=True, max_output_length=20, remove_newline=True)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.1"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "981b6680a42bdb5eb22187741e1607b3aae2cf73db800d1af1f268d1de6a1f70"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@ -0,0 +1,102 @@
|
||||
"""Loader that loads .ipynb notebook files."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, List
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from langchain.docstore.document import Document
|
||||
from langchain.document_loaders.base import BaseLoader
|
||||
|
||||
|
||||
def concatenate_cells(
|
||||
cell: dict, include_outputs: bool, max_output_length: int, traceback: bool
|
||||
) -> str:
|
||||
"""Combine cells information in a readable format ready to be used."""
|
||||
cell_type = cell["cell_type"]
|
||||
source = cell["source"]
|
||||
output = cell["outputs"]
|
||||
|
||||
if include_outputs and cell_type == "code" and output:
|
||||
if "ename" in output[0].keys():
|
||||
error_name = output[0]["ename"]
|
||||
error_value = output[0]["evalue"]
|
||||
if traceback:
|
||||
traceback = output[0]["traceback"]
|
||||
return (
|
||||
f"'{cell_type}' cell: '{source}'\n, gives error '{error_name}',"
|
||||
f" with description '{error_value}'\n"
|
||||
f"and traceback '{traceback}'\n\n"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"'{cell_type}' cell: '{source}'\n, gives error '{error_name}',"
|
||||
f"with description '{error_value}'\n\n"
|
||||
)
|
||||
elif output[0]["output_type"] == "stream":
|
||||
output = output[0]["text"]
|
||||
min_output = min(max_output_length, len(output))
|
||||
return (
|
||||
f"'{cell_type}' cell: '{source}'\n with "
|
||||
f"output: '{output[:min_output]}'\n\n"
|
||||
)
|
||||
else:
|
||||
return f"'{cell_type}' cell: '{source}'\n\n"
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def remove_newlines(x: Any) -> Any:
|
||||
"""Remove recursivelly newlines, no matter the data structure they are stored in."""
|
||||
if isinstance(x, str):
|
||||
return x.replace("\n", "")
|
||||
elif isinstance(x, list):
|
||||
return [remove_newlines(elem) for elem in x]
|
||||
elif isinstance(x, pd.DataFrame):
|
||||
return x.applymap(remove_newlines)
|
||||
else:
|
||||
return x
|
||||
|
||||
|
||||
class NotebookLoader(BaseLoader):
|
||||
"""Loader that loads .ipynb notebook files."""
|
||||
|
||||
def __init__(self, path: str):
|
||||
"""Initialize with path."""
|
||||
self.file_path = path
|
||||
|
||||
def load(
|
||||
self,
|
||||
include_outputs: bool = False,
|
||||
max_output_length: int = 10,
|
||||
remove_newline: bool = False,
|
||||
traceback: bool = False,
|
||||
) -> List[Document]:
|
||||
"""Load documents."""
|
||||
try:
|
||||
import pandas as pd
|
||||
except ImportError:
|
||||
raise ValueError(
|
||||
"pandas is needed for Notebook Loader, "
|
||||
"please install with `pip install pandas`"
|
||||
)
|
||||
p = Path(self.file_path)
|
||||
|
||||
with open(p, encoding="utf8") as f:
|
||||
d = json.load(f)
|
||||
|
||||
data = pd.json_normalize(d["cells"])
|
||||
filtered_data = data[["cell_type", "source", "outputs"]]
|
||||
if remove_newline:
|
||||
filtered_data = filtered_data.applymap(remove_newlines)
|
||||
|
||||
text = filtered_data.apply(
|
||||
lambda x: concatenate_cells(
|
||||
x, include_outputs, max_output_length, traceback
|
||||
),
|
||||
axis=1,
|
||||
).str.cat(sep=" ")
|
||||
|
||||
metadata = {"source": str(p)}
|
||||
|
||||
return [Document(page_content=text, metadata=metadata)]
|
||||
Loading…
Reference in New Issue