forked from Archives/langchain
JSON loader (#4067)
This implements a loader of text passages in JSON format. The `jq` syntax is used to define a schema for accessing the relevant contents from the JSON file. This requires dependency on the `jq` package: https://pypi.org/project/jq/. --------- Signed-off-by: Aivin V. Solatorio <avsolatorio@gmail.com>parallel_dir_loader
Aivin V. Solatorio
1 year ago
committed by
GitHub
parent
bb6d97c18c
commit
6567b73e1a
@ -0,0 +1,367 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# JSON Files\n",
|
||||
"\n",
|
||||
"The `JSONLoader` uses a specified [jq schema](https://en.wikipedia.org/wiki/Jq_(programming_language)) to parse the JSON files.\n",
|
||||
"\n",
|
||||
"This notebook shows how to use the `JSONLoader` to load [JSON](https://en.wikipedia.org/wiki/JSON) files into documents. A few examples of `jq` schema extracting different parts of a JSON file are also shown.\n",
|
||||
"\n",
|
||||
"Check this [manual](https://stedolan.github.io/jq/manual/#Basicfilters) for a detailed documentation of the `jq` syntax."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install jq"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"jupyter": {
|
||||
"outputs_hidden": true
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.document_loaders import JSONLoader"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"from pathlib import Path\n",
|
||||
"from pprint import pprint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"file_path='./example_data/facebook_chat.json'\n",
|
||||
"data = json.loads(Path(file_path).read_text())"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'image': {'creation_timestamp': 1675549016, 'uri': 'image_of_the_chat.jpg'},\n",
|
||||
" 'is_still_participant': True,\n",
|
||||
" 'joinable_mode': {'link': '', 'mode': 1},\n",
|
||||
" 'magic_words': [],\n",
|
||||
" 'messages': [{'content': 'Bye!',\n",
|
||||
" 'sender_name': 'User 2',\n",
|
||||
" 'timestamp_ms': 1675597571851},\n",
|
||||
" {'content': 'Oh no worries! Bye',\n",
|
||||
" 'sender_name': 'User 1',\n",
|
||||
" 'timestamp_ms': 1675597435669},\n",
|
||||
" {'content': 'No Im sorry it was my mistake, the blue one is not '\n",
|
||||
" 'for sale',\n",
|
||||
" 'sender_name': 'User 2',\n",
|
||||
" 'timestamp_ms': 1675596277579},\n",
|
||||
" {'content': 'I thought you were selling the blue one!',\n",
|
||||
" 'sender_name': 'User 1',\n",
|
||||
" 'timestamp_ms': 1675595140251},\n",
|
||||
" {'content': 'Im not interested in this bag. Im interested in the '\n",
|
||||
" 'blue one!',\n",
|
||||
" 'sender_name': 'User 1',\n",
|
||||
" 'timestamp_ms': 1675595109305},\n",
|
||||
" {'content': 'Here is $129',\n",
|
||||
" 'sender_name': 'User 2',\n",
|
||||
" 'timestamp_ms': 1675595068468},\n",
|
||||
" {'photos': [{'creation_timestamp': 1675595059,\n",
|
||||
" 'uri': 'url_of_some_picture.jpg'}],\n",
|
||||
" 'sender_name': 'User 2',\n",
|
||||
" 'timestamp_ms': 1675595060730},\n",
|
||||
" {'content': 'Online is at least $100',\n",
|
||||
" 'sender_name': 'User 2',\n",
|
||||
" 'timestamp_ms': 1675595045152},\n",
|
||||
" {'content': 'How much do you want?',\n",
|
||||
" 'sender_name': 'User 1',\n",
|
||||
" 'timestamp_ms': 1675594799696},\n",
|
||||
" {'content': 'Goodmorning! $50 is too low.',\n",
|
||||
" 'sender_name': 'User 2',\n",
|
||||
" 'timestamp_ms': 1675577876645},\n",
|
||||
" {'content': 'Hi! Im interested in your bag. Im offering $50. Let '\n",
|
||||
" 'me know if you are interested. Thanks!',\n",
|
||||
" 'sender_name': 'User 1',\n",
|
||||
" 'timestamp_ms': 1675549022673}],\n",
|
||||
" 'participants': [{'name': 'User 1'}, {'name': 'User 2'}],\n",
|
||||
" 'thread_path': 'inbox/User 1 and User 2 chat',\n",
|
||||
" 'title': 'User 1 and User 2 chat'}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pprint(data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Using `JSONLoader`\n",
|
||||
"\n",
|
||||
"Suppose we are interested in extracting the values under the `content` field within the `messages` key of the JSON data. This can easily be done through the `JSONLoader` as shown below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loader = JSONLoader(\n",
|
||||
" file_path='./example_data/facebook_chat.json',\n",
|
||||
" jq_schema='.messages[].content')\n",
|
||||
"\n",
|
||||
"data = loader.load()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[Document(page_content='Bye!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 1}),\n",
|
||||
" Document(page_content='Oh no worries! Bye', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 2}),\n",
|
||||
" Document(page_content='No Im sorry it was my mistake, the blue one is not for sale', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 3}),\n",
|
||||
" Document(page_content='I thought you were selling the blue one!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 4}),\n",
|
||||
" Document(page_content='Im not interested in this bag. Im interested in the blue one!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 5}),\n",
|
||||
" Document(page_content='Here is $129', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 6}),\n",
|
||||
" Document(page_content='', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 7}),\n",
|
||||
" Document(page_content='Online is at least $100', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 8}),\n",
|
||||
" Document(page_content='How much do you want?', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 9}),\n",
|
||||
" Document(page_content='Goodmorning! $50 is too low.', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 10}),\n",
|
||||
" Document(page_content='Hi! Im interested in your bag. Im offering $50. Let me know if you are interested. Thanks!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 11})]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pprint(data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Extracting metadata\n",
|
||||
"\n",
|
||||
"Generally, we want to include metadata available in the JSON file into the documents that we create from the content.\n",
|
||||
"\n",
|
||||
"The following demonstrates how metadata can be extracted using the `JSONLoader`.\n",
|
||||
"\n",
|
||||
"There are some key changes to be noted. In the previous example where we didn't collect the metadata, we managed to directly specify in the schema where the value for the `page_content` can be extracted from.\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
".messages[].content\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"In the current example, we have to tell the loader to iterate over the records in the `messages` field. The jq_schema then has to be:\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
".messages[]\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"This allows us to pass the records (dict) into the `metadata_func` that has to be implemented. The `metadata_func` is responsible for identifying which pieces of information in the record should be included in the metadata stored in the final `Document` object.\n",
|
||||
"\n",
|
||||
"Additionally, we now have to explicitly specify in the loader, via the `content_key` argument, the key from the record where the value for the `page_content` needs to be extracted from."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define the metadata extraction function.\n",
|
||||
"def metadata_func(record: dict, metadata: dict) -> dict:\n",
|
||||
"\n",
|
||||
" metadata[\"sender_name\"] = record.get(\"sender_name\")\n",
|
||||
" metadata[\"timestamp_ms\"] = record.get(\"timestamp_ms\")\n",
|
||||
"\n",
|
||||
" return metadata\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"loader = JSONLoader(\n",
|
||||
" file_path='./example_data/facebook_chat.json',\n",
|
||||
" jq_schema='.messages[]',\n",
|
||||
" content_key=\"content\",\n",
|
||||
" metadata_func=metadata_func\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"data = loader.load()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[Document(page_content='Bye!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 1, 'sender_name': 'User 2', 'timestamp_ms': 1675597571851}),\n",
|
||||
" Document(page_content='Oh no worries! Bye', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 2, 'sender_name': 'User 1', 'timestamp_ms': 1675597435669}),\n",
|
||||
" Document(page_content='No Im sorry it was my mistake, the blue one is not for sale', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 3, 'sender_name': 'User 2', 'timestamp_ms': 1675596277579}),\n",
|
||||
" Document(page_content='I thought you were selling the blue one!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 4, 'sender_name': 'User 1', 'timestamp_ms': 1675595140251}),\n",
|
||||
" Document(page_content='Im not interested in this bag. Im interested in the blue one!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 5, 'sender_name': 'User 1', 'timestamp_ms': 1675595109305}),\n",
|
||||
" Document(page_content='Here is $129', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 6, 'sender_name': 'User 2', 'timestamp_ms': 1675595068468}),\n",
|
||||
" Document(page_content='', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 7, 'sender_name': 'User 2', 'timestamp_ms': 1675595060730}),\n",
|
||||
" Document(page_content='Online is at least $100', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 8, 'sender_name': 'User 2', 'timestamp_ms': 1675595045152}),\n",
|
||||
" Document(page_content='How much do you want?', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 9, 'sender_name': 'User 1', 'timestamp_ms': 1675594799696}),\n",
|
||||
" Document(page_content='Goodmorning! $50 is too low.', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 10, 'sender_name': 'User 2', 'timestamp_ms': 1675577876645}),\n",
|
||||
" Document(page_content='Hi! Im interested in your bag. Im offering $50. Let me know if you are interested. Thanks!', metadata={'source': '/Users/avsolatorio/WBG/langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 11, 'sender_name': 'User 1', 'timestamp_ms': 1675549022673})]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pprint(data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now, you will see that the documents contain the metadata associated with the content we extracted."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## The `metadata_func`\n",
|
||||
"\n",
|
||||
"As shown above, the `metadata_func` accepts the default metadata generated by the `JSONLoader`. This allows full control to the user with respect to how the metadata is formatted.\n",
|
||||
"\n",
|
||||
"For example, the default metadata contains the `source` and the `seq_num` keys. However, it is possible that the JSON data contain these keys as well. The user can then exploit the `metadata_func` to rename the default keys and use the ones from the JSON data.\n",
|
||||
"\n",
|
||||
"The example below shows how we can modify the `source` to only contain information of the file source relative to the `langchain` directory."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define the metadata extraction function.\n",
|
||||
"def metadata_func(record: dict, metadata: dict) -> dict:\n",
|
||||
"\n",
|
||||
" metadata[\"sender_name\"] = record.get(\"sender_name\")\n",
|
||||
" metadata[\"timestamp_ms\"] = record.get(\"timestamp_ms\")\n",
|
||||
" \n",
|
||||
" if \"source\" in metadata:\n",
|
||||
" source = metadata[\"source\"].split(\"/\")\n",
|
||||
" source = source[source.index(\"langchain\"):]\n",
|
||||
" metadata[\"source\"] = \"/\".join(source)\n",
|
||||
"\n",
|
||||
" return metadata\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"loader = JSONLoader(\n",
|
||||
" file_path='./example_data/facebook_chat.json',\n",
|
||||
" jq_schema='.messages[]',\n",
|
||||
" content_key=\"content\",\n",
|
||||
" metadata_func=metadata_func\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"data = loader.load()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"[Document(page_content='Bye!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 1, 'sender_name': 'User 2', 'timestamp_ms': 1675597571851}),\n",
|
||||
" Document(page_content='Oh no worries! Bye', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 2, 'sender_name': 'User 1', 'timestamp_ms': 1675597435669}),\n",
|
||||
" Document(page_content='No Im sorry it was my mistake, the blue one is not for sale', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 3, 'sender_name': 'User 2', 'timestamp_ms': 1675596277579}),\n",
|
||||
" Document(page_content='I thought you were selling the blue one!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 4, 'sender_name': 'User 1', 'timestamp_ms': 1675595140251}),\n",
|
||||
" Document(page_content='Im not interested in this bag. Im interested in the blue one!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 5, 'sender_name': 'User 1', 'timestamp_ms': 1675595109305}),\n",
|
||||
" Document(page_content='Here is $129', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 6, 'sender_name': 'User 2', 'timestamp_ms': 1675595068468}),\n",
|
||||
" Document(page_content='', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 7, 'sender_name': 'User 2', 'timestamp_ms': 1675595060730}),\n",
|
||||
" Document(page_content='Online is at least $100', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 8, 'sender_name': 'User 2', 'timestamp_ms': 1675595045152}),\n",
|
||||
" Document(page_content='How much do you want?', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 9, 'sender_name': 'User 1', 'timestamp_ms': 1675594799696}),\n",
|
||||
" Document(page_content='Goodmorning! $50 is too low.', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 10, 'sender_name': 'User 2', 'timestamp_ms': 1675577876645}),\n",
|
||||
" Document(page_content='Hi! Im interested in your bag. Im offering $50. Let me know if you are interested. Thanks!', metadata={'source': 'langchain/docs/modules/indexes/document_loaders/examples/example_data/facebook_chat.json', 'seq_num': 11, 'sender_name': 'User 1', 'timestamp_ms': 1675549022673})]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pprint(data)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Common JSON structures with jq schema\n",
|
||||
"\n",
|
||||
"The list below provides a reference to the possible `jq_schema` the user can use to extract content from the JSON data depending on the structure.\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
"JSON -> [{\"text\": ...}, {\"text\": ...}, {\"text\": ...}]\n",
|
||||
"jq_schema -> \".[].text\"\n",
|
||||
" \n",
|
||||
"JSON -> {\"key\": [{\"text\": ...}, {\"text\": ...}, {\"text\": ...}]}\n",
|
||||
"jq_schema -> \".key[].text\"\n",
|
||||
"\n",
|
||||
"JSON -> [\"...\", \"...\", \"...\"]\n",
|
||||
"jq_schema -> \".[]\"\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.9.16"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
@ -0,0 +1,104 @@
|
||||
"""Loader that loads data from JSON."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, List, Optional, Union
|
||||
|
||||
from langchain.docstore.document import Document
|
||||
from langchain.document_loaders.base import BaseLoader
|
||||
|
||||
|
||||
class JSONLoader(BaseLoader):
|
||||
"""Loads a JSON file and references a jq schema provided to load the text into
|
||||
documents.
|
||||
|
||||
Example:
|
||||
[{"text": ...}, {"text": ...}, {"text": ...}] -> schema = .[].text
|
||||
{"key": [{"text": ...}, {"text": ...}, {"text": ...}]} -> schema = .key[].text
|
||||
["", "", ""] -> schema = .[]
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
file_path: Union[str, Path],
|
||||
jq_schema: str,
|
||||
content_key: Optional[str] = None,
|
||||
metadata_func: Optional[Callable[[Dict, Dict], Dict]] = None,
|
||||
):
|
||||
"""Initialize the JSONLoader.
|
||||
|
||||
Args:
|
||||
file_path (Union[str, Path]): The path to the JSON file.
|
||||
jq_schema (str): The jq schema to use to extract the data or text from
|
||||
the JSON.
|
||||
content_key (str): The key to use to extract the content from the JSON if
|
||||
the jq_schema results to a list of objects (dict).
|
||||
metadata_func (Callable[Dict, Dict]): A function that takes in the JSON
|
||||
object extracted by the jq_schema and the default metadata and returns
|
||||
a dict of the updated metadata.
|
||||
"""
|
||||
try:
|
||||
import jq # noqa:F401
|
||||
except ImportError:
|
||||
raise ValueError(
|
||||
"jq package not found, please install it with `pip install jq`"
|
||||
)
|
||||
|
||||
self.file_path = Path(file_path).resolve()
|
||||
self._jq_schema = jq.compile(jq_schema)
|
||||
self._content_key = content_key
|
||||
self._metadata_func = metadata_func
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
"""Load and return documents from the JSON file."""
|
||||
|
||||
data = self._jq_schema.input(json.loads(self.file_path.read_text()))
|
||||
|
||||
# Perform some validation
|
||||
# This is not a perfect validation, but it should catch most cases
|
||||
# and prevent the user from getting a cryptic error later on.
|
||||
if self._content_key is not None:
|
||||
sample = data.first()
|
||||
if not isinstance(sample, dict):
|
||||
raise ValueError(
|
||||
f"Expected the jq schema to result in a list of objects (dict), \
|
||||
so sample must be a dict but got `{type(sample)}`"
|
||||
)
|
||||
|
||||
if sample.get(self._content_key) is None:
|
||||
raise ValueError(
|
||||
f"Expected the jq schema to result in a list of objects (dict) \
|
||||
with the key `{self._content_key}`"
|
||||
)
|
||||
|
||||
if self._metadata_func is not None:
|
||||
sample_metadata = self._metadata_func(sample, {})
|
||||
if not isinstance(sample_metadata, dict):
|
||||
raise ValueError(
|
||||
f"Expected the metadata_func to return a dict but got \
|
||||
`{type(sample_metadata)}`"
|
||||
)
|
||||
|
||||
docs = []
|
||||
|
||||
for i, sample in enumerate(data, 1):
|
||||
metadata = dict(
|
||||
source=str(self.file_path),
|
||||
seq_num=i,
|
||||
)
|
||||
|
||||
if self._content_key is not None:
|
||||
text = sample.get(self._content_key)
|
||||
if self._metadata_func is not None:
|
||||
# We pass in the metadata dict to the metadata_func
|
||||
# so that the user can customize the default metadata
|
||||
# based on the content of the JSON object.
|
||||
metadata = self._metadata_func(sample, metadata)
|
||||
else:
|
||||
text = sample
|
||||
|
||||
# In case the text is None, set it to an empty string
|
||||
text = text or ""
|
||||
|
||||
docs.append(Document(page_content=text, metadata=metadata))
|
||||
|
||||
return docs
|
||||
@ -0,0 +1,16 @@
|
||||
from pathlib import Path
|
||||
|
||||
from langchain.document_loaders import JSONLoader
|
||||
|
||||
|
||||
def test_json_loader() -> None:
|
||||
"""Test unstructured loader."""
|
||||
file_path = Path(__file__).parent.parent / "examples/example.json"
|
||||
loader = JSONLoader(str(file_path), ".messages[].content")
|
||||
docs = loader.load()
|
||||
|
||||
# Check that the correct number of documents are loaded.
|
||||
assert len(docs) == 3
|
||||
|
||||
# Make sure that None content are converted to empty strings.
|
||||
assert docs[-1].page_content == ""
|
||||
@ -0,0 +1,25 @@
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"sender_name": "User 2",
|
||||
"timestamp_ms": 1675597571851,
|
||||
"content": "Bye!"
|
||||
},
|
||||
{
|
||||
"sender_name": "User 1",
|
||||
"timestamp_ms": 1675597435669,
|
||||
"content": "Oh no worries! Bye"
|
||||
},
|
||||
{
|
||||
"sender_name": "User 2",
|
||||
"timestamp_ms": 1675595060730,
|
||||
"photos": [
|
||||
{
|
||||
"uri": "url_of_some_picture.jpg",
|
||||
"creation_timestamp": 1675595059
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"title": "User 1 and User 2 chat"
|
||||
}
|
||||
Loading…
Reference in New Issue