mirror of https://github.com/hwchase17/langchain
Rspace doc loader (#11511)
**Description:** Add a document loader for the RSpace Electronic Lab Notebook (www.researchspace.com), so that scientific documents and research notes can be easily pulled into Langchain pipelines. **Issue** This is an new contribution, rather than an issue fix. **Dependencies:** There are no new required dependencies. In order to use the loader, clients will need to install rspace_client SDK using `pip install rspace_client` --------- Co-authored-by: richarda23 <richard.c.adams@infinityworks.com> Co-authored-by: Bagatur <baskaryan@gmail.com>pull/11311/head^2
Richard Adams
12 months ago
committed by
GitHub
parent
9d1867c77f
commit
35965df20d
@ -0,0 +1,127 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2f1572a5-9f8c-44f1-82f3-ddeee8f55145",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"This notebook shows how to use the RSpace document loader to import research notes and documents from RSpace Electronic\n",
|
||||
"Lab Notebook into Langchain pipelines.\n",
|
||||
"\n",
|
||||
"To start you'll need an RSpace account and an API key.\n",
|
||||
"\n",
|
||||
"You can set up a free account at [https://community.researchspace.com](https://community.researchspace.com) or use your institutional RSpace.\n",
|
||||
"\n",
|
||||
"You can get an RSpace API token from your account's profile page. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9e5310d2-a864-4464-bdca-81f30c9d0bdb",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install rspace_client"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "61b1d1b7-a28c-4fba-83a3-df64baa8b6b8",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"It's best to store your RSpace API key as an environment variable. \n",
|
||||
"\n",
|
||||
" RSPACE_API_KEY=<YOUR_KEY>\n",
|
||||
"\n",
|
||||
"You'll also need to set the URL of your RSpace installation e.g.\n",
|
||||
"\n",
|
||||
" RSPACE_URL=https://community.researchspace.com\n",
|
||||
"\n",
|
||||
"If you use these exact environment variable names, they will be detected automatically. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "13c19ea4-100f-417e-b52f-7e8730c7c1d1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.document_loaders.rspace import RSpaceLoader"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4fd42831-0e79-4068-a5e1-7e2cfc242789",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"You can import various items from RSpace:\n",
|
||||
"\n",
|
||||
"* A single RSpace structured or basic document. This will map 1-1 to a Langchain document.\n",
|
||||
"* A folder or noteook. All documents inside the notebook or folder are imported as Langchain documents. \n",
|
||||
"* If you have PDF files in the RSpace Gallery, these can be imported individually as well. Under the hood, Langchain's PDF loader will be used and this creates one Langchain document per PDF page. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8e614357-5eca-401b-ab98-ea55b0465009",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## replace these ids with some from your own research notes.\n",
|
||||
"## Make sure to use global ids (with the 2 character prefix). This helps the loader know which API calls to make \n",
|
||||
"## to RSpace API.\n",
|
||||
"\n",
|
||||
"rspace_ids = [\"NB1932027\", \"FL1921314\", \"SD1932029\", \"GL1932384\"]\n",
|
||||
"for rs_id in rspace_ids:\n",
|
||||
" loader = RSpaceLoader(global_id=rs_id)\n",
|
||||
" docs = loader.load()\n",
|
||||
" for doc in docs:\n",
|
||||
" ## the name and ID are added to the 'source' metadata property.\n",
|
||||
" print (doc.metadata)\n",
|
||||
" print(doc.page_content[:500])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "1b41758d-24e0-4994-a30f-3acccc7795e4",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"If you don't want to use the environment variables as above, you can pass these into the RSpaceLoader"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "aa079ca6-439d-4010-9edd-cd77d8884fab",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"loader = RSpaceLoader(global_id=rs_id, api_key=\"MY_API_KEY\", url=\"https://my.researchspace.com\")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@ -0,0 +1,130 @@
|
||||
import os
|
||||
from typing import Any, Dict, Iterator, List, Optional, Union
|
||||
|
||||
from langchain.docstore.document import Document
|
||||
from langchain.document_loaders import PyPDFLoader
|
||||
from langchain.document_loaders.base import BaseLoader
|
||||
from langchain.utils import get_from_dict_or_env
|
||||
|
||||
|
||||
class RSpaceLoader(BaseLoader):
|
||||
"""
|
||||
Loads content from RSpace notebooks, folders, documents or PDF Gallery files into
|
||||
Langchain documents.
|
||||
|
||||
Maps RSpace document <-> Langchain Document in 1-1. PDFs are imported using PyPDF.
|
||||
|
||||
Requirements are rspace_client (`pip install rspace_client`) and PyPDF if importing
|
||||
PDF docs (`pip install pypdf`).
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, global_id: str, api_key: Optional[str] = None, url: Optional[str] = None
|
||||
):
|
||||
"""api_key: RSpace API key - can also be supplied as environment variable
|
||||
'RSPACE_API_KEY'
|
||||
url: str
|
||||
The URL of your RSpace instance - can also be supplied as environment
|
||||
variable 'RSPACE_URL'
|
||||
global_id: str
|
||||
The global ID of the resource to load,
|
||||
e.g. 'SD12344' (a single document); 'GL12345'(A PDF file in the gallery);
|
||||
'NB4567' (a notebook); 'FL12244' (a folder)
|
||||
"""
|
||||
args: Dict[str, Optional[str]] = {
|
||||
"api_key": api_key,
|
||||
"url": url,
|
||||
"global_id": global_id,
|
||||
}
|
||||
verified_args: Dict[str, str] = RSpaceLoader.validate_environment(args)
|
||||
self.api_key = verified_args["api_key"]
|
||||
self.url = verified_args["url"]
|
||||
self.global_id: str = verified_args["global_id"]
|
||||
|
||||
@classmethod
|
||||
def validate_environment(cls, values: Dict) -> Dict:
|
||||
"""Validate that API key and URL exists in environment."""
|
||||
values["api_key"] = get_from_dict_or_env(values, "api_key", "RSPACE_API_KEY")
|
||||
values["url"] = get_from_dict_or_env(values, "url", "RSPACE_URL")
|
||||
if "global_id" not in values or values["global_id"] is None:
|
||||
raise ValueError(
|
||||
"No value supplied for global_id. Please supply an RSpace global ID"
|
||||
)
|
||||
return values
|
||||
|
||||
def _create_rspace_client(self) -> Any:
|
||||
"""Create a RSpace client."""
|
||||
try:
|
||||
from rspace_client.eln import eln, field_content
|
||||
|
||||
except ImportError:
|
||||
raise ImportError("You must run " "`pip install rspace_client`")
|
||||
|
||||
try:
|
||||
eln = eln.ELNClient(self.url, self.api_key)
|
||||
eln.get_status()
|
||||
|
||||
except Exception:
|
||||
raise Exception(
|
||||
f"Unable to initialise client - is url {self.url} or "
|
||||
f"api key correct?"
|
||||
)
|
||||
|
||||
return eln, field_content.FieldContent
|
||||
|
||||
def _get_doc(self, cli: Any, field_content: Any, d_id: Union[str, int]) -> Document:
|
||||
content = ""
|
||||
doc = cli.get_document(d_id)
|
||||
content += f"<h2>{doc['name']}<h2/>"
|
||||
for f in doc["fields"]:
|
||||
content += f"{f['name']}\n"
|
||||
fc = field_content(f["content"])
|
||||
content += fc.get_text()
|
||||
content += "\n"
|
||||
return Document(
|
||||
metadata={"source": f"rspace: {doc['name']}-{doc['globalId']}"},
|
||||
page_content=content,
|
||||
)
|
||||
|
||||
def _load_structured_doc(self) -> Iterator[Document]:
|
||||
cli, field_content = self._create_rspace_client()
|
||||
yield self._get_doc(cli, field_content, self.global_id)
|
||||
|
||||
def _load_folder_tree(self) -> Iterator[Document]:
|
||||
cli, field_content = self._create_rspace_client()
|
||||
if self.global_id:
|
||||
docs_in_folder = cli.list_folder_tree(
|
||||
folder_id=self.global_id[2:], typesToInclude=["document"]
|
||||
)
|
||||
doc_ids: List[int] = [d["id"] for d in docs_in_folder["records"]]
|
||||
for doc_id in doc_ids:
|
||||
yield self._get_doc(cli, field_content, doc_id)
|
||||
|
||||
def _load_pdf(self) -> Iterator[Document]:
|
||||
cli, field_content = self._create_rspace_client()
|
||||
file_info = cli.get_file_info(self.global_id)
|
||||
_, ext = os.path.splitext(file_info["name"])
|
||||
if ext.lower() == ".pdf":
|
||||
outfile = f"{self.global_id}.pdf"
|
||||
cli.download_file(self.global_id, outfile)
|
||||
pdf_loader = PyPDFLoader(outfile)
|
||||
for pdf in pdf_loader.lazy_load():
|
||||
pdf.metadata["rspace_src"] = self.global_id
|
||||
yield pdf
|
||||
|
||||
def lazy_load(self) -> Iterator[Document]:
|
||||
if self.global_id and "GL" in self.global_id:
|
||||
for d in self._load_pdf():
|
||||
yield d
|
||||
elif self.global_id and "SD" in self.global_id:
|
||||
for d in self._load_structured_doc():
|
||||
yield d
|
||||
elif self.global_id and self.global_id[0:2] in ["FL", "NB"]:
|
||||
for d in self._load_folder_tree():
|
||||
yield d
|
||||
else:
|
||||
raise ValueError("Unknown global ID type")
|
||||
|
||||
def load(self) -> List[Document]:
|
||||
return list(self.lazy_load())
|
||||
@ -0,0 +1,35 @@
|
||||
import unittest
|
||||
|
||||
from langchain.document_loaders.rspace import RSpaceLoader
|
||||
|
||||
|
||||
class TestRSpaceLoader(unittest.TestCase):
|
||||
url = "https://community.researchspace.com"
|
||||
api_key = "myapikey"
|
||||
global_id = "SD12345"
|
||||
|
||||
def test_valid_arguments(self) -> None:
|
||||
loader = RSpaceLoader(
|
||||
url=TestRSpaceLoader.url,
|
||||
api_key=TestRSpaceLoader.api_key,
|
||||
global_id=TestRSpaceLoader.global_id,
|
||||
)
|
||||
self.assertEqual(TestRSpaceLoader.url, loader.url) # add assertion here
|
||||
self.assertEqual(TestRSpaceLoader.api_key, loader.api_key) # add assertion here
|
||||
self.assertEqual(
|
||||
TestRSpaceLoader.global_id, loader.global_id
|
||||
) # add assertion here
|
||||
|
||||
def test_missing_apikey_raises_validation_error(self) -> None:
|
||||
with self.assertRaises(ValueError) as cm:
|
||||
RSpaceLoader(url=TestRSpaceLoader.url, global_id=TestRSpaceLoader.global_id)
|
||||
e = cm.exception
|
||||
self.assertRegex(str(e), r"Did not find api_key")
|
||||
|
||||
def test_missing_url_raises_validation_error(self) -> None:
|
||||
with self.assertRaises(ValueError) as cm:
|
||||
RSpaceLoader(
|
||||
api_key=TestRSpaceLoader.api_key, global_id=TestRSpaceLoader.global_id
|
||||
)
|
||||
e = cm.exception
|
||||
self.assertRegex(str(e), r"Did not find url")
|
||||
Loading…
Reference in New Issue