forked from Archives/langchain
`Arxiv` document loader (#3627)
It makes sense to use `arxiv` as another source of the documents for downloading. - Added the `arxiv` document_loader, based on the `utilities/arxiv.py:ArxivAPIWrapper` - added tests - added an example notebook - sorted `__all__` in `__init__.py` (otherwise it is hard to find a class in the very long list)fix_agent_callbacks
parent
539142f8d5
commit
36c59e0c25
@ -0,0 +1,177 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "bda1f3f5",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Arxiv\n",
|
||||||
|
"\n",
|
||||||
|
"[arXiv](https://arxiv.org/) is an open-access archive for 2 million scholarly articles in the fields of physics, mathematics, computer science, quantitative biology, quantitative finance, statistics, electrical engineering and systems science, and economics.\n",
|
||||||
|
"\n",
|
||||||
|
"This notebook shows how to load scientific articles from `Arxiv.org` into a document format that we can use downstream."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "1b7a1eef-7bf7-4e7d-8bfc-c4e27c9488cb",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Installation"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "2abd5578-aa3d-46b9-99af-8b262f0b3df8",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"First, you need to install `arxiv` python package."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "b674aaea-ed3a-4541-8414-260a8f67f623",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!pip install arxiv"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "094b5f13-7e54-4354-9d83-26d6926ecaa0",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Second, you need to install `PyMuPDF` python package which transform PDF files from the `arxiv.org` site into the text fromat."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "7cd91121-2e96-43ba-af50-319853695f86",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!pip install pymupdf"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "95f05e1c-195e-4e2b-ae8e-8d6637f15be6",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## Examples"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"id": "e29b954c-1407-4797-ae21-6ba8937156be",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"`ArxivLoader` has these arguments:\n",
|
||||||
|
"- `query`: free text which used to find documents in the Arxiv\n",
|
||||||
|
"- optional `load_max_docs`: default=100. Use it to limit number of downloaded documents. It takes time to download all 100 documents, so use a small number for experiments.\n",
|
||||||
|
"- optional `load_all_available_meta`: default=False. By defaul only the most important fields downloaded: `Published` (date when document was published/last updated), `Title`, `Authors`, `Summary`. If True, other fields also downloaded."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "9bfd5e46",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"from langchain.document_loaders.base import Document\n",
|
||||||
|
"from langchain.document_loaders import ArxivLoader"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"id": "700e4ef2",
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"docs = ArxivLoader(query=\"1605.08386\", load_max_docs=2).load()\n",
|
||||||
|
"len(docs)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"id": "8977bac0-0042-4f23-9754-247dbd32439b",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"{'Published': '2016-05-26',\n",
|
||||||
|
" 'Title': 'Heat-bath random walks with Markov bases',\n",
|
||||||
|
" 'Authors': 'Caprice Stanley, Tobias Windisch',\n",
|
||||||
|
" 'Summary': 'Graphs on lattice points are studied whose edges come from a finite set of\\nallowed moves of arbitrary length. We show that the diameter of these graphs on\\nfibers of a fixed integer matrix can be bounded from above by a constant. We\\nthen study the mixing behaviour of heat-bath random walks on these graphs. We\\nalso state explicit conditions on the set of moves so that the heat-bath random\\nwalk, a generalization of the Glauber dynamics, is an expander in fixed\\ndimension.'}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"doc[0].metadata # meta-information of the Document"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"id": "46969806-45a9-4c4d-a61b-cfb9658fc9de",
|
||||||
|
"metadata": {
|
||||||
|
"tags": []
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"'arXiv:1605.08386v1 [math.CO] 26 May 2016\\nHEAT-BATH RANDOM WALKS WITH MARKOV BASES\\nCAPRICE STANLEY AND TOBIAS WINDISCH\\nAbstract. Graphs on lattice points are studied whose edges come from a finite set of\\nallowed moves of arbitrary length. We show that the diameter of these graphs on fibers of a\\nfixed integer matrix can be bounded from above by a constant. We then study the mixing\\nbehaviour of heat-b'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"output_type": "execute_result"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"doc[0].page_content[:400] # all pages of the Document content\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3 (ipykernel)",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.10.6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 5
|
||||||
|
}
|
@ -0,0 +1,31 @@
|
|||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from langchain.docstore.document import Document
|
||||||
|
from langchain.document_loaders.base import BaseLoader
|
||||||
|
from langchain.utilities.arxiv import ArxivAPIWrapper
|
||||||
|
|
||||||
|
|
||||||
|
class ArxivLoader(BaseLoader):
|
||||||
|
"""Loads a query result from arxiv.org into a list of Documents.
|
||||||
|
|
||||||
|
Each document represents one Document.
|
||||||
|
The loader converts the original PDF format into the text.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
load_max_docs: Optional[int] = 100,
|
||||||
|
load_all_available_meta: Optional[bool] = False,
|
||||||
|
):
|
||||||
|
self.query = query
|
||||||
|
self.load_max_docs = load_max_docs
|
||||||
|
self.load_all_available_meta = load_all_available_meta
|
||||||
|
|
||||||
|
def load(self) -> List[Document]:
|
||||||
|
arxiv_client = ArxivAPIWrapper(
|
||||||
|
load_max_docs=self.load_max_docs,
|
||||||
|
load_all_available_meta=self.load_all_available_meta,
|
||||||
|
)
|
||||||
|
docs = arxiv_client.load(self.query)
|
||||||
|
return docs
|
@ -0,0 +1,55 @@
|
|||||||
|
from typing import List
|
||||||
|
|
||||||
|
from langchain.document_loaders.arxiv import ArxivLoader
|
||||||
|
from langchain.schema import Document
|
||||||
|
|
||||||
|
|
||||||
|
def assert_docs(docs: List[Document]) -> None:
|
||||||
|
for doc in docs:
|
||||||
|
assert doc.page_content
|
||||||
|
assert doc.metadata
|
||||||
|
assert set(doc.metadata) == {"Published", "Title", "Authors", "Summary"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_success() -> None:
|
||||||
|
"""Test that returns one document"""
|
||||||
|
loader = ArxivLoader(query="1605.08386", load_max_docs=2)
|
||||||
|
|
||||||
|
docs = loader.load()
|
||||||
|
assert len(docs) == 1
|
||||||
|
print(docs[0].metadata)
|
||||||
|
print(docs[0].page_content)
|
||||||
|
assert_docs(docs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_returns_no_result() -> None:
|
||||||
|
"""Test that returns no docs"""
|
||||||
|
loader = ArxivLoader(query="1605.08386WWW", load_max_docs=2)
|
||||||
|
docs = loader.load()
|
||||||
|
|
||||||
|
assert len(docs) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_returns_limited_docs() -> None:
|
||||||
|
"""Test that returns several docs"""
|
||||||
|
expected_docs = 2
|
||||||
|
loader = ArxivLoader(query="ChatGPT", load_max_docs=expected_docs)
|
||||||
|
docs = loader.load()
|
||||||
|
|
||||||
|
assert len(docs) == expected_docs
|
||||||
|
assert_docs(docs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_returns_full_set_of_metadata() -> None:
|
||||||
|
"""Test that returns several docs"""
|
||||||
|
loader = ArxivLoader(query="ChatGPT", load_max_docs=1, load_all_available_meta=True)
|
||||||
|
docs = loader.load()
|
||||||
|
assert len(docs) == 1
|
||||||
|
for doc in docs:
|
||||||
|
assert doc.page_content
|
||||||
|
assert doc.metadata
|
||||||
|
assert set(doc.metadata).issuperset(
|
||||||
|
{"Published", "Title", "Authors", "Summary"}
|
||||||
|
)
|
||||||
|
print(doc.metadata)
|
||||||
|
assert len(set(doc.metadata)) > 4
|
Loading…
Reference in New Issue