Harrison/spreedly (#3937)

Co-authored-by: Esmit Pérez <esmitperez@users.noreply.github.com>
This commit is contained in:
Harrison Chase 2023-05-01 20:56:56 -07:00 committed by GitHub
parent cd3f8582cb
commit f04faf8496
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 189 additions and 0 deletions

View File

@ -0,0 +1,129 @@
{
"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"# Spreedly\n",
"\n",
"This notebook covers how to load data from the [Spreedly REST API](https://docs.spreedly.com/reference/api/v1/) into a format that can be ingested into LangChain, along with example usage for vectorization.\n",
"\n",
"Note: this notebook assumes the following packages are installed: `openai`, `chromadb`, and `tiktoken`."
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"from langchain.document_loaders import SpreedlyLoader\n",
"from langchain.indexes import VectorstoreIndexCreator"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Spreedly API requires an access token, which can be found inside the Spreedly Admin Console.\n",
"\n",
"This document loader does not currently support pagination, nor access to more complex objects which require additional parameters. It also requires a `resource` option which defines what objects you want to load.\n",
"\n",
"Following resources are available:\n",
"- `gateways_options`: [Documentation](https://docs.spreedly.com/reference/api/v1/#list-supported-gateways)\n",
"- `gateways`: [Documentation](https://docs.spreedly.com/reference/api/v1/#list-created-gateways)\n",
"- `receivers_options`: [Documentation](https://docs.spreedly.com/reference/api/v1/#list-supported-receivers)\n",
"- `receivers`: [Documentation](https://docs.spreedly.com/reference/api/v1/#list-created-receivers)\n",
"- `payment_methods`: [Documentation](https://docs.spreedly.com/reference/api/v1/#list)\n",
"- `certificates`: [Documentation](https://docs.spreedly.com/reference/api/v1/#list-certificates)\n",
"- `transactions`: [Documentation](https://docs.spreedly.com/reference/api/v1/#list49)\n",
"- `environments`: [Documentation](https://docs.spreedly.com/reference/api/v1/#list-environments)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"spreedly_loader = SpreedlyLoader(os.environ[\"SPREEDLY_ACCESS_TOKEN\"], \"gateways_options\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using embedded DuckDB without persistence: data will be transient\n"
]
}
],
"source": [
"# Create a vectorstore retriver from the loader\n",
"# see https://python.langchain.com/en/latest/modules/indexes/getting_started.html for more details\n",
"\n",
"index = VectorstoreIndexCreator().from_loaders([spreedly_loader])\n",
"spreedly_doc_retriever = index.vectorstore.as_retriever()"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[Document(page_content='installment_grace_period_duration\\nreference_data_code\\ninvoice_number\\ntax_management_indicator\\noriginal_amount\\ninvoice_amount\\nvat_tax_rate\\nmobile_remote_payment_type\\ngratuity_amount\\nmdd_field_1\\nmdd_field_2\\nmdd_field_3\\nmdd_field_4\\nmdd_field_5\\nmdd_field_6\\nmdd_field_7\\nmdd_field_8\\nmdd_field_9\\nmdd_field_10\\nmdd_field_11\\nmdd_field_12\\nmdd_field_13\\nmdd_field_14\\nmdd_field_15\\nmdd_field_16\\nmdd_field_17\\nmdd_field_18\\nmdd_field_19\\nmdd_field_20\\nsupported_countries: US\\nAE\\nBR\\nCA\\nCN\\nDK\\nFI\\nFR\\nDE\\nIN\\nJP\\nMX\\nNO\\nSE\\nGB\\nSG\\nLB\\nPK\\nsupported_cardtypes: visa\\nmaster\\namerican_express\\ndiscover\\ndiners_club\\njcb\\ndankort\\nmaestro\\nelo\\nregions: asia_pacific\\neurope\\nlatin_america\\nnorth_america\\nhomepage: http://www.cybersource.com\\ndisplay_api_url: https://ics2wsa.ic3.com/commerce/1.x/transactionProcessor\\ncompany_name: CyberSource', metadata={'source': 'https://core.spreedly.com/v1/gateways_options.json'}),\n",
" Document(page_content='BG\\nBH\\nBI\\nBJ\\nBM\\nBN\\nBO\\nBR\\nBS\\nBT\\nBW\\nBY\\nBZ\\nCA\\nCC\\nCF\\nCH\\nCK\\nCL\\nCM\\nCN\\nCO\\nCR\\nCV\\nCX\\nCY\\nCZ\\nDE\\nDJ\\nDK\\nDO\\nDZ\\nEC\\nEE\\nEG\\nEH\\nES\\nET\\nFI\\nFJ\\nFK\\nFM\\nFO\\nFR\\nGA\\nGB\\nGD\\nGE\\nGF\\nGG\\nGH\\nGI\\nGL\\nGM\\nGN\\nGP\\nGQ\\nGR\\nGT\\nGU\\nGW\\nGY\\nHK\\nHM\\nHN\\nHR\\nHT\\nHU\\nID\\nIE\\nIL\\nIM\\nIN\\nIO\\nIS\\nIT\\nJE\\nJM\\nJO\\nJP\\nKE\\nKG\\nKH\\nKI\\nKM\\nKN\\nKR\\nKW\\nKY\\nKZ\\nLA\\nLC\\nLI\\nLK\\nLS\\nLT\\nLU\\nLV\\nMA\\nMC\\nMD\\nME\\nMG\\nMH\\nMK\\nML\\nMN\\nMO\\nMP\\nMQ\\nMR\\nMS\\nMT\\nMU\\nMV\\nMW\\nMX\\nMY\\nMZ\\nNA\\nNC\\nNE\\nNF\\nNG\\nNI\\nNL\\nNO\\nNP\\nNR\\nNU\\nNZ\\nOM\\nPA\\nPE\\nPF\\nPH\\nPK\\nPL\\nPN\\nPR\\nPT\\nPW\\nPY\\nQA\\nRE\\nRO\\nRS\\nRU\\nRW\\nSA\\nSB\\nSC\\nSE\\nSG\\nSI\\nSK\\nSL\\nSM\\nSN\\nST\\nSV\\nSZ\\nTC\\nTD\\nTF\\nTG\\nTH\\nTJ\\nTK\\nTM\\nTO\\nTR\\nTT\\nTV\\nTW\\nTZ\\nUA\\nUG\\nUS\\nUY\\nUZ\\nVA\\nVC\\nVE\\nVI\\nVN\\nVU\\nWF\\nWS\\nYE\\nYT\\nZA\\nZM\\nsupported_cardtypes: visa\\nmaster\\namerican_express\\ndiscover\\njcb\\nmaestro\\nelo\\nnaranja\\ncabal\\nunionpay\\nregions: asia_pacific\\neurope\\nmiddle_east\\nnorth_america\\nhomepage: http://worldpay.com\\ndisplay_api_url: https://secure.worldpay.com/jsp/merchant/xml/paymentService.jsp\\ncompany_name: WorldPay', metadata={'source': 'https://core.spreedly.com/v1/gateways_options.json'}),\n",
" Document(page_content='gateway_specific_fields: receipt_email\\nradar_session_id\\nskip_radar_rules\\napplication_fee\\nstripe_account\\nmetadata\\nidempotency_key\\nreason\\nrefund_application_fee\\nrefund_fee_amount\\nreverse_transfer\\naccount_id\\ncustomer_id\\nvalidate\\nmake_default\\ncancellation_reason\\ncapture_method\\nconfirm\\nconfirmation_method\\ncustomer\\ndescription\\nmoto\\noff_session\\non_behalf_of\\npayment_method_types\\nreturn_email\\nreturn_url\\nsave_payment_method\\nsetup_future_usage\\nstatement_descriptor\\nstatement_descriptor_suffix\\ntransfer_amount\\ntransfer_destination\\ntransfer_group\\napplication_fee_amount\\nrequest_three_d_secure\\nerror_on_requires_action\\nnetwork_transaction_id\\nclaim_without_transaction_id\\nfulfillment_date\\nevent_type\\nmodal_challenge\\nidempotent_request\\nmerchant_reference\\ncustomer_reference\\nshipping_address_zip\\nshipping_from_zip\\nshipping_amount\\nline_items\\nsupported_countries: AE\\nAT\\nAU\\nBE\\nBG\\nBR\\nCA\\nCH\\nCY\\nCZ\\nDE\\nDK\\nEE\\nES\\nFI\\nFR\\nGB\\nGR\\nHK\\nHU\\nIE\\nIN\\nIT\\nJP\\nLT\\nLU\\nLV\\nMT\\nMX\\nMY\\nNL\\nNO\\nNZ\\nPL\\nPT\\nRO\\nSE\\nSG\\nSI\\nSK\\nUS\\nsupported_cardtypes: visa', metadata={'source': 'https://core.spreedly.com/v1/gateways_options.json'}),\n",
" Document(page_content='mdd_field_57\\nmdd_field_58\\nmdd_field_59\\nmdd_field_60\\nmdd_field_61\\nmdd_field_62\\nmdd_field_63\\nmdd_field_64\\nmdd_field_65\\nmdd_field_66\\nmdd_field_67\\nmdd_field_68\\nmdd_field_69\\nmdd_field_70\\nmdd_field_71\\nmdd_field_72\\nmdd_field_73\\nmdd_field_74\\nmdd_field_75\\nmdd_field_76\\nmdd_field_77\\nmdd_field_78\\nmdd_field_79\\nmdd_field_80\\nmdd_field_81\\nmdd_field_82\\nmdd_field_83\\nmdd_field_84\\nmdd_field_85\\nmdd_field_86\\nmdd_field_87\\nmdd_field_88\\nmdd_field_89\\nmdd_field_90\\nmdd_field_91\\nmdd_field_92\\nmdd_field_93\\nmdd_field_94\\nmdd_field_95\\nmdd_field_96\\nmdd_field_97\\nmdd_field_98\\nmdd_field_99\\nmdd_field_100\\nsupported_countries: US\\nAE\\nBR\\nCA\\nCN\\nDK\\nFI\\nFR\\nDE\\nIN\\nJP\\nMX\\nNO\\nSE\\nGB\\nSG\\nLB\\nPK\\nsupported_cardtypes: visa\\nmaster\\namerican_express\\ndiscover\\ndiners_club\\njcb\\nmaestro\\nelo\\nunion_pay\\ncartes_bancaires\\nmada\\nregions: asia_pacific\\neurope\\nlatin_america\\nnorth_america\\nhomepage: http://www.cybersource.com\\ndisplay_api_url: https://api.cybersource.com\\ncompany_name: CyberSource REST', metadata={'source': 'https://core.spreedly.com/v1/gateways_options.json'})]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Test the retriever\n",
"spreedly_doc_retriever.get_relevant_documents(\"CRC\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"language": "python",
"name": "python2"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.11"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

View File

@ -71,6 +71,7 @@ from langchain.document_loaders.s3_directory import S3DirectoryLoader
from langchain.document_loaders.s3_file import S3FileLoader from langchain.document_loaders.s3_file import S3FileLoader
from langchain.document_loaders.sitemap import SitemapLoader from langchain.document_loaders.sitemap import SitemapLoader
from langchain.document_loaders.slack_directory import SlackDirectoryLoader from langchain.document_loaders.slack_directory import SlackDirectoryLoader
from langchain.document_loaders.spreedly import SpreedlyLoader
from langchain.document_loaders.srt import SRTLoader from langchain.document_loaders.srt import SRTLoader
from langchain.document_loaders.stripe import StripeLoader from langchain.document_loaders.stripe import StripeLoader
from langchain.document_loaders.telegram import TelegramChatLoader from langchain.document_loaders.telegram import TelegramChatLoader
@ -106,6 +107,7 @@ __all__ = [
"ApifyDatasetLoader", "ApifyDatasetLoader",
"ArxivLoader", "ArxivLoader",
"StripeLoader", "StripeLoader",
"SpreedlyLoader",
"AzureBlobStorageContainerLoader", "AzureBlobStorageContainerLoader",
"AzureBlobStorageFileLoader", "AzureBlobStorageFileLoader",
"BSHTMLLoader", "BSHTMLLoader",

View File

@ -0,0 +1,47 @@
"""Loader that fetches data from Spreedly API."""
import json
import urllib.request
from typing import List
from langchain.docstore.document import Document
from langchain.document_loaders.base import BaseLoader
from langchain.utils import stringify_dict
SPREEDLY_ENDPOINTS = {
"gateways_options": "https://core.spreedly.com/v1/gateways_options.json",
"gateways": "https://core.spreedly.com/v1/gateways.json",
"receivers_options": "https://core.spreedly.com/v1/receivers_options.json",
"receivers": "https://core.spreedly.com/v1/receivers.json",
"payment_methods": "https://core.spreedly.com/v1/payment_methods.json",
"certificates": "https://core.spreedly.com/v1/certificates.json",
"transactions": "https://core.spreedly.com/v1/transactions.json",
"environments": "https://core.spreedly.com/v1/environments.json",
}
class SpreedlyLoader(BaseLoader):
def __init__(self, access_token: str, resource: str) -> None:
self.access_token = access_token
self.resource = resource
self.headers = {
"Authorization": f"Bearer {self.access_token}",
"Accept": "application/json",
}
def _make_request(self, url: str) -> List[Document]:
request = urllib.request.Request(url, headers=self.headers)
with urllib.request.urlopen(request) as response:
json_data = json.loads(response.read().decode())
text = stringify_dict(json_data)
metadata = {"source": url}
return [Document(page_content=text, metadata=metadata)]
def _get_resource(self) -> List[Document]:
endpoint = SPREEDLY_ENDPOINTS.get(self.resource)
if endpoint is None:
return []
return self._make_request(endpoint)
def load(self) -> List[Document]:
return self._get_resource()

View File

@ -0,0 +1,11 @@
from langchain.document_loaders.spreedly import SpreedlyLoader
def test_spreedly_loader() -> None:
"""Test Spreedly Loader."""
access_token = ""
resource = "gateways_options"
spreedly_loader = SpreedlyLoader(access_token, resource)
documents = spreedly_loader.load()
assert len(documents) == 1