mirror of https://github.com/hwchase17/langchain
You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
204 lines
7.6 KiB
Python
204 lines
7.6 KiB
Python
9 months ago
|
import os
|
||
|
import re
|
||
|
from typing import Iterator, List
|
||
|
|
||
|
import requests
|
||
|
from langchain_core.documents import Document
|
||
|
|
||
|
from langchain_community.document_loaders.base import BaseLoader
|
||
|
|
||
|
|
||
|
class EtherscanLoader(BaseLoader):
|
||
|
"""Load transactions from `Ethereum` mainnet.
|
||
|
|
||
|
The Loader use Etherscan API to interact with Ethereum mainnet.
|
||
|
|
||
|
ETHERSCAN_API_KEY environment variable must be set use this loader.
|
||
|
"""
|
||
|
|
||
|
def __init__(
|
||
|
self,
|
||
|
account_address: str,
|
||
|
api_key: str = "docs-demo",
|
||
|
filter: str = "normal_transaction",
|
||
|
page: int = 1,
|
||
|
offset: int = 10,
|
||
|
start_block: int = 0,
|
||
|
end_block: int = 99999999,
|
||
|
sort: str = "desc",
|
||
|
):
|
||
|
self.account_address = account_address
|
||
|
self.api_key = os.environ.get("ETHERSCAN_API_KEY") or api_key
|
||
|
self.filter = filter
|
||
|
self.page = page
|
||
|
self.offset = offset
|
||
|
self.start_block = start_block
|
||
|
self.end_block = end_block
|
||
|
self.sort = sort
|
||
|
|
||
|
if not self.api_key:
|
||
|
raise ValueError("Etherscan API key not provided")
|
||
|
|
||
|
if not re.match(r"^0x[a-fA-F0-9]{40}$", self.account_address):
|
||
|
raise ValueError(f"Invalid contract address {self.account_address}")
|
||
|
if filter not in [
|
||
|
"normal_transaction",
|
||
|
"internal_transaction",
|
||
|
"erc20_transaction",
|
||
|
"eth_balance",
|
||
|
"erc721_transaction",
|
||
|
"erc1155_transaction",
|
||
|
]:
|
||
|
raise ValueError(f"Invalid filter {filter}")
|
||
|
|
||
|
def lazy_load(self) -> Iterator[Document]:
|
||
|
"""Lazy load Documents from table."""
|
||
|
result = []
|
||
|
if self.filter == "normal_transaction":
|
||
|
result = self.getNormTx()
|
||
|
elif self.filter == "internal_transaction":
|
||
|
result = self.getInternalTx()
|
||
|
elif self.filter == "erc20_transaction":
|
||
|
result = self.getERC20Tx()
|
||
|
elif self.filter == "eth_balance":
|
||
|
result = self.getEthBalance()
|
||
|
elif self.filter == "erc721_transaction":
|
||
|
result = self.getERC721Tx()
|
||
|
elif self.filter == "erc1155_transaction":
|
||
|
result = self.getERC1155Tx()
|
||
|
else:
|
||
|
raise ValueError(f"Invalid filter {filter}")
|
||
|
for doc in result:
|
||
|
yield doc
|
||
|
|
||
|
def load(self) -> List[Document]:
|
||
|
"""Load transactions from spcifc account by Etherscan."""
|
||
|
return list(self.lazy_load())
|
||
|
|
||
|
def getNormTx(self) -> List[Document]:
|
||
|
url = (
|
||
|
f"https://api.etherscan.io/api?module=account&action=txlist&address={self.account_address}"
|
||
|
f"&startblock={self.start_block}&endblock={self.end_block}&page={self.page}"
|
||
|
f"&offset={self.offset}&sort={self.sort}&apikey={self.api_key}"
|
||
|
)
|
||
|
try:
|
||
|
response = requests.get(url)
|
||
|
response.raise_for_status()
|
||
|
except requests.exceptions.RequestException as e:
|
||
|
print("Error occurred while making the request:", e)
|
||
|
items = response.json()["result"]
|
||
|
result = []
|
||
|
if len(items) == 0:
|
||
|
return [Document(page_content="")]
|
||
|
for item in items:
|
||
|
content = str(item)
|
||
|
metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
|
||
|
result.append(Document(page_content=content, metadata=metadata))
|
||
|
print(len(result))
|
||
|
return result
|
||
|
|
||
|
def getEthBalance(self) -> List[Document]:
|
||
|
url = (
|
||
|
f"https://api.etherscan.io/api?module=account&action=balance"
|
||
|
f"&address={self.account_address}&tag=latest&apikey={self.api_key}"
|
||
|
)
|
||
|
|
||
|
try:
|
||
|
response = requests.get(url)
|
||
|
response.raise_for_status()
|
||
|
except requests.exceptions.RequestException as e:
|
||
|
print("Error occurred while making the request:", e)
|
||
|
return [Document(page_content=response.json()["result"])]
|
||
|
|
||
|
def getInternalTx(self) -> List[Document]:
|
||
|
url = (
|
||
|
f"https://api.etherscan.io/api?module=account&action=txlistinternal"
|
||
|
f"&address={self.account_address}&startblock={self.start_block}"
|
||
|
f"&endblock={self.end_block}&page={self.page}&offset={self.offset}"
|
||
|
f"&sort={self.sort}&apikey={self.api_key}"
|
||
|
)
|
||
|
|
||
|
try:
|
||
|
response = requests.get(url)
|
||
|
response.raise_for_status()
|
||
|
except requests.exceptions.RequestException as e:
|
||
|
print("Error occurred while making the request:", e)
|
||
|
items = response.json()["result"]
|
||
|
result = []
|
||
|
if len(items) == 0:
|
||
|
return [Document(page_content="")]
|
||
|
for item in items:
|
||
|
content = str(item)
|
||
|
metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
|
||
|
result.append(Document(page_content=content, metadata=metadata))
|
||
|
return result
|
||
|
|
||
|
def getERC20Tx(self) -> List[Document]:
|
||
|
url = (
|
||
|
f"https://api.etherscan.io/api?module=account&action=tokentx"
|
||
|
f"&address={self.account_address}&startblock={self.start_block}"
|
||
|
f"&endblock={self.end_block}&page={self.page}&offset={self.offset}"
|
||
|
f"&sort={self.sort}&apikey={self.api_key}"
|
||
|
)
|
||
|
|
||
|
try:
|
||
|
response = requests.get(url)
|
||
|
response.raise_for_status()
|
||
|
except requests.exceptions.RequestException as e:
|
||
|
print("Error occurred while making the request:", e)
|
||
|
items = response.json()["result"]
|
||
|
result = []
|
||
|
if len(items) == 0:
|
||
|
return [Document(page_content="")]
|
||
|
for item in items:
|
||
|
content = str(item)
|
||
|
metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
|
||
|
result.append(Document(page_content=content, metadata=metadata))
|
||
|
return result
|
||
|
|
||
|
def getERC721Tx(self) -> List[Document]:
|
||
|
url = (
|
||
|
f"https://api.etherscan.io/api?module=account&action=tokennfttx"
|
||
|
f"&address={self.account_address}&startblock={self.start_block}"
|
||
|
f"&endblock={self.end_block}&page={self.page}&offset={self.offset}"
|
||
|
f"&sort={self.sort}&apikey={self.api_key}"
|
||
|
)
|
||
|
|
||
|
try:
|
||
|
response = requests.get(url)
|
||
|
response.raise_for_status()
|
||
|
except requests.exceptions.RequestException as e:
|
||
|
print("Error occurred while making the request:", e)
|
||
|
items = response.json()["result"]
|
||
|
result = []
|
||
|
if len(items) == 0:
|
||
|
return [Document(page_content="")]
|
||
|
for item in items:
|
||
|
content = str(item)
|
||
|
metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
|
||
|
result.append(Document(page_content=content, metadata=metadata))
|
||
|
return result
|
||
|
|
||
|
def getERC1155Tx(self) -> List[Document]:
|
||
|
url = (
|
||
|
f"https://api.etherscan.io/api?module=account&action=token1155tx"
|
||
|
f"&address={self.account_address}&startblock={self.start_block}"
|
||
|
f"&endblock={self.end_block}&page={self.page}&offset={self.offset}"
|
||
|
f"&sort={self.sort}&apikey={self.api_key}"
|
||
|
)
|
||
|
|
||
|
try:
|
||
|
response = requests.get(url)
|
||
|
response.raise_for_status()
|
||
|
except requests.exceptions.RequestException as e:
|
||
|
print("Error occurred while making the request:", e)
|
||
|
items = response.json()["result"]
|
||
|
result = []
|
||
|
if len(items) == 0:
|
||
|
return [Document(page_content="")]
|
||
|
for item in items:
|
||
|
content = str(item)
|
||
|
metadata = {"from": item["from"], "tx_hash": item["hash"], "to": item["to"]}
|
||
|
result.append(Document(page_content=content, metadata=metadata))
|
||
|
return result
|