Some notebook and client fixes (add retries, clean up docs, etc) (#4820)

# Your PR Title (What it does)

<!--
Thank you for contributing to LangChain! Your PR will appear in our next
release under the title you set. Please make sure it highlights your
valuable contribution.

Replace this with a description of the change, the issue it fixes (if
applicable), and relevant context. List any dependencies required for
this change.

After you're done, someone will review your PR. They may suggest
improvements. If no one reviews your PR within a few days, feel free to
@-mention the same people again, as notifications can get lost.
-->

<!-- Remove if not applicable -->

Fixes # (issue)

## Before submitting

<!-- If you're adding a new integration, include an integration test and
an example notebook showing its use! -->

## Who can review?

Community members can review the PR once tests pass. Tag
maintainers/contributors who might be interested:

<!-- For a quicker response, figure out the right person to tag with @

        @hwchase17 - project lead

        Tracing / Callbacks
        - @agola11

        Async
        - @agola11

        DataLoaders
        - @eyurtsev

        Models
        - @hwchase17
        - @agola11

        Agents / Tools / Toolkits
        - @vowelparrot
        
        VectorStores / Retrievers / Memory
        - @dev2049
        
 -->
dynamic_agent_tools
Ankush Gola 1 year ago committed by GitHub
parent 0a591da6db
commit aa73a888fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -323,10 +323,122 @@
"await task"
]
},
{
"cell_type": "markdown",
"id": "c552a5dd-cbca-48b9-90e6-930076006f78",
"metadata": {},
"source": [
"## [Beta] Tracing V2\n",
"\n",
"We are rolling out a newer version of our tracing service with more features coming soon. Here are the instructions on how to use it to trace your runs.\n",
"\n",
"To use, you can use the `tracing_v2_enabled` context manager or set `LANGCHAIN_TRACING_V2 = 'true'`\n",
"\n",
"**Option 1 (Local)**: \n",
"* Run the local LangChainPlus Server\n",
"```\n",
"pip install --upgrade langchain\n",
"langchain plus start\n",
"```\n",
"\n",
"**Option 2 (Hosted)**:\n",
"* After making an account an grabbing a LangChainPlus API Key, set the `LANGCHAIN_ENDPOINT` and `LANGCHAIN_API_KEY` environment variables"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "87027b0d-3a61-47cf-8a65-3002968be7f9",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"# os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://langchainpro-api-gateway-12bfv6cf.uc.gateway.dev\" # Uncomment this line if you want to use the hosted version\n",
"# os.environ[\"LANGCHAIN_API_KEY\"] = \"<YOUR-LANGCHAINPLUS-API-KEY>\" # Uncomment this line if you want to use the hosted version."
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "5b4f49a2-7d09-4601-a8ba-976f0517c64c",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import langchain\n",
"from langchain.agents import Tool, initialize_agent, load_tools\n",
"from langchain.agents import AgentType\n",
"from langchain.callbacks import tracing_enabled\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.llms import OpenAI"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "029b4a57-dc49-49de-8f03-53c292144e09",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Agent run with tracing. Ensure that OPENAI_API_KEY is set appropriately to run this example.\n",
"\n",
"llm = OpenAI(temperature=0)\n",
"tools = load_tools([\"llm-math\"], llm=llm)\n",
"agent = initialize_agent(\n",
" tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "91a85fb2-6027-4bd0-b1fe-2a3b3b79e2dd",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
"\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
"\u001b[32;1m\u001b[1;3m I need to use a calculator to solve this.\n",
"Action: Calculator\n",
"Action Input: 2^.123243\u001b[0m\n",
"Observation: \u001b[36;1m\u001b[1;3mAnswer: 1.0891804557407723\u001b[0m\n",
"Thought:\u001b[32;1m\u001b[1;3m I now know the final answer.\n",
"Final Answer: 1.0891804557407723\u001b[0m\n",
"\n",
"\u001b[1m> Finished chain.\u001b[0m\n"
]
},
{
"data": {
"text/plain": [
"'1.0891804557407723'"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"agent.run(\"What is 2 raised to .123243 power?\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4e46c85b-2ac0-4661-abed-9c2bf3036820",
"id": "f2291e9f-02f3-4b55-bd3d-d719de815df1",
"metadata": {},
"outputs": [],
"source": []

@ -24,6 +24,7 @@ from uuid import UUID
import requests
from pydantic import BaseSettings, Field, root_validator
from requests import Response
from tenacity import retry, stop_after_attempt, wait_fixed
from langchain.base_language import BaseLanguageModel
from langchain.callbacks.tracers.langchain import LangChainTracer
@ -69,8 +70,8 @@ class LangChainPlusClient(BaseSettings):
"""Client for interacting with the LangChain+ API."""
api_key: Optional[str] = Field(default=None, env="LANGCHAIN_API_KEY")
api_url: str = Field(..., env="LANGCHAIN_ENDPOINT")
tenant_id: str = Field(..., env="LANGCHAIN_TENANT_ID")
api_url: str = Field(default="http://localhost:8000", env="LANGCHAIN_ENDPOINT")
tenant_id: Optional[str] = None
@root_validator(pre=True)
def validate_api_key_if_hosted(cls, values: Dict[str, Any]) -> Dict[str, Any]:
@ -82,25 +83,25 @@ class LangChainPlusClient(BaseSettings):
raise ValueError(
"API key must be provided when using hosted LangChain+ API"
)
else:
tenant_id = values.get("tenant_id")
if not tenant_id:
values["tenant_id"] = LangChainPlusClient._get_seeded_tenant_id(
api_url, api_key
)
tenant_id = values.get("tenant_id")
if not tenant_id:
values["tenant_id"] = LangChainPlusClient._get_seeded_tenant_id(
api_url, api_key
)
return values
@staticmethod
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
def _get_seeded_tenant_id(api_url: str, api_key: Optional[str]) -> str:
"""Get the tenant ID from the seeded tenant."""
url = f"{api_url}/tenants"
headers = {"authorization": f"Bearer {api_key}"} if api_key else {}
headers = {"x-api-key": api_key} if api_key else {}
response = requests.get(url, headers=headers)
try:
raise_for_status_with_text(response)
except Exception as e:
raise ValueError(
"Unable to get seeded tenant ID. Please manually provide."
"Unable to get default tenant ID. Please manually provide."
) from e
results: List[dict] = response.json()
if len(results) == 0:
@ -124,7 +125,12 @@ class LangChainPlusClient(BaseSettings):
def _repr_html_(self) -> str:
"""Return an HTML representation of the instance with a link to the URL."""
link = _get_link_stem(self.api_url)
if _is_localhost(self.api_url):
link = "http://localhost"
elif "dev" in self.api_url:
link = "https://dev.langchain.plus"
else:
link = "https://www.langchain.plus"
return f'<a href="{link}", target="_blank" rel="noopener">LangChain+ Client</a>'
def __repr__(self) -> str:
@ -136,11 +142,11 @@ class LangChainPlusClient(BaseSettings):
"""Get the headers for the API request."""
headers = {}
if self.api_key:
headers["authorization"] = f"Bearer {self.api_key}"
headers["x-api-key"] = self.api_key
return headers
@property
def query_params(self) -> Dict[str, str]:
def query_params(self) -> Dict[str, Any]:
"""Get the headers for the API request."""
return {"tenant_id": self.tenant_id}
@ -199,12 +205,14 @@ class LangChainPlusClient(BaseSettings):
raise ValueError(f"Dataset {file_name} already exists")
return Dataset(**result)
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
def read_run(self, run_id: str) -> Run:
"""Read a run from the LangChain+ API."""
response = self._get(f"/runs/{run_id}")
raise_for_status_with_text(response)
return Run(**response.json())
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
def list_runs(
self,
*,
@ -228,6 +236,7 @@ class LangChainPlusClient(BaseSettings):
raise_for_status_with_text(response)
return [Run(**run) for run in response.json()]
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
@xor_args(("session_id", "session_name"))
def read_session(
self, *, session_id: Optional[str] = None, session_name: Optional[str] = None
@ -258,6 +267,7 @@ class LangChainPlusClient(BaseSettings):
return TracerSession(**result[0])
return TracerSession(**response.json())
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
def list_sessions(self) -> List[TracerSession]:
"""List sessions from the LangChain+ API."""
response = self._get("/sessions")
@ -279,6 +289,7 @@ class LangChainPlusClient(BaseSettings):
raise_for_status_with_text(response)
return Dataset(**response.json())
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
@xor_args(("dataset_name", "dataset_id"))
def read_dataset(
self, *, dataset_name: Optional[str] = None, dataset_id: Optional[str] = None
@ -303,6 +314,7 @@ class LangChainPlusClient(BaseSettings):
return Dataset(**result[0])
return Dataset(**result)
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
def list_datasets(self, limit: int = 100) -> Iterable[Dataset]:
"""List the datasets on the LangChain+ API."""
response = self._get("/datasets", params={"limit": limit})
@ -353,12 +365,14 @@ class LangChainPlusClient(BaseSettings):
result = response.json()
return Example(**result)
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
def read_example(self, example_id: str) -> Example:
"""Read an example from the LangChain+ API."""
response = self._get(f"/examples/{example_id}")
raise_for_status_with_text(response)
return Example(**response.json())
@retry(stop=stop_after_attempt(3), wait=wait_fixed(0.5))
def list_examples(
self, dataset_id: Optional[str] = None, dataset_name: Optional[str] = None
) -> Iterable[Example]:

@ -7,9 +7,9 @@
"tags": []
},
"source": [
"# Tracing and Datasets\n",
"# Tracing and Datasets with LangChainPlus\n",
"\n",
"LangChain makes it easy to get started with Agents and other LLM applications. However, it can be tricky to get right, especially when you need to deliver a full product. To speed up your application development process, and to help monitor your applications in production, LangChain offers additional tracing tooling.\n",
"LangChain makes it easy to get started with Agents and other LLM applications. However, it can be tricky to get right, especially when you need to deliver a full product. To speed up your application development process, and to help monitor your applications in production, LangChain offers additional tracing and tooling.\n",
"\n",
"When might you want to use tracing? Some situations we've found it useful include:\n",
"- Quickly debugging a new chain, agent, or set of tools\n",
@ -20,7 +20,7 @@
"In this notebook, we'll show how to enable tracing in your LangChain applications and walk you a couple common ways to evaluate your agents.\n",
"We'll focus on using Datasets to benchmark Chain behavior.\n",
"\n",
"Bear in mind that this notebook is designed under the assumption that you're running LangChain+ server locally in the background, and it's set up to work specifically with the V2 endpoints. This is done using the folowing command in your terminal:\n",
"**Bear in mind that this notebook is designed under the assumption that you're running the latest LangChain+ server locally in the background. This is done using the folowing command in your terminal:**\n",
"\n",
"\n",
"```\n",
@ -28,9 +28,46 @@
"langchain plus start\n",
"```\n",
"\n",
"We also have a hosted version which is in private beta. We will share more details as it progresses.\n",
"\n",
"Now, let's get started by creating a client to connect to LangChain+."
]
},
{
"cell_type": "markdown",
"id": "2d77d064-41b4-41fb-82e6-2d16461269ec",
"metadata": {
"tags": []
},
"source": [
"## Setting up Tracing\n",
"\n",
"The V2 tracing API can be activated by setting the `LANGCHAIN_TRACING_V2` environment variable to true. Assuming you've successfully initiated the server as described earlier, running LangChain Agents, Chains, LLMs, and other primitives will automatically start capturing traces. Let's begin our exploration with a straightforward math example.\n",
"\n",
"**NOTE**: You must also set your `OPENAI_API_KEY` and `SERPAPI_API_KEY` environment variables in order to run the following tutorial.\n"
]
},
{
"cell_type": "markdown",
"id": "7935e832-9ae1-4557-8d08-890c425f18e2",
"metadata": {},
"source": [
"**NOTE:** You can also use the `tracing_v2_enabled` context manager to capture sessions within a given context:\n",
"```\n",
"from langchain.callbacks.manager import tracing_v2_enabled\n",
"with tracing_v2_enabled(\"My Session Name\"):\n",
" ...\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "221b638a-2ae4-46ef-bf6a-d59bf85d587f",
"metadata": {},
"source": [
"**NOTE:** You can optionally set the `LANGCHAIN_ENDPOINT` and `LANGCHAIN_API_KEY` environment variables if using the hosted version which is in private beta."
]
},
{
"cell_type": "code",
"execution_count": 1,
@ -61,61 +98,23 @@
}
],
"source": [
"import os\n",
"from langchain.client import LangChainPlusClient\n",
"\n",
"client = LangChainPlusClient(\n",
" api_url=\"http://localhost:8000\",\n",
" api_key=None,\n",
" # tenant_id=\"your_tenant_uuid\", # This is required when connecting to a hosted LangChain instance\n",
")\n",
"print(\"You can click the link below to view the UI\")\n",
"client"
]
},
{
"cell_type": "markdown",
"id": "2d77d064-41b4-41fb-82e6-2d16461269ec",
"metadata": {
"tags": []
},
"source": [
"## Tracing Runs\n",
"\n",
"The V2 tracing API can be activated by setting the `LANGCHAIN_TRACING_V2` environment variable to true. Assuming you've successfully initiated the server as described earlier, running LangChain Agents, Chains, LLMs, and other primitives will automatically start capturing traces. Let's begin our exploration with a straightforward math example.\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "4417e0b8-a26f-4a11-b7eb-ba7a18e73885",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import os\n",
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"os.environ[\"LANGCHAIN_SESSION\"] = \"Tracing Walkthrough\"\n",
"# os.environ[\"LANGCHAIN_ENDPOINT\"] = \"http://localhost:8000\" # The default. Update this if you wish to connect to a hosted LangChain instance\n",
"# os.environ[\"LANGCHAIN_API_KEY\"] = None # Update if you wish to authenticate with a hosted LangChain instance"
]
},
{
"cell_type": "markdown",
"id": "7935e832-9ae1-4557-8d08-890c425f18e2",
"metadata": {},
"source": [
"**Note** You can also use the `tracing_v2_enabled` context manager to capture sessions within a given context:\n",
"```\n",
"from langchain.callbacks.manager import tracing_v2_enabled\n",
"with tracing_v2_enabled(\"My Session Name\"):\n",
" ...\n",
"```"
"# os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.langchain.plus\" # Uncomment this line if you want to use the hosted version\n",
"# os.environ[\"LANGCHAIN_API_KEY\"] = \"<YOUR-LANGCHAINPLUS-API-KEY>\" # Uncomment this line if you want to use the hosted version.\n",
"\n",
"client = LangChainPlusClient()\n",
"print(\"You can click the link below to view the UI\")\n",
"client"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 2,
"id": "7c801853-8e96-404d-984c-51ace59cbbef",
"metadata": {
"tags": []
@ -133,7 +132,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 3,
"id": "19537902-b95c-4390-80a4-f6c9a937081e",
"metadata": {
"tags": []
@ -146,16 +145,30 @@
"39,566,248\n",
"Anwar Hadid is Dua Lipa's boyfriend and his age raised to the 0.43 power is approximately 3.87.\n",
"LLMMathChain._evaluate(\"\n",
"(age)**0.43\n",
"(age ** 0.43)\n",
"\") raised error: 'age'. Please try again with a valid numerical expression\n",
"The distance between Paris and Boston is 3448 miles.\n",
"unknown format from LLM: Assuming we don't have any information about the actual number of points scored in the 2023 super bowl, we cannot provide a mathematical expression to solve this problem.\n",
"The total number of points scored in the 2023 super bowl raised to the .23 power is approximately 3.457460415669602.\n",
"LLMMathChain._evaluate(\"\n",
"(total number of points scored in the 2023 super bowl)**0.23\n",
"\") raised error: invalid syntax. Perhaps you forgot a comma? (<expr>, line 1). Please try again with a valid numerical expression\n",
"15 points were scored more in the 2023 Super Bowl than in the 2022 Super Bowl.\n",
"\") raised error: invalid syntax. Perhaps you forgot a comma? (<expr>, line 1). Please try again with a valid numerical expression\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 63c89b8bad9b172227d890620cdec651 in your message.).\n",
"Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID e3dd37877de500d7defe699f8411b3dd in your message.).\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"0\n",
"1.9347796717823205\n",
"77\n",
"1.2600907451828602 (inches)\n",
"LLMMathChain._evaluate(\"\n",
"round(0.2791714614499425, 2)\n",
"\") raised error: 'VariableNode' object is not callable. Please try again with a valid numerical expression\n"
@ -197,7 +210,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 4,
"id": "d14a9881-2a01-404c-8c56-0b78565c3ff4",
"metadata": {
"tags": []
@ -209,7 +222,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 5,
"id": "c0e12629-bca5-4438-8665-890d0cb9cc4a",
"metadata": {
"tags": []
@ -223,7 +236,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 6,
"id": "17580c4b-bd04-4dde-9d21-9d4edd25b00d",
"metadata": {
"tags": []
@ -250,7 +263,7 @@
"tags": []
},
"source": [
"**Creating a Dataset in the UI** \n",
"**Alternative: Creating a Dataset in the UI** \n",
"\n",
"Alternatively, you could create or edit the dataset in the UI using the following steps:\n",
"\n",
@ -273,7 +286,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 7,
"id": "1baa677c-5642-4378-8e01-3aa1647f19d6",
"metadata": {
"tags": []
@ -286,7 +299,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 8,
"id": "60d14593-c61f-449f-a38f-772ca43707c2",
"metadata": {
"tags": []
@ -304,7 +317,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 9,
"id": "52a7ea76-79ca-4765-abf7-231e884040d6",
"metadata": {
"tags": []
@ -340,7 +353,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 10,
"id": "c2b59104-b90e-466a-b7ea-c5bd0194263b",
"metadata": {
"tags": []
@ -368,7 +381,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 11,
"id": "112d7bdf-7e50-4c1a-9285-5bac8473f2ee",
"metadata": {
"tags": []
@ -405,7 +418,7 @@
"\n",
"Returns:\n",
" A dictionary mapping example ids to the model outputs.\n",
"\u001b[0;31mFile:\u001b[0m ~/code/lc/lckg/langchain/client/langchain.py\n",
"\u001b[0;31mFile:\u001b[0m ~/Code/langchain/langchain/client/langchain.py\n",
"\u001b[0;31mType:\u001b[0m method"
]
},
@ -419,7 +432,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 12,
"id": "6e10f823",
"metadata": {
"tags": []
@ -438,7 +451,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 13,
"id": "a8088b7d-3ab6-4279-94c8-5116fe7cee33",
"metadata": {
"tags": []
@ -455,22 +468,8 @@
"name": "stderr",
"output_type": "stream",
"text": [
"Chain failed for example 8d4ff5b4-41fb-4986-80f1-025e6fec96b0. Error: unknown format from LLM: It is impossible to accurately predict the total number of points scored in a future event. Therefore, a mathematical expression cannot be provided.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processed examples: 2\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Chain failed for example 178081fb-a44a-46d5-a23b-74a830da65f3. Error: LLMMathChain._evaluate(\"\n",
"(age)**0.43\n",
"Chain failed for example 604fbd32-7cbe-4dd4-9ddd-fd5ab5c01566. Error: LLMMathChain._evaluate(\"\n",
"(age ** 0.43)\n",
"\") raised error: 'age'. Please try again with a valid numerical expression\n"
]
},
@ -478,14 +477,14 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Processed examples: 5\r"
"Processed examples: 4\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Chain failed for example 7de97d34-50e2-4ec5-bc49-c8e6287ae73e. Error: LLMMathChain._evaluate(\"\n",
"Chain failed for example 4c82b6a4-d8ce-4129-8229-7f4e2f76294c. Error: LLMMathChain._evaluate(\"\n",
"(total number of points scored in the 2023 super bowl)**0.23\n",
"\") raised error: invalid syntax. Perhaps you forgot a comma? (<expr>, line 1). Please try again with a valid numerical expression\n"
]
@ -1047,7 +1046,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.2"
"version": "3.10.9"
}
},
"nbformat": 4,

@ -72,7 +72,7 @@ def test_headers() -> None:
LangChainPlusClient, "_get_seeded_tenant_id", new=mock_get_seeded_tenant_id
):
client = LangChainPlusClient(api_url="http://localhost:8000", api_key="123")
assert client._headers == {"authorization": "Bearer 123"}
assert client._headers == {"x-api-key": "123"}
with mock.patch.object(
LangChainPlusClient, "_get_seeded_tenant_id", new=mock_get_seeded_tenant_id

Loading…
Cancel
Save