Archives/langchain
2
0
Fork 1
mirror of https://github.com/hwchase17/langchain synced 2024-11-10 01:10:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

core[patch]: support oai dicts as messages (#25621)

Browse Source 
and update langsmtih example selector docs
This commit is contained in:
Bagatur 2024-08-21 09:13:15 -07:00 committed by GitHub
parent a78843bb77
commit 0bc3845e1e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 87 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
docs/docs/how_to/example_selectors_langsmith.ipynb
View File

@ -18,7 +18,7 @@
"\n",
"\n",
"<Compatibility packagesAndVersions={[\n",
" [\"langsmith\", \"0.1.101\"],\n",
" [\"langsmith\", \"0.1.100\"],\n",
"]} />\n",
"\n",
"\n",
@ -33,7 +33,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 2,
"id": "85445e0e",
"metadata": {},
"outputs": [
@ -72,7 +72,7 @@
"metadata": {},
"outputs": [],
"source": [
"%pip install -qU langsmith>=0.1.101 langchain langchain-openai langchain-benchmarks"
"%pip install -qU langsmith>=0.1.100 langchain langchain-openai langchain-benchmarks"
]
},
{
@ -84,7 +84,7 @@
"\n",
"We'll clone the [Multiverse math few shot example dataset](https://blog.langchain.dev/few-shot-prompting-to-improve-tool-calling-performance/).\n",
"\n",
"This enables searching over the dataset, and will make sure that anytime we update/add examples they are also indexed."
"This enables searching over the dataset and will make sure that anytime we update/add examples they are also indexed."
]
},
{
@ -94,20 +94,19 @@
"metadata": {},
"outputs": [],
"source": [
"from langsmith import AsyncClient as AsyncLangSmith\n",
"from langsmith import Client as LangSmith\n",
"\n",
"ls_client = LangSmith()\n",
"async_ls_client = AsyncLangSmith()\n",
"\n",
"dataset_name = \"multiverse-math-examples-for-few-shot\"\n",
"dataset_name = \"multiverse-math-few-shot-examples-v2\"\n",
"dataset_public_url = (\n",
" \"https://smith.langchain.com/public/0df59e49-d226-4ef2-9ecd-8c0fc9cd0288/d\"\n",
" \"https://smith.langchain.com/public/620596ee-570b-4d2b-8c8f-f828adbe5242/d\"\n",
")\n",
"\n",
"ls_client.clone_public_dataset(dataset_public_url)\n",
"\n",
"dataset_id = ls_client.read_dataset(dataset_name=dataset_name).id\n",
"\n",
"ls_client.index_dataset(dataset_id=dataset_id)"
]
},
@ -116,12 +115,12 @@
"id": "5767d171",
"metadata": {},
"source": [
"Indexing can take a few seconds. Once the dataset is indexed, we can search for similar examples like so:"
"Indexing can take a few seconds. Once the dataset is indexed, we can search for similar examples. Note that the input to the `similar_examples` method must have the same schema as the examples inputs. In this case our example inputs are a dictionary with a \"question\" key:"
]
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 12,
"id": "5013a56f",
"metadata": {},
"outputs": [
@ -131,14 +130,14 @@
"3"
]
},
"execution_count": 29,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"examples = ls_client.similar_examples(\n",
" {\"input\": \"whats the negation of the negation of the negation of 3\"},\n",
" {\"question\": \"whats the negation of the negation of the negation of 3\"},\n",
" limit=3,\n",
" dataset_id=dataset_id,\n",
")\n",
@ -147,7 +146,7 @@
},
{
"cell_type": "code",
"execution_count": 34,
"execution_count": 13,
"id": "a142db06",
"metadata": {},
"outputs": [
@ -157,13 +156,13 @@
"'evaluate the negation of -100'"
]
},
"execution_count": 34,
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"examples[0].inputs[\"input\"]"
"examples[0].inputs[\"question\"]"
]
},
{
@ -171,28 +170,51 @@
"id": "d2627125",
"metadata": {},
"source": [
"For this dataset the outputs are an entire chat history:"
"For this dataset, the outputs are the conversation that followed the question in OpenAI message format:"
]
},
{
"cell_type": "code",
"execution_count": 33,
"execution_count": 14,
"id": "af5b9191",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"9"
"[{'role': 'assistant',\n",
" 'content': None,\n",
" 'tool_calls': [{'id': 'toolu_01HTpq4cYNUac6F7omUc2Wz3',\n",
" 'type': 'function',\n",
" 'function': {'name': 'negate', 'arguments': '{\"a\": -100}'}}]},\n",
" {'role': 'tool',\n",
" 'content': '-100.0',\n",
" 'tool_call_id': 'toolu_01HTpq4cYNUac6F7omUc2Wz3'},\n",
" {'role': 'assistant', 'content': 'So the answer is 100.'},\n",
" {'role': 'user',\n",
" 'content': '100 is incorrect. Please refer to the output of your tool call.'},\n",
" {'role': 'assistant',\n",
" 'content': [{'text': \"You're right, my previous answer was incorrect. Let me re-evaluate using the tool output:\",\n",
" 'type': 'text'}],\n",
" 'tool_calls': [{'id': 'toolu_01XsJQboYghGDygQpPjJkeRq',\n",
" 'type': 'function',\n",
" 'function': {'name': 'negate', 'arguments': '{\"a\": -100}'}}]},\n",
" {'role': 'tool',\n",
" 'content': '-100.0',\n",
" 'tool_call_id': 'toolu_01XsJQboYghGDygQpPjJkeRq'},\n",
" {'role': 'assistant', 'content': 'The answer is -100.0'},\n",
" {'role': 'user',\n",
" 'content': 'You have the correct numerical answer but are returning additional text. Please only respond with the numerical answer.'},\n",
" {'role': 'assistant', 'content': '-100.0'}]"
]
},
"execution_count": 33,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(examples[1].outputs[\"output\"])"
"examples[0].outputs[\"conversation\"]"
]
},
{
@ -205,7 +227,7 @@
},
{
"cell_type": "code",
"execution_count": 50,
"execution_count": 20,
"id": "12cba1e1",
"metadata": {},
"outputs": [],
@ -223,8 +245,10 @@
" sin,\n",
" subtract,\n",
")\n",
"from langchain_core.messages import HumanMessage, SystemMessage, convert_to_messages\n",
"from langchain_core.runnables import RunnableLambda\n",
"from langsmith import AsyncClient as AsyncLangSmith\n",
"\n",
"async_ls_client = AsyncLangSmith()\n",
"\n",
"\n",
"def similar_examples(input_: dict) -> dict:\n",
@ -241,20 +265,24 @@
"\n",
"def construct_prompt(input_: dict) -> list:\n",
" instructions = \"\"\"You are great at using mathematical tools.\"\"\"\n",
" messages = []\n",
" examples = []\n",
" for ex in input_[\"examples\"]:\n",
" # For this dataset, a multi-turn conversation is stored as output.\n",
" messages.extend(convert_to_messages(ex.outputs[\"output\"]))\n",
" examples = [msg for msg in messages if not isinstance(msg, SystemMessage)]\n",
" for ex in examples:\n",
" ex.name = (\n",
" \"example_user\" if isinstance(ex, HumanMessage) else \"example_assistant\"\n",
" )\n",
" return [SystemMessage(instructions), *examples, HumanMessage(input_[\"input\"])]\n",
" examples.append({\"role\": \"user\", \"content\": ex.inputs[\"question\"]})\n",
" for msg in ex.outputs[\"conversation\"]:\n",
" if msg[\"role\"] == \"assistant\":\n",
" msg[\"name\"] = \"example_assistant\"\n",
" if msg[\"role\"] == \"user\":\n",
" msg[\"name\"] = \"example_user\"\n",
" examples.append(msg)\n",
" return [\n",
" {\"role\": \"system\", \"content\": instructions},\n",
" *examples,\n",
" {\"role\": \"user\", \"content\": input_[\"question\"]},\n",
" ]\n",
"\n",
"\n",
"tools = [add, cos, divide, log, multiply, negate, pi, power, sin, subtract]\n",
"llm = init_chat_model(\"gpt-4o\")\n",
"llm = init_chat_model(\"gpt-4o-2024-08-06\")\n",
"llm_with_tools = llm.bind_tools(tools)\n",
"\n",
"example_selector = RunnableLambda(func=similar_examples, afunc=asimilar_examples)\n",
@ -264,7 +292,7 @@
},
{
"cell_type": "code",
"execution_count": 52,
"execution_count": 21,
"id": "c423b367",
"metadata": {},
"outputs": [
@ -273,17 +301,17 @@
"text/plain": [
"[{'name': 'negate',\n",
" 'args': {'a': 3},\n",
" 'id': 'call_ehmx3Z4Cj6HFpI8FV4pYZ5Oo',\n",
" 'id': 'call_uMSdoTl6ehfHh5a6JQUb2NoZ',\n",
" 'type': 'tool_call'}]"
]
},
"execution_count": 52,
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ai_msg = await chain.ainvoke({\"input\": \"whats the negation of the negation of 3\"})\n",
"ai_msg = await chain.ainvoke({\"question\": \"whats the negation of the negation of 3\"})\n",
"ai_msg.tool_calls"
]
},
@ -292,7 +320,7 @@
"id": "94489b4a",
"metadata": {},
"source": [
"Looking at the LangSmith trace, we can see that relevant examples were pulled in in the `similar_examples` step and passed as messages to ChatOpenAI: https://smith.langchain.com/public/05af2ce8-1a45-4f3a-8d54-6524ff919279/r."
"Looking at the LangSmith trace, we can see that relevant examples were pulled in in the `similar_examples` step and passed as messages to ChatOpenAI: https://smith.langchain.com/public/9585e30f-765a-4ed9-b964-2211420cd2f8/r."
]
}
],

22
libs/core/langchain_core/messages/utils.py
View File

@ -10,6 +10,7 @@ Some examples of what you can do with these functions include:
from __future__ import annotations
import inspect
import json
from functools import partial
from typing import (
TYPE_CHECKING,
@ -213,7 +214,23 @@ def _create_message_from_message_type(
if id is not None:
kwargs["id"] = id
if tool_calls is not None:
kwargs["tool_calls"] = tool_calls
kwargs["tool_calls"] = []
for tool_call in tool_calls:
# Convert OpenAI-format tool call to LangChain format.
if "function" in tool_call:
args = tool_call["function"]["arguments"]
if isinstance(args, str):
args = json.loads(args, strict=False)
kwargs["tool_calls"].append(
{
"name": tool_call["function"]["name"],
"args": args,
"id": tool_call["id"],
"type": "tool_call",
}
)
else:
kwargs["tool_calls"].append(tool_call)
if message_type in ("human", "user"):
message: BaseMessage = HumanMessage(content=content, **kwargs)
elif message_type in ("ai", "assistant"):
@ -271,7 +288,8 @@ def _convert_to_message(message: MessageLikeRepresentation) -> BaseMessage:
msg_type = msg_kwargs.pop("role")
except KeyError:
msg_type = msg_kwargs.pop("type")
msg_content = msg_kwargs.pop("content")
# None msg content is not allowed
msg_content = msg_kwargs.pop("content") or ""
except KeyError:
raise ValueError(
f"Message dict must contain 'role' and 'content' keys, got {message}"

8
libs/core/langchain_core/output_parsers/openai_tools.py
View File

@ -5,12 +5,8 @@ from typing import Any, Dict, List, Optional
from langchain_core.exceptions import OutputParserException
from langchain_core.messages import AIMessage, InvalidToolCall
from langchain_core.messages.tool import (
invalid_tool_call,
)
from langchain_core.messages.tool import (
tool_call as create_tool_call,
)
from langchain_core.messages.tool import invalid_tool_call
from langchain_core.messages.tool import tool_call as create_tool_call
from langchain_core.output_parsers.transform import BaseCumulativeTransformOutputParser
from langchain_core.outputs import ChatGeneration, Generation
from langchain_core.pydantic_v1 import ValidationError