blob42
/
langchain
forked from Archives/langchain
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Dereference Messages (#4557)

Browse Source 
Update how we parse the messages now that the server splits prompts /
messages up
parallel_dir_loader
Zander Chase 1 year ago committed by GitHub
parent e17d0319d5
commit 258c319855
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 126 additions and 100 deletions
Show Stats Download Patch File Download Diff File

30
langchain/client/langchain.py
Unescape Escape View File

@ -294,16 +294,16 @@ class LangChainPlusClient(BaseSettings):
langchain_tracer: LangChainTracerV2,
) -> Union[LLMResult, ChatResult]:
if isinstance(llm, BaseLLM):
if "prompts" not in inputs:
raise ValueError(f"LLM Run requires 'prompts' input. Got {inputs}")
llm_prompts: List[str] = inputs["prompts"]
llm_output = await llm.agenerate(llm_prompts, callbacks=[langchain_tracer])
if "prompt" not in inputs:
raise ValueError(f"LLM Run requires 'prompt' input. Got {inputs}")
llm_prompt: str = inputs["prompt"]
llm_output = await llm.agenerate([llm_prompt], callbacks=[langchain_tracer])
elif isinstance(llm, BaseChatModel):
if "messages" not in inputs:
raise ValueError(f"Chat Run requires 'messages' input. Got {inputs}")
raw_messages: List[List[dict]] = inputs["messages"]
messages = [messages_from_dict(batch) for batch in raw_messages]
llm_output = await llm.agenerate(messages, callbacks=[langchain_tracer])
raw_messages: List[dict] = inputs["messages"]
messages = messages_from_dict(raw_messages)
llm_output = await llm.agenerate([messages], callbacks=[langchain_tracer])
else:
raise ValueError(f"Unsupported LLM type {type(llm)}")
return llm_output
@ -454,18 +454,18 @@ class LangChainPlusClient(BaseSettings):
) -> Union[LLMResult, ChatResult]:
"""Run the language model on the example."""
if isinstance(llm, BaseLLM):
if "prompts" not in inputs:
raise ValueError(f"LLM Run must contain 'prompts' key. Got {inputs}")
llm_prompts: List[str] = inputs["prompts"]
llm_output = llm.generate(llm_prompts, callbacks=[langchain_tracer])
if "prompt" not in inputs:
raise ValueError(f"LLM Run must contain 'prompt' key. Got {inputs}")
llm_prompt: str = inputs["prompt"]
llm_output = llm.generate([llm_prompt], callbacks=[langchain_tracer])
elif isinstance(llm, BaseChatModel):
if "messages" not in inputs:
raise ValueError(
f"Chat Model Run must contain 'messages' key. Got {inputs}"
)
raw_messages: List[List[dict]] = inputs["messages"]
messages = [messages_from_dict(batch) for batch in raw_messages]
llm_output = llm.generate(messages, callbacks=[langchain_tracer])
raw_messages: List[dict] = inputs["messages"]
messages = messages_from_dict(raw_messages)
llm_output = llm.generate([messages], callbacks=[langchain_tracer])
else:
raise ValueError(f"Unsupported LLM type {type(llm)}")
return llm_output
@ -529,7 +529,7 @@ class LangChainPlusClient(BaseSettings):
f"{dataset_name}-{llm_or_chain.__class__.__name__}-{current_time}"
)
dataset = self.read_dataset(dataset_name=dataset_name)
examples = self.list_examples(dataset_id=str(dataset.id))
examples = list(self.list_examples(dataset_id=str(dataset.id)))
results: Dict[str, Any] = {}
with tracing_v2_enabled(session_name=session_name) as session:
tracer = LangChainTracerV2()

196
langchain/experimental/client/tracing_datasets.ipynb
Unescape Escape View File

@ -124,7 +124,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:65: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n",
"/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:78: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n",
" warnings.warn(\n"
]
},
@ -132,16 +132,22 @@
"name": "stdout",
"output_type": "stream",
"text": [
"The current population of Canada as of 2023 is 38,681,797.\n",
"Anwar Hadid is Dua Lipa's boyfriend and his age raised to the 0.43 power is approximately 3.87.\n",
"'age'. Please try again with a valid numerical expression\n",
"The distance between Paris and Boston is approximately 3448 miles.\n",
"The current population of Canada as of 2023 is 39,566,248.\n",
"Anwar Hadid's age raised to the 0.43 power is approximately 3.87.\n",
"LLMMathChain._evaluate(\"\n",
"(age)**0.43\n",
"\") raised error: 'age'. Please try again with a valid numerical expression\n",
"The distance between Paris and Boston is 3448 miles.\n",
"unknown format from LLM: Assuming we don't have any information about the actual number of points scored in the 2023 super bowl, we cannot provide a mathematical expression to solve this problem.\n",
"invalid syntax. Perhaps you forgot a comma? (<expr>, line 1). Please try again with a valid numerical expression\n",
"0\n",
"LLMMathChain._evaluate(\"\n",
"(total number of points scored in the 2023 super bowl)**0.23\n",
"\") raised error: invalid syntax. Perhaps you forgot a comma? (<expr>, line 1). Please try again with a valid numerical expression\n",
"3 points were scored more in the 2023 Super Bowl than in the 2022 Super Bowl.\n",
"1.9347796717823205\n",
"1.275494929129063\n",
"1213 divided by 4345 is approximately 0.2791.\n"
"81\n",
"LLMMathChain._evaluate(\"\n",
"round(0.2791714614499425, 2)\n",
"\") raised error: 'VariableNode' object is not callable. Please try again with a valid numerical expression\n"
]
}
],
@ -186,7 +192,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 5,
"id": "d14a9881-2a01-404c-8c56-0b78565c3ff4",
"metadata": {
"tags": []
@ -198,7 +204,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 6,
"id": "7bfb4699-62c3-400a-b3e7-7fb8ad3a68ad",
"metadata": {
"tags": []
@ -213,7 +219,7 @@
"LangChainPlusClient (API URL: http://localhost:8000)"
]
},
"execution_count": 5,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@ -235,7 +241,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"id": "1baa677c-5642-4378-8e01-3aa1647f19d6",
"metadata": {
"tags": []
@ -248,7 +254,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"id": "60d14593-c61f-449f-a38f-772ca43707c2",
"metadata": {
"tags": []
@ -266,7 +272,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"id": "52a7ea76-79ca-4765-abf7-231e884040d6",
"metadata": {
"tags": []
@ -302,7 +308,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 11,
"id": "c2b59104-b90e-466a-b7ea-c5bd0194263b",
"metadata": {
"tags": []
@ -330,7 +336,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 12,
"id": "112d7bdf-7e50-4c1a-9285-5bac8473f2ee",
"metadata": {
"tags": []
@ -378,7 +384,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 13,
"id": "a8088b7d-3ab6-4279-94c8-5116fe7cee33",
"metadata": {
"tags": []
@ -388,28 +394,48 @@
"name": "stderr",
"output_type": "stream",
"text": [
"Chain failed for example bb17c39c-c90c-4706-8410-e501e89a81fb. Error: 'age'. Please try again with a valid numerical expression\n"
"/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:78: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n",
" warnings.warn(\n",
"Chain failed for example 92c75ce4-f807-4d44-8f7e-027610f7fcbd. Error: unknown format from LLM: Sorry, I cannot answer this question as it requires information from the future.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processed examples: 1\r"
"Processed examples: 2\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Chain failed for example e66a366d-f5d3-4dcc-9ec4-aae88369f45e. Error: invalid syntax. Perhaps you forgot a comma? (<expr>, line 1). Please try again with a valid numerical expression\n"
"Chain failed for example 9f5d1426-3e21-4628-b5f9-d2ad354bfa8d. Error: LLMMathChain._evaluate(\"\n",
"(age ** 0.43)\n",
"\") raised error: 'age'. Please try again with a valid numerical expression\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processed examples: 6\r"
"Processed examples: 4\r"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Chain failed for example e480f086-6d3f-4659-8669-26316db7e772. Error: LLMMathChain._evaluate(\"\n",
"(total number of points scored in the 2023 super bowl)**0.23\n",
"\") raised error: invalid syntax. Perhaps you forgot a comma? (<expr>, line 1). Please try again with a valid numerical expression\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Processed examples: 10\r"
]
}
],
@ -437,7 +463,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 14,
"id": "136db492-d6ca-4215-96f9-439c23538241",
"metadata": {
"tags": []
@ -452,7 +478,7 @@
"LangChainPlusClient (API URL: http://localhost:8000)"
]
},
"execution_count": 11,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
@ -482,7 +508,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 8,
"id": "64490d7c-9a18-49ed-a3ac-36049c522cb4",
"metadata": {
"tags": []
@ -492,13 +518,13 @@
"name": "stderr",
"output_type": "stream",
"text": [
"Found cached dataset parquet (/Users/wfh/.cache/huggingface/datasets/LangChainDatasets___parquet/LangChainDatasets--two-player-dnd-2e84407830cdedfc/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n"
"Found cached dataset parquet (/Users/wfh/.cache/huggingface/datasets/LangChainDatasets___parquet/LangChainDatasets--two-player-dnd-cc62c3037e2d9250/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "57ffb582606f47e9a3e96629a6d417c2",
"model_id": "31a576ae98634602b349046ec0821c0d",
"version_major": 2,
"version_minor": 0
},
@ -530,57 +556,57 @@
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>prompts</th>\n",
" <th>generations</th>\n",
" <th>messages</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>[System: Here is the topic for a Dungeons &amp; Dr...</td>\n",
" <td>[[{'generation_info': None, 'message': {'conte...</td>\n",
" <td>[{'data': {'content': 'Here is the topic for a...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>[System: Here is the topic for a Dungeons &amp; Dr...</td>\n",
" <td>[[{'generation_info': None, 'message': {'conte...</td>\n",
" <td>[{'data': {'content': 'Here is the topic for a...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>[System: Here is the topic for a Dungeons &amp; Dr...</td>\n",
" <td>[[{'generation_info': None, 'message': {'conte...</td>\n",
" <td>[{'data': {'content': 'Here is the topic for a...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>[System: Here is the topic for a Dungeons &amp; Dr...</td>\n",
" <td>[[{'generation_info': None, 'message': {'conte...</td>\n",
" <td>[{'data': {'content': 'Here is the topic for a...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>[System: Here is the topic for a Dungeons &amp; Dr...</td>\n",
" <td>[[{'generation_info': None, 'message': {'conte...</td>\n",
" <td>[{'data': {'content': 'Here is the topic for a...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" prompts \\\n",
"0 [System: Here is the topic for a Dungeons & Dr... \n",
"1 [System: Here is the topic for a Dungeons & Dr... \n",
"2 [System: Here is the topic for a Dungeons & Dr... \n",
"3 [System: Here is the topic for a Dungeons & Dr... \n",
"4 [System: Here is the topic for a Dungeons & Dr... \n",
" generations \\\n",
"0 [[{'generation_info': None, 'message': {'conte... \n",
"1 [[{'generation_info': None, 'message': {'conte... \n",
"2 [[{'generation_info': None, 'message': {'conte... \n",
"3 [[{'generation_info': None, 'message': {'conte... \n",
"4 [[{'generation_info': None, 'message': {'conte... \n",
"\n",
" generations \n",
"0 [[{'generation_info': None, 'message': {'conte... \n",
"1 [[{'generation_info': None, 'message': {'conte... \n",
"2 [[{'generation_info': None, 'message': {'conte... \n",
"3 [[{'generation_info': None, 'message': {'conte... \n",
"4 [[{'generation_info': None, 'message': {'conte... "
" messages \n",
"0 [{'data': {'content': 'Here is the topic for a... \n",
"1 [{'data': {'content': 'Here is the topic for a... \n",
"2 [{'data': {'content': 'Here is the topic for a... \n",
"3 [{'data': {'content': 'Here is the topic for a... \n",
"4 [{'data': {'content': 'Here is the topic for a... "
]
},
"execution_count": 12,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
@ -596,7 +622,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 16,
"id": "348acd86-a927-4d60-8d52-02e64585e4fc",
"metadata": {
"tags": []
@ -609,7 +635,7 @@
" client.upload_dataframe(chat_df, \n",
" name=chat_dataset_name,\n",
" description=\"An example dataset traced from chat models in a multiagent bidding dialogue\",\n",
" input_keys=[\"prompts\"],\n",
" input_keys=[\"messages\"],\n",
" output_keys=[\"generations\"],\n",
" )"
]
@ -626,7 +652,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 17,
"id": "a69dd183-ad5e-473d-b631-db90706e837f",
"metadata": {
"tags": []
@ -640,7 +666,7 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 18,
"id": "063da2a9-3692-4b7b-8edb-e474824fe416",
"metadata": {
"tags": []
@ -650,7 +676,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:65: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n",
"/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:78: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n",
" warnings.warn(\n"
]
},
@ -690,7 +716,7 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 19,
"id": "5b7a81f2-d19d-438b-a4bb-5678f746b965",
"metadata": {
"tags": []
@ -705,7 +731,7 @@
"LangChainPlusClient (API URL: http://localhost:8000)"
]
},
"execution_count": 16,
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
@ -726,7 +752,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 10,
"id": "d6805d0b-4612-4671-bffb-e6978992bd40",
"metadata": {
"tags": []
@ -740,7 +766,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 11,
"id": "5d7cb243-40c3-44dd-8158-a7b910441e9f",
"metadata": {
"tags": []
@ -750,13 +776,13 @@
"name": "stderr",
"output_type": "stream",
"text": [
"Found cached dataset parquet (/Users/wfh/.cache/huggingface/datasets/LangChainDatasets___parquet/LangChainDatasets--state-of-the-union-completions-ae7542e7bbd0ae0a/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n"
"Found cached dataset parquet (/Users/wfh/.cache/huggingface/datasets/LangChainDatasets___parquet/LangChainDatasets--state-of-the-union-completions-a7eb4af13453cd35/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "17b798a06afb49e3af24ca62ee867e82",
"model_id": "189832bd50114f129fb58e590d6e8267",
"version_major": 2,
"version_minor": 0
},
@ -788,54 +814,47 @@
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>prompts</th>\n",
" <th>generations</th>\n",
" <th>ground_truth</th>\n",
" <th>prompt</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>[Putin may circle Kyiv with tanks, but he will...</td>\n",
" <td>[[{'generation_info': {'finish_reason': 'stop'...</td>\n",
" <td>The pandemic has been punishing. \\n\\nAnd so ma...</td>\n",
" <td>Putin may circle Kyiv with tanks, but he will ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>[Madam Speaker, Madam Vice President, our Firs...</td>\n",
" <td>[[]]</td>\n",
" <td>With a duty to one another to the American peo...</td>\n",
" <td>Madam Speaker, Madam Vice President, our First...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>[With a duty to one another to the American pe...</td>\n",
" <td>[[{'generation_info': {'finish_reason': 'stop'...</td>\n",
" <td>He thought he could roll into Ukraine and the ...</td>\n",
" <td>With a duty to one another to the American peo...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>[Madam Speaker, Madam Vice President, our Firs...</td>\n",
" <td>[[{'generation_info': {'finish_reason': 'lengt...</td>\n",
" <td>With a duty to one another to the American peo...</td>\n",
" <td>Madam Speaker, Madam Vice President, our First...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>[Please rise if you are able and show that, Ye...</td>\n",
" <td>[[]]</td>\n",
" <td>And the costs and the threats to America and t...</td>\n",
" <td>Please rise if you are able and show that, Yes...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" prompts \\\n",
"0 [Putin may circle Kyiv with tanks, but he will... \n",
"1 [Madam Speaker, Madam Vice President, our Firs... \n",
"2 [With a duty to one another to the American pe... \n",
"3 [Madam Speaker, Madam Vice President, our Firs... \n",
"4 [Please rise if you are able and show that, Ye... \n",
"\n",
" generations \\\n",
"0 [[{'generation_info': {'finish_reason': 'stop'... \n",
"1 [[]] \n",
@ -843,15 +862,22 @@
"3 [[{'generation_info': {'finish_reason': 'lengt... \n",
"4 [[]] \n",
"\n",
" ground_truth \n",
"0 The pandemic has been punishing. \\n\\nAnd so ma... \n",
"1 With a duty to one another to the American peo... \n",
"2 He thought he could roll into Ukraine and the ... \n",
"3 With a duty to one another to the American peo... \n",
"4 And the costs and the threats to America and t... "
" ground_truth \\\n",
"0 The pandemic has been punishing. \\n\\nAnd so ma... \n",
"1 With a duty to one another to the American peo... \n",
"2 He thought he could roll into Ukraine and the ... \n",
"3 With a duty to one another to the American peo... \n",
"4 And the costs and the threats to America and t... \n",
"\n",
" prompt \n",
"0 Putin may circle Kyiv with tanks, but he will ... \n",
"1 Madam Speaker, Madam Vice President, our First... \n",
"2 With a duty to one another to the American peo... \n",
"3 Madam Speaker, Madam Vice President, our First... \n",
"4 Please rise if you are able and show that, Yes... "
]
},
"execution_count": 18,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
@ -864,7 +890,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 12,
"id": "c7dcc1b2-7aef-44c0-ba0f-c812279099a5",
"metadata": {
"tags": []
@ -877,14 +903,14 @@
" client.upload_dataframe(completions_df, \n",
" name=completions_dataset_name,\n",
" description=\"An example dataset traced from completion endpoints over the state of the union address\",\n",
" input_keys=[\"prompts\"],\n",
" input_keys=[\"prompt\"],\n",
" output_keys=[\"generations\"],\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": 13,
"id": "e946138e-bf7c-43d7-861d-9c5740c933fa",
"metadata": {
"tags": []
@ -894,7 +920,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:65: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n",
"/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:78: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n",
" warnings.warn(\n"
]
},
@ -907,7 +933,7 @@
}
],
"source": [
"# We also offer a synchronous method for running examples if a cahin's async methods aren't yet implemented\n",
"# We also offer a synchronous method for running examples if a chain or llm's async methods aren't yet implemented\n",
"completions_model_results = client.run_on_dataset(\n",
" dataset_name=completions_dataset_name,\n",
" llm_or_chain=llm,\n",
@ -928,7 +954,7 @@
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 24,
"id": "2bf96f17-74c1-4f7d-8458-ae5ab5c6bd36",
"metadata": {
"tags": []
@ -943,7 +969,7 @@
"LangChainPlusClient (API URL: http://localhost:8000)"
]
},
"execution_count": 21,
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
@ -977,7 +1003,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
"version": "3.11.2"
}
},
"nbformat": 4,

Write Preview
Loading…
Cancel
Save