From 258c3198559da5844be3f78680f42b2930e5b64b Mon Sep 17 00:00:00 2001 From: Zander Chase <130414180+vowelparrot@users.noreply.github.com> Date: Fri, 12 May 2023 00:12:43 -0700 Subject: [PATCH] Dereference Messages (#4557) Update how we parse the messages now that the server splits prompts / messages up --- langchain/client/langchain.py | 30 +-- .../client/tracing_datasets.ipynb | 196 ++++++++++-------- 2 files changed, 126 insertions(+), 100 deletions(-) diff --git a/langchain/client/langchain.py b/langchain/client/langchain.py index e197f055..9d65716c 100644 --- a/langchain/client/langchain.py +++ b/langchain/client/langchain.py @@ -294,16 +294,16 @@ class LangChainPlusClient(BaseSettings): langchain_tracer: LangChainTracerV2, ) -> Union[LLMResult, ChatResult]: if isinstance(llm, BaseLLM): - if "prompts" not in inputs: - raise ValueError(f"LLM Run requires 'prompts' input. Got {inputs}") - llm_prompts: List[str] = inputs["prompts"] - llm_output = await llm.agenerate(llm_prompts, callbacks=[langchain_tracer]) + if "prompt" not in inputs: + raise ValueError(f"LLM Run requires 'prompt' input. Got {inputs}") + llm_prompt: str = inputs["prompt"] + llm_output = await llm.agenerate([llm_prompt], callbacks=[langchain_tracer]) elif isinstance(llm, BaseChatModel): if "messages" not in inputs: raise ValueError(f"Chat Run requires 'messages' input. Got {inputs}") - raw_messages: List[List[dict]] = inputs["messages"] - messages = [messages_from_dict(batch) for batch in raw_messages] - llm_output = await llm.agenerate(messages, callbacks=[langchain_tracer]) + raw_messages: List[dict] = inputs["messages"] + messages = messages_from_dict(raw_messages) + llm_output = await llm.agenerate([messages], callbacks=[langchain_tracer]) else: raise ValueError(f"Unsupported LLM type {type(llm)}") return llm_output @@ -454,18 +454,18 @@ class LangChainPlusClient(BaseSettings): ) -> Union[LLMResult, ChatResult]: """Run the language model on the example.""" if isinstance(llm, BaseLLM): - if "prompts" not in inputs: - raise ValueError(f"LLM Run must contain 'prompts' key. Got {inputs}") - llm_prompts: List[str] = inputs["prompts"] - llm_output = llm.generate(llm_prompts, callbacks=[langchain_tracer]) + if "prompt" not in inputs: + raise ValueError(f"LLM Run must contain 'prompt' key. Got {inputs}") + llm_prompt: str = inputs["prompt"] + llm_output = llm.generate([llm_prompt], callbacks=[langchain_tracer]) elif isinstance(llm, BaseChatModel): if "messages" not in inputs: raise ValueError( f"Chat Model Run must contain 'messages' key. Got {inputs}" ) - raw_messages: List[List[dict]] = inputs["messages"] - messages = [messages_from_dict(batch) for batch in raw_messages] - llm_output = llm.generate(messages, callbacks=[langchain_tracer]) + raw_messages: List[dict] = inputs["messages"] + messages = messages_from_dict(raw_messages) + llm_output = llm.generate([messages], callbacks=[langchain_tracer]) else: raise ValueError(f"Unsupported LLM type {type(llm)}") return llm_output @@ -529,7 +529,7 @@ class LangChainPlusClient(BaseSettings): f"{dataset_name}-{llm_or_chain.__class__.__name__}-{current_time}" ) dataset = self.read_dataset(dataset_name=dataset_name) - examples = self.list_examples(dataset_id=str(dataset.id)) + examples = list(self.list_examples(dataset_id=str(dataset.id))) results: Dict[str, Any] = {} with tracing_v2_enabled(session_name=session_name) as session: tracer = LangChainTracerV2() diff --git a/langchain/experimental/client/tracing_datasets.ipynb b/langchain/experimental/client/tracing_datasets.ipynb index 4c6e5678..7330d663 100644 --- a/langchain/experimental/client/tracing_datasets.ipynb +++ b/langchain/experimental/client/tracing_datasets.ipynb @@ -124,7 +124,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:65: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n", + "/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:78: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n", " warnings.warn(\n" ] }, @@ -132,16 +132,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "The current population of Canada as of 2023 is 38,681,797.\n", - "Anwar Hadid is Dua Lipa's boyfriend and his age raised to the 0.43 power is approximately 3.87.\n", - "'age'. Please try again with a valid numerical expression\n", - "The distance between Paris and Boston is approximately 3448 miles.\n", + "The current population of Canada as of 2023 is 39,566,248.\n", + "Anwar Hadid's age raised to the 0.43 power is approximately 3.87.\n", + "LLMMathChain._evaluate(\"\n", + "(age)**0.43\n", + "\") raised error: 'age'. Please try again with a valid numerical expression\n", + "The distance between Paris and Boston is 3448 miles.\n", "unknown format from LLM: Assuming we don't have any information about the actual number of points scored in the 2023 super bowl, we cannot provide a mathematical expression to solve this problem.\n", - "invalid syntax. Perhaps you forgot a comma? (, line 1). Please try again with a valid numerical expression\n", - "0\n", + "LLMMathChain._evaluate(\"\n", + "(total number of points scored in the 2023 super bowl)**0.23\n", + "\") raised error: invalid syntax. Perhaps you forgot a comma? (, line 1). Please try again with a valid numerical expression\n", + "3 points were scored more in the 2023 Super Bowl than in the 2022 Super Bowl.\n", "1.9347796717823205\n", - "1.275494929129063\n", - "1213 divided by 4345 is approximately 0.2791.\n" + "81\n", + "LLMMathChain._evaluate(\"\n", + "round(0.2791714614499425, 2)\n", + "\") raised error: 'VariableNode' object is not callable. Please try again with a valid numerical expression\n" ] } ], @@ -186,7 +192,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "id": "d14a9881-2a01-404c-8c56-0b78565c3ff4", "metadata": { "tags": [] @@ -198,7 +204,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "7bfb4699-62c3-400a-b3e7-7fb8ad3a68ad", "metadata": { "tags": [] @@ -213,7 +219,7 @@ "LangChainPlusClient (API URL: http://localhost:8000)" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -235,7 +241,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "1baa677c-5642-4378-8e01-3aa1647f19d6", "metadata": { "tags": [] @@ -248,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "60d14593-c61f-449f-a38f-772ca43707c2", "metadata": { "tags": [] @@ -266,7 +272,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "52a7ea76-79ca-4765-abf7-231e884040d6", "metadata": { "tags": [] @@ -302,7 +308,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 11, "id": "c2b59104-b90e-466a-b7ea-c5bd0194263b", "metadata": { "tags": [] @@ -330,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 12, "id": "112d7bdf-7e50-4c1a-9285-5bac8473f2ee", "metadata": { "tags": [] @@ -378,7 +384,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 13, "id": "a8088b7d-3ab6-4279-94c8-5116fe7cee33", "metadata": { "tags": [] @@ -388,28 +394,48 @@ "name": "stderr", "output_type": "stream", "text": [ - "Chain failed for example bb17c39c-c90c-4706-8410-e501e89a81fb. Error: 'age'. Please try again with a valid numerical expression\n" + "/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:78: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n", + " warnings.warn(\n", + "Chain failed for example 92c75ce4-f807-4d44-8f7e-027610f7fcbd. Error: unknown format from LLM: Sorry, I cannot answer this question as it requires information from the future.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Processed examples: 1\r" + "Processed examples: 2\r" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "Chain failed for example e66a366d-f5d3-4dcc-9ec4-aae88369f45e. Error: invalid syntax. Perhaps you forgot a comma? (, line 1). Please try again with a valid numerical expression\n" + "Chain failed for example 9f5d1426-3e21-4628-b5f9-d2ad354bfa8d. Error: LLMMathChain._evaluate(\"\n", + "(age ** 0.43)\n", + "\") raised error: 'age'. Please try again with a valid numerical expression\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Processed examples: 6\r" + "Processed examples: 4\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Chain failed for example e480f086-6d3f-4659-8669-26316db7e772. Error: LLMMathChain._evaluate(\"\n", + "(total number of points scored in the 2023 super bowl)**0.23\n", + "\") raised error: invalid syntax. Perhaps you forgot a comma? (, line 1). Please try again with a valid numerical expression\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processed examples: 10\r" ] } ], @@ -437,7 +463,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 14, "id": "136db492-d6ca-4215-96f9-439c23538241", "metadata": { "tags": [] @@ -452,7 +478,7 @@ "LangChainPlusClient (API URL: http://localhost:8000)" ] }, - "execution_count": 11, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -482,7 +508,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "id": "64490d7c-9a18-49ed-a3ac-36049c522cb4", "metadata": { "tags": [] @@ -492,13 +518,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "Found cached dataset parquet (/Users/wfh/.cache/huggingface/datasets/LangChainDatasets___parquet/LangChainDatasets--two-player-dnd-2e84407830cdedfc/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n" + "Found cached dataset parquet (/Users/wfh/.cache/huggingface/datasets/LangChainDatasets___parquet/LangChainDatasets--two-player-dnd-cc62c3037e2d9250/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "57ffb582606f47e9a3e96629a6d417c2", + "model_id": "31a576ae98634602b349046ec0821c0d", "version_major": 2, "version_minor": 0 }, @@ -530,57 +556,57 @@ " \n", " \n", " \n", - " prompts\n", " generations\n", + " messages\n", " \n", " \n", " \n", " \n", " 0\n", - " [System: Here is the topic for a Dungeons & Dr...\n", " [[{'generation_info': None, 'message': {'conte...\n", + " [{'data': {'content': 'Here is the topic for a...\n", " \n", " \n", " 1\n", - " [System: Here is the topic for a Dungeons & Dr...\n", " [[{'generation_info': None, 'message': {'conte...\n", + " [{'data': {'content': 'Here is the topic for a...\n", " \n", " \n", " 2\n", - " [System: Here is the topic for a Dungeons & Dr...\n", " [[{'generation_info': None, 'message': {'conte...\n", + " [{'data': {'content': 'Here is the topic for a...\n", " \n", " \n", " 3\n", - " [System: Here is the topic for a Dungeons & Dr...\n", " [[{'generation_info': None, 'message': {'conte...\n", + " [{'data': {'content': 'Here is the topic for a...\n", " \n", " \n", " 4\n", - " [System: Here is the topic for a Dungeons & Dr...\n", " [[{'generation_info': None, 'message': {'conte...\n", + " [{'data': {'content': 'Here is the topic for a...\n", " \n", " \n", "\n", "" ], "text/plain": [ - " prompts \\\n", - "0 [System: Here is the topic for a Dungeons & Dr... \n", - "1 [System: Here is the topic for a Dungeons & Dr... \n", - "2 [System: Here is the topic for a Dungeons & Dr... \n", - "3 [System: Here is the topic for a Dungeons & Dr... \n", - "4 [System: Here is the topic for a Dungeons & Dr... \n", + " generations \\\n", + "0 [[{'generation_info': None, 'message': {'conte... \n", + "1 [[{'generation_info': None, 'message': {'conte... \n", + "2 [[{'generation_info': None, 'message': {'conte... \n", + "3 [[{'generation_info': None, 'message': {'conte... \n", + "4 [[{'generation_info': None, 'message': {'conte... \n", "\n", - " generations \n", - "0 [[{'generation_info': None, 'message': {'conte... \n", - "1 [[{'generation_info': None, 'message': {'conte... \n", - "2 [[{'generation_info': None, 'message': {'conte... \n", - "3 [[{'generation_info': None, 'message': {'conte... \n", - "4 [[{'generation_info': None, 'message': {'conte... " + " messages \n", + "0 [{'data': {'content': 'Here is the topic for a... \n", + "1 [{'data': {'content': 'Here is the topic for a... \n", + "2 [{'data': {'content': 'Here is the topic for a... \n", + "3 [{'data': {'content': 'Here is the topic for a... \n", + "4 [{'data': {'content': 'Here is the topic for a... " ] }, - "execution_count": 12, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -596,7 +622,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 16, "id": "348acd86-a927-4d60-8d52-02e64585e4fc", "metadata": { "tags": [] @@ -609,7 +635,7 @@ " client.upload_dataframe(chat_df, \n", " name=chat_dataset_name,\n", " description=\"An example dataset traced from chat models in a multiagent bidding dialogue\",\n", - " input_keys=[\"prompts\"],\n", + " input_keys=[\"messages\"],\n", " output_keys=[\"generations\"],\n", " )" ] @@ -626,7 +652,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 17, "id": "a69dd183-ad5e-473d-b631-db90706e837f", "metadata": { "tags": [] @@ -640,7 +666,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 18, "id": "063da2a9-3692-4b7b-8edb-e474824fe416", "metadata": { "tags": [] @@ -650,7 +676,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:65: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n", + "/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:78: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n", " warnings.warn(\n" ] }, @@ -690,7 +716,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 19, "id": "5b7a81f2-d19d-438b-a4bb-5678f746b965", "metadata": { "tags": [] @@ -705,7 +731,7 @@ "LangChainPlusClient (API URL: http://localhost:8000)" ] }, - "execution_count": 16, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -726,7 +752,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 10, "id": "d6805d0b-4612-4671-bffb-e6978992bd40", "metadata": { "tags": [] @@ -740,7 +766,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 11, "id": "5d7cb243-40c3-44dd-8158-a7b910441e9f", "metadata": { "tags": [] @@ -750,13 +776,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "Found cached dataset parquet (/Users/wfh/.cache/huggingface/datasets/LangChainDatasets___parquet/LangChainDatasets--state-of-the-union-completions-ae7542e7bbd0ae0a/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n" + "Found cached dataset parquet (/Users/wfh/.cache/huggingface/datasets/LangChainDatasets___parquet/LangChainDatasets--state-of-the-union-completions-a7eb4af13453cd35/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "17b798a06afb49e3af24ca62ee867e82", + "model_id": "189832bd50114f129fb58e590d6e8267", "version_major": 2, "version_minor": 0 }, @@ -788,54 +814,47 @@ " \n", " \n", " \n", - " prompts\n", " generations\n", " ground_truth\n", + " prompt\n", " \n", " \n", " \n", " \n", " 0\n", - " [Putin may circle Kyiv with tanks, but he will...\n", " [[{'generation_info': {'finish_reason': 'stop'...\n", " The pandemic has been punishing. \\n\\nAnd so ma...\n", + " Putin may circle Kyiv with tanks, but he will ...\n", " \n", " \n", " 1\n", - " [Madam Speaker, Madam Vice President, our Firs...\n", " [[]]\n", " With a duty to one another to the American peo...\n", + " Madam Speaker, Madam Vice President, our First...\n", " \n", " \n", " 2\n", - " [With a duty to one another to the American pe...\n", " [[{'generation_info': {'finish_reason': 'stop'...\n", " He thought he could roll into Ukraine and the ...\n", + " With a duty to one another to the American peo...\n", " \n", " \n", " 3\n", - " [Madam Speaker, Madam Vice President, our Firs...\n", " [[{'generation_info': {'finish_reason': 'lengt...\n", " With a duty to one another to the American peo...\n", + " Madam Speaker, Madam Vice President, our First...\n", " \n", " \n", " 4\n", - " [Please rise if you are able and show that, Ye...\n", " [[]]\n", " And the costs and the threats to America and t...\n", + " Please rise if you are able and show that, Yes...\n", " \n", " \n", "\n", "" ], "text/plain": [ - " prompts \\\n", - "0 [Putin may circle Kyiv with tanks, but he will... \n", - "1 [Madam Speaker, Madam Vice President, our Firs... \n", - "2 [With a duty to one another to the American pe... \n", - "3 [Madam Speaker, Madam Vice President, our Firs... \n", - "4 [Please rise if you are able and show that, Ye... \n", - "\n", " generations \\\n", "0 [[{'generation_info': {'finish_reason': 'stop'... \n", "1 [[]] \n", @@ -843,15 +862,22 @@ "3 [[{'generation_info': {'finish_reason': 'lengt... \n", "4 [[]] \n", "\n", - " ground_truth \n", - "0 The pandemic has been punishing. \\n\\nAnd so ma... \n", - "1 With a duty to one another to the American peo... \n", - "2 He thought he could roll into Ukraine and the ... \n", - "3 With a duty to one another to the American peo... \n", - "4 And the costs and the threats to America and t... " + " ground_truth \\\n", + "0 The pandemic has been punishing. \\n\\nAnd so ma... \n", + "1 With a duty to one another to the American peo... \n", + "2 He thought he could roll into Ukraine and the ... \n", + "3 With a duty to one another to the American peo... \n", + "4 And the costs and the threats to America and t... \n", + "\n", + " prompt \n", + "0 Putin may circle Kyiv with tanks, but he will ... \n", + "1 Madam Speaker, Madam Vice President, our First... \n", + "2 With a duty to one another to the American peo... \n", + "3 Madam Speaker, Madam Vice President, our First... \n", + "4 Please rise if you are able and show that, Yes... " ] }, - "execution_count": 18, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -864,7 +890,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 12, "id": "c7dcc1b2-7aef-44c0-ba0f-c812279099a5", "metadata": { "tags": [] @@ -877,14 +903,14 @@ " client.upload_dataframe(completions_df, \n", " name=completions_dataset_name,\n", " description=\"An example dataset traced from completion endpoints over the state of the union address\",\n", - " input_keys=[\"prompts\"],\n", + " input_keys=[\"prompt\"],\n", " output_keys=[\"generations\"],\n", " )" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 13, "id": "e946138e-bf7c-43d7-861d-9c5740c933fa", "metadata": { "tags": [] @@ -894,7 +920,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:65: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n", + "/Users/wfh/code/lc/lckg/langchain/callbacks/manager.py:78: UserWarning: The experimental tracing v2 is in development. This is not yet stable and may change in the future.\n", " warnings.warn(\n" ] }, @@ -907,7 +933,7 @@ } ], "source": [ - "# We also offer a synchronous method for running examples if a cahin's async methods aren't yet implemented\n", + "# We also offer a synchronous method for running examples if a chain or llm's async methods aren't yet implemented\n", "completions_model_results = client.run_on_dataset(\n", " dataset_name=completions_dataset_name,\n", " llm_or_chain=llm,\n", @@ -928,7 +954,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 24, "id": "2bf96f17-74c1-4f7d-8458-ae5ab5c6bd36", "metadata": { "tags": [] @@ -943,7 +969,7 @@ "LangChainPlusClient (API URL: http://localhost:8000)" ] }, - "execution_count": 21, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -977,7 +1003,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.11.2" } }, "nbformat": 4,