From 0fc3af89327b89e2d5c164525d914487dec4738b Mon Sep 17 00:00:00 2001 From: Bagatur <22008038+baskaryan@users.noreply.github.com> Date: Fri, 17 Nov 2023 11:58:54 -0800 Subject: [PATCH] IMPROVEMENT: update assistants output and doc (#13480) --- cookbook/Multi_modal_RAG.ipynb | 15 +- .../integrations/vectorstores/astradb.ipynb | 1 - .../agent_types/openai_assistants.ipynb | 130 +++++++++++++++--- .../langchain/agents/openai_assistant/base.py | 15 +- 4 files changed, 133 insertions(+), 28 deletions(-) diff --git a/cookbook/Multi_modal_RAG.ipynb b/cookbook/Multi_modal_RAG.ipynb index afef516f58..ca6f879c70 100644 --- a/cookbook/Multi_modal_RAG.ipynb +++ b/cookbook/Multi_modal_RAG.ipynb @@ -116,8 +116,8 @@ "metadata": {}, "outputs": [], "source": [ - "from unstructured.partition.pdf import partition_pdf\n", "from langchain.text_splitter import CharacterTextSplitter\n", + "from unstructured.partition.pdf import partition_pdf\n", "\n", "\n", "# Extract elements from PDF\n", @@ -267,8 +267,9 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", "import base64\n", + "import os\n", + "\n", "from langchain.schema.messages import HumanMessage\n", "\n", "\n", @@ -351,11 +352,12 @@ "outputs": [], "source": [ "import uuid\n", - "from langchain.vectorstores import Chroma\n", - "from langchain.storage import InMemoryStore\n", - "from langchain.schema.document import Document\n", + "\n", "from langchain.embeddings import OpenAIEmbeddings\n", "from langchain.retrievers.multi_vector import MultiVectorRetriever\n", + "from langchain.schema.document import Document\n", + "from langchain.storage import InMemoryStore\n", + "from langchain.vectorstores import Chroma\n", "\n", "\n", "def create_multi_vector_retriever(\n", @@ -438,9 +440,10 @@ "source": [ "import io\n", "import re\n", - "from PIL import Image\n", + "\n", "from IPython.display import HTML, display\n", "from langchain.schema.runnable import RunnableLambda, RunnablePassthrough\n", + "from PIL import Image\n", "\n", "\n", "def plt_img_base64(img_base64):\n", diff --git a/docs/docs/integrations/vectorstores/astradb.ipynb b/docs/docs/integrations/vectorstores/astradb.ipynb index fc3ebac563..4518265a9f 100644 --- a/docs/docs/integrations/vectorstores/astradb.ipynb +++ b/docs/docs/integrations/vectorstores/astradb.ipynb @@ -64,7 +64,6 @@ "from langchain.document_loaders import PyPDFLoader\n", "from langchain.embeddings import OpenAIEmbeddings\n", "from langchain.prompts import ChatPromptTemplate\n", - "\n", "from langchain.schema import Document\n", "from langchain.schema.output_parser import StrOutputParser\n", "from langchain.schema.runnable import RunnablePassthrough\n", diff --git a/docs/docs/modules/agents/agent_types/openai_assistants.ipynb b/docs/docs/modules/agents/agent_types/openai_assistants.ipynb index e5389ca96a..152d41f90a 100644 --- a/docs/docs/modules/agents/agent_types/openai_assistants.ipynb +++ b/docs/docs/modules/agents/agent_types/openai_assistants.ipynb @@ -25,7 +25,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "aa761a93-caa1-4e56-b901-5ff50a89bc82", "metadata": {}, "outputs": [], @@ -35,10 +35,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "5944a18a-95eb-44ce-a66f-5f50db1d3e1f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[ThreadMessage(id='msg_qgxkD5kvkZyl0qOaL4czPFkZ', assistant_id='asst_0T8S7CJuUa4Y4hm1PF6n62v7', content=[MessageContentText(text=Text(annotations=[], value='The result of the calculation \\\\(10 - 4^{2.7}\\\\) is approximately \\\\(-32.224\\\\).'), type='text')], created_at=1700169519, file_ids=[], metadata={}, object='thread.message', role='assistant', run_id='run_aH3ZgSWNk3vYIBQm3vpE8tr4', thread_id='thread_9K6cYfx1RBh0pOWD8SxwVWW9')]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "interpreter_assistant = OpenAIAssistantRunnable.create_assistant(\n", " name=\"langchain assistant\",\n", @@ -72,19 +83,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "cc0cba70-8507-498d-92ac-fe47133db200", "metadata": {}, "outputs": [], "source": [ + "import getpass\n", + "\n", "from langchain.tools import DuckDuckGoSearchRun, E2BDataAnalysisTool\n", "\n", - "tools = [E2BDataAnalysisTool(api_key=\"...\"), DuckDuckGoSearchRun()]" + "tools = [E2BDataAnalysisTool(api_key=getpass.getpass()), DuckDuckGoSearchRun()]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "91e6973d-3d9a-477f-99e2-4aaad16004ec", "metadata": {}, "outputs": [], @@ -103,15 +116,31 @@ "id": "78fa9320-06fc-4cbc-a3cf-39aaf2427080", "metadata": {}, "source": [ - "#### Using AgentExecutor" + "#### Using AgentExecutor\n", + "\n", + "The OpenAIAssistantRunnable is compatible with the AgentExecutor, so we can pass it in as an agent directly to the executor. The AgentExecutor handles calling the invoked tools and uploading the tool outputs back to the Assistants API. Plus it comes with built-in LangSmith tracing." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "e38007a4-fcc1-419b-9ae4-70d36c3fc1cd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'content': \"What's the weather in SF today divided by 2.7\",\n", + " 'output': \"The search results indicate that the weather in San Francisco is 67 °F. Now I will divide this temperature by 2.7 and provide you with the result. Please note that this is a mathematical operation and does not represent a meaningful physical quantity.\\n\\nLet's calculate 67 °F divided by 2.7.\\nThe result of dividing the current temperature in San Francisco, which is 67 °F, by 2.7 is approximately 24.815.\",\n", + " 'thread_id': 'thread_hcpYI0tfpB9mHa9d95W7nK2B',\n", + " 'run_id': 'run_qOuVmPXS9xlV3XNPcfP8P9W2'}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from langchain.agents import AgentExecutor\n", "\n", @@ -119,17 +148,28 @@ "agent_executor.invoke({\"content\": \"What's the weather in SF today divided by 2.7\"})" ] }, + { + "cell_type": "markdown", + "id": "db6b9cbf-dd54-4346-be6c-842e08756ccc", + "metadata": {}, + "source": [ + ":::tip [LangSmith trace](https://smith.langchain.com/public/6750972b-0849-4beb-a8bb-353d424ffade/r)\n", + ":::" + ] + }, { "cell_type": "markdown", "id": "6bf4199a-eed1-485a-8da3-aed948c0e1e2", "metadata": {}, "source": [ - "#### Custom execution" + "#### Custom execution\n", + "\n", + "Or with LCEL we can easily write our own execution loop for running the assistant. This gives us full control over execution." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "357361ff-f54d-4fd0-b69b-77689f56f40e", "metadata": {}, "outputs": [], @@ -145,7 +185,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "864e7f9b-0501-4bb7-8aad-a7aa19b601af", "metadata": {}, "outputs": [], @@ -177,34 +217,86 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "5ad6bb07-aac4-4b71-9e67-cc177fcbc537", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "e2b_data_analysis {'python_code': 'result = 10 - 4 ** 2.7\\nprint(result)'} {\"stdout\": \"-32.22425314473263\", \"stderr\": \"\", \"artifacts\": []}\n", + "\n", + "\\( 10 - 4^{2.7} \\) equals approximately -32.224.\n" + ] + } + ], "source": [ "response = execute_agent(agent, tools, {\"content\": \"What's 10 - 4 raised to the 2.7\"})\n", "print(response.return_values[\"output\"])" ] }, + { + "cell_type": "markdown", + "id": "6fd9f9c0-4b07-4f71-a784-88ee7bd4b089", + "metadata": {}, + "source": [ + "## Using existing Thread\n", + "\n", + "To use an existing thread we just need to pass the \"thread_id\" in when invoking the agent." + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "f55a3a3a-8169-491e-aa15-cf30a2b230df", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "e2b_data_analysis {'python_code': 'result = 10 - 4 ** 2.7 + 17.241\\nprint(result)'} {\"stdout\": \"-14.983253144732629\", \"stderr\": \"\", \"artifacts\": []}\n", + "\n", + "\\( 10 - 4^{2.7} + 17.241 \\) equals approximately -14.983.\n" + ] + } + ], "source": [ "next_response = execute_agent(\n", - " agent, tools, {\"content\": \"now add 17.241\", \"thread_id\": response.thread_id}\n", + " agent,\n", + " tools,\n", + " {\"content\": \"now add 17.241\", \"thread_id\": response.return_values[\"thread_id\"]},\n", ")\n", "print(next_response.return_values[\"output\"])" ] + }, + { + "cell_type": "markdown", + "id": "1b97ee01-a657-452c-ba7f-95227ec7056e", + "metadata": {}, + "source": [ + "## Using existing Assistant\n", + "\n", + "To use an existing Assistant we can initialize the `OpenAIAssistantRunnable` directly with an `assistant_id`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08ef6ef5-e8bc-4c69-882d-65273655f6a7", + "metadata": {}, + "outputs": [], + "source": [ + "agent = OpenAIAssistantRunnable(assistant_id=\"\", as_agent=True)" + ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "poetry-venv", "language": "python", - "name": "python3" + "name": "poetry-venv" }, "language_info": { "codemirror_mode": { diff --git a/libs/langchain/langchain/agents/openai_assistant/base.py b/libs/langchain/langchain/agents/openai_assistant/base.py index 5494d3c7e9..66323125c0 100644 --- a/libs/langchain/langchain/agents/openai_assistant/base.py +++ b/libs/langchain/langchain/agents/openai_assistant/base.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from json import JSONDecodeError from time import sleep from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union @@ -335,7 +336,11 @@ class OpenAIAssistantRunnable(RunnableSerializable[Dict, OutputType]): ): answer = "\n".join(content.text.value for content in answer) return OpenAIAssistantFinish( - return_values={"output": answer}, + return_values={ + "output": answer, + "thread_id": run.thread_id, + "run_id": run.id, + }, log="", run_id=run.id, thread_id=run.thread_id, @@ -346,7 +351,13 @@ class OpenAIAssistantRunnable(RunnableSerializable[Dict, OutputType]): actions = [] for tool_call in run.required_action.submit_tool_outputs.tool_calls: function = tool_call.function - args = json.loads(function.arguments) + try: + args = json.loads(function.arguments, strict=False) + except JSONDecodeError as e: + raise ValueError( + f"Received invalid JSON function arguments: " + f"{function.arguments} for function {function.name}" + ) from e if len(args) == 1 and "__arg1" in args: args = args["__arg1"] actions.append(