diff --git a/docs/docs/integrations/chat/anthropic.ipynb b/docs/docs/integrations/chat/anthropic.ipynb index f08351d5ab..ea72b7faac 100644 --- a/docs/docs/integrations/chat/anthropic.ipynb +++ b/docs/docs/integrations/chat/anthropic.ipynb @@ -80,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "238bdbaa-526a-4130-89e9-523aa44bb196", "metadata": {}, "outputs": [], @@ -217,88 +217,6 @@ " print(chunk.content, end=\"\", flush=True)" ] }, - { - "cell_type": "markdown", - "id": "70d5e0fb", - "metadata": {}, - "source": [ - "## Multimodal\n", - "\n", - "Anthropic's Claude-3 models are compatible with both image and text inputs. You can use this as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "3e9d1ab5", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# open ../../../static/img/brand/wordmark.png as base64 str\n", - "import base64\n", - "from pathlib import Path\n", - "\n", - "from IPython.display import HTML\n", - "\n", - "img_path = Path(\"../../../static/img/brand/wordmark.png\")\n", - "img_base64 = base64.b64encode(img_path.read_bytes()).decode(\"utf-8\")\n", - "\n", - "# display b64 image in notebook\n", - "HTML(f'')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "b6bb2aa2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "AIMessage(content='This logo is for LangChain, which appears to be some kind of software or technology platform based on the name and minimalist design style of the logo featuring a silhouette of a bird (likely an eagle or hawk) and the company name in a simple, modern font.')" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from langchain_core.messages import HumanMessage\n", - "\n", - "chat = ChatAnthropic(model=\"claude-3-opus-20240229\")\n", - "messages = [\n", - " HumanMessage(\n", - " content=[\n", - " {\n", - " \"type\": \"image_url\",\n", - " \"image_url\": {\n", - " # langchain logo\n", - " \"url\": f\"data:image/png;base64,{img_base64}\", # noqa: E501\n", - " },\n", - " },\n", - " {\"type\": \"text\", \"text\": \"What is this logo for?\"},\n", - " ]\n", - " )\n", - "]\n", - "chat.invoke(messages)" - ] - }, { "cell_type": "markdown", "id": "ab0174d8-7140-413c-80a9-7cf3a8b81bb4", @@ -329,16 +247,23 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "42f87466-cb8e-490d-a9f8-aa0f8e9b4217", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/bagatur/langchain/libs/core/langchain_core/_api/beta_decorator.py:87: LangChainBetaWarning: The function `bind_tools` is in beta. It is actively being worked on, so the API may change.\n", + " warn_beta(\n" + ] + } + ], "source": [ "from langchain_core.pydantic_v1 import BaseModel, Field\n", "\n", - "llm = ChatAnthropic(\n", - " model=\"claude-3-opus-20240229\",\n", - ")\n", + "llm = ChatAnthropic(model=\"claude-3-opus-20240229\", temperature=0)\n", "\n", "\n", "class GetWeather(BaseModel):\n", @@ -352,17 +277,17 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "997be6ff-3fd3-4b1c-b7e3-2e5fed4ac964", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "AIMessage(content=[{'text': '\\nBased on the user\\'s question, the relevant function to call is GetWeather, which requires the \"location\" parameter.\\n\\nThe user has directly specified the location as \"San Francisco\". Since San Francisco is a well known city, I can reasonably infer they mean San Francisco, CA without needing the state specified.\\n\\nAll the required parameters are provided, so I can proceed with the API call.\\n', 'type': 'text'}, {'text': None, 'type': 'tool_use', 'id': 'toolu_01SCgExKzQ7eqSkMHfygvYuu', 'name': 'GetWeather', 'input': {'location': 'San Francisco, CA'}}], response_metadata={'id': 'msg_01GM3zQtoFv8jGQMW7abLnhi', 'model': 'claude-3-opus-20240229', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': {'input_tokens': 487, 'output_tokens': 145}}, id='run-87b1331e-9251-4a68-acef-f0a018b639cc-0')" + "AIMessage(content=[{'text': '\\nThe user is asking about the current weather in a specific location, San Francisco. The relevant tool to answer this is the GetWeather function.\\n\\nLooking at the parameters for GetWeather:\\n- location (required): The user directly provided the location in the query - \"San Francisco\"\\n\\nSince the required \"location\" parameter is present, we can proceed with calling the GetWeather function.\\n', 'type': 'text'}, {'id': 'toolu_01StzxdWQSZhAMbR1CCchQV9', 'input': {'location': 'San Francisco, CA'}, 'name': 'GetWeather', 'type': 'tool_use'}], response_metadata={'id': 'msg_01HepCTzqXJed5iNuLgV1VCZ', 'model': 'claude-3-opus-20240229', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': {'input_tokens': 487, 'output_tokens': 143}}, id='run-1a1b3289-ba2c-47ae-8be1-8929d7cc547e-0', tool_calls=[{'name': 'GetWeather', 'args': {'location': 'San Francisco, CA'}, 'id': 'toolu_01StzxdWQSZhAMbR1CCchQV9'}])" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -384,15 +309,50 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "id": "7c4cd4c4-1c78-4d6c-8607-759e32a8903b", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'text': '\\nBased on the user\\'s question, the relevant function to call is GetWeather, which requires the \"location\" parameter.\\n\\nThe user has directly specified the location as \"San Francisco\". Since San Francisco is a well known city, I can reasonably infer they mean San Francisco, CA without needing the state specified.\\n\\nAll the required parameters are provided, so I can proceed with the API call.\\n',\n", - " 'type': 'text'}" + "[{'text': '\\nThe user is asking about the current weather in a specific location, San Francisco. The relevant tool to answer this is the GetWeather function.\\n\\nLooking at the parameters for GetWeather:\\n- location (required): The user directly provided the location in the query - \"San Francisco\"\\n\\nSince the required \"location\" parameter is present, we can proceed with calling the GetWeather function.\\n',\n", + " 'type': 'text'},\n", + " {'id': 'toolu_01StzxdWQSZhAMbR1CCchQV9',\n", + " 'input': {'location': 'San Francisco, CA'},\n", + " 'name': 'GetWeather',\n", + " 'type': 'tool_use'}]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ai_msg.content" + ] + }, + { + "cell_type": "markdown", + "id": "d446bd0f-06cc-4aa6-945d-74335d5a8780", + "metadata": {}, + "source": [ + "Crucially, the tool calls are also extracted into the `tool_calls` where they are in a standardized, model-agnostic format:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e36f254e-bb89-4978-9351-a463b13eb3c7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'GetWeather',\n", + " 'args': {'location': 'San Francisco, CA'},\n", + " 'id': 'toolu_01StzxdWQSZhAMbR1CCchQV9'}]" ] }, "execution_count": 7, @@ -401,32 +361,7 @@ } ], "source": [ - "ai_msg.content[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "5b92d91d-37cb-4843-8b2e-e337d2eec53e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'text': None,\n", - " 'type': 'tool_use',\n", - " 'id': 'toolu_01SCgExKzQ7eqSkMHfygvYuu',\n", - " 'name': 'GetWeather',\n", - " 'input': {'location': 'San Francisco, CA'}}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ai_msg.content[1]" + "ai_msg.tool_calls" ] }, { @@ -448,57 +383,17 @@ "source": [ "### Parsing tool calls\n", "\n", - "The `langchain_anthropic.output_parsers.ToolsOutputParser` makes it easy to extract just the tool calls from an Anthropic AI message:" + "The `langchain_anthropic.output_parsers.ToolsOutputParser` makes it easy to parse the tool calls from an Anthropic AI message into Pydantic objects if we'd like:" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "59c175b1-0929-4ed4-a608-f0006031a3c2", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'name': 'GetWeather',\n", - " 'args': {'location': 'New York City, NY'},\n", - " 'id': 'toolu_01UK2AEWa75PUGA3DpiaHfBN',\n", - " 'index': 1},\n", - " {'name': 'GetWeather',\n", - " 'args': {'location': 'Los Angeles, CA'},\n", - " 'id': 'toolu_01M84DY7xWg9bLoX6JCArczx',\n", - " 'index': 2},\n", - " {'name': 'GetWeather',\n", - " 'args': {'location': 'San Francisco, CA'},\n", - " 'id': 'toolu_01FEasmxGpxFPwf9SF3nCTeb',\n", - " 'index': 3},\n", - " {'name': 'GetWeather',\n", - " 'args': {'location': 'Cleveland, OH'},\n", - " 'id': 'toolu_01B8fZdiyPbzWyj5cDCzGSTe',\n", - " 'index': 4}]" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "from langchain_anthropic.output_parsers import ToolsOutputParser\n", - "\n", - "parser = ToolsOutputParser()\n", - "chain = llm_with_tools | parser\n", - "chain.invoke(\"What is the weather like in nyc, la, sf and cleveland\")" - ] - }, - { - "cell_type": "markdown", - "id": "c4394c23-8d79-4f2c-b0fe-7b877eaac7c7", - "metadata": {}, - "source": [ - "The `index` tells us where in the original list of content blocks each tool call was.\n", - "\n", - "We can pass in Pydantic classes to parse our tool calls into pydantic objects:" + "from langchain_anthropic.output_parsers import ToolsOutputParser" ] }, { @@ -527,40 +422,6 @@ "chain.invoke(\"What is the weather like in nyc, la, sf and cleveland\")" ] }, - { - "cell_type": "markdown", - "id": "8ccdc039-d8ce-4460-bb2f-543753aac016", - "metadata": {}, - "source": [ - "If we want we can return only the first tool call:" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "7746c643-851f-4908-ac34-8ddbb949454d", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'name': 'GetWeather',\n", - " 'args': {'location': 'New York City, NY'},\n", - " 'id': 'toolu_01EjFAADbpdrML1uaSMr9tN3',\n", - " 'index': 1}" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "parser = ToolsOutputParser(first_tool_only=True)\n", - "chain = llm_with_tools | parser\n", - "chain.invoke(\"What is the weather like in nyc\")" - ] - }, { "cell_type": "markdown", "id": "ab05dd51-0a9e-4b7b-b182-65cec44941ac", @@ -595,6 +456,23 @@ ")" ] }, + { + "cell_type": "markdown", + "id": "2d74b83e-bcd3-47e6-911e-82b5dcfbd20e", + "metadata": {}, + "source": [ + "The main difference between using \n", + "```python\n", + "llm.with_structured_output(GetWeather)\n", + "``` \n", + "vs \n", + "\n", + "```python\n", + "llm.bind_tools([GetWeather]) | ToolsOutputParser(pydantic_schemas=[GetWeather])\n", + "``` \n", + "is that it will return only the first GetWeather call, whereas the second approach will return a list." + ] + }, { "cell_type": "markdown", "id": "5b61884e-3e4e-4145-b10d-188987ae1eb6", @@ -692,6 +570,88 @@ "source": [ "list(llm_with_tools.stream(\"What's the weather in san francisco\"))" ] + }, + { + "cell_type": "markdown", + "id": "70d5e0fb", + "metadata": {}, + "source": [ + "## Multimodal\n", + "\n", + "Anthropic's Claude-3 models are compatible with both image and text inputs. You can use this as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "3e9d1ab5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# open ../../../static/img/brand/wordmark.png as base64 str\n", + "import base64\n", + "from pathlib import Path\n", + "\n", + "from IPython.display import HTML\n", + "\n", + "img_path = Path(\"../../../static/img/brand/wordmark.png\")\n", + "img_base64 = base64.b64encode(img_path.read_bytes()).decode(\"utf-8\")\n", + "\n", + "# display b64 image in notebook\n", + "HTML(f'')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b6bb2aa2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "AIMessage(content='This logo is for LangChain, which appears to be some kind of software or technology platform based on the name and minimalist design style of the logo featuring a silhouette of a bird (likely an eagle or hawk) and the company name in a simple, modern font.')" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain_core.messages import HumanMessage\n", + "\n", + "chat = ChatAnthropic(model=\"claude-3-opus-20240229\")\n", + "messages = [\n", + " HumanMessage(\n", + " content=[\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " # langchain logo\n", + " \"url\": f\"data:image/png;base64,{img_base64}\", # noqa: E501\n", + " },\n", + " },\n", + " {\"type\": \"text\", \"text\": \"What is this logo for?\"},\n", + " ]\n", + " )\n", + "]\n", + "chat.invoke(messages)" + ] } ], "metadata": {