diff --git a/cookbook/anthropic_structured_outputs.ipynb b/cookbook/anthropic_structured_outputs.ipynb
index 2d095cfa8b..e952e8afcf 100644
--- a/cookbook/anthropic_structured_outputs.ipynb
+++ b/cookbook/anthropic_structured_outputs.ipynb
@@ -60,7 +60,9 @@
"cell_type": "code",
"execution_count": null,
"id": "9caa2aaf-1918-4a8a-982d-f8052b92ed44",
- "metadata": {},
+ "metadata": {
+ "scrolled": true
+ },
"outputs": [],
"source": [
"from langchain_anthropic import ChatAnthropic\n",
@@ -92,18 +94,18 @@
},
{
"cell_type": "code",
- "execution_count": 197,
+ "execution_count": 2,
"id": "9025bfdc-6060-4042-9a61-4e361dda7087",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "{'text': \"\\n- To answer this request, we need to use the `code` function, which takes 3 required parameters: `prefix`, `imports`, and `code`.\\n- The user has provided enough context to generate the full code snippet and explanation, so we have values for all the required parameters:\\n - `prefix`: A brief description of what the code does\\n - `imports`: No imports needed for this simple program \\n - `code`: A Python print statement to print 'hello world'\\n\",\n",
+ "{'text': \"\\nThe tool 'code' is relevant for writing a Python program to print a string.\\n\\nTo use the 'code' tool, I need values for these required parameters:\\nprefix: A description of the problem and approach. I can provide this based on the request.\\nimports: The import statements needed for the code. For this simple program, no imports are needed, so I can leave this blank.\\ncode: The actual Python code, not including imports. I can write a simple print statement to output the string.\\n\\nI have all the required parameters, so I can proceed with calling the 'code' tool.\\n\",\n",
" 'type': 'text'}"
]
},
- "execution_count": 197,
+ "execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
@@ -115,7 +117,7 @@
},
{
"cell_type": "code",
- "execution_count": 198,
+ "execution_count": 3,
"id": "2393d9b6-67a2-41ea-ac01-dc038b4800f5",
"metadata": {},
"outputs": [
@@ -124,14 +126,14 @@
"text/plain": [
"{'text': None,\n",
" 'type': 'tool_use',\n",
- " 'id': 'toolu_012Dsij8yn87daCuGPpgkwas',\n",
+ " 'id': 'toolu_01UwZVQub6vL36wiBww6CU7a',\n",
" 'name': 'code',\n",
- " 'input': {'prefix': \"This Python program prints the string 'hello world' to the console.\",\n",
+ " 'input': {'prefix': \"To print the string 'hello world' in Python:\",\n",
" 'imports': '',\n",
" 'code': \"print('hello world')\"}}"
]
},
- "execution_count": 198,
+ "execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
@@ -143,19 +145,19 @@
},
{
"cell_type": "code",
- "execution_count": 199,
+ "execution_count": 4,
"id": "f4f390ac-fbda-4173-892a-ffd12844228c",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "{'prefix': \"This Python program prints the string 'hello world' to the console.\",\n",
+ "{'prefix': \"To print the string 'hello world' in Python:\",\n",
" 'imports': '',\n",
" 'code': \"print('hello world')\"}"
]
},
- "execution_count": 199,
+ "execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
@@ -167,7 +169,7 @@
},
{
"cell_type": "code",
- "execution_count": 200,
+ "execution_count": 5,
"id": "ba77d0f8-f79b-4656-9023-085ffdaf35f5",
"metadata": {},
"outputs": [],
@@ -179,7 +181,7 @@
},
{
"cell_type": "code",
- "execution_count": 201,
+ "execution_count": 6,
"id": "cd854451-68d7-43df-bcae-4f3c3565536a",
"metadata": {},
"outputs": [],
@@ -190,17 +192,17 @@
},
{
"cell_type": "code",
- "execution_count": 202,
+ "execution_count": 7,
"id": "47b3405f-0aea-460e-8603-f6092019fcd4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "\"This Python program prints the string 'hello world' to the console.\""
+ "\"To print the string 'hello world' in Python:\""
]
},
- "execution_count": 202,
+ "execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
@@ -211,7 +213,7 @@
},
{
"cell_type": "code",
- "execution_count": 203,
+ "execution_count": 8,
"id": "85b16b62-1b72-4b6e-81fa-b1d707b728fa",
"metadata": {},
"outputs": [
@@ -221,7 +223,7 @@
"''"
]
},
- "execution_count": 203,
+ "execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
@@ -232,7 +234,7 @@
},
{
"cell_type": "code",
- "execution_count": 204,
+ "execution_count": 9,
"id": "23857441-3e67-460c-b6be-b57cf0dd17ad",
"metadata": {},
"outputs": [
@@ -242,7 +244,7 @@
"\"print('hello world')\""
]
},
- "execution_count": 204,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
@@ -278,7 +280,7 @@
},
{
"cell_type": "code",
- "execution_count": 205,
+ "execution_count": 10,
"id": "97dd1b8c-724a-436a-88b1-b38204fc81f5",
"metadata": {},
"outputs": [],
@@ -310,18 +312,22 @@
"\n",
"`What if we want to enforce tool use?`\n",
"\n",
- "We can use fallbacks."
+ "We can use fallbacks.\n",
+ "\n",
+ "Let's select a code gen prompt that -- from some of my testing -- does not correctly invoke the tool.\n",
+ "\n",
+ "We can see if we can correct from this."
]
},
{
"cell_type": "code",
- "execution_count": 206,
+ "execution_count": 12,
"id": "94e77be5-dddb-4386-b523-6f1136150bbd",
"metadata": {},
"outputs": [],
"source": [
- "# Code solution prompt\n",
- "code_gen_prompt = ChatPromptTemplate.from_messages(\n",
+ "# This code gen prompt invokes tool use\n",
+ "code_gen_prompt_working = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\n",
" \"system\",\n",
@@ -335,6 +341,21 @@
" ]\n",
")\n",
"\n",
+ "# This code gen prompt does not invoke tool use\n",
+ "code_gen_prompt_bad = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\n",
+ " \"system\",\n",
+ " \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
+ " Here is a full set of LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user \n",
+ " question based on the above provided documentation. Ensure any code you provide can be executed \\n \n",
+ " with all required imports and variables defined. Structure your answer with a description of the code solution. \\n\n",
+ " Then list the imports. And finally list the functioning code block. Here is the user question:\"\"\",\n",
+ " ),\n",
+ " (\"placeholder\", \"{messages}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
"\n",
"# Data model\n",
"class code(BaseModel):\n",
@@ -358,21 +379,30 @@
"\n",
"\n",
"# Check for errors\n",
- "def check(tool_output):\n",
- " \"\"\"Check for errors\"\"\"\n",
+ "def check_claude_output(tool_output):\n",
+ " \"\"\"Check for parse error or failure to call the tool\"\"\"\n",
+ "\n",
+ " # Error with parsing\n",
" if tool_output[\"parsing_error\"]:\n",
" # Report back output and parsing errors\n",
+ " print(\"Parsing error!\")\n",
" raw_output = str(code_output[\"raw\"].content)\n",
" error = tool_output[\"parsing_error\"]\n",
" raise ValueError(\n",
" f\"Error parsing your output! Be sure to invoke the tool. Output: {raw_output}. \\n Parse error: {error}\"\n",
" )\n",
"\n",
+ " # Tool was not invoked\n",
+ " elif not tool_output[\"parsed\"]:\n",
+ " print(\"Failed to invoke tool!\")\n",
+ " raise ValueError(\n",
+ " \"You did not use the provided tool! Be sure to invoke the tool to structure the output.\"\n",
+ " )\n",
" return tool_output\n",
"\n",
"\n",
"# Chain with output check\n",
- "code_chain = code_gen_prompt | structured_llm | check"
+ "code_chain = code_gen_prompt_bad | structured_llm | check_claude_output"
]
},
{
@@ -385,12 +415,15 @@
},
{
"cell_type": "code",
- "execution_count": 207,
+ "execution_count": 13,
"id": "efae1ff7-4413-4c47-a403-1630dd453219",
"metadata": {},
"outputs": [],
"source": [
"def insert_errors(inputs):\n",
+ " \"\"\"Insert errors in the messages\"\"\"\n",
+ "\n",
+ " # Get errors\n",
" error = inputs[\"error\"]\n",
" messages = inputs[\"messages\"]\n",
" messages += [\n",
@@ -401,8 +434,7 @@
" ]\n",
" return {\n",
" \"messages\": messages,\n",
- " \"document\": inputs[\"document\"],\n",
- " \"question\": inputs[\"question\"],\n",
+ " \"context\": inputs[\"context\"],\n",
" }\n",
"\n",
"\n",
@@ -416,13 +448,21 @@
},
{
"cell_type": "code",
- "execution_count": 208,
+ "execution_count": 14,
"id": "c7712c49-ee8c-4a61-927e-3c0beb83782b",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Failed to invoke tool!\n"
+ ]
+ }
+ ],
"source": [
"# Test\n",
- "messages = [(\"user\", \"Write a RAG chain in LCEL.\")]\n",
+ "messages = [(\"user\", \"How do I build a RAG chain in LCEL?\")]\n",
"code_output_lcel = code_chain_re_try.invoke(\n",
" {\"context\": concatenated_content, \"messages\": messages}\n",
")"
@@ -430,7 +470,7 @@
},
{
"cell_type": "code",
- "execution_count": 209,
+ "execution_count": 15,
"id": "c8027a6f-6992-4bb4-9d6e-9d0778b04e28",
"metadata": {},
"outputs": [],
@@ -440,17 +480,17 @@
},
{
"cell_type": "code",
- "execution_count": 210,
+ "execution_count": 16,
"id": "209186ac-3121-43a9-8358-86ace7e07f61",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "\"Here's how you can write a retrieval-augmented generation (RAG) chain in LCEL:\""
+ "\"To build a RAG chain using LCEL, we'll use a vector store to retrieve relevant documents, a prompt template that incorporates the retrieved context, a chat model (like OpenAI) to generate a response based on the prompt, and an output parser to clean up the model output.\""
]
},
- "execution_count": 210,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
@@ -461,17 +501,17 @@
},
{
"cell_type": "code",
- "execution_count": 211,
+ "execution_count": 17,
"id": "b8d6d189-e5df-49b6-ada8-83f6c0b26886",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "'from langchain_community.vectorstores import DocArrayInMemorySearch\\nfrom langchain_core.output_parsers import StrOutputParser\\nfrom langchain_core.prompts import ChatPromptTemplate\\nfrom langchain_core.runnables import RunnableParallel, RunnablePassthrough\\nfrom langchain_openai import OpenAIEmbeddings'"
+ "'from langchain_community.vectorstores import DocArrayInMemorySearch\\nfrom langchain_core.output_parsers import StrOutputParser\\nfrom langchain_core.prompts import ChatPromptTemplate\\nfrom langchain_core.runnables import RunnablePassthrough\\nfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings'"
]
},
- "execution_count": 211,
+ "execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
@@ -482,17 +522,17 @@
},
{
"cell_type": "code",
- "execution_count": 212,
+ "execution_count": 18,
"id": "e3822253-d28b-4f7e-9364-79974d04eff1",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "'# Create a vector store retriever\\nvectorstore = DocArrayInMemorySearch.from_texts(\\n [\"harrison worked at kensho\", \"bears like to eat honey\"], \\n embedding=OpenAIEmbeddings(),\\n)\\nretriever = vectorstore.as_retriever()\\n\\n# Define a prompt template\\ntemplate = \"\"\"Answer the question based only on the following context:\\n{context}\\n\\nQuestion: {question}\"\"\"\\nprompt = ChatPromptTemplate.from_template(template)\\n\\n# Define the model and output parser \\nmodel = ChatOpenAI()\\noutput_parser = StrOutputParser()\\n\\n# Compose the chain using LCEL\\nsetup_and_retrieval = RunnableParallel(\\n {\"context\": retriever, \"question\": RunnablePassthrough()}\\n)\\nchain = setup_and_retrieval | prompt | model | output_parser\\n\\n# Test the chain\\nchain.invoke(\"where did harrison work?\")'"
+ "'vectorstore = DocArrayInMemorySearch.from_texts(\\n [\"harrison worked at kensho\", \"bears like to eat honey\"], \\n embedding=OpenAIEmbeddings(),\\n)\\n\\nretriever = vectorstore.as_retriever()\\n\\ntemplate = \"\"\"Answer the question based only on the following context:\\n{context}\\nQuestion: {question}\"\"\"\\nprompt = ChatPromptTemplate.from_template(template)\\n\\noutput_parser = StrOutputParser()\\n\\nrag_chain = (\\n {\"context\": retriever, \"question\": RunnablePassthrough()} \\n | prompt \\n | ChatOpenAI()\\n | output_parser\\n)\\n\\nprint(rag_chain.invoke(\"where did harrison work?\"))'"
]
},
- "execution_count": 212,
+ "execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
@@ -502,19 +542,19 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
- "id": "e27cd768-a274-4ded-b7ba-14aaf6c57164",
+ "cell_type": "markdown",
+ "id": "80d63a3d-bad8-4385-bd85-40ca95c260c6",
"metadata": {},
- "outputs": [],
"source": [
- "exec(parsed_result_lcel.code)"
+ "Example trace catching an error and correcting:\n",
+ "\n",
+ "https://smith.langchain.com/public/f06e62cb-2fac-46ae-80cd-0470b3155eae/r"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "17b44e14-af01-45cd-9c37-c4105e2ddd40",
+ "id": "5f70e45c-eb68-4679-979c-0c04502affd1",
"metadata": {},
"outputs": [],
"source": []