Archives
/
openai-cookbook
mirror of https://github.com/openai/openai-cookbook
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor instructor schema

Browse Source
pull/1139/head
Shyamal H Anadkat 2 months ago
parent fd6ff47510
commit 7ace042428
1 changed files with 86 additions and 89 deletions
Show Stats Download Patch File Download Diff File

175
examples/multimodal/Using_GPT4_Vision_With_Function_Calling.ipynb
Unescape Escape View File

@ -44,13 +44,13 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 44,
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2024-04-10T02:49:55.186528Z",
"start_time": "2024-04-10T02:49:33.180461Z"
"end_time": "2024-04-10T03:50:37.564145Z",
"start_time": "2024-04-10T03:50:37.560040Z"
}
},
"outputs": [],
@ -60,9 +60,11 @@
"from enum import Enum\n",
"from io import BytesIO\n",
"from typing import Iterable\n",
"from typing import List, Optional\n",
"from typing import List\n",
"from typing import Literal, Optional\n",
"\n",
"import fitz\n",
"# Instructor is powered by Pydantic, which is powered by type hints. Schema validation, prompting is controlled by type annotations\n",
"import instructor\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
@ -157,12 +159,12 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-04-10T02:49:55.934227Z",
"start_time": "2024-04-10T02:49:55.187781Z"
"end_time": "2024-04-10T03:50:38.120670Z",
"start_time": "2024-04-10T03:50:37.565555Z"
}
},
"id": "2f2066d40dbbfbe8",
"execution_count": 3
"execution_count": 45
},
{
"cell_type": "markdown",
@ -182,78 +184,81 @@
"source": [
"MODEL = \"gpt-4-turbo-2024-04-09\"\n",
"\n",
"\n",
"class Order(BaseModel):\n",
" \"\"\"Represents an order with details such as order ID, customer name, product name, price, status, and delivery date.\"\"\"\n",
" order_id: str = Field(..., description=\"The unique identifier of the order\")\n",
" product_name: str = Field(..., description=\"The name of the product\")\n",
" price: float = Field(..., description=\"The price of the product\")\n",
" status: str = Field(..., description=\"The status of the order\")\n",
" delivery_date: str = Field(..., description=\"The delivery date of the order\")\n",
"# Placeholder functions for order processing\n",
"\n",
"def get_order_details(order_id):\n",
" # Placeholder function to retrieve order details based on the order ID\n",
" return {\n",
" \"order_id\": order_id,\n",
" \"customer_name\": \"John Doe\",\n",
" \"product_name\": \"Product X\",\n",
" \"price\": 100.0,\n",
" \"status\": \"Delivered\",\n",
" \"delivery_date\": \"2024-01-01\",\n",
" }\n",
"\n",
" return Order(\n",
" order_id=order_id,\n",
" product_name=\"Product X\",\n",
" price=100.0,\n",
" status=\"Delivered\",\n",
" delivery_date=\"2024-04-10\",\n",
" )\n",
"\n",
"def escalate_to_agent(order):\n",
"def escalate_to_agent(order: Order, message: str):\n",
" # Placeholder function to escalate the order to a human agent\n",
" return f\"Order {order['order_id']} has been escalated to an agent for further assistance.\"\n",
"\n",
" return f\"Order {order.order_id} has been escalated to an agent with message: `{message}`\"\n",
"\n",
"def refund_order(order):\n",
"def refund_order(order: Order):\n",
" # Placeholder function to process a refund for the order\n",
" return f\"Order {order['order_id']} has been refunded successfully.\"\n",
"\n",
" return f\"Order {order.order_id} has been refunded successfully.\"\n",
"\n",
"def replace_order(order):\n",
"def replace_order(order: Order):\n",
" # Placeholder function to replace the order with a new one\n",
" return f\"Order {order['order_id']} has been replaced with a new order.\"\n",
" return f\"Order {order.order_id} has been replaced with a new order.\"\n",
"\n",
"\n",
"def process_action(tool_call: str, order_id: str):\n",
" action_map = {\n",
" \"escalate_to_agent\": lambda order_id: escalate_to_agent(get_order_details(order_id)),\n",
" \"replace_order\": lambda order_id: replace_order(get_order_details(order_id)),\n",
" \"refund_order\": lambda order_id: refund_order(get_order_details(order_id)),\n",
" }\n",
" # Execute the corresponding function from the action_map based on the tool_call\n",
" action_function = action_map.get(tool_call)\n",
" if action_function:\n",
" return action_function(order_id)\n",
" else:\n",
" raise ValueError(\"Invalid tool_call provided\")\n",
"\n",
"\n",
"# Define the base model for function calls\n",
"class FunctionCallBase(BaseModel):\n",
" image_description: Optional[str] = Field(..., description=\"The detailed description of the package image.\")\n",
" rationale: Optional[str] = Field(..., description=\"The reason for the action.\")\n",
"\n",
" image_description: Optional[str] = Field(\n",
" ..., description=\"The detailed description of the package image.\"\n",
" )\n",
" action: Literal[\"escalate_to_agent\", \"replace_order\", \"refund_order\"]\n",
" message: Optional[str] = Field(\n",
" ...,\n",
" description=\"The message to be escalated to the agent if action is escalate_to_agent\",\n",
" )\n",
" # Placeholder functions to process the action based on the order ID\n",
" def __call__(self, order_id):\n",
" order: Order = get_order_details(order_id=order_id)\n",
" if self.action == \"escalate_to_agent\":\n",
" return escalate_to_agent(order, self.message)\n",
" if self.action == \"replace_order\":\n",
" return replace_order(order)\n",
" if self.action == \"refund_order\":\n",
" return refund_order(order)\n",
"\n",
"class EscalateToAgent(FunctionCallBase):\n",
" tool_call: str = Field(\"escalate_to_agent\", description=\"The tool call for escalating to an agent.\")\n",
" message: str = Field(..., description=\"The message to be escalated to the agent.\")\n",
"\n",
" \"\"\"Escalate to an agent for further assistance.\"\"\"\n",
" pass\n",
"\n",
"class OrderActionBase(FunctionCallBase):\n",
" pass\n",
"\n",
"\n",
"class ReplaceOrder(OrderActionBase):\n",
" tool_call: str = Field(\"replace_order\", description=\"The tool call for replacing the order.\")\n",
" \"\"\"Tool call to replace an order.\"\"\"\n",
" pass\n",
"\n",
"\n",
"class RefundOrder(OrderActionBase):\n",
" tool_call: str = Field(\"refund_order\", description=\"The tool call for refunding the order.\")\n",
" \"\"\"Tool call to refund an order.\"\"\"\n",
" pass"
],
"metadata": {
"collapsed": false
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-04-10T03:50:38.135826Z",
"start_time": "2024-04-10T03:50:38.122420Z"
}
},
"id": "115b5a0bf208b054",
"execution_count": 4
"execution_count": 46
},
{
"cell_type": "markdown",
@ -277,21 +282,18 @@
"Processing delivery exception support for different package images...\n",
"\n",
"===================== Simulating user message 1 =====================\n",
"Tool call identified: refund_order for provided img: damaged_package\n",
"Parameters: image_description='The package in the image appears heavily damaged. It is significantly crushed and torn on multiple sides, with visible creases and holes in the packaging material.' rationale='The package is clearly damaged, which likely compromised the contents. According to policy, a refund is processed for damaged packages.' tool_call='refund_order'\n",
"Processing action...\n",
"- Tool call: refund_order for provided img: damaged_package\n",
"- Parameters: rationale='The package is visibly damaged with significant tears and crushing, indicating potential harm to the contents.' image_description='The package in the image shows extensive damage, including deep creases and tears in the cardboard. The package is also wrapped with extra tape, suggesting prior attempts to secure it after damage.' action='refund_order' message=None\n",
">> Action result: Order 12345 has been refunded successfully.\n",
"\n",
"===================== Simulating user message 2 =====================\n",
"Tool call identified: escalate_to_agent for provided img: normal_package\n",
"Parameters: image_description='A normal, undamaged package on a wooden floor.' rationale='The package appears normal and not damaged, requiring further assistance.' tool_call='escalate_to_agent' message='Customer has submitted an image of a package that appears normal and undamaged. Please review and provide further instructions or assistance as needed.'\n",
"Processing action...\n",
">> Action result: Order 12345 has been escalated to an agent for further assistance.\n",
"- Tool call: escalate_to_agent for provided img: normal_package\n",
"- Parameters: rationale='The package appears normal and not damaged, requiring further assistance for any potential issues not visible in the image.' image_description='A cardboard box on a wooden floor, appearing intact and undamaged, with no visible signs of wear, tear, or wetness.' action='escalate_to_agent' message='Please review this package for any issues not visible in the image. The package appears normal and undamaged.'\n",
">> Action result: Order 12345 has been escalated to an agent with message: `Please review this package for any issues not visible in the image. The package appears normal and undamaged.`\n",
"\n",
"===================== Simulating user message 3 =====================\n",
"Tool call identified: replace_order for provided img: wet_package\n",
"Parameters: image_description=\"The package appears wet as indicated by the dark, damp patch on the cardboard. The box is labeled 'FRAGILE'.\" rationale='The package is wet, which could compromise the integrity of the contents, especially since it is marked as fragile. A replacement is necessary to ensure customer satisfaction and product safety.' tool_call='replace_order'\n",
"Processing action...\n",
"- Tool call: replace_order for provided img: wet_package\n",
"- Parameters: rationale='The package appears wet, which may have compromised the contents, especially since it is labeled as fragile.' image_description=\"The package in the image shows significant wetness on the top surface, indicating potential water damage. The box is labeled 'FRAGILE', which suggests that the contents are delicate and may be more susceptible to damage from moisture.\" action='replace_order' message=None\n",
">> Action result: Order 12345 has been replaced with a new order.\n"
]
}
@ -299,7 +301,7 @@
"source": [
"# extract the tool call from the response\n",
"ORDER_ID = \"12345\" # Placeholder order ID for testing\n",
"\n",
"INSTRUCTION_PROMPT = \"You are a customer service assistant for a delivery service, equipped to analyze images of packages. If a package appears damaged in the image, automatically process a refund according to policy. If the package looks wet, initiate a replacement. If the package appears normal and not damaged, escalate to agent. For any other issues or unclear images, escalate to agent. You must always use tools!\"\n",
"\n",
"def delivery_exception_support_handler(test_image: str):\n",
" payload = {\n",
@ -312,7 +314,7 @@
" payload[\"messages\"] = [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": 'You are a customer service assistant for a delivery service, equipped to analyze images of packages. If a package appears damaged in the image, automatically process a refund according to policy. If the package looks wet, initiate a replacement. If the package appears normal and not damaged, escalate to agent. For any other issues or unclear images, escalate to agent. You must always use tools',\n",
" \"content\": INSTRUCTION_PROMPT,\n",
" },\n",
" {\n",
" \"role\": \"user\",\n",
@ -330,34 +332,32 @@
" OpenAI(), mode=instructor.Mode.PARALLEL_TOOLS\n",
" ).chat.completions.create(**payload)\n",
" for tool in function_calls:\n",
" tool_call_name = tool.tool_call\n",
" print(f\"Tool call identified: {tool_call_name} for provided img: {test_image}\")\n",
" print(f\"Parameters: {tool}\")\n",
" print(\"Processing action...\")\n",
" print(f\">> Action result: {process_action(tool_call_name, ORDER_ID)}\")\n",
" print(f\"- Tool call: {tool.action} for provided img: {test_image}\")\n",
" print(f\"- Parameters: {tool}\")\n",
" print(f\">> Action result: {tool(ORDER_ID)}\")\n",
" return tool\n",
"\n",
"\n",
"print(\"Processing delivery exception support for different package images...\")\n",
"\n",
"print(\"\\n===================== Simulating user message 1 =====================\")\n",
"assert delivery_exception_support_handler(\"damaged_package\").tool_call == \"refund_order\"\n",
"assert delivery_exception_support_handler(\"damaged_package\").action == \"refund_order\"\n",
"\n",
"print(\"\\n===================== Simulating user message 2 =====================\")\n",
"assert delivery_exception_support_handler(\"normal_package\").tool_call == \"escalate_to_agent\"\n",
"assert delivery_exception_support_handler(\"normal_package\").action == \"escalate_to_agent\"\n",
"\n",
"print(\"\\n===================== Simulating user message 3 =====================\")\n",
"assert delivery_exception_support_handler(\"wet_package\").tool_call == \"replace_order\""
"assert delivery_exception_support_handler(\"wet_package\").action == \"replace_order\""
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-04-10T02:50:09.862020Z",
"start_time": "2024-04-10T02:49:55.948210Z"
"end_time": "2024-04-10T03:50:53.927476Z",
"start_time": "2024-04-10T03:50:38.136857Z"
}
},
"id": "dacd602d9e2728d9",
"execution_count": 5
"execution_count": 47
},
{
"cell_type": "markdown",
@ -412,12 +412,12 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-04-10T02:50:09.974027Z",
"start_time": "2024-04-10T02:50:09.863129Z"
"end_time": "2024-04-10T03:50:54.033536Z",
"start_time": "2024-04-10T03:50:53.929732Z"
}
},
"id": "c3fc6f64b2a2f8e6",
"execution_count": 6
"execution_count": 48
},
{
"cell_type": "markdown",
@ -435,7 +435,6 @@
"source": [
"base64_img = encode_image(output_path)\n",
"\n",
"\n",
"class RoleEnum(str, Enum):\n",
" \"\"\"Defines possible roles within an organization.\"\"\"\n",
" CEO = \"CEO\"\n",
@ -447,7 +446,6 @@
" INTERN = \"Intern\"\n",
" OTHER = \"Other\"\n",
"\n",
"\n",
"class Employee(BaseModel):\n",
" \"\"\"Represents an employee, including their name, role, and optional manager information.\"\"\"\n",
" employee_name: str = Field(..., description=\"The name of the employee\")\n",
@ -460,7 +458,6 @@
" \"\"\"A list of employees within the organizational structure.\"\"\"\n",
" employees: List[Employee] = Field(..., description=\"A list of employees\")\n",
"\n",
"\n",
"def parse_orgchart(base64_img: str) -> EmployeeList:\n",
" response = instructor.from_openai(OpenAI()).chat.completions.create(\n",
" model='gpt-4-turbo',\n",
@ -488,12 +485,12 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-04-10T02:50:09.983135Z",
"start_time": "2024-04-10T02:50:09.974980Z"
"end_time": "2024-04-10T03:50:54.043273Z",
"start_time": "2024-04-10T03:50:54.034475Z"
}
},
"id": "b11469486fc95c23",
"execution_count": 7
"execution_count": 49
},
{
"cell_type": "markdown",
@ -510,8 +507,8 @@
"outputs": [
{
"data": {
"text/plain": " employee_name role manager_name manager_role\n0 Juliana Silva CEO None None\n1 Kim Chun Hei CFO Juliana Silva CEO\n2 Chad Gibbons CTO Juliana Silva CEO\n3 Chiaki Sato COO Juliana Silva CEO\n4 Cahaya Dewi Manager Kim Chun Hei CFO\n5 Shawn Garcia Manager Chad Gibbons CTO\n6 Aaron Loeb Manager Chiaki Sato COO\n7 Drew Feig Employee Cahaya Dewi Manager\n8 Richard Sanchez Employee Cahaya Dewi Manager\n9 Olivia Wilson Employee Shawn Garcia Manager\n10 Avery Davis Employee Shawn Garcia Manager\n11 Harper Russo Employee Aaron Loeb Manager\n12 Sacha Dubois Intern Cahaya Dewi Manager\n13 Matt Zhang Intern Shawn Garcia Manager\n14 Taylor Alonso Intern Aaron Loeb Manager",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>employee_name</th>\n <th>role</th>\n <th>manager_name</th>\n <th>manager_role</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>Juliana Silva</td>\n <td>CEO</td>\n <td>None</td>\n <td>None</td>\n </tr>\n <tr>\n <th>1</th>\n <td>Kim Chun Hei</td>\n <td>CFO</td>\n <td>Juliana Silva</td>\n <td>CEO</td>\n </tr>\n <tr>\n <th>2</th>\n <td>Chad Gibbons</td>\n <td>CTO</td>\n <td>Juliana Silva</td>\n <td>CEO</td>\n </tr>\n <tr>\n <th>3</th>\n <td>Chiaki Sato</td>\n <td>COO</td>\n <td>Juliana Silva</td>\n <td>CEO</td>\n </tr>\n <tr>\n <th>4</th>\n <td>Cahaya Dewi</td>\n <td>Manager</td>\n <td>Kim Chun Hei</td>\n <td>CFO</td>\n </tr>\n <tr>\n <th>5</th>\n <td>Shawn Garcia</td>\n <td>Manager</td>\n <td>Chad Gibbons</td>\n <td>CTO</td>\n </tr>\n <tr>\n <th>6</th>\n <td>Aaron Loeb</td>\n <td>Manager</td>\n <td>Chiaki Sato</td>\n <td>COO</td>\n </tr>\n <tr>\n <th>7</th>\n <td>Drew Feig</td>\n <td>Employee</td>\n <td>Cahaya Dewi</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>8</th>\n <td>Richard Sanchez</td>\n <td>Employee</td>\n <td>Cahaya Dewi</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>9</th>\n <td>Olivia Wilson</td>\n <td>Employee</td>\n <td>Shawn Garcia</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>10</th>\n <td>Avery Davis</td>\n <td>Employee</td>\n <td>Shawn Garcia</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>11</th>\n <td>Harper Russo</td>\n <td>Employee</td>\n <td>Aaron Loeb</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>12</th>\n <td>Sacha Dubois</td>\n <td>Intern</td>\n <td>Cahaya Dewi</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>13</th>\n <td>Matt Zhang</td>\n <td>Intern</td>\n <td>Shawn Garcia</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>14</th>\n <td>Taylor Alonso</td>\n <td>Intern</td>\n <td>Aaron Loeb</td>\n <td>Manager</td>\n </tr>\n </tbody>\n</table>\n</div>"
"text/plain": " employee_name role manager_name manager_role\n0 Juliana Silva CEO None None\n1 Kim Chun Hei CFO Juliana Silva CEO\n2 Chad Gibbons CTO Juliana Silva CEO\n3 Chiaki Sato COO Juliana Silva CEO\n4 Cahaya Dewi Manager Kim Chun Hei CFO\n5 Shawn Garcia Manager Chad Gibbons CTO\n6 Aaron Loeb Manager Chiaki Sato COO\n7 Drew Feig Employee Cahaya Dewi Manager\n8 Richard Sanchez Employee Cahaya Dewi Manager\n9 Sacha Dubois Intern Cahaya Dewi Manager\n10 Olivia Wilson Employee Shawn Garcia Manager\n11 Matt Zhang Intern Shawn Garcia Manager\n12 Avery Davis Employee Aaron Loeb Manager\n13 Harper Russo Employee Aaron Loeb Manager\n14 Taylor Alonso Intern Aaron Loeb Manager",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>employee_name</th>\n <th>role</th>\n <th>manager_name</th>\n <th>manager_role</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>Juliana Silva</td>\n <td>CEO</td>\n <td>None</td>\n <td>None</td>\n </tr>\n <tr>\n <th>1</th>\n <td>Kim Chun Hei</td>\n <td>CFO</td>\n <td>Juliana Silva</td>\n <td>CEO</td>\n </tr>\n <tr>\n <th>2</th>\n <td>Chad Gibbons</td>\n <td>CTO</td>\n <td>Juliana Silva</td>\n <td>CEO</td>\n </tr>\n <tr>\n <th>3</th>\n <td>Chiaki Sato</td>\n <td>COO</td>\n <td>Juliana Silva</td>\n <td>CEO</td>\n </tr>\n <tr>\n <th>4</th>\n <td>Cahaya Dewi</td>\n <td>Manager</td>\n <td>Kim Chun Hei</td>\n <td>CFO</td>\n </tr>\n <tr>\n <th>5</th>\n <td>Shawn Garcia</td>\n <td>Manager</td>\n <td>Chad Gibbons</td>\n <td>CTO</td>\n </tr>\n <tr>\n <th>6</th>\n <td>Aaron Loeb</td>\n <td>Manager</td>\n <td>Chiaki Sato</td>\n <td>COO</td>\n </tr>\n <tr>\n <th>7</th>\n <td>Drew Feig</td>\n <td>Employee</td>\n <td>Cahaya Dewi</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>8</th>\n <td>Richard Sanchez</td>\n <td>Employee</td>\n <td>Cahaya Dewi</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>9</th>\n <td>Sacha Dubois</td>\n <td>Intern</td>\n <td>Cahaya Dewi</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>10</th>\n <td>Olivia Wilson</td>\n <td>Employee</td>\n <td>Shawn Garcia</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>11</th>\n <td>Matt Zhang</td>\n <td>Intern</td>\n <td>Shawn Garcia</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>12</th>\n <td>Avery Davis</td>\n <td>Employee</td>\n <td>Aaron Loeb</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>13</th>\n <td>Harper Russo</td>\n <td>Employee</td>\n <td>Aaron Loeb</td>\n <td>Manager</td>\n </tr>\n <tr>\n <th>14</th>\n <td>Taylor Alonso</td>\n <td>Intern</td>\n <td>Aaron Loeb</td>\n <td>Manager</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"metadata": {},
"output_type": "display_data"
@ -534,12 +531,12 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2024-04-10T02:50:18.781678Z",
"start_time": "2024-04-10T02:50:09.984142Z"
"end_time": "2024-04-10T03:51:03.255172Z",
"start_time": "2024-04-10T03:50:54.044659Z"
}
},
"id": "962c5ea24b8a9dc9",
"execution_count": 8
"execution_count": 50
},
{
"cell_type": "markdown",

Write Preview
Loading…
Cancel
Save