From 027ad902afe00ee341a1911d06b957ed676dde89 Mon Sep 17 00:00:00 2001 From: royziv11 <103690170+royziv11@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:08:28 -0700 Subject: [PATCH] o1 Cookbooks (#1404) --- authors.yaml | 2 +- examples/data/helpcenter_articles.csv | 93 +++ .../Using_reasoning_for_data_validation.ipynb | 737 ++++++++++++++++++ ...ing_reasoning_for_routine_generation.ipynb | 294 +++++++ registry.yaml | 18 + 5 files changed, 1143 insertions(+), 1 deletion(-) create mode 100644 examples/data/helpcenter_articles.csv create mode 100644 examples/o1/Using_reasoning_for_data_validation.ipynb create mode 100644 examples/o1/Using_reasoning_for_routine_generation.ipynb diff --git a/authors.yaml b/authors.yaml index 50831f26..768c70e5 100644 --- a/authors.yaml +++ b/authors.yaml @@ -91,7 +91,7 @@ dylanra-openai: royziv11: name: "Roy Ziv" website: "https://www.linkedin.com/in/roy-ziv-a46001149/" - avatar: "https://media.licdn.com/dms/image/D5603AQHkaEOOGZWtbA/profile-displayphoto-shrink_200_200/0/1699500606122?e=1723075200&v=beta&t=2h6JQpVYm0YncvJRCxCHMB-PmGLg2m3UNbfOB0Yc8rM" + avatar: "https://avatars.githubusercontent.com/u/103690170?v=4" justonf: name: "Juston Forte" diff --git a/examples/data/helpcenter_articles.csv b/examples/data/helpcenter_articles.csv new file mode 100644 index 00000000..c671df17 --- /dev/null +++ b/examples/data/helpcenter_articles.csv @@ -0,0 +1,93 @@ +policy,content +"Delete Payment Method","How do I delete my payment method? +Updated over a week ago +We keep your payment method on file to cover any outstanding charges on your account. To stop charges to your payment method, please follow the steps below. + +## ChatGPT +You can cancel your ChatGPT Plus subscription to stop further charges at any time: +Click on 'My Plan' in the ChatGPT sidebar. +Click on 'Manage my subscription' in the pop-up window. +Select 'Cancel Plan'. +Please note that your cancellation will take effect the day after the next billing date, and you can continue using our services until then. To avoid being charged for your next billing period, please cancel your subscription at least 24 hours before your next billing date. + +## API +We'll need to keep a payment method on file to account for any outstanding usage costs. You're welcome to cancel your pay-as-you-go service, by clicking 'Cancel paid account' in your billing overview. After the current month's invoice has been issued, the current card will no longer be charged. +If you'd like to continue using the service, add a new payment method in the billing overview page and select 'Set as default'. You'll then be able to delete the old payment method. +" +"Business Associate Agreement","How can I get a Business Associate Agreement (BAA) with OpenAI? +Information about HIPAA compliance for healthcare companies + +The Health Insurance Portability and Accountability Act (HIPAA) is a U.S. federal law that requires privacy and security protections for protected health information (PHI). Our API platform can be a great fit for any covered entity or business associate looking to process protected health information, and we’d be happy to assist you in fulfilling your HIPAA compliance. To use our API platform, you’ll first need a BAA with OpenAI. + + +How do I get started? +If you require a BAA before you can use our API, email us at baa@openai.com with details about your company and use case. + +Our team will respond within 1-2 business days. We review each BAA request on a case-by-case basis and may need additional information. The process is usually completed within a few business days. + + +Can I get a BAA for ChatGPT? +If you're interested in exploring a BAA for ChatGPT Enterprise, please contact sales. + + +What happens if I’m not approved? +We are able to approve most customers that request BAAs, but occasionally a use case doesn’t pass our team's evaluation. In that case, we’ll give feedback and context as to why that is and give you the opportunity to update your intended use of our API and re-apply. + + +Are all API services covered by the BAA? +No, only endpoints that are eligible for zero retention are covered by the BAA. You can see a list of those endpoints. + + +Is an enterprise agreement requirement to sign a BAA? +No, an enterprise agreement is not required to sign a BAA. +" +"Set up prepaid billing","How can I set up prepaid billing? + +How it works +Prepaid billing allows API users to pre-purchase usage. The credits you've bought will be applied to your monthly invoice. This means that any API usage you incur will first be deducted from the prepaid credits. If your usage exceeds the credits you've purchased, you'll then be billed for the additional amount. +Prepaid billing helps developers know what they are committing to upfront which can provide more predictability for budgeting and spend management. + + +Setting up prepaid billing +If you're on a Monthly Billing plan, you may also choose to switch to prepaid billing and purchase credits upfront for API usage. +- Go to your billing overview in your account settings +- Click ""Start payment plan"" (you may see variations like ""Buy credits"") + Note: If you previously had an arrears billing plan, you'll need to cancel this existing payment plan first. +- Choose the initial amount of credits you want to purchase. The minimum purchase is $5. The maximum purchase will be based on your trust tier. +- Confirm and purchase your initial amount of credits. +- Use auto-recharge to set an automatic recharge amount, which is the amount of credits that will be added to your account when your balance falls below a set threshold. + +Please note that any purchased credits will expire after 1 year and they are non-refundable. +After you’ve purchased credits, you should be able to start using the API. Note that there may be a couple minutes of delay while our systems update to reflect your credit balance. + + +Purchasing additional credits +Once you’ve consumed all your credits, your API requests will start returning an error letting you know you’ve hit your billing quota. If you’d like to continue your API usage, you can return to the billing portal and use the “Add to balance” button to purchase additional credits. + + +Delayed billing +Due to the complexity of our billing and processing systems, there may be delays in our ability to cut off access after you consume all of your credits. This excess usage may appear as a negative credit balance in your billing dashboard, and will be deducted from your next credit purchase. +" +"VAT Exemption request","How do I submit a VAT exemption request? +Updated over a week ago +If you are eligible for a tax exemption and would like to apply it to your account, please follow these steps: + +Depending on the state and if required: +1. Obtain a current tax exemption certificate from your state or local government and/or your fully completed non-profit sales tax exemption forms. The certificate/forms should include: + Your name and address + Tax exemption number + Signatures and dates, etc. +2. Send us a copy of your certificate using the chat widget in the bottom-right corner. Please include your organization id, invoice number or email address associated with the account, so we can easily find you. Instructions on how to find these items are below. + +3. Once we receive your certificate/form, we will review it and apply the tax exemption to your account. You will receive a confirmation email once the exemption has been applied. + +Please note that the tax exemption will only apply to future purchases. We cannot apply VAT exemptions retroactively. + + + +Where to find your account data +In order to submit a Value Added Tax ('VAT') exemption request you will need the following from your organization Billing preferences: + 1. Company name + 2. Billing email + 3. Primary business address + 4. Business tax ID" \ No newline at end of file diff --git a/examples/o1/Using_reasoning_for_data_validation.ipynb b/examples/o1/Using_reasoning_for_data_validation.ipynb new file mode 100644 index 00000000..0d953d1a --- /dev/null +++ b/examples/o1/Using_reasoning_for_data_validation.ipynb @@ -0,0 +1,737 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using reasoning for data validation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this guide, we’ll explore how to use the o1 model, specifically o1-preview, to perform data validation through reasoning. We’ll walk through a practical example involving a synthetic medical dataset and demonstrate how to assess the model’s accuracy in identifying issues within the data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n", + "\n", + "Data validation is a critical step in ensuring the quality and reliability of datasets, especially in sensitive fields like healthcare. Traditional validation methods often rely on predefined rules and patterns. However, advanced models like o1 can understand context and reason about data, offering a more flexible and intelligent approach to validation.\n", + "\n", + "In this tutorial, we will:\n", + "- Generate a synthetic dataset of medical data that contains inconsistencies.\n", + "- Define a function that takes in a row of data and validates its accuracy\n", + "- Run the validation process and compute accuracy metrics.\n", + "- Analyze and interpret the results." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "import json\n", + "from IPython.display import display, HTML\n", + "from sklearn.metrics import precision_score, recall_score, f1_score\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "import csv\n", + "import pandas as pd\n", + "\n", + "client = OpenAI()\n", + "MODEL = 'o1-preview'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Synthetic Data Generation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will use a lot of the principles described in the [Synthetic Data Generation](https://cookbook.openai.com/examples/sdg1) cookbook to create the foundation of our dataset.\n", + "\n", + "We will prompt the model to generate sets of medical data for our use case. We have provide detailed instructions to the model on how to create the dataset, what format to follow, and how to fill it with inaccuracies. We also provide a few rows of sample data to get the model started. \n", + "\n", + "Each row in the dataset will have the following fields:\n", + "- Patient ID: A randomly generated patient id\n", + "- Date of Birth: Date of birth of the patient\n", + "- Gender: M/F\n", + "- Medical History: Past diagnoses\n", + "- Current Medications: Medication the patient is taking\n", + "- Allergies: Identified allergies\n", + "- Lab Results (Glucose mg/dL)\n", + "- Diagnoses: Current diagnosis\n", + "- Treatment Plan: Current treatment plan\n", + "- Is Valid: Whether or not the current row of data is valid (True/False)\n", + "- Issue: If the row of data is not valid, what the issue is\n", + "\n", + "Some examples of inaccuracies that may be present in the data are:\n", + "- Prescribing medications that the patient is allergic to\n", + "- Current medications do not match medical history\n", + "- Treatment plan does not match diagnosis" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_data():\n", + " messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"\"\"\n", + "You are a helpful assistant designed to generate data. You will be given a format for the data to generate and some examples of the data.\n", + "\n", + "When generating Patient IDs, use the format 'P' followed by a three-digit number (e.g., P006, P941, P319).\n", + "\n", + "Intentionally make some mistakes in the data generation and document them in the appropriate columns ('Is Valid' and 'Issue') if the row of data is invalid.\n", + "\n", + "The types of mistakes to include are:\n", + "\n", + "- **Allergy Contradictions**: Prescribing a medication that the patient is allergic to (e.g., prescribing Penicillin to a patient allergic to Penicillin).\n", + "- **Medical History and Medication Mismatch**: A patient with a medical condition not receiving appropriate medication (e.g., a diabetic patient not prescribed any diabetes medication).\n", + "- **Lab Results and Diagnosis Mismatch**: Lab results that do not support the diagnosis (e.g., normal glucose levels but diagnosed with Diabetes Type 2).\n", + "- **Other Plausible Mistakes**: Any other realistic errors that could occur in medical records, such as incorrect gender entries, impossible dates of birth, or inconsistent treatment plans.\n", + "\n", + "Ensure that when 'Is Valid' is 'False', the 'Issue' column clearly explains the problem.\n", + "\n", + "Return 100 rows of data for the user. Your response should strictly be in the format of a valid CSV.\n", + "\n", + "Generate Synthetic Medical Records Dataset with the following columns:\n", + " - Patient ID: A randomly generated patient id\n", + " - Date of Birth: Date of birth of the patient\n", + " - Gender: M/F\n", + " - Medical History: Past diagnoses\n", + " - Current Medications: Medication the patient is taking\n", + " - Allergies: Identified allergies\n", + " - Lab Results (Glucose mg/dL)\n", + " - Diagnoses: Current diagnosis\n", + " - Treatment Plan: Current treatment plan\n", + " - Is Valid: Whether or not the current row of data is valid (True/False)\n", + " - Issue: If the row of data is not valid, what the issue is\n", + "\n", + "Patient ID,Date of Birth,Gender,Medical History,Current Medications,Allergies,Lab Results (Glucose mg/dL),Diagnoses,Treatment Plan,Is Valid,Issue\n", + "P001,1980-05-14,M,Hypertension,Lisinopril,None,110,Hypertension,Continue Lisinopril,True,\n", + "P002,1975-11-30,F,Diabetes Type 2,Metformin,Penicillin,90,Diabetes Type 2,Continue Metformin,True,\n", + "P003,1990-07-22,F,Asthma,Albuterol,Aspirin,85,Asthma,Prescribe Albuterol,True,\n", + "P004,2000-03-10,M,None,Amoxicillin,Penicillin,95,Infection,Prescribe Amoxicillin,False,Prescribed Amoxicillin despite Penicillin allergy\n", + "P005,1985-09-18,F,Hyperlipidemia,Atorvastatin,None,200,Hyperlipidemia,Continue Atorvastatin,True,\n", + "P006,1978-12-05,M,Hypertension; Diabetes Type 2,Lisinopril; Insulin,None,55,Diabetes Type 2,Adjust insulin dosage,False,Low glucose level not properly addressed\n", + " \"\"\"\n", + " }\n", + " ]\n", + "\n", + " response = client.chat.completions.create(\n", + " model=MODEL,\n", + " messages=messages\n", + " )\n", + "\n", + " return response.choices[0].message.content.replace('```csv', '').replace('```', '')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Synthetic data generation and appending completed.\n" + ] + } + ], + "source": [ + "# Generate data three times using the existing dataGeneration function\n", + "generated_data = []\n", + "data = generate_data()\n", + "generated_data.extend(data.strip().split('\\n'))\n", + "\n", + "# Append the generated data to the medicalData.csv file\n", + "with open('../data/medicalData.csv', 'a', newline='') as csvfile:\n", + " csvwriter = csv.writer(csvfile)\n", + " for row in generated_data:\n", + " csvwriter.writerow(row.split(','))\n", + "\n", + "print(\"Synthetic data generation and appending completed.\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Validation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have our dataset prepared, we will prompt the reasoning model to review each row of data and determine whether or not it contains an issue. We will ask the model to output whether or not there is an issue in the data and then offer an explanation of the issue.\n", + "\n", + "Once we have the model determine its list of invalid data, we will pass those results on to a model grader to assess two metrics:\n", + "- Accuracy of the model's ability correctly identify issues with the data\n", + "- For the subset of data that issues have been correctly identified, what is the accuracy of the model in identifying the issue at hand\n", + "\n", + "Given that this task is much more narrow, we can use the faster gpt-4o model to calculate the accuracy.\n", + "\n", + "REMINDER: Given that these models are still in beta, rate limits will be significantly reduced. Please adjust the number of concurrent workers accordingly." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def validate_data(input_data):\n", + " messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": f\"\"\"\n", + "You are a helpful assistant designed to validate the quality of medical datasets. You will be given a single row of medical data, and your task is to determine whether the data is valid.\n", + "\n", + "- Carefully analyze the data for any inconsistencies, contradictions, missing values, or implausible information.\n", + "- Consider the logical relationships between different fields (e.g., treatments should be appropriate for the diagnoses, medications should not conflict with allergies, lab results should be consistent with diagnoses, etc.).\n", + "- Use your general medical knowledge to assess the validity of the data.\n", + "- Focus solely on the information provided without making assumptions beyond the given data.\n", + "\n", + "**Return only a JSON object** with the following two properties:\n", + "\n", + "- `\"is_valid\"`: a boolean (`true` or `false`) indicating whether the data is valid.\n", + "- `\"issue\"`: if `\"is_valid\"` is `false`, provide a brief explanation of the issue; if `\"is_valid\"` is `true`, set `\"issue\"` to `null`.\n", + "\n", + "Both JSON properties must always be present.\n", + "\n", + "Do not include any additional text or explanations outside the JSON object.\n", + "\n", + "MEDICAL DATA:\n", + "{input_data}\n", + " \"\"\"\n", + " }\n", + " ]\n", + "\n", + " response = client.chat.completions.create(\n", + " model=MODEL,\n", + " messages=messages\n", + " )\n", + "\n", + " response_content = response.choices[0].message.content.replace('```json', '').replace('```', '').strip()\n", + " \n", + " try:\n", + " if isinstance(response_content, dict):\n", + " response_dict = response_content\n", + " else:\n", + " response_dict = json.loads(response_content)\n", + " return response_dict\n", + " except json.JSONDecodeError as e:\n", + " print(f\"Failed to decode JSON response: {response_content}\")\n", + " raise e" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Read the CSV file and exclude the last two columns\n", + "input_data = []\n", + "with open('../data/medicalData.csv', 'r') as file:\n", + " reader = csv.reader(file)\n", + " headers = next(reader)\n", + " for row in reader:\n", + " input_data.append(row[:-2]) # Exclude \"Is Valid\" and \"Issue\" columns\n", + "\n", + "# Initialize lists to store true labels\n", + "true_is_valid = []\n", + "true_issues = []\n", + "\n", + "# Extract true labels from the CSV file\n", + "with open('../data/medicalData.csv', 'r') as file:\n", + " reader = csv.reader(file)\n", + " headers = next(reader)\n", + " for row in reader:\n", + " true_is_valid.append(row[-2] == 'True')\n", + " true_issues.append(row[-1])\n", + "\n", + "# Function to validate a single row of data\n", + "def validate_row(row):\n", + " input_str = ','.join(row)\n", + " result_json = validate_data(input_str)\n", + " return result_json\n", + "\n", + "# Validate data rows and collect results\n", + "pred_is_valid = [False] * len(input_data)\n", + "pred_issues = [''] * len(input_data)\n", + "\n", + "with ThreadPoolExecutor() as executor:\n", + " futures = {executor.submit(validate_row, row): i for i, row in enumerate(input_data)}\n", + " \n", + " for future in as_completed(futures):\n", + " i = futures[future] # Get the index of the current row\n", + " result_json = future.result()\n", + " pred_is_valid[i] = result_json['is_valid']\n", + " pred_issues[i] = result_json['issue']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have the model's results, we can compare it against the source of truth and determine the system's accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Convert predicted and true 'is_valid' labels to boolean if they aren't already\n", + "pred_is_valid_bool = [bool(val) if isinstance(val, bool) else val == 'True' for val in pred_is_valid]\n", + "true_is_valid_bool = [bool(val) if isinstance(val, bool) else val == 'True' for val in true_is_valid]\n", + "\n", + "# Calculate precision, recall, and f1 score for the 'is_valid' prediction\n", + "precision = precision_score(true_is_valid_bool, pred_is_valid_bool, pos_label=True)\n", + "recall = recall_score(true_is_valid_bool, pred_is_valid_bool, pos_label=True)\n", + "f1 = f1_score(true_is_valid_bool, pred_is_valid_bool, pos_label=True)\n", + "\n", + "# Initialize issue_matches_full with False\n", + "issue_matches_full = [False] * len(true_is_valid)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Precision: 0.82\n", + "Recall: 0.87\n", + "F1: 0.84\n" + ] + } + ], + "source": [ + "print(f\"Precision: {precision:.2f}\")\n", + "print(f\"Recall: {recall:.2f}\")\n", + "print(f\"F1: {f1:.2f}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Issue Identification" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will now determine the model's ability to accurately classify the issue in the data" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def validate_issue(model_generated_answer, correct_answer):\n", + " messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": f\"\"\"\n", + "You are a medical expert assistant designed to validate the quality of an LLM-generated answer.\n", + "\n", + "The model was asked to review a medical dataset row to determine if the data is valid. If the data is not valid, it should provide a justification explaining why.\n", + "\n", + "Your task:\n", + "\n", + " •\tCompare the model-generated justification with the correct reason provided.\n", + " •\tDetermine if they address the same underlying medical issue or concern, even if phrased differently.\n", + " •\tFocus on the intent, medical concepts, and implications rather than exact wording.\n", + "\n", + "Instructions:\n", + "\n", + " •\tIf the justifications have the same intent or address the same medical issue, return True.\n", + " •\tIf they address different issues or concerns, return False.\n", + " •\tOnly respond with a single word: True or False.\n", + "\n", + "Examples:\n", + "\n", + " 1.\tExample 1:\n", + " •\tModel Generated Response: “The patient is allergic to penicillin”\n", + " •\tCorrect Response: “The patient was prescribed penicillin despite being allergic”\n", + " •\tAnswer: True\n", + " 2.\tExample 2:\n", + " •\tModel Generated Response: “The date of birth of the patient is incorrect”\n", + " •\tCorrect Response: “The patient was prescribed penicillin despite being allergic”\n", + " •\tAnswer: False\n", + "\n", + "\n", + "Model Generated Response: {model_generated_answer}\n", + "Correct Response: {correct_answer}\n", + " \"\"\"\n", + " }\n", + " ]\n", + "\n", + " response = client.chat.completions.create(\n", + " model=\"o1-preview\",\n", + " messages=messages\n", + " )\n", + "\n", + " result = response.choices[0].message.content\n", + "\n", + " return result" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# Validate issues for rows where both true and predicted 'is_valid' are False\n", + "validation_results = []\n", + "\n", + "with ThreadPoolExecutor() as executor:\n", + " futures = {\n", + " executor.submit(validate_issue, pred_issues[i], true_issues[i]): i\n", + " for i in range(len(pred_is_valid_bool))\n", + " if not pred_is_valid_bool[i] and not true_is_valid_bool[i]\n", + " }\n", + " \n", + " for future in as_completed(futures):\n", + " i = futures[future] # Get the original index\n", + " issue_match = future.result()\n", + " issue_matches_full[i] = (issue_match == 'True')\n", + " validation_results.append({\n", + " \"index\": i,\n", + " \"predicted_issue\": pred_issues[i],\n", + " \"true_issue\": true_issues[i],\n", + " \"issue_match\": issue_matches_full[i]\n", + " })\n", + " \n", + " # Calculate issue accuracy\n", + " issue_accuracy = sum([i['issue_match'] for i in validation_results]) / len(validation_results)\n", + " \n", + " # Store the results in the dictionary\n", + " model_results = {\n", + " \"precision\": precision,\n", + " \"recall\": recall,\n", + " \"f1\": f1,\n", + " \"issue_accuracy\": issue_accuracy\n", + " }\n", + "\n", + "# Create a DataFrame to store the results\n", + "df_results = pd.DataFrame([model_results])\n", + "\n", + "# Create a DataFrame to store the validation results for each row\n", + "df_validation_results = pd.DataFrame(validation_results)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Below we'll display the subset of rows that we correctly identified contained an issue. For each row, we'll show the predicted vs. true issue and whether or not there is a match" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
indexpredicted_issuetrue_issueissue_match
039Amoxicillin is prescribed to a patient with Penicillin allergy.Prescribed Amoxicillin despite Penicillin allergyTrue
150Patient diagnosed with Type 1 Diabetes is not on any medications and the treatment field lists the diagnosis instead of appropriate treatment.Diabetes Type 1 patient not receiving insulinTrue
251Lab result of 300 indicates hyperglycemia but no diagnosis or treatment is recorded.Extremely high glucose level not diagnosed or treatedTrue
326The patient is being prescribed penicillin despite having an allergy to penicillin.Prescribed Penicillin despite Penicillin allergyTrue
431The patient's age (88) is inconsistent with the date of birth (1996-11-05).Osteoporosis patient not receiving treatmentFalse
524The 'Treatment Plan' field should not be 'Depression'; it should specify the treatment prescribed for depression.Depression patient not receiving treatmentTrue
63Patient is allergic to Penicillin but is prescribed Amoxicillin.Prescribed Amoxicillin despite Penicillin allergyTrue
728The treatment field contains 'Asthma', which is a diagnosis, not a treatment.Asthma patient not prescribed any medicationFalse
87Patient with asthma and low lab result (100) is treated only with lifestyle modifications without medications, which is inappropriate.Asthma patient not prescribed any medicationTrue
916The patient's age (86) does not match the date of birth (1955-10-10).COPD patient not receiving treatmentFalse
1053The age provided (92) is inconsistent with the date of birth (1983-08-19).Depression patient not receiving treatmentFalse
1123Treatment field incorrectly lists 'Hyperlipidemia' instead of an appropriate treatment for the diagnosis.Hyperlipidemia patient not prescribed any medicationTrue
1213Patient is allergic to sulfa drugs but is prescribed Sulfamethoxazole, which is a sulfa drug.Prescribed Sulfa drug despite Sulfa allergyTrue
1398The patient is prescribed Penicillin despite having a Penicillin allergy.Prescribed Penicillin despite Penicillin allergyTrue
149Patient has a medication allergy to Penicillin but is prescribed Penicillin.Prescribed Penicillin despite Penicillin allergyTrue
1585Treatment field contains 'Hyperlipidemia', which is a diagnosis, not a treatment.Hyperlipidemia patient not prescribed any medicationFalse
1618Prescribed treatment (Aspirin) is not appropriate for the diagnosis of infection.Prescribed Aspirin despite Aspirin allergy; high glucose level not addressedFalse
1770Treatment field contains a diagnosis 'Osteoporosis' instead of a treatment.Osteoporosis patient not receiving treatmentTrue
1857Patient is allergic to Penicillin but is being prescribed Amoxicillin, which is contraindicated.Prescribed Amoxicillin despite Penicillin allergyTrue
1980Treatment field incorrectly lists 'Diabetes Type 2' instead of a valid treatment plan.Diabetes Type 2 patient not receiving medicationTrue
2087Treatment plan includes prescribing Amoxicillin, which the patient is allergic to.Prescribed Amoxicillin despite Penicillin allergyTrue
2137Treatment field contains 'Hyperlipidemia', which is a diagnosis, not a treatment.Hyperlipidemia patient not prescribed any medicationFalse
2295Treatment is listed as 'Asthma', which is not an appropriate treatment for the diagnosis.Asthma patient not prescribed any medicationTrue
2396Treatment field lists 'Hyperlipidemia', which is not an appropriate treatment.Hyperlipidemia patient not prescribed any medicationFalse
2459Treatment field contains 'Anemia', which is not a valid treatment.Anemia patient not receiving treatmentFalse
255Age does not match date of birthLow glucose level not properly addressedFalse
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def display_formatted_dataframe(df):\n", + " def format_text(text):\n", + " return text.replace('\\n', '
')\n", + "\n", + " df_formatted = df.copy()\n", + " df_formatted['predicted_issue'] = df_formatted['predicted_issue'].apply(format_text)\n", + " df_formatted['true_issue'] = df_formatted['true_issue'].apply(format_text)\n", + " \n", + " display(HTML(df_formatted.to_html(escape=False, justify='left')))\n", + " \n", + "display_formatted_dataframe(pd.DataFrame(validation_results))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1 issue_accuracy\n", + "0 0.818182 0.870968 0.84375 0.615385\n" + ] + } + ], + "source": [ + "# Display the DataFrame\n", + "print(df_results)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "We can see from the results here that we're able to generate a high precision/recall for issue identification as well as decent accuracy for pinpointing the exact issue in the data.\n", + "\n", + "This should help streamline data validation for eval sets across a variety of domains." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/o1/Using_reasoning_for_routine_generation.ipynb b/examples/o1/Using_reasoning_for_routine_generation.ipynb new file mode 100644 index 00000000..8c76dfd8 --- /dev/null +++ b/examples/o1/Using_reasoning_for_routine_generation.ipynb @@ -0,0 +1,294 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using reasoning for routine generation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When developing customer service solutions, one of the initial steps involves transforming knowledge base articles into a set of routines that an LLM can comprehend and follow. A routine, in this context, refers to a set of step-by-step instructions designed specifically for the LLM to execute efficiently. Each routine is carefully structured so that a step corresponds to a clear action. Actions can include responding to a user, triggering a function call, or retrieving additional relevant knowledge.\n", + "\n", + "Most internal knowledge base articles are complex and structured for human interpretation. They often include intricate diagrams, multi-step processes, and decision trees that pose challenges for LLM-based solutions to reason through in a meaningful way. By breaking down these documents into routines, each instruction can be simplified and formatted in a way that guides the LLM through a series of small, manageable tasks. This granular approach reduces ambiguity, allowing the LLM to process the information methodically and reducing the risk of hallucination or deviation from the expected path.\n", + "\n", + "Converting these knowledge base articles into routines can be time-consuming and challenging, especially for companies attempting to build an automated pipeline. Each routine must account for various user scenarios, where actions need to be clearly defined. For instance, when a function call is necessary, the routine must specify the exact information to retrieve or the action to execute—whether it’s triggering an API, retrieving external data, or pulling in additional context. While automating this process with traditional GPT-class models can significantly reduce the manual effort involved, it often introduces new challenges. Some challenges include designing robust instructions that are specific enough for the LLM to follow consistently, capturing unique edge cases that may arise during customer interactions, providing high-quality few-shot examples to guide the model’s behavior, and in some cases, fine-tuning the model to achieve more reliable or specialized outcomes.\n", + "\n", + "o1 has demonstrated the capability to efficiently deconstruct these articles and convert them into sets of routines zero-shot, meaning that the LLM can understand and follow the instructions without extensive examples or prior training on similar tasks. This minimizes the prompting effort required, as the routine structure itself provides the necessary guidance for the LLM to complete each step. By breaking down tasks into specific actions and integrating function calls where needed, o1’s approach ensures that even complex workflows can be handled seamlessly by the LLM, leading to more effective and scalable customer service solutions." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Selecting Knowledge Base Articles\n", + "\n", + "In this example, we will use a set of publicly available Help Center articles from the OpenAI website and convert them into internal routines that an LLM can execute. Besides transforming the policies into routines, we will also have the model generate functions that allow the LLM to perform actions on behalf of the user. This is necessary to allow the LLM to execute the same actions that human agents have, and access additional information that may not be immediately available just from the policy documentation.\n", + "\n", + "We will begin by using the following Help Center articles for conversion into routines:\n", + "- [How do I delete my payment method](https://help.openai.com/en/articles/8156070-how-do-i-delete-my-payment-method)\n", + "- [How can I get a Business Associate Agreement (BAA) with OpenAI?](https://help.openai.com/en/articles/8660679-how-can-i-get-a-business-associate-agreement-baa-with-openai)\n", + "- [How can I set up prepaid billing?](https://help.openai.com/en/articles/8264644-how-can-i-set-up-prepaid-billing)\n", + "- [How do I submit a VAT exemption request](https://help.openai.com/en/articles/7232908-how-do-i-submit-a-vat-exemption-request)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "from IPython.display import display, HTML\n", + "import pandas as pd\n", + "from concurrent.futures import ThreadPoolExecutor\n", + "import csv\n", + "\n", + "client = OpenAI()\n", + "MODEL = 'o1-preview'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We have our articles stored in an accessible csv. We will take the articles and pass them to o1-preview in parallel and generate the initial routines.\n", + "\n", + "Our instructions for converting to policy to a routine include:\n", + "- Converting the policy from an external facing document to an internal SOP routine\n", + "- Breaking down the policy in specific actions and sub-actions\n", + "- Outlining specific conditions for moving between steps\n", + "- Determing where external knowledge/actions may be required, and defining functions that we could use to get that information" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "articles = []\n", + "\n", + "with open('../data/helpcenter_articles.csv', mode='r', encoding='utf-8') as file:\n", + " reader = csv.DictReader(file)\n", + " for row in reader:\n", + " articles.append({\n", + " \"policy\": row[\"policy\"],\n", + " \"content\": row[\"content\"]\n", + " })" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "CONVERSION_PROMPT = \"\"\"\n", + "You are a helpful assistant tasked with taking an external facing help center article and converting it into a internal-facing programmatically executable routine optimized for an LLM. \n", + "The LLM using this routine will be tasked with reading the policy, answering incoming questions from customers, and helping drive the case toward resolution.\n", + "\n", + "Please follow these instructions:\n", + "1. **Review the customer service policy carefully** to ensure every step is accounted for. It is crucial not to skip any steps or policies.\n", + "2. **Organize the instructions into a logical, step-by-step order**, using the specified format.\n", + "3. **Use the following format**:\n", + " - **Main actions are numbered** (e.g., 1, 2, 3).\n", + " - **Sub-actions are lettered** under their relevant main actions (e.g., 1a, 1b).\n", + " **Sub-actions should start on new lines**\n", + " - **Specify conditions using clear 'if...then...else' statements** (e.g., 'If the product was purchased within 30 days, then...').\n", + " - **For instructions that require more information from the customer**, provide polite and professional prompts to ask for additional information.\n", + " - **For actions that require data from external systems**, write a step to call a function using backticks for the function name (e.g., `call the check_delivery_date function`).\n", + " - **If a step requires the customer service agent to take an action** (e.g., process a refund), generate a function call for this action (e.g., `call the process_refund function`).\n", + " - **Define any new functions** by providing a brief description of their purpose and required parameters.\n", + " - **If there is an action an assistant can performon behalf of the user**, include a function call for this action (e.g., `call the change_email_address function`), and ensure the function is defined with its purpose and required parameters.\n", + " - This action may not be explicitly defined in the help center article, but can be done to help the user resolve their inquiry faster\n", + " - **The step prior to case resolution should always be to ask if there is anything more you can assist with**.\n", + " - **End with a final action for case resolution**: calling the `case_resolution` function should always be the final step.\n", + "4. **Ensure compliance** by making sure all steps adhere to company policies, privacy regulations, and legal requirements.\n", + "5. **Handle exceptions or escalations** by specifying steps for scenarios that fall outside the standard policy.\n", + "\n", + "**Important**: If at any point you are uncertain, respond with \"I don't know.\"\n", + "\n", + "Please convert the customer service policy into the formatted routine, ensuring it is easy to follow and execute programmatically.\n", + "\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def generate_routine(policy):\n", + " try:\n", + " messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": f\"\"\"\n", + " {CONVERSION_PROMPT}\n", + "\n", + " POLICY:\n", + " {policy}\n", + " \"\"\"\n", + " }\n", + " ]\n", + "\n", + " response = client.chat.completions.create(\n", + " model=MODEL,\n", + " messages=messages\n", + " )\n", + " \n", + "\n", + " return response.choices[0].message.content \n", + " except Exception as e:\n", + " print(f\"An error occurred: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def process_article(article):\n", + " routine = generate_routine(article['content'])\n", + " return {\"policy\": article['policy'], \"content\": article['content'], \"routine\": routine}\n", + "\n", + "\n", + "with ThreadPoolExecutor() as executor:\n", + " results = list(executor.map(process_article, articles))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll store the results of our routines in a dataframe and print them out so we can get an initial look." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
policycontentroutine
0Delete Payment MethodHow do I delete my payment method?
Updated over a week ago
We keep your payment method on file to cover any outstanding charges on your account. To stop charges to your payment method, please follow the steps below.

## ChatGPT
You can cancel your ChatGPT Plus subscription to stop further charges at any time:
Click on 'My Plan' in the ChatGPT sidebar.
Click on 'Manage my subscription' in the pop-up window.
Select 'Cancel Plan'.
Please note that your cancellation will take effect the day after the next billing date, and you can continue using our services until then. To avoid being charged for your next billing period, please cancel your subscription at least 24 hours before your next billing date.

## API
We'll need to keep a payment method on file to account for any outstanding usage costs. You're welcome to cancel your pay-as-you-go service, by clicking 'Cancel paid account' in your billing overview. After the current month's invoice has been issued, the current card will no longer be charged.
If you'd like to continue using the service, add a new payment method in the billing overview page and select 'Set as default'. You'll then be able to delete the old payment method.
1. Verify the customer's account.
a. Politely ask the customer for their email address or account ID to locate their account.
b. `call the verify_customer_account(email_or_account_id)`.

2. Verify the customer's identity.
a. Politely ask the customer to provide security information to confirm their identity (e.g., the last four digits of the payment method on file).
b. `call the verify_customer_identity(account_id, security_information)`.
c. If the customer's identity cannot be verified, then:
- Inform the customer that we are unable to proceed without identity verification for security reasons.
- Provide guidance on how they can verify their identity.
- Proceed to step 6.

3. Determine the customer's account type.
a. `call the check_account_type(account_id)`.

4. If the customer is a ChatGPT Plus subscriber, then:
a. Ask the customer if they would like assistance with canceling their ChatGPT Plus subscription.
b. If the customer agrees, then:
- `call the cancel_subscription(account_id)`.
- Inform the customer that their subscription has been canceled and the cancellation will take effect the day after the next billing date.
- Remind the customer that they can continue using our services until then.
c. Else:
- Provide the following steps for the customer to cancel their subscription:
- Click on **'My Plan'** in the ChatGPT sidebar.
- Click on **'Manage my subscription'** in the pop-up window.
- Select **'Cancel Plan'**.
- Inform the customer about the cancellation effective date and continued access until then.
- Advise the customer to cancel at least 24 hours before their next billing date to avoid being charged for the next billing period.

5. Else if the customer is an API user, then:
a. Inform the customer that we need to keep a payment method on file to account for any outstanding usage costs.
b. Ask the customer if they would like assistance with canceling their pay-as-you-go service.
c. If the customer agrees, then:
- `call the cancel_paid_account(account_id)`.
- Inform the customer that after the current month's invoice has been issued, the current card will no longer be charged.
d. Else:
- Provide the following steps for the customer to cancel their pay-as-you-go service:
- Go to the **billing overview** page.
- Click on **'Cancel paid account'**.
- Inform the customer that after the current month's invoice has been issued, the current card will no longer be charged.
e. If the customer wants to continue using the service but change the payment method:
- Ask the customer if they would like assistance with adding a new payment method and setting it as default.
- If the customer agrees:
- Politely request the new payment method details.
- `call the add_payment_method(account_id, payment_details)`.
- `call the set_default_payment_method(account_id, new_payment_method_id)`.
- `call the delete_payment_method(account_id, old_payment_method_id)`.
- Inform the customer that the old payment method has been deleted and the new one is set as default.
- Else:
- Instruct the customer to add a new payment method in the billing overview page.
- Ask them to select **'Set as default'** for the new payment method.
- Inform them that they can then delete the old payment method.

6. Ask the customer if there is anything else you can assist them with.

7. `call the case_resolution()`.

---

**Function Definitions:**

- `verify_customer_account(email_or_account_id)`: Verifies the customer's account using their email address or account ID.
**Parameters:** `email_or_account_id`

- `verify_customer_identity(account_id, security_information)`: Confirms the customer's identity using security information.
**Parameters:** `account_id`, `security_information`

- `check_account_type(account_id)`: Retrieves the customer's account type (ChatGPT Plus subscriber or API user).
**Parameters:** `account_id`

- `cancel_subscription(account_id)`: Cancels the ChatGPT Plus subscription for the customer.
**Parameters:** `account_id`

- `cancel_paid_account(account_id)`: Cancels the API pay-as-you-go service for the customer.
**Parameters:** `account_id`

- `add_payment_method(account_id, payment_details)`: Adds a new payment method to the customer's account.
**Parameters:** `account_id`, `payment_details`

- `set_default_payment_method(account_id, payment_method_id)`: Sets a payment method as the default for the customer.
**Parameters:** `account_id`, `payment_method_id`

- `delete_payment_method(account_id, payment_method_id)`: Deletes a payment method from the customer's account.
**Parameters:** `account_id`, `payment_method_id`

- `case_resolution()`: Resolves the case and marks it as completed.
1Business Associate AgreementHow can I get a Business Associate Agreement (BAA) with OpenAI?
Information about HIPAA compliance for healthcare companies

The Health Insurance Portability and Accountability Act (HIPAA) is a U.S. federal law that requires privacy and security protections for protected health information (PHI). Our API platform can be a great fit for any covered entity or business associate looking to process protected health information, and we’d be happy to assist you in fulfilling your HIPAA compliance. To use our API platform, you’ll first need a BAA with OpenAI.


How do I get started?
If you require a BAA before you can use our API, email us at baa@openai.com with details about your company and use case.

Our team will respond within 1-2 business days. We review each BAA request on a case-by-case basis and may need additional information. The process is usually completed within a few business days.


Can I get a BAA for ChatGPT?
If you're interested in exploring a BAA for ChatGPT Enterprise, please contact sales.


What happens if I’m not approved?
We are able to approve most customers that request BAAs, but occasionally a use case doesn’t pass our team's evaluation. In that case, we’ll give feedback and context as to why that is and give you the opportunity to update your intended use of our API and re-apply.


Are all API services covered by the BAA?
No, only endpoints that are eligible for zero retention are covered by the BAA. You can see a list of those endpoints.


Is an enterprise agreement requirement to sign a BAA?
No, an enterprise agreement is not required to sign a BAA.
1. Thank the customer and ask for clarification:
a. \"Thank you for reaching out! Could you please specify whether you require a Business Associate Agreement (BAA) for using our API or for ChatGPT Enterprise?\"

2. If the customer requires a BAA for the API, then:
a. Inform the customer: \"To obtain a BAA for our API, please email baa@openai.com with details about your company and use case.\"
b. Inform the customer: \"Our team will respond within 1-2 business days.\"
c. Inform the customer: \"We review each BAA request on a case-by-case basis and may need additional information.\"
d. Inform the customer: \"The process is usually completed within a few business days.\"
e. Inform the customer: \"Please note that only endpoints eligible for zero data retention are covered by the BAA.\"
i. Call the `provide_list_of_zero_retention_endpoints` function.
f. Inform the customer: \"An enterprise agreement is not required to sign a BAA.\"

3. If the customer requires a BAA for ChatGPT Enterprise, then:
a. Inform the customer: \"To explore a BAA for ChatGPT Enterprise, please contact our sales team.\"
i. Call the `provide_sales_contact_information` function.

4. If the customer is not approved, then:
a. Inform the customer: \"We are able to approve most customers that request BAAs, but occasionally a use case doesn't pass our team's evaluation.\"
b. Inform the customer: \"In that case, we'll provide feedback and context as to why and give you the opportunity to update your intended use of our API and re-apply.\"

5. Ask the customer if there is anything else you can assist with:
a. \"Is there anything else I can assist you with today?\"

6. Call the `case_resolution` function.

---

**Function Definitions:**

- `provide_list_of_zero_retention_endpoints`:
- **Purpose**: Provides the customer with a list of API endpoints that are eligible for zero data retention under the BAA.
- **Parameters**: None.

- `provide_sales_contact_information`:
- **Purpose**: Provides the customer with contact information to reach our sales team for ChatGPT Enterprise inquiries.
- **Parameters**: None.

- `case_resolution`:
- **Purpose**: Finalizes the case and marks it as resolved.
- **Parameters**: None.
2Set up prepaid billingHow can I set up prepaid billing?

How it works
Prepaid billing allows API users to pre-purchase usage. The credits you've bought will be applied to your monthly invoice. This means that any API usage you incur will first be deducted from the prepaid credits. If your usage exceeds the credits you've purchased, you'll then be billed for the additional amount.
Prepaid billing helps developers know what they are committing to upfront which can provide more predictability for budgeting and spend management.


Setting up prepaid billing
If you're on a Monthly Billing plan, you may also choose to switch to prepaid billing and purchase credits upfront for API usage.
- Go to your billing overview in your account settings
- Click \"Start payment plan\" (you may see variations like \"Buy credits\")
Note: If you previously had an arrears billing plan, you'll need to cancel this existing payment plan first.
- Choose the initial amount of credits you want to purchase. The minimum purchase is $5. The maximum purchase will be based on your trust tier.
- Confirm and purchase your initial amount of credits.
- Use auto-recharge to set an automatic recharge amount, which is the amount of credits that will be added to your account when your balance falls below a set threshold.

Please note that any purchased credits will expire after 1 year and they are non-refundable.
After you’ve purchased credits, you should be able to start using the API. Note that there may be a couple minutes of delay while our systems update to reflect your credit balance.


Purchasing additional credits
Once you’ve consumed all your credits, your API requests will start returning an error letting you know you’ve hit your billing quota. If you’d like to continue your API usage, you can return to the billing portal and use the “Add to balance” button to purchase additional credits.


Delayed billing
Due to the complexity of our billing and processing systems, there may be delays in our ability to cut off access after you consume all of your credits. This excess usage may appear as a negative credit balance in your billing dashboard, and will be deducted from your next credit purchase.
1. `call check_billing_plan(user_id)`
- **Function:** `check_billing_plan(user_id)`
- **Purpose:** Retrieves the user's current billing plan (e.g., Monthly Billing, Prepaid Billing, or Arrears Billing).
- **Parameters:**
- `user_id`: The unique identifier of the user.

2. If the user has an arrears billing plan:
2a. Inform the user: \"Please note that since you have an arrears billing plan, you'll need to cancel your existing payment plan before switching to prepaid billing. Would you like assistance with cancelling your current plan?\"
2b. If the user agrees, `call cancel_payment_plan(user_id)`
- **Function:** `cancel_payment_plan(user_id)`
- **Purpose:** Cancels the user's current arrears billing plan.
- **Parameters:**
- `user_id`: The unique identifier of the user.

3. Guide the user to set up prepaid billing:
3a. Instruct the user: \"Please go to your billing overview in your account settings.\"
3b. Instruct the user: \"Click on 'Start payment plan' (you may see variations like 'Buy credits').\"
3c. Inform the user: \"Choose the initial amount of credits you want to purchase. The minimum purchase is $5, and the maximum purchase will be based on your trust tier.\"
3d. Instruct the user: \"Confirm and purchase your initial amount of credits.\"
3e. Suggest to the user: \"You can set up auto-recharge to automatically add credits to your account when your balance falls below a set threshold.\"

4. Inform the user about credit expiration and refund policy:
4a. Inform the user: \"Please note that any purchased credits will expire after 1 year and they are non-refundable.\"

5. Inform the user about activation time:
5a. Inform the user: \"After you’ve purchased credits, you should be able to start using the API. Note that there may be a couple of minutes delay while our systems update to reflect your credit balance.\"

6. Ask the user: \"Is there anything else I can assist you with today?\"

7. If the user has no further questions, `call case_resolution()`
- **Function:** `case_resolution()`
- **Purpose:** Marks the case as resolved and ends the interaction.
3VAT Exemption requestHow do I submit a VAT exemption request?
Updated over a week ago
If you are eligible for a tax exemption and would like to apply it to your account, please follow these steps:

Depending on the state and if required:
1. Obtain a current tax exemption certificate from your state or local government and/or your fully completed non-profit sales tax exemption forms. The certificate/forms should include:
Your name and address
Tax exemption number
Signatures and dates, etc.
2. Send us a copy of your certificate using the chat widget in the bottom-right corner. Please include your organization id, invoice number or email address associated with the account, so we can easily find you. Instructions on how to find these items are below.

3. Once we receive your certificate/form, we will review it and apply the tax exemption to your account. You will receive a confirmation email once the exemption has been applied.

Please note that the tax exemption will only apply to future purchases. We cannot apply VAT exemptions retroactively.



Where to find your account data
In order to submit a Value Added Tax ('VAT') exemption request you will need the following from your organization Billing preferences:
1. Company name
2. Billing email
3. Primary business address
4. Business tax ID		1. Greet the customer and acknowledge their request.
1a. Say: \"Certainly, I'd be happy to assist you with submitting a VAT exemption request.\"

2. Request necessary information from the customer.
2a. Politely ask for the following:
- \"Could you please provide your **company name**?\"
- \"May I have the **billing email** associated with your account?\"
- \"Could you provide your **primary business address**?\"
- \"Please provide your **business tax ID**.\"
2b. If the customer needs assistance finding this information, provide guidance.
- Say: \"You can find this information in your organization's billing preferences.\"

3. Request a copy of their current tax exemption certificate.
3a. Say: \"Could you please send us a copy of your current **tax exemption certificate**? It should include your name and address, tax exemption number, signatures, and dates.\"

4. Instruct the customer on how to send the certificate.
4a. Say: \"You can send us a copy of your certificate using the **chat widget in the bottom-right corner**.\"
4b. Say: \"Please include your **organization ID**, **invoice number**, or the **email address associated with your account** so we can easily find you.\"

5. Once the customer has provided the required information and certificate:
5a. `call the process_vat_exemption_request function` with parameters: company_name, billing_email, business_address, business_tax_id, tax_exemption_certificate, account_identifier.
5b. **Define `process_vat_exemption_request function`**:
- **Purpose**: To review and apply the VAT exemption to the customer's account based on the provided information and certificate.
- **Parameters**:
- company_name
- billing_email
- business_address
- business_tax_id
- tax_exemption_certificate
- account_identifier (organization ID/invoice number/email address)

6. Inform the customer:
6a. Say: \"Thank you. Once we have reviewed your certificate, we will apply the VAT exemption to your account.\"
6b. Say: \"You will receive a confirmation email once the exemption has been applied.\"
6c. Say: \"Please note that the tax exemption will only apply to future purchases. We cannot apply VAT exemptions retroactively.\"
6d. If the customer requests to apply the VAT exemption to past purchases:
- Say: \"I apologize, but we're unable to apply VAT exemptions to past purchases. The tax exemption will only apply to future purchases once it's applied to your account.\"

7. Ask if there is anything more you can assist with.
7a. Say: \"Is there anything else I can assist you with today?\"

8. `call the case_resolution function`
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "df = pd.DataFrame(results)\n", + "\n", + "# Set display options to show all text in the dataframe cells\n", + "pd.set_option('display.max_colwidth', None)\n", + "\n", + "# Function to display formatted text in HTML\n", + "def display_formatted_dataframe(df):\n", + " def format_text(text):\n", + " return text.replace('\\n', '
')\n", + "\n", + " df_formatted = df.copy()\n", + " df_formatted['content'] = df_formatted['content'].apply(format_text)\n", + " df_formatted['routine'] = df_formatted['routine'].apply(format_text)\n", + " \n", + " display(HTML(df_formatted.to_html(escape=False, justify='left')))\n", + "\n", + "display_formatted_dataframe(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Results\n", + "\n", + "Upon reviewing the generated routines, we can derive several insights:\n", + "- Sample Responses: The model effectively generates sample responses that the LLM can utilize when executing the policy (e.g., “Instruct the user: ‘Confirm and purchase your initial amount of credits.’”).\n", + "- Discrete Steps: The model excels at decomposing the problem into discrete actions that the LLM needs to execute. Each instruction is clearly defined and easy to interpret.\n", + "- Function Definitions: The routines’ outputs include clearly defined functions to retrieve external information or trigger actions (e.g., `review_and_apply_tax_exemption`, `get_billing_plan`, `update_payment_method`).\n", + " - This is crucial for any successful routine because LLMs often need to interact with external systems. Leveraging function calls is an effective way to interact with those systems and execute actions.\n", + "- IFTTT Logic: The model effectively employs IFTTT (If This, Then That) logic, which is ideal for an LLM (e.g., “If the customer requests assistance, proceed to step 3f.”).\n", + " - This type of translation becomes extremely valuable when the original knowledge base articles contain complex workflows and diagrams. Such complexity may not be easily understood by humans, and even less so by an LLM. IFTTT logic is easily comprehensible and works well for customer service solution" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Where do we go from here?\n", + "\n", + "These routines can now be integrated into agentic systems to address specific customer issues. When a customer requests assistance with tasks such as setting up prepaid billing, we can employ a classifier to determine the appropriate routine to retrieve and provide that to the LLM to interact directly with the customer. Beyond providing instructions to the user on *how* to set up billing, the system can also perform the action on their behalf.\n", + "\n", + "Before deploying these routines into production, we should develop comprehensive evaluations to test and validate the quality of the model’s responses. This process may necessitate adjusting the routines to ensure compliance and effectiveness." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/registry.yaml b/registry.yaml index 1b507d14..5b00ef3b 100644 --- a/registry.yaml +++ b/registry.yaml @@ -1568,3 +1568,21 @@ tags: - gpt-actions-library - chatgpt + +- title: Using reasoning for data validation + path: examples/o1/Using_reasoning_for_data_validation.ipynb + date: 2024-09-12 + authors: + - royziv11 + tags: + - completions + - reasoning + +- title: Using reasoning for routine generation + path: examples/o1/Using_reasoning_for_routine_generation.ipynb + date: 2024-09-12 + authors: + - royziv11 + tags: + - completions + - reasoning