From a9f8eb35189e19e804a4106004fa5d9b5b9c04c4 Mon Sep 17 00:00:00 2001 From: Avery Khoo Date: Sat, 24 Jun 2023 07:10:30 +0800 Subject: [PATCH] Update How_to_count_tokens_with_tiktoken.ipynb (#511) Add support for gpt-3.5-*-0613 models --- examples/How_to_count_tokens_with_tiktoken.ipynb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/How_to_count_tokens_with_tiktoken.ipynb b/examples/How_to_count_tokens_with_tiktoken.ipynb index e14073dd..f4bcd7da 100644 --- a/examples/How_to_count_tokens_with_tiktoken.ipynb +++ b/examples/How_to_count_tokens_with_tiktoken.ipynb @@ -441,13 +441,16 @@ " if model == \"gpt-3.5-turbo\":\n", " print(\"Warning: gpt-3.5-turbo may change over time. Returning num tokens assuming gpt-3.5-turbo-0301.\")\n", " return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-0301\")\n", + " elif model == \"gpt-3.5-turbo-16k\":\n", + " print(\"Warning: gpt-3.5-turbo-16k may change over time. Returning num tokens assuming gpt-3.5-turbo-16k-0613.\")\n", + " return num_tokens_from_messages(messages, model=\"gpt-3.5-turbo-16k-0613\")\n", " elif model == \"gpt-4\":\n", " print(\"Warning: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.\")\n", " return num_tokens_from_messages(messages, model=\"gpt-4-0314\")\n", " elif model == \"gpt-3.5-turbo-0301\":\n", " tokens_per_message = 4 # every message follows <|start|>{role/name}\\n{content}<|end|>\\n\n", " tokens_per_name = -1 # if there's a name, the role is omitted\n", - " elif model == \"gpt-4-0314\":\n", + " elif model in {\"gpt-4-0314\", \"gpt-3.5-turbo-0613\", \"gpt-3.5-turbo-16k-0613\"}:\n", " tokens_per_message = 3\n", " tokens_per_name = 1\n", " else:\n",