Merge pull request #92 from Androbin/patch-1

Encode separator for completions endpoint
pull/114/head
Ted Sanders 1 year ago committed by GitHub
commit 0acccd2ad4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -554,7 +554,7 @@
"source": [
"MAX_SECTION_LEN = 500\n",
"SEPARATOR = \"\\n* \"\n",
"ENCODING = \"cl100k_base\" # encoding for text-embedding-ada-002\n",
"ENCODING = \"gpt2\" # encoding for text-davinci-003\n",
"\n",
"encoding = tiktoken.get_encoding(ENCODING)\n",
"separator_len = len(encoding.encode(SEPARATOR))\n",

Loading…
Cancel
Save