Encode separator for completions endpoint

`SEPARATOR` is being used to construct a prompt for use with the completions endpoint
pull/1077/head
Robin Richtsfeld 2 years ago committed by GitHub
parent d063f1d86d
commit b13e040443

@ -554,7 +554,7 @@
"source": [ "source": [
"MAX_SECTION_LEN = 500\n", "MAX_SECTION_LEN = 500\n",
"SEPARATOR = \"\\n* \"\n", "SEPARATOR = \"\\n* \"\n",
"ENCODING = \"cl100k_base\" # encoding for text-embedding-ada-002\n", "ENCODING = \"gpt2\" # encoding for text-davinci-003\n",
"\n", "\n",
"encoding = tiktoken.get_encoding(ENCODING)\n", "encoding = tiktoken.get_encoding(ENCODING)\n",
"separator_len = len(encoding.encode(SEPARATOR))\n", "separator_len = len(encoding.encode(SEPARATOR))\n",

Loading…
Cancel
Save