|
|
|
@ -673,17 +673,13 @@
|
|
|
|
|
" # get texts to encode\n",
|
|
|
|
|
" texts = [x['text'] for x in meta_batch]\n",
|
|
|
|
|
" # create embeddings (try-except added to avoid RateLimitError)\n",
|
|
|
|
|
" try:\n",
|
|
|
|
|
" res = openai.Embedding.create(input=texts, engine=embed_model)\n",
|
|
|
|
|
" except:\n",
|
|
|
|
|
" done = False\n",
|
|
|
|
|
" while not done:\n",
|
|
|
|
|
" sleep(5)\n",
|
|
|
|
|
" try:\n",
|
|
|
|
|
" res = openai.Embedding.create(input=texts, engine=embed_model)\n",
|
|
|
|
|
" done = True\n",
|
|
|
|
|
" except:\n",
|
|
|
|
|
" pass\n",
|
|
|
|
|
" done = False\n",
|
|
|
|
|
" while not done:\n",
|
|
|
|
|
" try:\n",
|
|
|
|
|
" res = openai.Embedding.create(input=texts, engine=embed_model)\n",
|
|
|
|
|
" done = True\n",
|
|
|
|
|
" except:\n",
|
|
|
|
|
" sleep(5)\n",
|
|
|
|
|
" embeds = [record['embedding'] for record in res['data']]\n",
|
|
|
|
|
" # cleanup metadata\n",
|
|
|
|
|
" meta_batch = [{\n",
|
|
|
|
|