|
|
@ -204,8 +204,9 @@
|
|
|
|
" chunk_embeddings.append(get_embedding(chunk, model=model))\n",
|
|
|
|
" chunk_embeddings.append(get_embedding(chunk, model=model))\n",
|
|
|
|
"\n",
|
|
|
|
"\n",
|
|
|
|
" if average:\n",
|
|
|
|
" if average:\n",
|
|
|
|
" chunk_embeddings = np.average(chunk_embeddings, axis=0, weights=[len(c) for c in chunk_embeddings]).tolist()\n",
|
|
|
|
" chunk_embeddings = np.average(chunk_embeddings, axis=0, weights=[len(c) for c in chunk_embeddings])\n",
|
|
|
|
" chunk_embeddings = chunk_embeddings / np.linalg.norm(chunk_embeddings) # normalizes length to 1\n",
|
|
|
|
" chunk_embeddings = chunk_embeddings / np.linalg.norm(chunk_embeddings) # normalizes length to 1\n",
|
|
|
|
|
|
|
|
" chunk_embeddings = chunk_embeddings.tolist()\n",
|
|
|
|
" return chunk_embeddings"
|
|
|
|
" return chunk_embeddings"
|
|
|
|
]
|
|
|
|
]
|
|
|
|
},
|
|
|
|
},
|
|
|
|