@ -178,7 +178,7 @@ class OpenAIEmbeddings(BaseModel, Embeddings):
openai . api_type = openai_api_type
openai . api_type = openai_api_type
values [ " client " ] = openai . Embedding
values [ " client " ] = openai . Embedding
except ImportError :
except ImportError :
raise Value Error(
raise Import Error(
" Could not import openai python package. "
" Could not import openai python package. "
" Please install it with `pip install openai`. "
" Please install it with `pip install openai`. "
)
)
@ -192,6 +192,12 @@ class OpenAIEmbeddings(BaseModel, Embeddings):
embeddings : List [ List [ float ] ] = [ [ ] for _ in range ( len ( texts ) ) ]
embeddings : List [ List [ float ] ] = [ [ ] for _ in range ( len ( texts ) ) ]
try :
try :
import tiktoken
import tiktoken
except ImportError :
raise ImportError (
" Could not import tiktoken python package. "
" This is needed in order to for OpenAIEmbeddings. "
" Please install it with `pip install tiktoken`. "
)
tokens = [ ]
tokens = [ ]
indices = [ ]
indices = [ ]
@ -239,20 +245,11 @@ class OpenAIEmbeddings(BaseModel, Embeddings):
headers = self . headers ,
headers = self . headers ,
) [ " data " ] [ 0 ] [ " embedding " ]
) [ " data " ] [ 0 ] [ " embedding " ]
else :
else :
average = np . average (
average = np . average ( _result , axis = 0 , weights = num_tokens_in_batch [ i ] )
_result , axis = 0 , weights = num_tokens_in_batch [ i ]
)
embeddings [ i ] = ( average / np . linalg . norm ( average ) ) . tolist ( )
embeddings [ i ] = ( average / np . linalg . norm ( average ) ) . tolist ( )
return embeddings
return embeddings
except ImportError :
raise ValueError (
" Could not import tiktoken python package. "
" This is needed in order to for OpenAIEmbeddings. "
" Please install it with `pip install tiktoken`. "
)
def _embedding_func ( self , text : str , * , engine : str ) - > List [ float ] :
def _embedding_func ( self , text : str , * , engine : str ) - > List [ float ] :
""" Call out to OpenAI ' s embedding endpoint. """
""" Call out to OpenAI ' s embedding endpoint. """
# handle large input text
# handle large input text