@ -1,9 +1,13 @@
""" Wrapper around the C Transformers library. """
""" Wrapper around the C Transformers library. """
from typing import Any , Dict , Optional , Sequence
from functools import partial
from typing import Any , Dict , List , Optional , Sequence
from pydantic import root_validator
from pydantic import root_validator
from langchain . callbacks . manager import CallbackManagerForLLMRun
from langchain . callbacks . manager import (
AsyncCallbackManagerForLLMRun ,
CallbackManagerForLLMRun ,
)
from langchain . llms . base import LLM
from langchain . llms . base import LLM
@ -103,3 +107,36 @@ class CTransformers(LLM):
text . append ( chunk )
text . append ( chunk )
_run_manager . on_llm_new_token ( chunk , verbose = self . verbose )
_run_manager . on_llm_new_token ( chunk , verbose = self . verbose )
return " " . join ( text )
return " " . join ( text )
async def _acall (
self ,
prompt : str ,
stop : Optional [ List [ str ] ] = None ,
run_manager : Optional [ AsyncCallbackManagerForLLMRun ] = None ,
* * kwargs : Any ,
) - > str :
""" Asynchronous Call out to CTransformers generate method.
Very helpful when streaming ( like with websockets ! )
Args :
prompt : The prompt to pass into the model .
stop : A list of strings to stop generation when encountered .
Returns :
The string generated by the model .
Example :
. . code - block : : python
response = llm ( " Once upon a time, " )
"""
text_callback = None
if run_manager :
text_callback = partial ( run_manager . on_llm_new_token , verbose = self . verbose )
text = " "
for token in self . client ( prompt , stop = stop , stream = True ) :
if text_callback :
await text_callback ( token )
text + = token
return text