mirror of https://github.com/xtekky/gpt4free
Add local models to gui, Fix You Provider, add AsyncClient
Heiner Lohaus
2 months ago
parent
674ba8f2c5
commit
b35dfcd1b0
@ -0,0 +1,42 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from ..locals.models import get_models
|
||||||
|
try:
|
||||||
|
from ..locals.provider import LocalProvider
|
||||||
|
has_requirements = True
|
||||||
|
except ModuleNotFoundError:
|
||||||
|
has_requirements = False
|
||||||
|
|
||||||
|
from ..typing import Messages, CreateResult
|
||||||
|
from ..providers.base_provider import AbstractProvider, ProviderModelMixin
|
||||||
|
from ..errors import MissingRequirementsError
|
||||||
|
|
||||||
|
class Local(AbstractProvider, ProviderModelMixin):
|
||||||
|
working = True
|
||||||
|
supports_message_history = True
|
||||||
|
supports_system_message = True
|
||||||
|
supports_stream = True
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_models(cls):
|
||||||
|
if not cls.models:
|
||||||
|
cls.models = list(get_models())
|
||||||
|
cls.default_model = cls.models[0]
|
||||||
|
return cls.models
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create_completion(
|
||||||
|
cls,
|
||||||
|
model: str,
|
||||||
|
messages: Messages,
|
||||||
|
stream: bool,
|
||||||
|
**kwargs
|
||||||
|
) -> CreateResult:
|
||||||
|
if not has_requirements:
|
||||||
|
raise MissingRequirementsError('Install "gpt4all" package | pip install -U g4f[local]')
|
||||||
|
return LocalProvider.create_completion(
|
||||||
|
cls.get_model(model),
|
||||||
|
messages,
|
||||||
|
stream,
|
||||||
|
**kwargs
|
||||||
|
)
|
||||||
@ -0,0 +1,70 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from ...requests import StreamSession, raise_for_status
|
||||||
|
|
||||||
|
class NoValidHarFileError(Exception):
|
||||||
|
...
|
||||||
|
|
||||||
|
class arkReq:
|
||||||
|
def __init__(self, arkURL, arkHeaders, arkBody, arkCookies, userAgent):
|
||||||
|
self.arkURL = arkURL
|
||||||
|
self.arkHeaders = arkHeaders
|
||||||
|
self.arkBody = arkBody
|
||||||
|
self.arkCookies = arkCookies
|
||||||
|
self.userAgent = userAgent
|
||||||
|
|
||||||
|
arkPreURL = "https://telemetry.stytch.com/submit"
|
||||||
|
chatArks: list = None
|
||||||
|
|
||||||
|
def readHAR():
|
||||||
|
dirPath = "./"
|
||||||
|
harPath = []
|
||||||
|
chatArks = []
|
||||||
|
for root, dirs, files in os.walk(dirPath):
|
||||||
|
for file in files:
|
||||||
|
if file.endswith(".har"):
|
||||||
|
harPath.append(os.path.join(root, file))
|
||||||
|
if harPath:
|
||||||
|
break
|
||||||
|
if not harPath:
|
||||||
|
raise NoValidHarFileError("No .har file found")
|
||||||
|
for path in harPath:
|
||||||
|
with open(path, 'rb') as file:
|
||||||
|
try:
|
||||||
|
harFile = json.load(file)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
# Error: not a HAR file!
|
||||||
|
continue
|
||||||
|
for v in harFile['log']['entries']:
|
||||||
|
if arkPreURL in v['request']['url']:
|
||||||
|
chatArks.append(parseHAREntry(v))
|
||||||
|
if not chatArks:
|
||||||
|
raise NoValidHarFileError("No telemetry in .har files found")
|
||||||
|
return chatArks
|
||||||
|
|
||||||
|
def parseHAREntry(entry) -> arkReq:
|
||||||
|
tmpArk = arkReq(
|
||||||
|
arkURL=entry['request']['url'],
|
||||||
|
arkHeaders={h['name'].lower(): h['value'] for h in entry['request']['headers'] if h['name'].lower() not in ['content-length', 'cookie'] and not h['name'].startswith(':')},
|
||||||
|
arkBody=entry['request']['postData']['text'],
|
||||||
|
arkCookies={c['name']: c['value'] for c in entry['request']['cookies']},
|
||||||
|
userAgent=""
|
||||||
|
)
|
||||||
|
tmpArk.userAgent = tmpArk.arkHeaders.get('user-agent', '')
|
||||||
|
return tmpArk
|
||||||
|
|
||||||
|
async def sendRequest(tmpArk: arkReq, proxy: str = None):
|
||||||
|
async with StreamSession(headers=tmpArk.arkHeaders, cookies=tmpArk.arkCookies, proxies={"all": proxy}) as session:
|
||||||
|
async with session.post(tmpArk.arkURL, data=tmpArk.arkBody) as response:
|
||||||
|
await raise_for_status(response)
|
||||||
|
return await response.text()
|
||||||
|
|
||||||
|
async def get_dfp_telemetry_id(proxy: str = None):
|
||||||
|
return str(uuid.uuid4())
|
||||||
|
global chatArks
|
||||||
|
if chatArks is None:
|
||||||
|
chatArks = readHAR()
|
||||||
|
return await sendRequest(random.choice(chatArks), proxy)
|
||||||
@ -0,0 +1,3 @@
|
|||||||
|
from .stubs import ChatCompletion, ChatCompletionChunk, ImagesResponse
|
||||||
|
from .client import Client
|
||||||
|
from .async_client import AsyncClient
|
||||||
@ -0,0 +1,114 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
|
from .. import debug, version
|
||||||
|
from ..errors import ProviderNotFoundError, ModelNotFoundError, ProviderNotWorkingError, StreamNotSupportedError
|
||||||
|
from ..models import Model, ModelUtils
|
||||||
|
from ..Provider import ProviderUtils
|
||||||
|
from ..providers.types import BaseRetryProvider, ProviderType
|
||||||
|
from ..providers.retry_provider import IterProvider
|
||||||
|
|
||||||
|
def convert_to_provider(provider: str) -> ProviderType:
|
||||||
|
if " " in provider:
|
||||||
|
provider_list = [ProviderUtils.convert[p] for p in provider.split() if p in ProviderUtils.convert]
|
||||||
|
if not provider_list:
|
||||||
|
raise ProviderNotFoundError(f'Providers not found: {provider}')
|
||||||
|
provider = IterProvider(provider_list)
|
||||||
|
elif provider in ProviderUtils.convert:
|
||||||
|
provider = ProviderUtils.convert[provider]
|
||||||
|
elif provider:
|
||||||
|
raise ProviderNotFoundError(f'Provider not found: {provider}')
|
||||||
|
return provider
|
||||||
|
|
||||||
|
def get_model_and_provider(model : Union[Model, str],
|
||||||
|
provider : Union[ProviderType, str, None],
|
||||||
|
stream : bool,
|
||||||
|
ignored : list[str] = None,
|
||||||
|
ignore_working: bool = False,
|
||||||
|
ignore_stream: bool = False) -> tuple[str, ProviderType]:
|
||||||
|
"""
|
||||||
|
Retrieves the model and provider based on input parameters.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
model (Union[Model, str]): The model to use, either as an object or a string identifier.
|
||||||
|
provider (Union[ProviderType, str, None]): The provider to use, either as an object, a string identifier, or None.
|
||||||
|
stream (bool): Indicates if the operation should be performed as a stream.
|
||||||
|
ignored (list[str], optional): List of provider names to be ignored.
|
||||||
|
ignore_working (bool, optional): If True, ignores the working status of the provider.
|
||||||
|
ignore_stream (bool, optional): If True, ignores the streaming capability of the provider.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
tuple[str, ProviderType]: A tuple containing the model name and the provider type.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ProviderNotFoundError: If the provider is not found.
|
||||||
|
ModelNotFoundError: If the model is not found.
|
||||||
|
ProviderNotWorkingError: If the provider is not working.
|
||||||
|
StreamNotSupportedError: If streaming is not supported by the provider.
|
||||||
|
"""
|
||||||
|
if debug.version_check:
|
||||||
|
debug.version_check = False
|
||||||
|
version.utils.check_version()
|
||||||
|
|
||||||
|
if isinstance(provider, str):
|
||||||
|
provider = convert_to_provider(provider)
|
||||||
|
|
||||||
|
if isinstance(model, str):
|
||||||
|
if model in ModelUtils.convert:
|
||||||
|
model = ModelUtils.convert[model]
|
||||||
|
|
||||||
|
if not provider:
|
||||||
|
if isinstance(model, str):
|
||||||
|
raise ModelNotFoundError(f'Model not found: {model}')
|
||||||
|
provider = model.best_provider
|
||||||
|
|
||||||
|
if not provider:
|
||||||
|
raise ProviderNotFoundError(f'No provider found for model: {model}')
|
||||||
|
|
||||||
|
if isinstance(model, Model):
|
||||||
|
model = model.name
|
||||||
|
|
||||||
|
if not ignore_working and not provider.working:
|
||||||
|
raise ProviderNotWorkingError(f'{provider.__name__} is not working')
|
||||||
|
|
||||||
|
if not ignore_working and isinstance(provider, BaseRetryProvider):
|
||||||
|
provider.providers = [p for p in provider.providers if p.working]
|
||||||
|
|
||||||
|
if ignored and isinstance(provider, BaseRetryProvider):
|
||||||
|
provider.providers = [p for p in provider.providers if p.__name__ not in ignored]
|
||||||
|
|
||||||
|
if not ignore_stream and not provider.supports_stream and stream:
|
||||||
|
raise StreamNotSupportedError(f'{provider.__name__} does not support "stream" argument')
|
||||||
|
|
||||||
|
if debug.logging:
|
||||||
|
if model:
|
||||||
|
print(f'Using {provider.__name__} provider and {model} model')
|
||||||
|
else:
|
||||||
|
print(f'Using {provider.__name__} provider')
|
||||||
|
|
||||||
|
debug.last_provider = provider
|
||||||
|
debug.last_model = model
|
||||||
|
|
||||||
|
return model, provider
|
||||||
|
|
||||||
|
def get_last_provider(as_dict: bool = False) -> Union[ProviderType, dict[str, str]]:
|
||||||
|
"""
|
||||||
|
Retrieves the last used provider.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
as_dict (bool, optional): If True, returns the provider information as a dictionary.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Union[ProviderType, dict[str, str]]: The last used provider, either as an object or a dictionary.
|
||||||
|
"""
|
||||||
|
last = debug.last_provider
|
||||||
|
if isinstance(last, BaseRetryProvider):
|
||||||
|
last = last.last_provider
|
||||||
|
if last and as_dict:
|
||||||
|
return {
|
||||||
|
"name": last.__name__,
|
||||||
|
"url": last.url,
|
||||||
|
"model": debug.last_model,
|
||||||
|
}
|
||||||
|
return last
|
||||||
@ -1,42 +0,0 @@
|
|||||||
import os
|
|
||||||
|
|
||||||
from gpt4all import GPT4All
|
|
||||||
from ._models import models
|
|
||||||
|
|
||||||
class LocalProvider:
|
|
||||||
@staticmethod
|
|
||||||
def create_completion(model, messages, stream, **kwargs):
|
|
||||||
if model not in models:
|
|
||||||
raise ValueError(f"Model '{model}' not found / not yet implemented")
|
|
||||||
|
|
||||||
model = models[model]
|
|
||||||
model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models/')
|
|
||||||
full_model_path = os.path.join(model_dir, model['path'])
|
|
||||||
|
|
||||||
if not os.path.isfile(full_model_path):
|
|
||||||
print(f"Model file '{full_model_path}' not found.")
|
|
||||||
download = input(f'Do you want to download {model["path"]} ? [y/n]')
|
|
||||||
|
|
||||||
if download in ['y', 'Y']:
|
|
||||||
GPT4All.download_model(model['path'], model_dir)
|
|
||||||
else:
|
|
||||||
raise ValueError(f"Model '{model['path']}' not found.")
|
|
||||||
|
|
||||||
model = GPT4All(model_name=model['path'],
|
|
||||||
#n_threads=8,
|
|
||||||
verbose=False,
|
|
||||||
allow_download=False,
|
|
||||||
model_path=model_dir)
|
|
||||||
|
|
||||||
system_template = next((message['content'] for message in messages if message['role'] == 'system'),
|
|
||||||
'A chat between a curious user and an artificial intelligence assistant.')
|
|
||||||
|
|
||||||
prompt_template = 'USER: {0}\nASSISTANT: '
|
|
||||||
conversation = '\n'.join(f"{msg['role'].upper()}: {msg['content']}" for msg in messages) + "\nASSISTANT: "
|
|
||||||
|
|
||||||
with model.chat_session(system_template, prompt_template):
|
|
||||||
if stream:
|
|
||||||
for token in model.generate(conversation, streaming=True):
|
|
||||||
yield token
|
|
||||||
else:
|
|
||||||
yield model.generate(conversation)
|
|
||||||
@ -1,86 +0,0 @@
|
|||||||
models = {
|
|
||||||
"mistral-7b": {
|
|
||||||
"path": "mistral-7b-openorca.gguf2.Q4_0.gguf",
|
|
||||||
"ram": "8",
|
|
||||||
"prompt": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n",
|
|
||||||
"system": "<|im_start|>system\nYou are MistralOrca, a large language model trained by Alignment Lab AI. For multi-step problems, write out your reasoning for each step.\n<|im_end|>"
|
|
||||||
},
|
|
||||||
"mistral-7b-instruct": {
|
|
||||||
"path": "mistral-7b-instruct-v0.1.Q4_0.gguf",
|
|
||||||
"ram": "8",
|
|
||||||
"prompt": "[INST] %1 [/INST]",
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"gpt4all-falcon": {
|
|
||||||
"path": "gpt4all-falcon-newbpe-q4_0.gguf",
|
|
||||||
"ram": "8",
|
|
||||||
"prompt": "### Instruction:\n%1\n### Response:\n",
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"orca-2": {
|
|
||||||
"path": "orca-2-13b.Q4_0.gguf",
|
|
||||||
"ram": "16",
|
|
||||||
"prompt": None,
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"wizardlm-13b": {
|
|
||||||
"path": "wizardlm-13b-v1.2.Q4_0.gguf",
|
|
||||||
"ram": "16",
|
|
||||||
"prompt": None,
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"nous-hermes-llama2": {
|
|
||||||
"path": "nous-hermes-llama2-13b.Q4_0.gguf",
|
|
||||||
"ram": "16",
|
|
||||||
"prompt": "### Instruction:\n%1\n### Response:\n",
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"gpt4all-13b-snoozy": {
|
|
||||||
"path": "gpt4all-13b-snoozy-q4_0.gguf",
|
|
||||||
"ram": "16",
|
|
||||||
"prompt": None,
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"mpt-7b-chat": {
|
|
||||||
"path": "mpt-7b-chat-newbpe-q4_0.gguf",
|
|
||||||
"ram": "8",
|
|
||||||
"prompt": "<|im_start|>user\n%1<|im_end|>\n<|im_start|>assistant\n",
|
|
||||||
"system": "<|im_start|>system\n- You are a helpful assistant chatbot trained by MosaicML.\n- You answer questions.\n- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.\n- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.<|im_end|>"
|
|
||||||
},
|
|
||||||
"orca-mini-3b": {
|
|
||||||
"path": "orca-mini-3b-gguf2-q4_0.gguf",
|
|
||||||
"ram": "4",
|
|
||||||
"prompt": "### User:\n%1\n### Response:\n",
|
|
||||||
"system": "### System:\nYou are an AI assistant that follows instruction extremely well. Help as much as you can.\n\n"
|
|
||||||
},
|
|
||||||
"replit-code-3b": {
|
|
||||||
"path": "replit-code-v1_5-3b-newbpe-q4_0.gguf",
|
|
||||||
"ram": "4",
|
|
||||||
"prompt": "%1",
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"starcoder": {
|
|
||||||
"path": "starcoder-newbpe-q4_0.gguf",
|
|
||||||
"ram": "4",
|
|
||||||
"prompt": "%1",
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"rift-coder-7b": {
|
|
||||||
"path": "rift-coder-v0-7b-q4_0.gguf",
|
|
||||||
"ram": "8",
|
|
||||||
"prompt": "%1",
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"all-MiniLM-L6-v2": {
|
|
||||||
"path": "all-MiniLM-L6-v2-f16.gguf",
|
|
||||||
"ram": "1",
|
|
||||||
"prompt": None,
|
|
||||||
"system": None
|
|
||||||
},
|
|
||||||
"mistral-7b-german": {
|
|
||||||
"path": "em_german_mistral_v01.Q4_0.gguf",
|
|
||||||
"ram": "8",
|
|
||||||
"prompt": "USER: %1 ASSISTANT: ",
|
|
||||||
"system": "Du bist ein hilfreicher Assistent. "
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1 +0,0 @@
|
|||||||
.
|
|
||||||
@ -0,0 +1,50 @@
|
|||||||
|
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
|
||||||
|
from ..requests.raise_for_status import raise_for_status
|
||||||
|
|
||||||
|
def load_models():
|
||||||
|
response = requests.get("https://gpt4all.io/models/models3.json")
|
||||||
|
raise_for_status(response)
|
||||||
|
return format_models(response.json())
|
||||||
|
|
||||||
|
def get_model_name(filename: str) -> str:
|
||||||
|
name = filename.split(".", 1)[0]
|
||||||
|
for replace in ["-v1_5", "-v1", "-q4_0", "_v01", "-v0", "-f16", "-gguf2", "-newbpe"]:
|
||||||
|
name = name.replace(replace, "")
|
||||||
|
return name
|
||||||
|
|
||||||
|
def format_models(models: list) -> dict:
|
||||||
|
return {get_model_name(model["filename"]): {
|
||||||
|
"path": model["filename"],
|
||||||
|
"ram": model["ramrequired"],
|
||||||
|
"prompt": model["promptTemplate"] if "promptTemplate" in model else None,
|
||||||
|
"system": model["systemPrompt"] if "systemPrompt" in model else None,
|
||||||
|
} for model in models}
|
||||||
|
|
||||||
|
def read_models(file_path: str):
|
||||||
|
with open(file_path, "rb") as f:
|
||||||
|
return json.load(f)
|
||||||
|
|
||||||
|
def save_models(file_path: str, data):
|
||||||
|
with open(file_path, 'w') as f:
|
||||||
|
json.dump(data, f, indent=4)
|
||||||
|
|
||||||
|
def get_model_dir() -> str:
|
||||||
|
local_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
project_dir = os.path.dirname(os.path.dirname(local_dir))
|
||||||
|
model_dir = os.path.join(project_dir, "models")
|
||||||
|
if os.path.exists(model_dir):
|
||||||
|
return model_dir
|
||||||
|
|
||||||
|
def get_models() -> dict[str, dict]:
|
||||||
|
model_dir = get_model_dir()
|
||||||
|
file_path = os.path.join(model_dir, "models.json")
|
||||||
|
if os.path.isfile(file_path):
|
||||||
|
return read_models(file_path)
|
||||||
|
else:
|
||||||
|
models = load_models()
|
||||||
|
save_models(file_path, models)
|
||||||
|
return models
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
|
from gpt4all import GPT4All
|
||||||
|
from .models import get_models
|
||||||
|
from ..typing import Messages
|
||||||
|
|
||||||
|
MODEL_LIST: dict[str, dict] = None
|
||||||
|
|
||||||
|
def find_model_dir(model_file: str) -> str:
|
||||||
|
local_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
project_dir = os.path.dirname(os.path.dirname(local_dir))
|
||||||
|
|
||||||
|
new_model_dir = os.path.join(project_dir, "models")
|
||||||
|
new_model_file = os.path.join(new_model_dir, model_file)
|
||||||
|
if os.path.isfile(new_model_file):
|
||||||
|
return new_model_dir
|
||||||
|
|
||||||
|
old_model_dir = os.path.join(local_dir, "models")
|
||||||
|
old_model_file = os.path.join(old_model_dir, model_file)
|
||||||
|
if os.path.isfile(old_model_file):
|
||||||
|
return old_model_dir
|
||||||
|
|
||||||
|
working_dir = "./"
|
||||||
|
for root, dirs, files in os.walk(working_dir):
|
||||||
|
if model_file in files:
|
||||||
|
return root
|
||||||
|
|
||||||
|
return new_model_dir
|
||||||
|
|
||||||
|
class LocalProvider:
|
||||||
|
@staticmethod
|
||||||
|
def create_completion(model: str, messages: Messages, stream: bool = False, **kwargs):
|
||||||
|
global MODEL_LIST
|
||||||
|
if MODEL_LIST is None:
|
||||||
|
MODEL_LIST = get_models()
|
||||||
|
if model not in MODEL_LIST:
|
||||||
|
raise ValueError(f'Model "{model}" not found / not yet implemented')
|
||||||
|
|
||||||
|
model = MODEL_LIST[model]
|
||||||
|
model_file = model["path"]
|
||||||
|
model_dir = find_model_dir(model_file)
|
||||||
|
if not os.path.isfile(os.path.join(model_dir, model_file)):
|
||||||
|
print(f'Model file "models/{model_file}" not found.')
|
||||||
|
download = input(f"Do you want to download {model_file}? [y/n]: ")
|
||||||
|
if download in ["y", "Y"]:
|
||||||
|
GPT4All.download_model(model_file, model_dir)
|
||||||
|
else:
|
||||||
|
raise ValueError(f'Model "{model_file}" not found.')
|
||||||
|
|
||||||
|
model = GPT4All(model_name=model_file,
|
||||||
|
#n_threads=8,
|
||||||
|
verbose=False,
|
||||||
|
allow_download=False,
|
||||||
|
model_path=model_dir)
|
||||||
|
|
||||||
|
system_message = "\n".join(message["content"] for message in messages if message["role"] == "system")
|
||||||
|
if system_message:
|
||||||
|
system_message = "A chat between a curious user and an artificial intelligence assistant."
|
||||||
|
|
||||||
|
prompt_template = "USER: {0}\nASSISTANT: "
|
||||||
|
conversation = "\n" . join(
|
||||||
|
f"{message['role'].upper()}: {message['content']}"
|
||||||
|
for message in messages
|
||||||
|
if message["role"] != "system"
|
||||||
|
) + "\nASSISTANT: "
|
||||||
|
|
||||||
|
with model.chat_session(system_message, prompt_template):
|
||||||
|
if stream:
|
||||||
|
for token in model.generate(conversation, streaming=True):
|
||||||
|
yield token
|
||||||
|
else:
|
||||||
|
yield model.generate(conversation)
|
||||||
Loading…
Reference in New Issue