From 8f99dca70feac094f7af03085e1db55f922bfb09 Mon Sep 17 00:00:00 2001 From: Adam Treat Date: Wed, 13 Sep 2023 10:32:08 -0400 Subject: [PATCH] Bring the vulkan backend to the GUI. --- gpt4all-backend/llama.cpp-mainline | 2 +- gpt4all-chat/chatllm.cpp | 34 ++++++++++++ gpt4all-chat/chatllm.h | 1 + gpt4all-chat/mysettings.cpp | 37 ++++++++++++- gpt4all-chat/mysettings.h | 10 ++++ gpt4all-chat/qml/ApplicationSettings.qml | 66 +++++++++++++++++++----- 6 files changed, 133 insertions(+), 17 deletions(-) diff --git a/gpt4all-backend/llama.cpp-mainline b/gpt4all-backend/llama.cpp-mainline index 9bee309a..2f7732b6 160000 --- a/gpt4all-backend/llama.cpp-mainline +++ b/gpt4all-backend/llama.cpp-mainline @@ -1 +1 @@ -Subproject commit 9bee309a7c8db77ca53fab49c2d896d486905617 +Subproject commit 2f7732b667b5c7786da0fa59fd612cc87b04b325 diff --git a/gpt4all-chat/chatllm.cpp b/gpt4all-chat/chatllm.cpp index e49c4207..58d68e60 100644 --- a/gpt4all-chat/chatllm.cpp +++ b/gpt4all-chat/chatllm.cpp @@ -81,6 +81,7 @@ ChatLLM::ChatLLM(Chat *parent, bool isServer) connect(parent, &Chat::idChanged, this, &ChatLLM::handleChatIdChanged); connect(&m_llmThread, &QThread::started, this, &ChatLLM::handleThreadStarted); connect(MySettings::globalInstance(), &MySettings::forceMetalChanged, this, &ChatLLM::handleForceMetalChanged); + connect(MySettings::globalInstance(), &MySettings::deviceChanged, this, &ChatLLM::handleDeviceChanged); // The following are blocking operations and will block the llm thread connect(this, &ChatLLM::requestRetrieveFromDB, LocalDocs::globalInstance()->database(), &Database::retrieveFromDB, @@ -124,6 +125,16 @@ void ChatLLM::handleForceMetalChanged(bool forceMetal) #endif } +void ChatLLM::handleDeviceChanged() +{ + if (isModelLoaded() && m_shouldBeLoaded) { + m_reloadingToChangeVariant = true; + unloadModel(); + reloadModel(); + m_reloadingToChangeVariant = false; + } +} + bool ChatLLM::loadDefaultModel() { ModelInfo defaultModel = ModelList::globalInstance()->defaultModelInfo(); @@ -250,7 +261,30 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo) #endif if (m_llModelInfo.model) { + // Update the settings that a model is being loaded and update the device list MySettings::globalInstance()->setAttemptModelLoad(filePath); + std::vector devices = m_llModelInfo.model->availableGPUDevices(0); + QVector deviceList{ "Auto" }; + for (LLModel::GPUDevice &d : devices) + deviceList << QString::fromStdString(d.name); + deviceList << "CPU"; + MySettings::globalInstance()->setDeviceList(deviceList); + + // Pick the best match for the device + const QString requestedDevice = MySettings::globalInstance()->device(); + if (requestedDevice != "CPU") { + const size_t requiredMemory = m_llModelInfo.model->requiredMem(filePath.toStdString()); + std::vector availableDevices = m_llModelInfo.model->availableGPUDevices(requiredMemory); + if (!availableDevices.empty() && requestedDevice == "Auto") { + m_llModelInfo.model->initializeGPUDevice(devices.front()); + } else { + for (LLModel::GPUDevice &d : availableDevices) { + if (QString::fromStdString(d.name) == requestedDevice) + m_llModelInfo.model->initializeGPUDevice(d); + } + } + } + bool success = m_llModelInfo.model->loadModel(filePath.toStdString()); MySettings::globalInstance()->setAttemptModelLoad(QString()); if (!success) { diff --git a/gpt4all-chat/chatllm.h b/gpt4all-chat/chatllm.h index be478c25..b15bfaa8 100644 --- a/gpt4all-chat/chatllm.h +++ b/gpt4all-chat/chatllm.h @@ -111,6 +111,7 @@ public Q_SLOTS: void handleShouldBeLoadedChanged(); void handleThreadStarted(); void handleForceMetalChanged(bool forceMetal); + void handleDeviceChanged(); void processSystemPrompt(); Q_SIGNALS: diff --git a/gpt4all-chat/mysettings.cpp b/gpt4all-chat/mysettings.cpp index 4c9e6e26..8db8c6eb 100644 --- a/gpt4all-chat/mysettings.cpp +++ b/gpt4all-chat/mysettings.cpp @@ -23,6 +23,7 @@ static bool default_localDocsShowReferences = true; static QString default_networkAttribution = ""; static bool default_networkIsActive = false; static bool default_networkUsageStatsActive = false; +static QString default_device = "Auto"; static QString defaultLocalModelsPath() { @@ -64,6 +65,17 @@ MySettings::MySettings() QSettings::setDefaultFormat(QSettings::IniFormat); } +Q_INVOKABLE QVector MySettings::deviceList() const +{ + return m_deviceList; +} + +void MySettings::setDeviceList(const QVector &deviceList) +{ + m_deviceList = deviceList; + emit deviceListChanged(); +} + void MySettings::restoreModelDefaults(const ModelInfo &model) { setModelTemperature(model, model.m_temperature); @@ -79,6 +91,9 @@ void MySettings::restoreModelDefaults(const ModelInfo &model) void MySettings::restoreApplicationDefaults() { + setChatTheme(default_chatTheme); + setFontSize(default_fontSize); + setDevice(default_device); setThreadCount(default_threadCount); setSaveChats(default_saveChats); setSaveChatGPTChats(default_saveChatGPTChats); @@ -485,7 +500,7 @@ QString MySettings::chatTheme() const void MySettings::setChatTheme(const QString &u) { - if(chatTheme() == u) + if (chatTheme() == u) return; QSettings setting; @@ -503,7 +518,7 @@ QString MySettings::fontSize() const void MySettings::setFontSize(const QString &u) { - if(fontSize() == u) + if (fontSize() == u) return; QSettings setting; @@ -512,6 +527,24 @@ void MySettings::setFontSize(const QString &u) emit fontSizeChanged(); } +QString MySettings::device() const +{ + QSettings setting; + setting.sync(); + return setting.value("device", default_device).toString(); +} + +void MySettings::setDevice(const QString &u) +{ + if (device() == u) + return; + + QSettings setting; + setting.setValue("device", u); + setting.sync(); + emit deviceChanged(); +} + bool MySettings::forceMetal() const { return m_forceMetal; diff --git a/gpt4all-chat/mysettings.h b/gpt4all-chat/mysettings.h index 7d77ede0..2f460864 100644 --- a/gpt4all-chat/mysettings.h +++ b/gpt4all-chat/mysettings.h @@ -25,6 +25,8 @@ class MySettings : public QObject Q_PROPERTY(QString networkAttribution READ networkAttribution WRITE setNetworkAttribution NOTIFY networkAttributionChanged) Q_PROPERTY(bool networkIsActive READ networkIsActive WRITE setNetworkIsActive NOTIFY networkIsActiveChanged) Q_PROPERTY(bool networkUsageStatsActive READ networkUsageStatsActive WRITE setNetworkUsageStatsActive NOTIFY networkUsageStatsActiveChanged) + Q_PROPERTY(QString device READ device WRITE setDevice NOTIFY deviceChanged) + Q_PROPERTY(QVector deviceList READ deviceList NOTIFY deviceListChanged) public: static MySettings *globalInstance(); @@ -78,6 +80,8 @@ public: void setFontSize(const QString &u); bool forceMetal() const; void setForceMetal(bool b); + QString device() const; + void setDevice(const QString &u); // Release/Download settings QString lastVersionStarted() const; @@ -102,6 +106,9 @@ public: QString attemptModelLoad() const; void setAttemptModelLoad(const QString &modelFile); + QVector deviceList() const; + void setDeviceList(const QVector &deviceList); + Q_SIGNALS: void nameChanged(const ModelInfo &model); void filenameChanged(const ModelInfo &model); @@ -131,9 +138,12 @@ Q_SIGNALS: void networkIsActiveChanged(); void networkUsageStatsActiveChanged(); void attemptModelLoadChanged(); + void deviceChanged(); + void deviceListChanged(); private: bool m_forceMetal; + QVector m_deviceList; private: explicit MySettings(); diff --git a/gpt4all-chat/qml/ApplicationSettings.qml b/gpt4all-chat/qml/ApplicationSettings.qml index 90408fca..dbfdddcd 100644 --- a/gpt4all-chat/qml/ApplicationSettings.qml +++ b/gpt4all-chat/qml/ApplicationSettings.qml @@ -88,17 +88,55 @@ MySettingsTab { MySettings.fontSize = fontBox.currentText } } + Label { + id: deviceLabel + text: qsTr("Device:") + color: theme.textColor + font.pixelSize: theme.fontSizeLarge + Layout.row: 3 + Layout.column: 0 + } + MyComboBox { + id: deviceBox + Layout.row: 3 + Layout.column: 1 + Layout.columnSpan: 1 + Layout.minimumWidth: 350 + Layout.fillWidth: false + model: MySettings.deviceList + Accessible.role: Accessible.ComboBox + Accessible.name: qsTr("ComboBox for displaying/picking the device") + Accessible.description: qsTr("Use this for picking the device of the chat client") + function updateModel() { + deviceBox.currentIndex = deviceBox.indexOfValue(MySettings.device); + } + Component.onCompleted: { + deviceBox.updateModel() + } + Connections { + target: MySettings + function onDeviceChanged() { + deviceBox.updateModel() + } + function onDeviceListChanged() { + deviceBox.updateModel() + } + } + onActivated: { + MySettings.device = deviceBox.currentText + } + } Label { id: defaultModelLabel text: qsTr("Default model:") color: theme.textColor font.pixelSize: theme.fontSizeLarge - Layout.row: 3 + Layout.row: 4 Layout.column: 0 } MyComboBox { id: comboBox - Layout.row: 3 + Layout.row: 4 Layout.column: 1 Layout.columnSpan: 2 Layout.minimumWidth: 350 @@ -128,7 +166,7 @@ MySettingsTab { text: qsTr("Download path:") color: theme.textColor font.pixelSize: theme.fontSizeLarge - Layout.row: 4 + Layout.row: 5 Layout.column: 0 } MyDirectoryField { @@ -136,7 +174,7 @@ MySettingsTab { text: MySettings.modelPath font.pixelSize: theme.fontSizeLarge implicitWidth: 300 - Layout.row: 4 + Layout.row: 5 Layout.column: 1 Layout.fillWidth: true ToolTip.text: qsTr("Path where model files will be downloaded to") @@ -153,7 +191,7 @@ MySettingsTab { } } MyButton { - Layout.row: 4 + Layout.row: 5 Layout.column: 2 text: qsTr("Browse") Accessible.description: qsTr("Opens a folder picker dialog to choose where to save model files") @@ -168,7 +206,7 @@ MySettingsTab { text: qsTr("CPU Threads:") color: theme.textColor font.pixelSize: theme.fontSizeLarge - Layout.row: 5 + Layout.row: 6 Layout.column: 0 } MyTextField { @@ -177,7 +215,7 @@ MySettingsTab { font.pixelSize: theme.fontSizeLarge ToolTip.text: qsTr("Amount of processing threads to use bounded by 1 and number of logical processors") ToolTip.visible: hovered - Layout.row: 5 + Layout.row: 6 Layout.column: 1 validator: IntValidator { bottom: 1 @@ -200,12 +238,12 @@ MySettingsTab { text: qsTr("Save chats to disk:") color: theme.textColor font.pixelSize: theme.fontSizeLarge - Layout.row: 6 + Layout.row: 7 Layout.column: 0 } MyCheckBox { id: saveChatsBox - Layout.row: 6 + Layout.row: 7 Layout.column: 1 checked: MySettings.saveChats onClicked: { @@ -220,12 +258,12 @@ MySettingsTab { text: qsTr("Save ChatGPT chats to disk:") color: theme.textColor font.pixelSize: theme.fontSizeLarge - Layout.row: 7 + Layout.row: 8 Layout.column: 0 } MyCheckBox { id: saveChatGPTChatsBox - Layout.row: 7 + Layout.row: 8 Layout.column: 1 checked: MySettings.saveChatGPTChats onClicked: { @@ -237,12 +275,12 @@ MySettingsTab { text: qsTr("Enable API server:") color: theme.textColor font.pixelSize: theme.fontSizeLarge - Layout.row: 8 + Layout.row: 9 Layout.column: 0 } MyCheckBox { id: serverChatBox - Layout.row: 8 + Layout.row: 9 Layout.column: 1 checked: MySettings.serverChat onClicked: { @@ -252,7 +290,7 @@ MySettingsTab { ToolTip.visible: hovered } Rectangle { - Layout.row: 9 + Layout.row: 10 Layout.column: 0 Layout.columnSpan: 3 Layout.fillWidth: true