Archives/gpt4all
2
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all synced 2024-11-08 07:10:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: use ccache to cache compiler output (#2942)

Browse Source 
Signed-off-by: Jared Van Bortel <jared@nomic.ai>
This commit is contained in:
Jared Van Bortel 2024-09-06 16:09:11 -04:00 committed by GitHub
parent facb706211
commit 1aae4ffe0a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 263 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

330
.circleci/continue_config.yml
View File

@ -29,25 +29,22 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- macos-qt-cache-v3
- ccache-gpt4all-macos-
- run:
name: Install Rosetta
command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW
- run:
name: Install dependencies
command: brew install ccache
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
/Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
fi
- save_cache: # this is the new step to save cache
key: macos-qt-cache-v3
paths:
- ~/Qt
- run:
name: Setup Keychain
command: |
@ -61,6 +58,7 @@ jobs:
name: Build
no_output_timeout: 30m
command: |
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
mkdir build
cd build
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
@ -69,6 +67,8 @@ jobs:
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \
-DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6 \
@ -78,11 +78,17 @@ jobs:
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
ccache -s
mkdir upload
cp gpt4all-installer-* upload
# persist the unsigned installer
- store_artifacts:
path: build/upload
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
# add workspace so signing jobs can connect & obtain dmg
- persist_to_workspace:
root: build
@ -161,25 +167,22 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- macos-qt-cache-v3
- ccache-gpt4all-macos-
- run:
name: Install Rosetta
command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW
- run:
name: Install dependencies
command: brew install ccache
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
/Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
fi
- save_cache: # this is the new step to save cache
key: macos-qt-cache-v3
paths:
- ~/Qt
- run:
name: Setup Keychain
command: |
@ -193,6 +196,7 @@ jobs:
name: Build
no_output_timeout: 30m
command: |
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
mkdir build
cd build
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
@ -201,6 +205,8 @@ jobs:
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \
-DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6 \
@ -210,12 +216,18 @@ jobs:
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target install
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target package
ccache -s
mkdir upload
cp gpt4all-installer-* upload
tar -cvzf upload/repository.tar.gz -C _CPack_Packages/Darwin/IFW/gpt4all-installer-darwin repository
# persist the unsigned installer
- store_artifacts:
path: build/upload
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
# add workspace so signing jobs can connect & obtain dmg
- persist_to_workspace:
root: build
@ -294,9 +306,9 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- linux-qt-cache-v2
- ccache-gpt4all-linux-amd64-
- run:
name: Setup Linux and Dependencies
command: |
@ -305,7 +317,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=(
bison build-essential cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
bison build-essential ccache cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6
libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
@ -317,15 +329,9 @@ jobs:
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
chmod +x qt-unified-linux-x64-4.6.0-online.run
./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
fi
- save_cache: # this is the new step to save cache
key: linux-qt-cache-v2
paths:
- ~/Qt
- run:
name: Build linuxdeployqt
command: |
@ -339,19 +345,30 @@ jobs:
export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
export PATH=$PATH:/usr/local/cuda/bin
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
mkdir build
cd build
mkdir upload
~/Qt/Tools/CMake/bin/cmake \
-S ../gpt4all-chat -B . \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON \
-DGPT4ALL_OFFLINE_INSTALLER=ON
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target all
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target install
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target package
ccache -s
cp gpt4all-installer-* upload
- store_artifacts:
path: build/upload
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
build-online-chat-installer-linux:
machine:
@ -363,9 +380,9 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- linux-qt-cache-v2
- ccache-gpt4all-linux-amd64-
- run:
name: Setup Linux and Dependencies
command: |
@ -374,7 +391,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=(
bison build-essential cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
bison build-essential ccache cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6
libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
@ -386,15 +403,9 @@ jobs:
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
chmod +x qt-unified-linux-x64-4.6.0-online.run
./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
fi
- save_cache: # this is the new step to save cache
key: linux-qt-cache-v2
paths:
- ~/Qt
- run:
name: Build linuxdeployqt
command: |
@ -408,20 +419,31 @@ jobs:
export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
export PATH=$PATH:$HOME/Qt/Tools/QtInstallerFramework/4.7/bin
export PATH=$PATH:/usr/local/cuda/bin
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
mkdir build
cd build
mkdir upload
~/Qt/Tools/CMake/bin/cmake \
-S ../gpt4all-chat -B . \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON \
-DGPT4ALL_OFFLINE_INSTALLER=OFF
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target all
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target install
~/Qt/Tools/CMake/bin/cmake --build . -j$(nproc) --target package
ccache -s
cp gpt4all-installer-* upload
tar -cvzf upload/repository.tar.gz -C _CPack_Packages/Linux/IFW/gpt4all-installer-linux repository
- store_artifacts:
path: build/upload
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
build-offline-chat-installer-windows:
machine:
@ -435,20 +457,17 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- windows-qt-cache-v2
- ccache-gpt4all-win-amd64-
- run:
name: Install dependencies
command: choco install -y ccache
- run:
name: Installing Qt
command: |
if (-not (Test-Path C:\Qt)) {
Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
& .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
}
- save_cache: # this is the new step to save cache
key: windows-qt-cache-v2
paths:
- C:\Qt
- run:
name: Install VulkanSDK
command: |
@ -483,29 +502,40 @@ jobs:
$Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
$Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
$Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
mkdir build
cd build
& "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-S ..\gpt4all-chat -B . -G Ninja `
"-DCMAKE_BUILD_TYPE=Release" `
-DCMAKE_BUILD_TYPE=Release `
"-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
"-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
"-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
"-DGPT4ALL_OFFLINE_INSTALLER=ON"
-DCMAKE_C_COMPILER_LAUNCHER=ccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON `
-DGPT4ALL_OFFLINE_INSTALLER=ON
& "C:\Qt\Tools\Ninja\ninja.exe"
& "C:\Qt\Tools\Ninja\ninja.exe" install
& "C:\Qt\Tools\Ninja\ninja.exe" package
ccache -s
mkdir upload
copy gpt4all-installer-win64.exe upload
- store_artifacts:
path: build/upload
# add workspace so signing jobs can connect & obtain dmg
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
- persist_to_workspace:
root: build
# specify path to only include components we want to persist
# accross builds
paths:
- upload
sign-offline-chat-installer-windows:
machine:
image: 'windows-server-2019-vs2019:2022.08.1'
@ -535,6 +565,7 @@ jobs:
AzureSignTool.exe sign -du "https://gpt4all.io/index.html" -kvu https://gpt4all.vault.azure.net -kvi "$Env:AZSignGUID" -kvs "$Env:AZSignPWD" -kvc "$Env:AZSignCertName" -kvt "$Env:AZSignTID" -tr http://timestamp.digicert.com -v "$($(Get-Location).Path)\build\upload\gpt4all-installer-win64.exe"
- store_artifacts:
path: build/upload
build-online-chat-installer-windows:
machine:
image: 'windows-server-2019-vs2019:2022.08.1'
@ -547,20 +578,17 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- windows-qt-cache-v2
- ccache-gpt4all-win-amd64-
- run:
name: Install dependencies
command: choco install -y ccache
- run:
name: Installing Qt
command: |
if (-not (Test-Path C:\Qt)) {
Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
& .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
}
- save_cache: # this is the new step to save cache
key: windows-qt-cache-v2
paths:
- C:\Qt
- run:
name: Install VulkanSDK
command: |
@ -600,24 +628,34 @@ jobs:
$Env:PATH = "${Env:PATH};C:\Qt\Tools\QtInstallerFramework\4.7\bin"
$Env:DOTNET_ROOT="$($(Get-Location).Path)\dotnet\dotnet-sdk-8.0.302-win-x64"
$Env:PATH="$Env:DOTNET_ROOT;$Env:PATH"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
mkdir build
cd build
& "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-S ..\gpt4all-chat -B . -G Ninja `
"-DCMAKE_BUILD_TYPE=Release" `
-DCMAKE_BUILD_TYPE=Release `
"-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
"-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
"-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON" `
"-DGPT4ALL_OFFLINE_INSTALLER=OFF"
-DCMAKE_C_COMPILER_LAUNCHER=ccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON `
-DGPT4ALL_OFFLINE_INSTALLER=OFF
& "C:\Qt\Tools\Ninja\ninja.exe"
& "C:\Qt\Tools\Ninja\ninja.exe" install
& "C:\Qt\Tools\Ninja\ninja.exe" package
ccache -s
mkdir upload
copy gpt4all-installer-win64.exe upload
Set-Location -Path "_CPack_Packages/win64/IFW/gpt4all-installer-win64"
Compress-Archive -Path 'repository' -DestinationPath '..\..\..\..\upload\repository.zip'
- store_artifacts:
path: build/upload
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
# add workspace so signing jobs can connect & obtain dmg
- persist_to_workspace:
root: build
@ -625,6 +663,7 @@ jobs:
# accross builds
paths:
- upload
sign-online-chat-installer-windows:
machine:
image: 'windows-server-2019-vs2019:2022.08.1'
@ -670,9 +709,9 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- linux-qt-cache-v2
- ccache-gpt4all-linux-amd64-
- run:
name: Setup Linux and Dependencies
command: |
@ -681,7 +720,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=(
bison build-essential cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
bison build-essential ccache cuda-compiler-11-8 flex gperf libcublas-dev-11-8 libfontconfig1 libfreetype6
libgl1-mesa-dev libmysqlclient21 libnvidia-compute-550-server libodbc2 libpq5 libwayland-dev libx11-6
libx11-xcb1 libxcb-cursor0 libxcb-glx0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0
libxcb-render-util0 libxcb-shape0 libxcb-shm0 libxcb-sync1 libxcb-util1 libxcb-xfixes0 libxcb-xinerama0
@ -693,24 +732,29 @@ jobs:
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run
chmod +x qt-unified-linux-x64-4.6.0-online.run
./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor
fi
- save_cache: # this is the new step to save cache
key: linux-qt-cache-v2
paths:
- ~/Qt
- run:
name: Build
command: |
export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake
export PATH=$PATH:/usr/local/cuda/bin
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
~/Qt/Tools/CMake/bin/cmake \
-S gpt4all-chat -B build \
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
~/Qt/Tools/CMake/bin/cmake --build build -j$(nproc) --target all
ccache -s
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
build-gpt4all-chat-windows:
machine:
@ -724,20 +768,17 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- windows-qt-cache-v2
- ccache-gpt4all-win-amd64-
- run:
name: Install dependencies
command: choco install -y ccache
- run:
name: Installing Qt
command: |
if (-not (Test-Path C:\Qt)) {
Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe
& .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
}
- save_cache: # this is the new step to save cache
key: windows-qt-cache-v2
paths:
- C:\Qt
- run:
name: Install VulkanSDK
command: |
@ -757,13 +798,23 @@ jobs:
$Env:PATH = "${Env:PATH};C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
& "C:\Qt\Tools\CMake_64\bin\cmake.exe" `
-S gpt4all-chat -B build -G Ninja `
"-DCMAKE_BUILD_TYPE=Release" `
-DCMAKE_BUILD_TYPE=Release `
"-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" `
"-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" `
"-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON"
-DCMAKE_C_COMPILER_LAUNCHER=ccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
& "C:\Qt\Tools\Ninja\ninja.exe" -C build
ccache -s
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
build-gpt4all-chat-macos:
macos:
@ -775,37 +826,44 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache: # this is the new step to restore cache
- restore_cache:
keys:
- macos-qt-cache-v3
- ccache-gpt4all-macos-
- run:
name: Install Rosetta
command: softwareupdate --install-rosetta --agree-to-license # needed for QtIFW
- run:
name: Install dependencies
command: brew install ccache
- run:
name: Installing Qt
command: |
if [ ! -d ~/Qt ]; then
curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg
hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg
/Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.47 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver
hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online
fi
- save_cache: # this is the new step to save cache
key: macos-qt-cache-v3
paths:
- ~/Qt
- run:
name: Build
command: |
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \
-S gpt4all-chat -B build -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake \
-DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6
~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build build --target all
ccache -s
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
build-ts-docs:
docker:
- image: cimg/base:stable
@ -824,6 +882,7 @@ jobs:
command: |
cd gpt4all-bindings/typescript
npm run docs:build
build-py-docs:
docker:
- image: circleci/python:3.8
@ -855,6 +914,9 @@ jobs:
image: ubuntu-2204:2023.04.2
steps:
- checkout
- restore_cache:
keys:
- ccache-gpt4all-linux-amd64-
- run:
name: Set Python Version
command: pyenv global 3.11.2
@ -866,7 +928,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=(
build-essential cmake cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server vulkan-sdk
build-essential ccache cmake cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server vulkan-sdk
)
sudo apt-get update
sudo apt-get install -y "${packages[@]}"
@ -876,12 +938,17 @@ jobs:
command: |
export PATH=$PATH:/usr/local/cuda/bin
git submodule update --init --recursive
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
cd gpt4all-backend
cmake -B build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON \
-DCMAKE_CUDA_ARCHITECTURES='52-virtual;61-virtual;70-virtual;75-virtual'
cmake --build build -j$(nproc)
ccache -s
- run:
name: Build wheel
command: |
@ -889,6 +956,11 @@ jobs:
python setup.py bdist_wheel --plat-name=manylinux1_x86_64
- store_artifacts:
path: gpt4all-bindings/python/dist
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
- persist_to_workspace:
root: gpt4all-bindings/python/dist
paths:
@ -900,22 +972,29 @@ jobs:
resource_class: macos.m1.large.gen1
steps:
- checkout
- restore_cache:
keys:
- ccache-gpt4all-macos-
- run:
name: Install dependencies
command: |
brew install cmake
brew install ccache cmake
pip install setuptools wheel cmake
- run:
name: Build C library
command: |
git submodule update --init # don't use --recursive because macOS doesn't use Kompute
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
cd gpt4all-backend
cmake -B build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6
cmake --build build --parallel
ccache -s
- run:
name: Build wheel
command: |
@ -923,6 +1002,11 @@ jobs:
python setup.py bdist_wheel --plat-name=macosx_10_15_universal2
- store_artifacts:
path: gpt4all-bindings/python/dist
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
- persist_to_workspace:
root: gpt4all-bindings/python/dist
paths:
@ -940,6 +1024,9 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache:
keys:
- ccache-gpt4all-win-amd64-
- run:
name: Install VulkanSDK
command: |
@ -953,7 +1040,7 @@ jobs:
- run:
name: Install dependencies
command:
choco install -y cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
choco install -y ccache cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
- run:
name: Install Python dependencies
command: pip install setuptools wheel cmake
@ -966,12 +1053,17 @@ jobs:
$Env:PATH += ";C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
cd gpt4all-backend
cmake -B build -G Ninja `
-DCMAKE_BUILD_TYPE=Release `
-DCMAKE_C_COMPILER_LAUNCHER=ccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON `
-DCMAKE_CUDA_ARCHITECTURES='52-virtual;61-virtual;70-virtual;75-virtual'
cmake --build build --parallel
ccache -s
- run:
name: Build wheel
command: |
@ -979,6 +1071,11 @@ jobs:
python setup.py bdist_wheel --plat-name=win_amd64
- store_artifacts:
path: gpt4all-bindings/python/dist
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
- persist_to_workspace:
root: gpt4all-bindings/python/dist
paths:
@ -1014,6 +1111,9 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache:
keys:
- ccache-gpt4all-linux-amd64-
- run:
name: Install dependencies
command: |
@ -1022,7 +1122,7 @@ jobs:
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
packages=(
build-essential cmake cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server vulkan-sdk
build-essential ccache cmake cuda-compiler-11-8 libcublas-dev-11-8 libnvidia-compute-550-server vulkan-sdk
)
sudo apt-get update
sudo apt-get install -y "${packages[@]}"
@ -1030,14 +1130,25 @@ jobs:
name: Build Libraries
command: |
export PATH=$PATH:/usr/local/cuda/bin
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
cd gpt4all-backend
mkdir -p runtimes/build
cd runtimes/build
cmake ../.. \
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
cmake --build . -j$(nproc)
ccache -s
mkdir ../linux-x64
cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks
- save_cache:
key: ccache-gpt4all-linux-amd64-{{ epoch }}
when: always
paths:
- ../.ccache
- persist_to_workspace:
root: gpt4all-backend
paths:
@ -1053,26 +1164,38 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache:
keys:
- ccache-gpt4all-macos-
- run:
name: Install dependencies
command: |
brew install cmake
brew install ccache cmake
- run:
name: Build Libraries
command: |
ccache -o "cache_dir=${PWD}/../.ccache" -o max_size=500M -p -z
cd gpt4all-backend
mkdir -p runtimes/build
cd runtimes/build
cmake ../.. \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DBUILD_UNIVERSAL=ON \
-DCMAKE_OSX_DEPLOYMENT_TARGET=12.6 \
-DGGML_METAL_MACOSX_VERSION_MIN=12.6
cmake --build . --parallel
ccache -s
mkdir ../osx-x64
cp -L *.dylib ../osx-x64
cp ../../llama.cpp-mainline/*.metal ../osx-x64
ls ../osx-x64
- save_cache:
key: ccache-gpt4all-macos-{{ epoch }}
when: always
paths:
- ../.ccache
- persist_to_workspace:
root: gpt4all-backend
paths:
@ -1091,6 +1214,9 @@ jobs:
command: |
git submodule sync
git submodule update --init --recursive
- restore_cache:
keys:
- ccache-gpt4all-win-amd64-
- run:
name: Install VulkanSDK
command: |
@ -1104,19 +1230,34 @@ jobs:
- run:
name: Install dependencies
command: |
choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
choco install -y ccache cmake ninja --installargs 'ADD_CMAKE_TO_PATH=System'
- run:
name: Build Libraries
command: |
$Env:Path += ";C:\Program Files\CMake\bin"
$vsInstallPath = & "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -property installationpath
Import-Module "${vsInstallPath}\Common7\Tools\Microsoft.VisualStudio.DevShell.dll"
Enter-VsDevShell -VsInstallPath "$vsInstallPath" -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -no_logo'
$Env:Path += ";C:\VulkanSDK\1.3.261.1\bin"
$Env:VULKAN_SDK = "C:\VulkanSDK\1.3.261.1"
ccache -o "cache_dir=${pwd}\..\.ccache" -o max_size=500M -p -z
cd gpt4all-backend
mkdir runtimes/win-x64_msvc
cd runtimes/win-x64_msvc
cmake -G "Visual Studio 17 2022" -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -A X64 ../..
cmake --build . --parallel --config Release
cmake -S ../.. -B . -G Ninja `
-DCMAKE_BUILD_TYPE=Release `
-DCMAKE_C_COMPILER_LAUNCHER=ccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache `
-DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON
cmake --build . --parallel
ccache -s
cp bin/Release/*.dll .
- save_cache:
key: ccache-gpt4all-win-amd64-{{ epoch }}
when: always
paths:
- ..\.ccache
- persist_to_workspace:
root: gpt4all-backend
paths:
@ -1153,6 +1294,7 @@ jobs:
paths:
- prebuilds/linux-x64/*.node
- runtimes/linux-x64/*-*.so
build-nodejs-macos:
macos:
xcode: 15.4.0