version: 2.1 orbs: win: circleci/windows@5.0 python: circleci/python@1.2 parameters: run-default-workflow: type: boolean default: false run-python-workflow: type: boolean default: false run-chat-workflow: type: boolean default: false run-csharp-workflow: type: boolean default: false jobs: default-job: docker: - image: circleci/python:3.7 steps: - run: echo "CircleCI pipeline triggered" build-gpt4all-chat-linux: machine: image: ubuntu-2204:2023.04.2 steps: - checkout - run: name: Update Submodules command: | git submodule sync git submodule update --init --recursive - restore_cache: # this is the new step to restore cache keys: - linux-qt-cache - run: name: Setup Linux and Dependencies command: | sudo apt update && sudo apt install -y libfontconfig1 libfreetype6 libx11-6 libx11-xcb1 libxext6 libxfixes3 libxi6 libxrender1 libxcb1 libxcb-cursor0 libxcb-glx0 libxcb-keysyms1 libxcb-image0 libxcb-shm0 libxcb-icccm4 libxcb-sync1 libxcb-xfixes0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 libxcb-util1 libxcb-xinerama0 libxcb-xkb1 libxkbcommon0 libxkbcommon-x11-0 bison build-essential flex gperf python3 gcc g++ libgl1-mesa-dev libwayland-dev - run: name: Installing Qt command: | if [ ! -d ~/Qt ]; then wget https://gpt4all.io/ci/qt-unified-linux-x64-4.6.0-online.run chmod +x qt-unified-linux-x64-4.6.0-online.run ./qt-unified-linux-x64-4.6.0-online.run --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.gcc_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver qt.qt6.651.qtwaylandcompositor fi - save_cache: # this is the new step to save cache key: linux-qt-cache paths: - ~/Qt - run: name: Build command: | export CMAKE_PREFIX_PATH=~/Qt/6.5.1/gcc_64/lib/cmake mkdir build cd build ~/Qt/Tools/CMake/bin/cmake -DCMAKE_BUILD_TYPE=Release -S ../gpt4all-chat -B . ~/Qt/Tools/CMake/bin/cmake --build . --target all build-gpt4all-chat-windows: machine: image: 'windows-server-2019-vs2019:2022.08.1' resource_class: windows.large shell: powershell.exe -ExecutionPolicy Bypass steps: - checkout - run: name: Update Submodules command: | git submodule sync git submodule update --init --recursive - restore_cache: # this is the new step to restore cache keys: - windows-qt-cache - run: name: Installing Qt command: | if (-not (Test-Path C:\Qt)) { Invoke-WebRequest -Uri https://gpt4all.io/ci/qt-unified-windows-x64-4.6.0-online.exe -OutFile qt-unified-windows-x64-4.6.0-online.exe & .\qt-unified-windows-x64-4.6.0-online.exe --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email ${Env:QT_EMAIL} --password ${Env:QT_PASSWORD} install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.win64_msvc2019_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver } - save_cache: # this is the new step to save cache key: windows-qt-cache paths: - C:\Qt - run: name: Build command: | $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\x64" $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Windows Kits\10\bin\10.0.22000.0\x64" $Env:PATH = "${Env:PATH};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64" $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\ucrt\x64" $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22000.0\um\x64" $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\lib\x64" $Env:LIB = "${Env:LIB};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\lib\x64" $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\ucrt" $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\um" $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\shared" $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\winrt" $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Windows Kits\10\include\10.0.22000.0\cppwinrt" $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\VS\include" $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\include" $Env:INCLUDE = "${Env:INCLUDE};C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.29.30133\ATLMFC\include" mkdir build cd build & "C:\Qt\Tools\CMake_64\bin\cmake.exe" ` "-DCMAKE_GENERATOR:STRING=Ninja" ` "-DCMAKE_BUILD_TYPE=Release" ` "-DCMAKE_PREFIX_PATH:PATH=C:\Qt\6.5.1\msvc2019_64" ` "-DCMAKE_MAKE_PROGRAM:FILEPATH=C:\Qt\Tools\Ninja\ninja.exe" ` "-S ..\gpt4all-chat" ` "-B ." & "C:\Qt\Tools\Ninja\ninja.exe" build-gpt4all-chat-macos: macos: xcode: 14.0.0 steps: - checkout - run: name: Update Submodules command: | git submodule sync git submodule update --init --recursive - restore_cache: # this is the new step to restore cache keys: - macos-qt-cache_v2 - run: name: Installing Qt command: | if [ ! -d ~/Qt ]; then curl -o qt-unified-macOS-x64-4.6.0-online.dmg https://gpt4all.io/ci/qt-unified-macOS-x64-4.6.0-online.dmg hdiutil attach qt-unified-macOS-x64-4.6.0-online.dmg /Volumes/qt-unified-macOS-x64-4.6.0-online/qt-unified-macOS-x64-4.6.0-online.app/Contents/MacOS/qt-unified-macOS-x64-4.6.0-online --no-force-installations --no-default-installations --no-size-checking --default-answer --accept-licenses --confirm-command --accept-obligations --email $QT_EMAIL --password $QT_PASSWORD install qt.tools.cmake qt.tools.ifw.46 qt.tools.ninja qt.qt6.651.clang_64 qt.qt6.651.qt5compat qt.qt6.651.debug_info qt.qt6.651.addons.qtpdf qt.qt6.651.addons.qthttpserver hdiutil detach /Volumes/qt-unified-macOS-x64-4.6.0-online fi - save_cache: # this is the new step to save cache key: macos-qt-cache_v2 paths: - ~/Qt - run: name: Build command: | mkdir build cd build ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake \ -DCMAKE_GENERATOR:STRING=Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH:PATH=~/Qt/6.5.1/macos/lib/cmake/Qt6 \ -DCMAKE_MAKE_PROGRAM:FILEPATH=~/Qt/Tools/Ninja/ninja \ -S ../gpt4all-chat \ -B . ~/Qt/Tools/CMake/CMake.app/Contents/bin/cmake --build . --target all build-py-docs: docker: - image: circleci/python:3.8 steps: - checkout - run: name: Install dependencies command: | sudo apt-get update sudo apt-get -y install python3 python3-pip sudo pip3 install awscli --upgrade sudo pip3 install mkdocs mkdocs-material mkautodoc 'mkdocstrings[python]' - run: name: Make Documentation command: | cd gpt4all-bindings/python/ mkdocs build - run: name: Deploy Documentation command: | cd gpt4all-bindings/python/ aws s3 cp ./site s3://docs.gpt4all.io/ --recursive | cat - run: name: Invalidate docs.gpt4all.io cloudfront command: aws cloudfront create-invalidation --distribution-id E1STQOW63QL2OH --paths "/*" build-py-linux: docker: - image: circleci/python:3.8 steps: - checkout - run: name: Install dependencies command: | sudo apt-get update sudo apt-get install -y cmake build-essential pip install setuptools wheel cmake - run: name: Build C library command: | git submodule init git submodule update cd gpt4all-backend mkdir build cd build cmake .. cmake --build . --parallel - run: name: Build wheel command: | cd gpt4all-bindings/python/ python setup.py bdist_wheel --plat-name=manylinux1_x86_64 - persist_to_workspace: root: gpt4all-bindings/python/dist paths: - "*.whl" build-py-macos: macos: xcode: "14.2.0" resource_class: macos.m1.large.gen1 steps: - checkout - run: name: Install dependencies command: | brew install cmake pip install setuptools wheel cmake - run: name: Build C library command: | git submodule init git submodule update cd gpt4all-backend mkdir build cd build cmake .. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64" cmake --build . --parallel - run: name: Build wheel command: | cd gpt4all-bindings/python python setup.py bdist_wheel --plat-name=macosx_10_9_universal2 - persist_to_workspace: root: gpt4all-bindings/python/dist paths: - "*.whl" build-py-windows: executor: name: win/default steps: - checkout - run: name: Install MinGW64 command: choco install -y mingw --force --no-progress - run: name: Add MinGW64 to PATH command: $env:Path += ";C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin" - run: name: Install dependencies command: choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' - run: name: Install Python dependencies command: pip install setuptools wheel cmake - run: name: Build C library command: | git submodule init git submodule update cd gpt4all-backend mkdir build cd build cmake -G "MinGW Makefiles" .. cmake --build . --parallel - run: name: Build wheel # TODO: As part of this task, we need to move mingw64 binaries into package. # This is terrible and needs a more robust solution eventually. command: | cd gpt4all-bindings/python cd gpt4all mkdir llmodel_DO_NOT_MODIFY mkdir llmodel_DO_NOT_MODIFY/build/ cp 'C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin\*dll' 'llmodel_DO_NOT_MODIFY/build/' cd .. python setup.py bdist_wheel --plat-name=win_amd64 - persist_to_workspace: root: gpt4all-bindings/python/dist paths: - "*.whl" store-and-upload-wheels: docker: - image: circleci/python:3.8 steps: - setup_remote_docker - attach_workspace: at: /tmp/workspace - run: name: Install dependencies command: | sudo apt-get update sudo apt-get install -y cmake build-essential pip install setuptools wheel twine - run: name: Upload Python package command: | twine upload /tmp/workspace/*.whl --username __token__ --password $PYPI_CRED - store_artifacts: path: /tmp/workspace build-bindings-backend-linux: machine: image: ubuntu-2204:2023.04.2 steps: - checkout - run: name: Update Submodules command: | git submodule sync git submodule update --init --recursive - run: name: Install dependencies command: | sudo apt-get update sudo apt-get install -y cmake build-essential - run: name: Build Libraries command: | cd gpt4all-backend mkdir -p runtimes/build cd runtimes/build cmake ../.. cmake --build . --parallel --config Release mkdir ../linux-x64 cp -L *.so ../linux-x64 # otherwise persist_to_workspace seems to mess symlinks - persist_to_workspace: root: gpt4all-backend paths: - runtimes/linux-x64/*.so build-bindings-backend-macos: macos: xcode: "14.0.0" steps: - checkout - run: name: Update Submodules command: | git submodule sync git submodule update --init --recursive - run: name: Install dependencies command: | brew install cmake - run: name: Build Libraries command: | cd gpt4all-backend mkdir -p runtimes/build cd runtimes/build cmake ../.. -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64" cmake --build . --parallel --config Release mkdir ../osx-x64 cp -L *.dylib ../osx-x64 cp ../../llama.cpp-mainline/*.metal ../osx-x64 ls ../osx-x64 - persist_to_workspace: root: gpt4all-backend paths: - runtimes/osx-x64/*.dylib - runtimes/osx-x64/*.metal build-bindings-backend-windows: executor: name: win/default size: large shell: powershell.exe -ExecutionPolicy Bypass steps: - checkout - run: name: Update Submodules command: | git submodule sync git submodule update --init --recursive - run: name: Install MinGW64 command: choco install -y mingw --force --no-progress - run: name: Install dependencies command: | choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' - run: name: Build Libraries command: | $MinGWBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin" $Env:Path += ";$MinGwBin" $Env:Path += ";C:\Program Files\CMake\bin" cd gpt4all-backend mkdir runtimes/win-x64 cd runtimes/win-x64 cmake -G "MinGW Makefiles" ../.. cmake --build . --parallel --config Release cp "$MinGWBin\libgcc*.dll" . cp "$MinGWBin\libstdc++*.dll" . cp "$MinGWBin\libwinpthread*.dll" . cp bin/*.dll . - persist_to_workspace: root: gpt4all-backend paths: - runtimes/win-x64/*.dll build-bindings-backend-windows-msvc: machine: image: 'windows-server-2022-gui:2023.03.1' resource_class: windows.large shell: powershell.exe -ExecutionPolicy Bypass steps: - checkout - run: name: Update Submodules command: | git submodule sync git submodule update --init --recursive - run: name: Install dependencies command: | choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' - run: name: Build Libraries command: | $Env:Path += ";C:\Program Files\CMake\bin" cd gpt4all-backend mkdir runtimes/win-x64_msvc cd runtimes/win-x64_msvc cmake -G "Visual Studio 17 2022" -A X64 ../.. cmake --build . --parallel --config Release cp bin/Release/*.dll . - persist_to_workspace: root: gpt4all-backend paths: - runtimes/win-x64_msvc/*.dll build-csharp-linux: docker: - image: mcr.microsoft.com/dotnet/sdk:7.0-jammy # Ubuntu 22.04 steps: - when: condition: << pipeline.parameters.run-csharp-workflow >> steps: - checkout - attach_workspace: at: /tmp/workspace - run: name: "Prepare Native Libs" command: | cd gpt4all-bindings/csharp mkdir -p runtimes/linux-x64/native cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/ ls -R runtimes - restore_cache: keys: - gpt4all-csharp-nuget-packages-nix - run: name: "Install project dependencies" command: | cd gpt4all-bindings/csharp dotnet restore Gpt4All - save_cache: paths: - ~/.nuget/packages key: gpt4all-csharp-nuget-packages-nix - run: name: Build C# Project command: | cd gpt4all-bindings/csharp dotnet build Gpt4All --configuration Release --nologo - run: name: "Run C# Tests" command: | cd gpt4all-bindings/csharp dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx" - run: name: Test results command: | cd gpt4all-bindings/csharp/Gpt4All.Tests dotnet tool install -g trx2junit export PATH="$PATH:$HOME/.dotnet/tools" trx2junit TestResults/*.trx - store_test_results: path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults build-csharp-windows: executor: name: win/default size: large shell: powershell.exe -ExecutionPolicy Bypass steps: - when: condition: << pipeline.parameters.run-csharp-workflow >> steps: - checkout - restore_cache: keys: - gpt4all-csharp-nuget-packages-win - attach_workspace: at: C:\Users\circleci\workspace - run: name: "Prepare Native Libs" command: | cd gpt4all-bindings/csharp mkdir -p runtimes\win-x64\native cp C:\Users\circleci\workspace\runtimes\win-x64\*.dll runtimes\win-x64\native\ ls -R runtimes - run: name: "Install project dependencies" command: | cd gpt4all-bindings/csharp dotnet.exe restore Gpt4All - save_cache: paths: - C:\Users\circleci\.nuget\packages key: gpt4all-csharp-nuget-packages-win - run: name: Build C# Project command: | cd gpt4all-bindings/csharp dotnet.exe build Gpt4All --configuration Release --nologo - run: name: "Run C# Tests" command: | cd gpt4all-bindings/csharp dotnet.exe test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx" - run: name: Test results command: | cd gpt4all-bindings/csharp/Gpt4All.Tests dotnet tool install -g trx2junit $Env:Path += ";$Env:USERPROFILE\.dotnet\tools" trx2junit TestResults/*.trx - store_test_results: path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults build-csharp-macos: macos: xcode: "14.0.0" steps: - when: condition: << pipeline.parameters.run-csharp-workflow >> steps: - checkout - restore_cache: keys: - gpt4all-csharp-nuget-packages-nix - run: name: Install dependencies command: | brew install --cask dotnet-sdk - attach_workspace: at: /tmp/workspace - run: name: "Prepare Native Libs" command: | cd gpt4all-bindings/csharp mkdir -p runtimes/osx/native cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/ cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/ ls -R runtimes - run: name: "Install project dependencies" command: | cd gpt4all-bindings/csharp dotnet restore Gpt4All - save_cache: paths: - ~/.nuget/packages key: gpt4all-csharp-nuget-packages-nix - run: name: Build C# Project command: | cd gpt4all-bindings/csharp dotnet build Gpt4All --configuration Release --nologo - run: name: "Run C# Tests" command: | cd gpt4all-bindings/csharp dotnet test Gpt4All.Tests -v n -c Release --filter "SKIP_ON_CI!=True" --logger "trx" - run: name: Test results command: | cd gpt4all-bindings/csharp/Gpt4All.Tests dotnet tool install -g trx2junit export PATH="$PATH:$HOME/.dotnet/tools" trx2junit TestResults/*.trx - store_test_results: path: gpt4all-bindings/csharp/Gpt4All.Tests/TestResults store-and-upload-nupkgs: docker: - image: mcr.microsoft.com/dotnet/sdk:6.0-jammy # Ubuntu 22.04 steps: - attach_workspace: at: /tmp/workspace - checkout - restore_cache: keys: - gpt4all-csharp-nuget-packages-nix - run: name: NuGet Pack command: | cd gpt4all-bindings/csharp mkdir -p runtimes/linux-x64/native cp /tmp/workspace/runtimes/linux-x64/*.so runtimes/linux-x64/native/ mkdir -p runtimes/win-x64/native cp /tmp/workspace/runtimes/win-x64/*.dll runtimes/win-x64/native/ mkdir -p runtimes/osx/native cp /tmp/workspace/runtimes/osx-x64/*.dylib runtimes/osx/native/ cp /tmp/workspace/runtimes/osx-x64/*.metal runtimes/osx/native/ dotnet pack ./Gpt4All/Gpt4All.csproj -p:IncludeSymbols=true -p:SymbolPackageFormat=snupkg -c Release dotnet nuget push ./Gpt4All/bin/Release/Gpt4All.*.nupkg -s $NUGET_URL -k $NUGET_TOKEN --skip-duplicate - store_artifacts: path: gpt4all-bindings/csharp/Gpt4All/bin/Release workflows: version: 2 default: when: << pipeline.parameters.run-default-workflow >> jobs: - default-job build-and-test-gpt4all-chat: when: << pipeline.parameters.run-chat-workflow >> jobs: - hold: type: approval - build-gpt4all-chat-linux: requires: - hold - build-gpt4all-chat-windows: requires: - hold - build-gpt4all-chat-macos: requires: - hold deploy-docs: when: << pipeline.parameters.run-python-workflow >> jobs: - build-py-docs: filters: branches: only: - main build-py-deploy: when: << pipeline.parameters.run-python-workflow >> jobs: - pypi-hold: type: approval - hold: type: approval - build-py-linux: filters: branches: only: requires: - hold - build-py-macos: filters: branches: only: requires: - hold - build-py-windows: filters: branches: only: requires: - hold - store-and-upload-wheels: filters: branches: only: requires: - pypi-hold - build-py-windows - build-py-linux - build-py-macos build-bindings: when: or: - << pipeline.parameters.run-python-workflow >> - << pipeline.parameters.run-csharp-workflow >> jobs: - hold: type: approval - nuget-hold: type: approval - build-bindings-backend-linux: filters: branches: only: requires: - hold - build-bindings-backend-macos: filters: branches: only: requires: - hold - build-bindings-backend-windows: filters: branches: only: requires: - hold - build-bindings-backend-windows-msvc: filters: branches: only: requires: - hold - build-csharp-linux: filters: branches: only: requires: - build-bindings-backend-linux - build-csharp-windows: filters: branches: only: requires: - build-bindings-backend-windows - build-csharp-macos: filters: branches: only: requires: - build-bindings-backend-macos - store-and-upload-nupkgs: filters: branches: only: requires: - nuget-hold - build-csharp-windows - build-csharp-linux - build-csharp-macos