You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
extract_otp_secret_keys/.github/workflows/ci_release.yml

411 lines
22 KiB
YAML

name: release
# https://data-dive.com/multi-os-deployment-in-cloud-using-pyinstaller-and-github-actions
# https://github.com/actions/upload-artifact
# https://github.com/actions/download-artifact
# https://github.com/docker/metadata-action
# https://github.com/marketplace/actions/generate-release-hashes
# https://github.com/oleksis/pyinstaller-manylinux
# https://github.com/pypa/manylinux
# https://github.com/batonogov/docker-pyinstaller
# https://docs.github.com/de/actions/using-workflows/workflow-syntax-for-github-actions
# https://docs.github.com/en/actions/using-workflows
# https://docs.github.com/en/actions/learn-github-actions/contexts
# https://docs.github.com/en/actions/learn-github-actions/expressions
# https://docs.github.com/en/rest/releases/releases
# https://peps.python.org/pep-0440/
# https://semver.org/
# macOS:
# https://pyinstaller.org/en/stable/usage.html#building-macos-app-bundles
# https://github.com/pyinstaller/pyinstaller/wiki/Recipe-OSX-Code-Signing
# Build matrix:
# - Linux x86_64 glibc 2.35: ubuntu-latest
# - Linux x86_64 glibc 2.31: extract_otp_secrets:bullseye
# - Linux x86_64 glibc 2.36: extract_otp_secrets:bookworm
# - Windows x86_64: windows-latest
# - MacOS x86_64: macos-11
# - Linux x86_64 glibc 2.28: extract_otp_secrets:buster
# - Linux aarch64 glibc 2.28: extract_otp_secrets:buster
# - MacOS universal2: macos-11
# - Windows arm64: [buildx + https://github.com/batonogov/docker-pyinstaller]
on:
push:
tags:
- 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
pull_request:
schedule:
# Run weekly on default branch
- cron: '47 4 * * 6'
jobs:
create-release:
name: Create Release
runs-on: ubuntu-latest
outputs:
date: ${{ steps.meta.outputs.date }}
version: ${{ steps.meta.outputs.version }}
inline_version: ${{ steps.meta.outputs.inline_version }}
tag_name: ${{ steps.meta.outputs.tag_name }}
tag_message: ${{ steps.meta.outputs.tag_message }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set meta data
id: meta
# Writing to env with >> $GITHUB_ENV is an alternative
run: |
echo "date=$(TZ=Europe/Zurich date +'%d.%m.%Y')" >> $GITHUB_OUTPUT
echo "version=${TAG_NAME/v/}" >> $GITHUB_OUTPUT
echo "inline_version=${TAG_NAME/v/_}" >> $GITHUB_OUTPUT
echo "tag_name=${{ github.ref_name }}" >> $GITHUB_OUTPUT
echo "tag_message=$(git tag -l --format='%(contents:subject)' ${{ github.ref_name }})" >> $GITHUB_OUTPUT
env:
TAG_NAME: ${{ github.ref_name }}
PYTHONHASHSEED: 31
- name: Create Release
id: create_release
if: startsWith(github.ref, 'refs/tags/v')
# https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release
run: |
echo "date: ${{ steps.meta.outputs.date }}"
echo "version: ${{ steps.meta.outputs.version }}"
echo "inline_version: ${{ steps.meta.outputs.inline_version }}"
echo "tag_name: ${{ steps.meta.outputs.tag_name }}"
echo "tag_message: ${{ steps.meta.outputs.tag_message }}"
response=$(curl \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/scito/extract_otp_secrets/releases \
--silent \
--show-error \
-d '{"tag_name":"${{ github.ref }}","target_commitish":"master","name":"${{ steps.meta.outputs.version }} - ${{ steps.meta.outputs.date }}","body":"${{ steps.meta.outputs.tag_message }}\n\n## Executables\n\nDownload the executable for your platform and execute it, see [README.md](https://github.com/scito/extract_otp_secrets#readme)\n\n | Executable | Description |\n | --- | --- |\n | extract_otp_secrets${{ steps.meta.outputs.inline_version }}_linux_x86_64 | Linux x86_64/amd64 (glibc >= 2.31) |\n | extract_otp_secrets${{ steps.meta.outputs.inline_version }}_linux_arm64 | Linux arm64 (glibc >= 2.31) |\n | extract_otp_secrets${{ steps.meta.outputs.inline_version }}_win_x86_64.exe | Windows x86_64/amd64/x64 |\n | extract_otp_secrets${{ steps.meta.outputs.inline_version }}_win_arm64.exe | N/A |\n | extract_otp_secrets${{ steps.meta.outputs.inline_version }}_macos_x86_64.dmg | N/A, see [README.md](https://github.com/scito/extract_otp_secrets#readme) |\n | extract_otp_secrets${{ steps.meta.outputs.inline_version }}_macos_x86_64.pkg | N/A, see [README.md](https://github.com/scito/extract_otp_secrets#readme) |\n | extract_otp_secrets${{ steps.meta.outputs.inline_version }}_macos_x86_64 | MacOS x86_64/amd64 (bare executable, see [README.md](https://github.com/scito/extract_otp_secrets#readme); optional libzbar must be installed manually, see [README.md](https://github.com/scito/extract_otp_secrets#readme)) |\n | extract_otp_secrets${{ steps.meta.outputs.inline_version }}_macos_arm64 | N/A |\n","draft":true,"prerelease":false,"generate_release_notes":true}')
echo upload_url=$(jq '.upload_url' <<< "$response") >> $GITHUB_OUTPUT
echo $(jq -r '.upload_url' <<< "$response") > release_url.txt
echo $(jq -r '.id' <<< "$response") > release_id.txt
- name: Save Release URL File for publish
if: startsWith(github.ref, 'refs/tags/v')
uses: actions/upload-artifact@v3
with:
name: release_url
path: release_url.txt
- name: Save asset upload id for publish
if: startsWith(github.ref, 'refs/tags/v')
uses: actions/upload-artifact@v3
with:
name: release_id
path: release_id.txt
build-linux-executable-in-docker:
name: Build ${{ matrix.PLATFORM }} release in docker container
# run only when code is compiling and tests are passing
runs-on: ubuntu-latest
needs: create-release
strategy:
matrix:
include:
- PLATFORM: linux/amd64
EXE: extract_otp_secrets_linux_x86_64
ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_linux_x86_64
- PLATFORM: linux/arm64
EXE: extract_otp_secrets_linux_arm64
ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_linux_arm64
# steps to perform in job
steps:
- name: Checkout code
uses: actions/checkout@v3
# avoid building if there are testing errors
- name: Run smoke test
run: |
sudo apt-get install -y libzbar0
python -m pip install --upgrade pip
pip install -U -r requirements-dev.txt
pip install -U .
pytest
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
# setup Docker build action
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
# Workaround for failing builds: https://github.com/docker/build-push-action/issues/761#issuecomment-1383822381
# TODO remove workaround when fixed
with:
driver-opts: |
image=moby/buildkit:v0.10.6
- name: Login to DockerHub
uses: docker/login-action@v2
if: github.secret_source == 'Actions'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Packages
uses: docker/login-action@v2
if: github.secret_source == 'Actions'
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- name: Image digest
# TODO upload digests to assets
run: |
echo "extract_otp_secrets: ${{ steps.docker_build_bullseye.outputs.digest }}"
# TODO use local docker image https://stackoverflow.com/a/61155718/1663871
# https://github.com/multiarch/qemu-user-static
# https://hub.docker.com/r/multiarch/qemu-user-static/
- name: Run Pyinstaller in container for ${{ matrix.EXE }}
run: |
docker run --pull always --rm --privileged multiarch/qemu-user-static --reset -p yes
docker run --platform ${{ matrix.PLATFORM }} --pull always --entrypoint /bin/bash --rm -v "$(pwd)":/files -w /files scit0/extract_otp_secrets:bullseye -c 'apt-get update && apt-get -y install binutils && pip install -U -r /files/requirements.txt && pip install pyinstaller && PYTHONHASHSEED=31 && pyinstaller -y --add-data /usr/local/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name ${{ matrix.EXE }} --distpath /files/dist/ /files/src/extract_otp_secrets.py'
- name: Smoke tests linux/amd64
if: matrix.PLATFORM == 'linux/amd64'
run: |
dist/${{ matrix.EXE }} -V
dist/${{ matrix.EXE }} -h
dist/${{ matrix.EXE }} --debug
dist/${{ matrix.EXE }} example_export.png
dist/${{ matrix.EXE }} - < example_export.txt
dist/${{ matrix.EXE }} --qr ZBAR example_export.png
dist/${{ matrix.EXE }} --qr QREADER example_export.png
dist/${{ matrix.EXE }} --qr QREADER_DEEP example_export.png
dist/${{ matrix.EXE }} --qr CV2 example_export.png
dist/${{ matrix.EXE }} --qr CV2_WECHAT example_export.png
- name: Smoke tests linux/arm64
if: matrix.PLATFORM == 'linux/arm64'
run: |
docker run --platform ${{ matrix.PLATFORM }} --pull always --entrypoint /bin/bash --rm -v "$(pwd)":/files -w /files scit0/extract_otp_secrets -c 'dist/${{ matrix.EXE }} -V && dist/${{ matrix.EXE }} -h && dist/${{ matrix.EXE }} example_export.png && dist/${{ matrix.EXE }} - < example_export.txt && dist/${{ matrix.EXE }} --qr ZBAR example_export.png && dist/${{ matrix.EXE }} --qr QREADER example_export.png && dist/${{ matrix.EXE }} --qr QREADER_DEEP example_export.png && dist/${{ matrix.EXE }} --qr CV2 example_export.png && dist/${{ matrix.EXE }} --qr CV2_WECHAT example_export.png'
- name: Load Release URL File from release job
if: startsWith(github.ref, 'refs/tags/v')
uses: actions/download-artifact@v3
with:
name: release_url
- name: Display structure of files
run: ls -R
- name: Upload EXE to artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.EXE }}
path: dist/${{ matrix.EXE }}
- name: Upload Release Asset
id: upload-release-asset
if: startsWith(github.ref, 'refs/tags/v')
run: |
response=$(curl \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Content-Type: application/x-executable" \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
-H "X-GitHub-Api-Version: 2022-11-28" \
--silent \
--show-error \
--data-binary @dist/${{ matrix.EXE }} \
$(cat release_url.txt)=${{ matrix.ASSET_NAME }})
build-native-executables:
name: Build native packages
needs: create-release
runs-on: ${{ matrix.os }}
strategy:
matrix:
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners
include:
- os: windows-latest
TARGET: windows
# TODO add --icon
# TODO add --manifest
# TODO find more elegant solution for pyzbar\libiconv.dll and pyzbar\libzbar-64.dll
# Files of Visual C++ 2013 Redistributable Package: https://support.microsoft.com/en-us/topic/update-for-visual-c-2013-redistributable-package-d8ccd6a5-4e26-c290-517b-8da6cfdf4f10
EXE: extract_otp_secrets.exe
ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_win_x86_64.exe
ASSET_MIME: application/vnd.microsoft.portable-executable
UPLOAD: true
CMD_BUILD: |
pyinstaller -y --add-data "$($Env:pythonLocation)\__yolo_v3_qr_detector:__yolo_v3_qr_detector" --add-binary "$($Env:pythonLocation)\Lib\site-packages\pyzbar\libiconv.dll:pyzbar" --add-binary "$($Env:pythonLocation)\Lib\site-packages\pyzbar\libzbar-64.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\msvcr120.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\msvcp120.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\vcamp120.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\vcomp120.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\vccorlib120.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120u.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120chs.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120cht.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120deu.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120enu.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120esn.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120fra.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120ita.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120jpn.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120kor.dll:pyzbar" --add-binary "$($Env:WinDir)\system32\mfc120rus.dll:pyzbar" --onefile --version-file build\win_file_version_info.txt --name extract_otp_secrets.exe src\extract_otp_secrets.py
- os: macos-12
TARGET: macos
# https://pyinstaller.org/en/stable/spec-files.html#spec-file-options-for-a-macos-bundle
EXE: extract_otp_secrets
ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_macos_x86_64
DMG: extract_otp_secrets.dmg
ASSET_NAME_DMG: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_macos_x86_64.dmg
ASSET_MIME: application/octet-stream
UPLOAD: true
CMD_BUILD: |
VERSION_STR=$(setuptools-git-versioning) COPYRIGHT_YEARS='2020-2023' envsubst < installer/extract_otp_secrets_macos_template.spec > extract_otp_secrets_macos.spec
pyinstaller -y extract_otp_secrets_macos.spec
installer/build_dmg.sh
- os: ubuntu-latest
TARGET: linux
EXE: extract_otp_secrets_ubuntu
ASSET_NAME: extract_otp_secrets${{ needs.create-release.outputs.inline_version }}_linux_x86_64_ubuntu_latest
ASSET_MIME: application/x-executable
UPLOAD: false
CMD_BUILD: |
pyinstaller -y --add-data $pythonLocation/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name extract_otp_secrets_ubuntu src/extract_otp_secrets.py
steps:
- name: Output path
if: runner.os == 'Windows'
run: echo "$($Env:Path)"
- name: List Windir
if: runner.os == 'Windows'
run: ls "$($Env:WinDir)\system32"
- uses: actions/checkout@v3
- name: Set macos macos_python_path
# TODO use variable for Python version
run: echo "macos_python_path=/Library/Frameworks/Python.framework/Versions/3.12" >> $GITHUB_ENV
- name: Set up Python 3.12
uses: actions/setup-python@v4
with:
python-version: 3.12
check-latest: true
- name: Install zbar shared lib for QReader (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get install -y libzbar0
- name: Install zbar shared lib for QReader (macOS)
if: runner.os == 'macOS'
run: |
brew install zbar create-dmg
- name: Install dependencies
# TODO fix --use-pep517
run: |
python -m pip install --upgrade pip
pip install -U -r requirements-dev.txt
pip install -U .
- name: Create Windows win_file_version_info.txt
shell: bash
run: |
mkdir -p build/
VERSION_STR=$(setuptools-git-versioning) VERSION_MAJOR=$(cut -d '.' -f 1 <<< "$(setuptools-git-versioning)") VERSION_MINOR=$(cut -d '.' -f 2 <<< "$(setuptools-git-versioning)") VERSION_PATCH=$(echo $(cut -d '.' -f 3 <<< "$(setuptools-git-versioning)") | sed -E -n "s/^([0-9]+).*/\1/p") VERSION_BUILD=$(echo $(cut -d '.' -f 3 <<< "$(setuptools-git-versioning)") | sed -E -n -e"s/^[0-9]+.+/99/p")$(($(git rev-list --count $(git tag | sort -V -r | sed '1!d')..HEAD))) COPYRIGHT_YEARS='2020-2023' envsubst < installer/win_file_version_info_template.txt > build/win_file_version_info.txt
- name: Build with pyinstaller for ${{ matrix.TARGET }}
env:
# Reproducible build: https://pyinstaller.org/en/stable/advanced-topics.html#creating-a-reproducible-build
PYTHONHASHSEED: 31
run: ${{ matrix.CMD_BUILD }}
- name: Smoke tests for generated exe (general)
run: |
dist/${{ matrix.EXE }} -V
dist/${{ matrix.EXE }} -h
dist/${{ matrix.EXE }} --debug
dist/${{ matrix.EXE }} example_export.png
dist/${{ matrix.EXE }} --qr ZBAR example_export.png
dist/${{ matrix.EXE }} --qr QREADER example_export.png
dist/${{ matrix.EXE }} --qr QREADER_DEEP example_export.png
dist/${{ matrix.EXE }} --qr CV2 example_export.png
dist/${{ matrix.EXE }} --qr CV2_WECHAT example_export.png
- name: Smoke tests for generated exe (stdin)
if: runner.os != 'Windows'
run: |
dist/${{ matrix.EXE }} - < example_export.txt
- name: Load Release URL File from release job
if: startsWith(github.ref, 'refs/tags/v')
uses: actions/download-artifact@v3
with:
name: release_url
- name: Load Release Id File from release job
if: startsWith(github.ref, 'refs/tags/v')
uses: actions/download-artifact@v3
with:
name: release_id
- name: Display structure of files
run: ls -R
- name: Set meta data
id: meta
if: startsWith(github.ref, 'refs/tags/v')
shell: bash
run: |
echo "release_id=$(cat release_id.txt)" >> $GITHUB_OUTPUT
echo "upload_url=https://uploads.github.com/repos/scito/extract_otp_secrets/releases/$(cat release_id.txt)/assets?name=" >> $GITHUB_OUTPUT
- name: Upload EXE to artifacts
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.EXE }}
path: dist/${{ matrix.EXE }}
- name: Upload DMG to artifacts
uses: actions/upload-artifact@v3
if: runner.os == 'macOS'
with:
name: ${{ matrix.DMG }}
path: dist/${{ matrix.DMG }}
- name: Upload Release Asset
id: upload-release-asset
if: matrix.UPLOAD && startsWith(github.ref, 'refs/tags/v')
run: |
curl -X POST -H "Accept: application/vnd.github+json" -H "Content-Type: ${{ matrix.ASSET_MIME }}" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "X-GitHub-Api-Version: 2022-11-28" --show-error --data-binary @dist/${{ matrix.EXE }} ${{ steps.meta.outputs.upload_url }}=${{ matrix.ASSET_NAME }}
- name: Upload Release Asset DMG (macOS)
id: upload-release-asset-dmg
if: false && matrix.UPLOAD && startsWith(github.ref, 'refs/tags/v') && runner.os == 'macOS'
run: |
curl -X POST -H "Accept: application/vnd.github+json" -H "Content-Type: ${{ matrix.ASSET_MIME }}" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "X-GitHub-Api-Version: 2022-11-28" --show-error --data-binary @dist/${{ matrix.DMG }} ${{ steps.meta.outputs.upload_url }}=${{ matrix.ASSET_NAME_DMG }}
upload-hashes:
name: Upload hashes
if: startsWith(github.ref, 'refs/tags/v')
needs:
- build-linux-executable-in-docker
- build-native-executables
runs-on: ubuntu-latest
steps:
- name: Load Release Id File from release job
uses: actions/download-artifact@v3
with:
name: release_id
- name: Set meta data
id: meta
run: |
echo "release_id=$(cat release_id.txt)" >> $GITHUB_OUTPUT
echo "upload_url=https://uploads.github.com/repos/scito/extract_otp_secrets/releases/$(cat release_id.txt)/assets?name=" >> $GITHUB_OUTPUT
- name: Calculate and upload hashes from assets
run: |
GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
for asset_url in $(curl \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $GITHUB_TOKEN"\
-H "X-GitHub-Api-Version: 2022-11-28" \
--silent \
--show-error \
https://api.github.com/repos/scito/extract_otp_secrets/releases/${{ steps.meta.outputs.release_id }}/assets |
jq -r '.[].url'); do
echo "Download $asset_url"
name=$(curl \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $GITHUB_TOKEN"\
-H "X-GitHub-Api-Version: 2022-11-28" \
--output-dir assets \
-L \
$asset_url |
jq -r '.name')
curl \
-H "Accept: application/octet-stream" \
-H "Authorization: Bearer $GITHUB_TOKEN"\
-H "X-GitHub-Api-Version: 2022-11-28" \
--create-dirs \
--output-dir assets \
-L \
-o $name \
$asset_url
done
(cd assets/ && sha256sum * > ../sha256_hashes.txt)
curl -X POST -H "Accept: application/vnd.github+json" -H "Content-Type: text/plain" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "X-GitHub-Api-Version: 2022-11-28" --show-error --data-binary @sha256_hashes.txt ${{ steps.meta.outputs.upload_url }}=sha256_hashes.txt
(cd assets/ && sha512sum * > ../sha512_hashes.txt)
curl -X POST -H "Accept: application/vnd.github+json" -H "Content-Type: text/plain" -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "X-GitHub-Api-Version: 2022-11-28" --show-error --data-binary @sha512_hashes.txt ${{ steps.meta.outputs.upload_url }}=sha512_hashes.txt