diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e4a2156..beafecb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -7,6 +7,8 @@ name: tests
on:
push:
+ paths-ignore:
+ - 'docs/**'
# pull_request:
schedule:
# Run daily on default branch
@@ -29,6 +31,7 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
+ check-latest: false
- name: Install zbar shared lib for QReader (Linux)
if: runner.os == 'Linux'
run: |
diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml
index a48c1b1..939bbc2 100644
--- a/.github/workflows/ci_docker.yml
+++ b/.github/workflows/ci_docker.yml
@@ -11,13 +11,19 @@ name: docker
on:
# run it on push to the default repository branch
push:
+ paths-ignore:
+ - 'docs/**'
+ tags-ignore:
+ - '**'
+ # branches is needed if tags-ignore is used
+ branches:
+ - '**'
schedule:
# Run weekly on default branch
- cron: '47 3 * * 6'
jobs:
- # define job to build and publish docker image
- build-and-push-docker-image:
+ build-and-push-docker-debian-image:
name: Build Docker image and push to repositories
# run only when code is compiling and tests are passing
runs-on: ubuntu-latest
@@ -57,40 +63,94 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- - name: "no_qr_reader: Build image and push to Docker Hub and GitHub Container Registry"
- id: docker_build_only_txt
- uses: docker/build-push-action@v2
+ - name: "qr_reader: Build image and push to Docker Hub and GitHub Container Registry"
+ id: docker_build_qr_reader_latest
+ uses: docker/build-push-action@v3
with:
- # relative path to the place where source code with Dockerfile is located
platforms: linux/amd64,linux/arm64
+ # relative path to the place where source code with Dockerfile is located
+ # TODO file:, move to docker/
context: .
- file: Dockerfile_only_txt
+ file: Dockerfile
# builder: ${{ steps.buildx.outputs.name }}
# Note: tags has to be all lower-case
+ pull: true
tags: |
- scit0/extract_otp_secrets:latest-only-txt
- ghcr.io/scito/extract_otp_secrets:latest-only-txt
+ scit0/extract_otp_secrets:latest
+ scit0/extract_otp_secrets:bullseye
+ ghcr.io/scito/extract_otp_secrets:latest
+ ghcr.io/scito/extract_otp_secrets:bullseye
# build on feature branches, push only on master branch
push: ${{ github.ref == 'refs/heads/master' }}
- build-args: |
- RUN_TESTS=true
- - name: "qr_reader: Build image and push to Docker Hub and GitHub Container Registry"
- id: docker_build_qr_reader
- uses: docker/build-push-action@v2
+ - name: Image digest
+ # TODO upload digests to assets
+ run: |
+ echo "extract_otp_secrets: ${{ steps.docker_build_qr_reader_latest.outputs.digest }}"
+
+ build-and-push-docker-alpine-image:
+ name: Build Docker image and push to repositories
+ # run only when code is compiling and tests are passing
+ runs-on: ubuntu-latest
+
+ # steps to perform in job
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v3
+
+ # avoid building if there are testing errors
+ - name: Run smoke test
+ run: |
+ sudo apt-get install -y libzbar0
+ python -m pip install --upgrade pip
+ pip install -U -r requirements-dev.txt
+ pip install -U .
+ pytest
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+
+ # setup Docker build action
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: docker/setup-buildx-action@v2
+
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Login to Github Packages
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GHCR_IO_TOKEN }}
+
+ - name: "only_txt: Build image and push to Docker Hub and GitHub Container Registry"
+ id: docker_build_only_txt
+ uses: docker/build-push-action@v3
with:
- platforms: linux/amd64,linux/arm64
# relative path to the place where source code with Dockerfile is located
+ platforms: linux/amd64,linux/arm64
context: .
+ file: Dockerfile_only_txt
# builder: ${{ steps.buildx.outputs.name }}
# Note: tags has to be all lower-case
+ pull: true
tags: |
- scit0/extract_otp_secrets:latest
- ghcr.io/scito/extract_otp_secrets:latest
+ scit0/extract_otp_secrets:only-txt
+ scit0/extract_otp_secrets:alpine
+ ghcr.io/scito/extract_otp_secrets:only-txt
+ ghcr.io/scito/extract_otp_secrets:alpine
# build on feature branches, push only on master branch
push: ${{ github.ref == 'refs/heads/master' }}
+ build-args: |
+ RUN_TESTS=true
+
- name: Image digest
+ # TODO upload digests to assets
run: |
- echo "extract_otp_secrets: ${{ steps.docker_build_qr_reader.outputs.digest }}"
- echo "extract_otp_secrets_only_txt: ${{ steps.docker_build_only_txt.outputs.digest }}"
+ echo "extract_otp_secrets:only-txt: ${{ steps.docker_build_only_txt.outputs.digest }}"
diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml
new file mode 100644
index 0000000..ec75240
--- /dev/null
+++ b/.github/workflows/ci_release.yml
@@ -0,0 +1,281 @@
+name: release
+
+# https://data-dive.com/multi-os-deployment-in-cloud-using-pyinstaller-and-github-actions
+# https://github.com/actions/create-release (archived)
+# https://github.com/actions/upload-artifact
+# https://github.com/actions/download-artifact
+# https://github.com/actions/upload-release-asset (archived)
+# https://github.com/docker/metadata-action
+# https://github.com/marketplace/actions/generate-release-hashes
+
+# https://github.com/oleksis/pyinstaller-manylinux
+# https://github.com/pypa/manylinux
+# https://github.com/batonogov/docker-pyinstaller
+
+# https://docs.github.com/de/actions/using-workflows/workflow-syntax-for-github-actions
+# https://docs.github.com/en/actions/using-workflows
+# https://docs.github.com/en/actions/learn-github-actions/contexts
+# https://docs.github.com/en/actions/learn-github-actions/expressions
+
+# https://docs.github.com/en/rest/releases/releases
+
+# https://peps.python.org/pep-0440/
+# https://semver.org/
+
+# Build matrix:
+# - Linux x86_64 glibc 2.35: ubuntu-latest
+# - Linux x86_64 glibc 2.34: extract_otp_secrets:buster
+# - Windows x86_64: windows-latest
+# - MacOS x86_64: macos-11
+# - Linux x86_64 glibc 2.28: extract_otp_secrets:buster
+# - Linux aarch64 glibc 2.28: extract_otp_secrets:buster
+# - MacOS universal2: macos-11
+# - Windows arm64: [buildx + https://github.com/batonogov/docker-pyinstaller]
+
+on:
+ push:
+ tags:
+ - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+
+jobs:
+
+ create-release:
+ name: Create Release
+ runs-on: ubuntu-latest
+ steps:
+ - name: Set meta data
+ id: meta
+ # Writing to env with >> $GITHUB_ENV is an alternative
+ run: |
+ echo "date=$(TZ=Europe/Bern date +'%d.%m.%Y')" >> $GITHUB_OUTPUT
+ echo "version=${TAG_NAME/v/}" >> $GITHUB_OUTPUT
+ echo "tag_name=${{ github.ref_name }}" >> $GITHUB_OUTPUT
+ echo "tag_message=$(git tag -l --format='%(contents:subject)' ${{ github.ref_name }})" >> $GITHUB_OUTPUT
+ env:
+ TAG_NAME: ${{ github.ref_name }}
+ - name: Create Release
+ id: create_release
+ run: |
+ # https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#create-a-release
+ response=$(curl \
+ -X POST \
+ -H "Accept: application/vnd.github+json" \
+ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ https://api.github.com/repos/scito/extract_otp_secrets/releases \
+ --silent \
+ --show-error \
+ -d '{"tag_name":"${{ github.ref }}","target_commitish":"master","name":"${{ steps.meta.outputs.version }} - ${{ steps.meta.outputs.date }}","body":"${{ steps.meta.outputs.tag_message }}","draft":true,"prerelease":false,"generate_release_notes":true}')
+ echo upload_url=$(jq '.upload_url' <<< "$response") >> $GITHUB_OUTPUT
+ echo $(jq -r '.upload_url' <<< "$response") > release_url.txt
+ - name: Save Release URL File for publish
+ uses: actions/upload-artifact@v3
+ with:
+ name: release_url
+ path: release_url.txt
+
+ build-and-push-docker-image:
+ name: Build Linux release in docker container
+ # run only when code is compiling and tests are passing
+ runs-on: ubuntu-latest
+ needs: create-release
+
+ # steps to perform in job
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v3
+
+ # avoid building if there are testing errors
+ - name: Run smoke test
+ run: |
+ sudo apt-get install -y libzbar0
+ python -m pip install --upgrade pip
+ pip install -U -r requirements-dev.txt
+ pip install -U .
+ pytest
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+
+ # setup Docker build action
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: docker/setup-buildx-action@v2
+
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+ - name: Login to Github Packages
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GHCR_IO_TOKEN }}
+
+ - name: "Build image from Buster and push to GitHub Container Registry"
+ id: docker_build_buster
+ uses: docker/build-push-action@v3
+ with:
+ platforms: linux/amd64,linux/arm64
+ # relative path to the place where source code with Dockerfile is located
+ # TODO file:, move to docker/
+ context: .
+ file: Dockerfile
+ # builder: ${{ steps.buildx.outputs.name }}
+ build-args: |
+ BASE_IMAGE=python:3.11-slim-buster
+ # Note: tags has to be all lower-case
+ pull: true
+ tags: |
+ ghcr.io/scito/extract_otp_secrets:buster
+ push: true
+
+ # # https://stackoverflow.com/a/61155718/1663871
+ # - name: Build docker images
+ # run: docker build -t local < .devcontainer/Dockerfile
+ # - name: Run tests
+ # run: docker run -it -v $PWD:/srv -w/srv local make test
+
+ - name: Image digest
+ # TODO upload digests to assets
+ run: |
+ echo "extract_otp_secrets: ${{ steps.docker_build_buster.outputs.digest }}"
+
+ - name: Run Pyinstaller in container
+ run: |
+ # TODO use local docker image https://stackoverflow.com/a/61155718/1663871
+ docker run --pull always --entrypoint /bin/bash --rm -v "$(pwd)":/files -w /files ghcr.io/scito/extract_otp_secrets:buster -c 'apt-get update && apt-get -y install binutils && pip install -U -r /files/requirements.txt && pip install pyinstaller && pyinstaller -y --add-data /usr/local/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name extract_otp_secrets_linux_x86_64 --distpath /files/dist/ /files/src/extract_otp_secrets.py'
+
+ - name: Smoke tests
+ run: |
+ dist/extract_otp_secrets_linux_x86_64 -h
+ dist/extract_otp_secrets_linux_x86_64 example_export.png
+ dist/extract_otp_secrets_linux_x86_64 - < example_export.txt
+ - name: Load Release URL File from release job
+ uses: actions/download-artifact@v3
+ with:
+ name: release_url
+ - name: Display structure of files
+ run: ls -R
+ - name: Upload Release Asset
+ id: upload-release-asset
+ # TODO only for tags
+ shell: bash
+ run: |
+ response=$(curl \
+ -X POST \
+ -H "Accept: application/vnd.github+json" \
+ -H "Content-Type: application/x-executable" \
+ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ --silent \
+ --show-error \
+ --data-binary @dist/extract_otp_secrets_linux_x86_64 \
+ $(cat release_url.txt)=extract_otp_secrets_linux_x86_64)
+
+ build:
+ name: Build packages
+ needs: create-release
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners
+ include:
+ - os: ubuntu-latest
+ TARGET: linux
+ CMD_BUILD: |
+ pyinstaller -y --add-data $pythonLocation/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile src/extract_otp_secrets.py
+ OUT_FILE_NAME: extract_otp_secrets
+ ASSET_NAME: extract_otp_secrets_linux_x86_64_ubuntu_latest
+ ASSET_MIME: application/x-executable
+ UPLOAD: false
+ - os: macos-11
+ TARGET: macos
+ # TODO add --icon
+ # TODO add --osx-bundle-identifier
+ # TODO add --codesign-identity
+ # TODO add --osx-entitlements-file
+ # TODO https://pyinstaller.org/en/stable/spec-files.html#spec-file-options-for-a-macos-bundle
+ # TODO --target-arch universal2
+ CMD_BUILD: |
+ pyinstaller -y --add-data $macos_python_path/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --argv-emulation src/extract_otp_secrets.py
+ OUT_FILE_NAME: extract_otp_secrets
+ ASSET_NAME: extract_otp_secrets_macos_x86_64
+ ASSET_MIME: application/x-newton-compatible-pkg
+ UPLOAD: true
+ - os: windows-latest
+ TARGET: windows
+ # TODO add --icon
+ # TODO add --manifest
+ # TODO find more elegant solution for pyzbar\libiconv.dll and pyzbar\libzbar-64.dll
+ CMD_BUILD: |
+ pyinstaller -y --add-data "$($Env:pythonLocation)\__yolo_v3_qr_detector;__yolo_v3_qr_detector" --add-binary "$($Env:pythonLocation)\Lib\site-packages\pyzbar\libiconv.dll;pyzbar" --add-binary "$($Env:pythonLocation)\Lib\site-packages\pyzbar\libzbar-64.dll;pyzbar" --onefile --version-file build\file_version_info.txt src\extract_otp_secrets.py
+ OUT_FILE_NAME: extract_otp_secrets.exe
+ ASSET_NAME: extract_otp_secrets_win_x86_64.exe
+ ASSET_MIME: application/vnd.microsoft.portable-executable
+ UPLOAD: true
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set macos macos_python_path
+ # TODO use variable for Python version
+ run: echo "macos_python_path=/Library/Frameworks/Python.framework/Versions/3.11" >> $GITHUB_ENV
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v4
+ with:
+ python-version: 3.11
+ check-latest: true
+ - name: Install zbar shared lib for QReader (Linux)
+ if: runner.os == 'Linux'
+ run: |
+ sudo apt-get install -y libzbar0
+ - name: Install zbar shared lib for QReader (macOS)
+ if: runner.os == 'macOS'
+ run: |
+ brew install zbar
+ - name: Install dependencies
+ # TODO fix --use-pep517
+ run: |
+ python -m pip install --upgrade pip
+ pip install -U -r requirements-dev.txt
+ pip install -U .
+ - name: Create Windows file_version_info.txt
+ shell: bash
+ run: |
+ mkdir -p build/
+ VERSION_STR=$(setuptools-git-versioning) VERSION_MAJOR=$(cut -d '.' -f 1 <<< "$(setuptools-git-versioning)") VERSION_MINOR=$(cut -d '.' -f 2 <<< "$(setuptools-git-versioning)") VERSION_PATCH=$(cut -d '.' -f 3 <<< "$(setuptools-git-versioning)") VERSION_BUILD=$(($(git rev-list --count $(git tag | sort -V -r | sed '1!d')..HEAD))) YEARS='2020-2023' envsubst < file_version_info_template.txt > build/file_version_info.txt
+ - name: Build with pyinstaller for ${{ matrix.TARGET }}
+ run: ${{ matrix.CMD_BUILD }}
+ - name: Smoke tests for generated exe (general)
+ run: |
+ dist/${{ matrix.OUT_FILE_NAME }} -h
+ dist/${{ matrix.OUT_FILE_NAME }} example_export.png
+ - name: Smoke tests for generated exe (stdin)
+ if: runner.os != 'Windows'
+ run: |
+ dist/${{ matrix.OUT_FILE_NAME }} - < example_export.txt
+ - name: Load Release URL File from release job
+ uses: actions/download-artifact@v3
+ with:
+ name: release_url
+ - name: Display structure of files
+ run: ls -R
+ - name: Upload Release Asset
+ id: upload-release-asset
+ # TODO only for tags
+ if: ${{ matrix.UPLOAD }}
+ shell: bash
+ run: |
+ response=$(curl \
+ -X POST \
+ -H "Accept: application/vnd.github+json" \
+ -H "Content-Type: ${{ matrix.ASSET_MIME }}" \
+ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
+ -H "X-GitHub-Api-Version: 2022-11-28" \
+ --silent \
+ --show-error \
+ --data-binary @dist/${{ matrix.OUT_FILE_NAME }} \
+ $(cat release_url.txt)=${{ matrix.ASSET_NAME }})
+
diff --git a/.gitignore b/.gitignore
index 19ff843..318a7a0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,9 @@ dist/
*.xml
pytest-coverage.txt
tests/reports/
+dist_*/
+*.spec
+
+file_version_info_python.txt
+file_version_info_explorer.txt
+file_version_info.txt
diff --git a/Dockerfile b/Dockerfile
index 7d5f5fa..43ae3f3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,6 @@
-FROM python:3.11-slim-bullseye
+# --build-arg BASE_IMAGE=python:3.11-slim-buster
+ARG BASE_IMAGE=python:3.11-slim-bullseye
+FROM $BASE_IMAGE
# https://docs.docker.com/engine/reference/builder/
@@ -10,7 +12,7 @@ FROM python:3.11-slim-bullseye
WORKDIR /extract
-COPY . .
+COPY requirements*.txt src/ run_pytest.sh pytest.ini tests/ example_*.txt example_*.png example_*.csv example*.json docker/.alias ./
ARG RUN_TESTS=true
@@ -20,13 +22,14 @@ RUN apt-get update && apt-get install -y \
libsm6 \
libzbar0 \
&& rm -rf /var/lib/apt/lists/* \
- && pip install --no-cache-dir -U -r \
- requirements.txt \
- && if [ "$RUN_TESTS" = "true" ]; then /extract/run_pytest.sh; else echo "Not running tests..."; fi
+ && pip install --no-cache-dir -U -r requirements.txt \
+ && if [ "$RUN_TESTS" = "true" ]; then /extract/run_pytest.sh; else echo "Not running tests..."; fi \
+ && echo 'test -s /extract/.alias && . /extract/.alias || true' >> ~/.bashrc
WORKDIR /files
-ENTRYPOINT ["python", "/extract/src/extract_otp_secrets.py"]
+ENTRYPOINT ["python", "/extract/extract_otp_secrets.py"]
LABEL org.opencontainers.image.source https://github.com/scito/extract_otp_secrets
LABEL org.opencontainers.image.license GPL-3.0+
+LABEL maintainer="Scito https://scito.ch, https://github.com/scito"
diff --git a/Dockerfile_only_txt b/Dockerfile_only_txt
index b44e665..f7a6f2a 100644
--- a/Dockerfile_only_txt
+++ b/Dockerfile_only_txt
@@ -1,16 +1,19 @@
-FROM python:3.11-alpine
+ARG BASE_IMAGE=python:3.11-alpine
+FROM $BASE_IMAGE
# https://docs.docker.com/engine/reference/builder/
# For debugging
# docker run --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt
# docker build . -t extract_otp_secrets_only_txt -f Dockerfile_only_txt --pull --build-arg RUN_TESTS=false
-# docker run --entrypoint /bin/sh -it --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt
-# docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt tests/extract_otp_secrets_test.py -k "not qreader" --relaxed -vvv -s
+# docker run --entrypoint /bin/ash -it --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt -l
+# docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt extract_otp_secrets_test.py -k "not qreader" --relaxed -vvv -s
+
+# https://github.com/pypa/manylinux/blob/main/docker/Dockerfile
WORKDIR /extract
-COPY . .
+COPY requirements*.txt src/ run_pytest.sh pytest.ini tests/ example_*.txt example_*.png example_*.csv example*.json docker/.alias ./
ARG RUN_TESTS=true
@@ -32,11 +35,13 @@ RUN apk add --no-cache \
protobuf \
qrcode \
&& if [[ "$(apk --print-arch)" == "aarch64" ]]; then apk del .build-deps; fi \
- && if [[ "$RUN_TESTS" == "true" ]]; then /extract/run_pytest.sh tests/extract_otp_secrets_test.py -k "not qreader" --relaxed; else echo "Not running tests..."; fi
+ && if [[ "$RUN_TESTS" == "true" ]]; then /extract/run_pytest.sh extract_otp_secrets_test.py -k "not qreader" --relaxed; else echo "Not running tests..."; fi \
+ && echo 'test -s /extract/.alias && . /extract/.alias || true' >> ~/.profile
WORKDIR /files
-ENTRYPOINT ["python", "/extract/src/extract_otp_secrets.py"]
+ENTRYPOINT ["python", "/extract/extract_otp_secrets.py"]
LABEL org.opencontainers.image.source https://github.com/scito/extract_otp_secrets
LABEL org.opencontainers.image.license GPL-3.0+
+LABEL maintainer="Scito https://scito.ch, https://github.com/scito"
diff --git a/Pipfile b/Pipfile
index e9b33bc..d4a844c 100644
--- a/Pipfile
+++ b/Pipfile
@@ -16,12 +16,14 @@ qreader = "<2.0.0"
[dev-packages]
build = "*"
flake8 = "*"
+gfm-toc = "*"
mypy = "*"
mypy-protobuf = "*"
pylint = "*"
pytest = "*"
pytest-cov = "*"
pytest-mock = "*"
+setuptools-git-versioning = "*"
types-protobuf = "*"
wheel = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
index 79aa9a6..ec57988 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "9b56c9708e464fbb035e36b29b0c89f0250bc50ac37234f3948a366136a8e6c9"
+ "sha256": "41edd4aebe075d6c39d035ec7cb10f0253a3ad21f9b4aa5b9c57deccca87874f"
},
"pipfile-spec": 6,
"requires": {
@@ -220,11 +220,11 @@
"develop": {
"astroid": {
"hashes": [
- "sha256:3bc7834720e1a24ca797fd785d77efb14f7a28ee8e635ef040b6e2d80ccb3303",
- "sha256:8f6a8d40c4ad161d6fc419545ae4b2f275ed86d1c989c97825772120842ee0d2"
+ "sha256:14c1603c41cc61aae731cad1884a073c4645e26f126d13ac8346113c95577f3b",
+ "sha256:6afc22718a48a689ca24a97981ad377ba7fb78c133f40335dfd16772f29bcfb1"
],
"markers": "python_full_version >= '3.7.2'",
- "version": "==2.13.2"
+ "version": "==2.13.3"
},
"attrs": {
"hashes": [
@@ -318,6 +318,14 @@
"index": "pypi",
"version": "==6.0.0"
},
+ "gfm-toc": {
+ "hashes": [
+ "sha256:247af7267a6cbbdd4213f8383157997bcb07e39e819db737bd2dbfbdb94ee7ae",
+ "sha256:c53ed0e2cd400e89051377017ca98c11c9cef628b2effddf787db4fc19ff343d"
+ ],
+ "index": "pypi",
+ "version": "==0.0.7"
+ },
"iniconfig": {
"hashes": [
"sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
@@ -535,6 +543,22 @@
"index": "pypi",
"version": "==3.10.0"
},
+ "setuptools": {
+ "hashes": [
+ "sha256:6f590d76b713d5de4e49fe4fbca24474469f53c83632d5d0fd056f7ff7e8112b",
+ "sha256:ac4008d396bc9cd983ea483cb7139c0240a07bbc74ffb6232fceffedc6cf03a8"
+ ],
+ "markers": "python_version >= '3.7'",
+ "version": "==66.1.1"
+ },
+ "setuptools-git-versioning": {
+ "hashes": [
+ "sha256:648481f7e1e9e12ccd2b069d616b909a338b4223956319649351751cbc0207f4",
+ "sha256:fde1a7cb3b2566979e5651cfca0d33cd5a82771711cd38a056216391936cf0ff"
+ ],
+ "index": "pypi",
+ "version": "==1.13.1"
+ },
"tomlkit": {
"hashes": [
"sha256:07de26b0d8cfc18f871aec595fda24d95b08fef89d147caa861939f37230bf4b",
@@ -545,11 +569,11 @@
},
"types-protobuf": {
"hashes": [
- "sha256:7df483d34ad3fcb1fa7fff1073560d596c9ac1f419cfa851b220c9a93386c998",
- "sha256:aeefcf39d637016998b3c7b699750847071b555f7c2e0c9873d42ab6103d1a39"
+ "sha256:6c87c7f8df61d57a53de8221777e4fcc3c7ed24419fbf43b8e9f50887f3773fa",
+ "sha256:824109e0fe87525a9d2da4cc4eec36ca004f1a0f3d1c0838cfd2873a484cffdd"
],
"index": "pypi",
- "version": "==4.21.0.2"
+ "version": "==4.21.0.3"
},
"typing-extensions": {
"hashes": [
diff --git a/README.md b/README.md
index d02800c..3a598a3 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,20 @@
# Extract TOTP/HOTP secrets from QR codes exported by two-factor authentication apps
[](https://github.com/scito/extract_otp_secrets/actions/workflows/ci.yml)
-
[](https://github.com/scito/extract_otp_secrets/actions/workflows/ci_docker.yml)
-
+
+
+
[](https://github.com/scito/extract_otp_secrets/blob/master/LICENSE)
-[](https://github.com/scito/extract_otp_secrets/tags)
+[](https://github.com/scito/extract_otp_secrets/releases/latest)
+[](https://github.com/scito/extract_otp_secrets/releases/latest)
+[](https://github.com/scito/extract_otp_secrets/releases/latest)
+[](https://github.com/scito/extract_otp_secrets/releases/latest)
+[](https://github.com/scito/extract_otp_secrets/releases/latest)
+[](https://hub.docker.com/repository/docker/scit0/extract_otp_secrets/general)
+
[](https://stand-with-ukraine.pp.ua)
---
@@ -23,6 +30,62 @@ The secrets can be exported to JSON or CSV, or printed as QR codes to console or
⚡ **This project/script was renamed from `extract_otp_secret_keys` to `extract_otp_secrets`.** ⚡
+
+Table of contents
+
+## Table of contents
+
+- [Usage](#usage)
+ - [Capture QR codes from camera (🆕 since version 2.0)](#capture-qr-codes-from-camera--since-version-20)
+ - [With builtin QR decoder from image files (🆕 since version 2.0)](#with-builtin-qr-decoder-from-image-files--since-version-20)
+ - [With external QR decoder app from text files](#with-external-qr-decoder-app-from-text-files)
+- [Installation](#installation)
+ - [Download binary executable (🆕 since v2.1)](#download-binary-executable--since-v21)
+ - [Run as script (recommend for developers or advanced users)](#run-as-script-recommend-for-developers-or-advanced-users)
+ - [Installation of shared system libraries](#installation-of-shared-system-libraries)
+- [Program help: arguments and options](#program-help-arguments-and-options)
+- [Examples](#examples)
+ - [Printing otp secrets form text file](#printing-otp-secrets-form-text-file)
+ - [Printing otp secrets from image file](#printing-otp-secrets-from-image-file)
+ - [Writing otp secrets to csv file](#writing-otp-secrets-to-csv-file)
+ - [Writing otp secrets to json file](#writing-otp-secrets-to-json-file)
+ - [Printing otp secrets multiple files](#printing-otp-secrets-multiple-files)
+ - [Printing otp secrets from stdin (text)](#printing-otp-secrets-from-stdin-text)
+ - [Printing otp secrets from stdin (image)](#printing-otp-secrets-from-stdin-image)
+ - [Printing otp secrets csv to stdout](#printing-otp-secrets-csv-to-stdout)
+ - [Printing otp secrets csv to stdout without header line](#printing-otp-secrets-csv-to-stdout-without-header-line)
+ - [Reading from stdin and printing to stdout](#reading-from-stdin-and-printing-to-stdout)
+- [Features](#features)
+- [KeePass](#keepass)
+- [How to export otp secrets from Google Authenticator app](#how-to-export-otp-secrets-from-google-authenticator-app)
+- [Glossary](#glossary)
+- [Alternative installation methods](#alternative-installation-methods)
+ - [pip using github](#pip-using-github)
+ - [local pip](#local-pip)
+ - [pipenv](#pipenv)
+ - [Visual Studio Code Remote - Containers / VSCode devcontainer](#visual-studio-code-remote---containers--vscode-devcontainer)
+ - [venv](#venv)
+ - [devbox](#devbox)
+ - [docker](#docker)
+ - [More docker examples](#more-docker-examples)
+- [Tests](#tests)
+ - [PyTest](#pytest)
+ - [unittest](#unittest)
+ - [VSCode Setup](#vscode-setup)
+- [Development](#development)
+ - [Build](#build)
+ - [Upgrade pip Packages](#upgrade-pip-packages)
+ - [Build docker images](#build-docker-images)
+ - [Create executables with pyinstaller](#create-executables-with-pyinstaller)
+ - [Full local build (bash)](#full-local-build-bash)
+- [Technical background](#technical-background)
+- [References](#references)
+- [Issues](#issues)
+- [Problems and Troubleshooting](#problems-and-troubleshooting)
+ - [Windows error message](#windows-error-message)
+- [Related projects](#related-projects)
+
+
## Usage
### Capture QR codes from camera (🆕 since version 2.0)
@@ -30,11 +93,16 @@ The secrets can be exported to JSON or CSV, or printed as QR codes to console or
1. Open "Google Authenticator" app on the mobile phone
2. Export the QR codes from "Google Authenticator" app (see [how to export](#how-to-export-otp-secrets-from-google-authenticator-app))
3. Point the exported QR codes to the camera of your computer
-4. Call this script without infile parameters:
-
- python src/extract_otp_secrets.py
+4. Run the program without infile parameters:
+```
+extract_otp_secrets
+```
+or
+```
+python src/extract_otp_secrets.py
+```
-
+
Detected QR codes are surrounded with a frame. The color of the frame indicates the extracting result:
@@ -48,13 +116,13 @@ The secrets are printed by default to the console. [Set program parameters](#pro
1. Open "Google Authenticator" app on the mobile phone
2. Export the QR codes from "Google Authenticator" app (see [how to export](#how-to-export-otp-secrets-from-google-authenticator-app))
-4. Save the QR code as image file, e.g. example_export.png
-5. Transfer the images files to the computer where his script is installed.
-6. Call this script with the file as input:
-
- python src/extract_otp_secrets.py example_export.png
-
-7. Remove unencrypted files with secrets from your computer and mobile.
+3. Save the QR code as image file, e.g. example_export.png
+4. Transfer the images files to the computer where his script is installed.
+5. Call this script with the file as input:
+```
+python src/extract_otp_secrets.py example_export.png
+```
+6. Remove unencrypted files with secrets from your computer and mobile.
### With external QR decoder app from text files
@@ -64,14 +132,25 @@ The secrets are printed by default to the console. [Set program parameters](#pro
4. Save the captured QR codes from the QR code reader to a text file, e.g. example_export.txt. Save each QR code on a new line. (The captured QR codes look like `otpauth-migration://offline?data=…`)
5. Transfer the file to the computer where his script is installed.
6. Call this script with the file as input:
-
- python src/extract_otp_secrets.py example_export.txt
-
+```
+python src/extract_otp_secrets.py example_export.txt
+```
7. Remove unencrypted files with secrets from your computer and mobile.
## Installation
-```
+### Download binary executable (🆕 since v2.1)
+
+1. Download executable for your platform from [latest release](https://github.com/scito/extract_otp_secrets/releases/latest), see assets
+2. Start executable by clicking or from command line
+
+✅ Everything is just packed in one executable.
+✅ No installation needed, neither Python nor dependencies have to be installed.
+✅ Easy and convenient
+
+### Run as script (recommend for developers or advanced users)
+
+```bash
git clone https://github.com/scito/extract_otp_secrets.git
cd extract_otp_secrets
pip install -U -r requirements.txt
@@ -120,7 +199,7 @@ OpenCV requires [Visual C++ redistributable 2015](https://www.microsoft.com/en-u
## Program help: arguments and options
-usage: extract_otp_secrets.py [-h] [--csv FILE] [--keepass FILE] [--json FILE] [--printqr] [--saveqr DIR] [--camera NUMBER] [--qr {ZBAR,QREADER,QREADER_DEEP,CV2,CV2_WECHAT}] [-i] [--no-color] [-d | -v | -q] [infile ...]
+usage: extract_otp_secrets.py [-h] [--csv FILE] [--keepass FILE] [--json FILE] [--printqr] [--saveqr DIR] [--camera NUMBER] [--qr {ZBAR,QREADER,QREADER_DEEP,CV2,CV2_WECHAT}] [-i] [--no-color] [--version] [-d | -v | -q] [infile ...]
Extracts one time password (OTP) secrets from QR codes exported by two-factor authentication (2FA) apps
If no infiles are provided, a GUI window starts and QR codes are captured from the camera.
@@ -141,6 +220,7 @@ options:
QR reader (default: ZBAR)
-i, --ignore ignore duplicate otps
--no-color, -n do not use ANSI colors in console output
+ --version, -V print version and quit
-d, --debug enter debug mode, do checks and quit
-v, --verbose verbose output
-q, --quiet no stdout output, except output set by -
@@ -226,6 +306,11 @@ python extract_otp_secrets.py = < example_export.png
* Portable image format - *.pbm, *.pgm, *.ppm *.pxm, *.pnm
* Prints errors and warnings to stderr (🆕 since v2.0)
* Prints colored output (🆕 since v2.0)
+* Startable as executable (script, Python, and all dependencies packed in one executable) (🆕 since v2.1)
+ * extract_otp_secrets_linux_x86_64 (requires glibc >= 2.28)
+ * extract_otp_secrets_win_x86_64.exe
+ * extract_otp_secrets_macos_x86_64 (untested)
+* Prebuilt Docker images provided for amd64 and arm64 (🆕 since v2.0)
* Many ways to run the script:
* Native Python
* pipenv
@@ -234,7 +319,6 @@ python extract_otp_secrets.py = < example_export.png
* Docker
* VSCode devcontainer
* devbox
-* Prebuilt Docker images provided for amd64 and arm64 (🆕 since v2.0)
* Compatible with major platforms:
* Linux
* macOS
@@ -308,10 +392,16 @@ KeePass can be used as a backup for one time passwords (second factor) from the
## Alternative installation methods
-### pip
+### pip using github
```
pip install -U git+https://github.com/scito/extract_otp_secrets
+extract_otp_secrets
+```
+
+or run it
+
+```
python -m extract_otp_secrets
```
@@ -319,7 +409,7 @@ or from a specific tag
```
pip install -U git+https://github.com/scito/extract_otp_secrets.git@v2.0.0
-python -m extract_otp_secrets
+extract_otp_secrets
curl -s https://raw.githubusercontent.com/scito/extract_otp_secrets/master/example_export.txt | python -m extract_otp_secrets -
```
@@ -328,6 +418,12 @@ curl -s https://raw.githubusercontent.com/scito/extract_otp_secrets/master/examp
```
git clone https://github.com/scito/extract_otp_secrets.git
pip install -U -e extract_otp_secrets
+extract_otp_secrets extract_otp_secrets/example_export.txt
+```
+
+or run it
+
+```
python -m extract_otp_secrets extract_otp_secrets/example_export.txt
```
@@ -500,16 +596,34 @@ docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extrac
#### Alpine (only text file processing)
```bash
-docker build . -t extract_otp_secrets_only_txt --pull -f Dockerfile_only_txt --build-arg RUN_TESTS=false
+docker build . -t extract_otp_secrets:only_txt --pull -f Dockerfile_only_txt --build-arg RUN_TESTS=false
```
Run tests in docker container:
```bash
-docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt tests/extract_otp_secrets_test.py -k "not qreader" --relaxed
+docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt extract_otp_secrets_test.py -k "not qreader" --relaxed
```
-### Full local build
+### Create executables with pyinstaller
+
+#### Linux
+
+```bash
+pyinstaller -y --add-data $pythonLocation/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile src/extract_otp_secrets.py
+```
+
+Output is executable `dist/extract_otp_secrets`
+
+#### Windows
+
+```
+pyinstaller -y --add-data "%pythonLocation%\__yolo_v3_qr_detector;__yolo_v3_qr_detector" --add-binary "%pythonLocation%\pyzbar\libiconv.dll;pyzbar" --add-binary "%pythonLocation%\pyzbar\libzbar-64.dll;pyzbar" --onefile --version-file build\file_version_info.txt src\extract_otp_secrets.py
+```
+
+Output is `dist\extract_otp_secrets.exe`
+
+### Full local build (bash)
There is a Bash script for a full local build including linting and type checking.
diff --git a/build.sh b/build.sh
index 3c8fd05..cf473ee 100755
--- a/build.sh
+++ b/build.sh
@@ -76,6 +76,7 @@ askContinueYn() {
interactive=false
ignore_version_check=true
clean=false
+clean_flag=""
build_docker=true
run_gui=true
generate_result_files=false
@@ -119,6 +120,7 @@ while test $# -gt 0; do
;;
-c)
clean=true
+ clean_flag="--clean"
shift
;;
esac
@@ -160,7 +162,7 @@ if $clean; then
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
- cmd="rm -r dist/ build/ *.whl pytest.xml pytest-coverage.txt .coverage tests/reports || true; find . -name '*.pyc' -type f -delete; find . -name '__pycache__' -type d -exec rm -r {} \; || true; find . -name '*.egg-info' -type d -exec rm -r {} \; || true; find . -name '*_cache' -type d -exec rm -r {} \; || true; mkdir -p tests/reports;"
+ cmd="sudo rm -rf dist/ build/ dist_*/ *.whl extracted_*.csv extracted_*.json pytest.xml pytest-coverage.txt .coverage tests/reports || true; find . -name '*.pyc' -type f -delete; find . -name '__pycache__' -type d -exec rm -r {} \; || true; find . -name '*.egg-info' -type d -exec rm -r {} \; || true; find . -name '*_cache' -type d -exec rm -r {} \; || true; mkdir -p tests/reports;"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
@@ -277,6 +279,15 @@ cmd="$MYPY --strict src/*.py tests/*.py | tee $TYPE_CHECK_OUT_FILE"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
+
+# Generate results files
+
+if $generate_result_files; then
+ cmd="for color in '' '-n'; do for level in '' '-v' '-vv' '-vvv'; do $PYTHON src/extract_otp_secrets.py example_export.txt \$color \$level > tests/data/print_verbose_output\$color\$level.txt; done; done"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+fi
+
# pip -e install
cmd="$PIP install -U -e ."
@@ -330,13 +341,26 @@ cmd="$PIP wheel ."
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
-# Generate results files
+# Build executable
-if $generate_result_files; then
- cmd="for color in '' '-n'; do for level in '' '-v' '-vv' '-vvv'; do $PYTHON src/extract_otp_secrets.py example_export.txt $color $level > tests/data/print_verbose_output$color$level.txt; done; done"
- if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
- eval "$cmd"
-fi
+cmd="LOCAL_GLIBC_VERSION=$(ldd --version | sed '1!d' | sed -E 's/.* ([[:digit:]]+\.[[:digit:]]+)$/\1/')"
+if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+eval "$cmd"
+echo "local glibc: $LOCAL_GLIBC_VERSION"
+
+cmd="pyinstaller -y --add-data $HOME/.local/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile $clean_flag src/extract_otp_secrets.py"
+if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+eval "$cmd"
+
+cmd="dist/extract_otp_secrets -h"
+if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+eval "$cmd"
+
+# Generate README.md TOC
+
+cmd="gfm-toc -s 2 -e 3 -t -o README.md > docs/README_TOC.md"
+if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+eval "$cmd"
# Update Code Coverage in README.md
@@ -349,7 +373,7 @@ if $build_docker; then
# Build docker
# Build Dockerfile_only_txt (Alpine)
- cmd="docker build . -t extract_otp_secrets_only_txt -f Dockerfile_only_txt --pull --build-arg RUN_TESTS=false"
+ cmd="docker build . -t extract_otp_secrets_only_txt -t extract_otp_secrets:only-txt -t extract_otp_secrets:alpine -f Dockerfile_only_txt --pull --build-arg RUN_TESTS=false"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
@@ -361,13 +385,12 @@ if $build_docker; then
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
- cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v \"$(pwd)\":/files:ro extract_otp_secrets_only_txt tests/extract_otp_secrets_test.py -k 'not qreader' -vvv --relaxed"
+ cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v \"$(pwd)\":/files:ro extract_otp_secrets_only_txt extract_otp_secrets_test.py -k 'not qreader' -vvv --relaxed"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
-
- # Build extract_otp_secrets (Debian)
- cmd="docker build . -t extract_otp_secrets --pull --build-arg RUN_TESTS=false"
+ # Build extract_otp_secrets (Debian Bullseye)
+ cmd="docker build . -t extract_otp_secrets -t extract_otp_secrets:bullseye --pull --build-arg RUN_TESTS=false"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
@@ -387,6 +410,58 @@ if $build_docker; then
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
+ # Build extract_otp_secrets (Debian Buster)
+ cmd="docker build . -t extract_otp_secrets:buster --pull --build-arg RUN_TESTS=false --build-arg BASE_IMAGE=python:3.11-slim-buster"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ cmd="docker run --rm -v \"$(pwd)\":/files:ro extract_otp_secrets:buster example_export.txt"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ cmd="cat example_export.txt | docker run --rm -i -v \"$(pwd)\":/files:ro extract_otp_secrets:buster - -c - > example_output.csv"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ cmd="docker run --rm -i -v \"$(pwd)\":/files:ro extract_otp_secrets:buster = < example_export.png"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v \"$(pwd)\":/files:ro extract_otp_secrets:buster"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ # Build executable from Docker latest
+ # sed "1!d" is workaround for head -n 1 since it head procduces exit code != 0
+ BULLSEYE_GLIBC_VERSION=$(docker run --entrypoint /bin/bash --rm extract_otp_secrets -c 'ldd --version | sed "1!d" | sed -E "s/.* ([[:digit:]]+\.[[:digit:]]+)$/\1/"')
+ echo "Bullseye glibc: $BULLSEYE_GLIBC_VERSION"
+
+ cmd="docker run --entrypoint /bin/bash --rm -v \"$(pwd)\":/files -w /files extract_otp_secrets -c 'apt-get update && apt-get -y install binutils && pip install -U -r /files/requirements.txt && pip install pyinstaller && pyinstaller -y --add-data /usr/local/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name extract_otp_secrets_linux_x86_64_bullseye --distpath /files/dist/ /files/src/extract_otp_secrets.py'"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ cmd="dist/extract_otp_secrets_linux_x86_64_bullseye -h"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ # Build executable from Docker buster
+ BUSTER_GLIBC_VERSION=$(docker run --entrypoint /bin/bash --rm extract_otp_secrets:buster -c 'ldd --version | sed "1!d" | sed -E "s/.* ([[:digit:]]+\.[[:digit:]]+)$/\1/"')
+ echo "Bullseye glibc: $BUSTER_GLIBC_VERSION"
+
+ cmd="docker run --entrypoint /bin/bash --rm -v \"$(pwd)\":/files -w /files extract_otp_secrets:buster -c 'apt-get update && apt-get -y install binutils && pip install -U -r /files/requirements.txt && pip install pyinstaller && pyinstaller -y --add-data /usr/local/__yolo_v3_qr_detector/:__yolo_v3_qr_detector/ --onefile --name extract_otp_secrets_linux_x86_64 --distpath /files/dist/ /files/src/extract_otp_secrets.py'"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ cmd="dist/extract_otp_secrets_linux_x86_64 -h"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ # create Windows file_version_info.txt
+ cmd="VERSION_STR=$(setuptools-git-versioning) VERSION_MAJOR=$(cut -d '.' -f 1 <<< "$(setuptools-git-versioning)") VERSION_MINOR=$(cut -d '.' -f 2 <<< "$(setuptools-git-versioning)") VERSION_PATCH=$(cut -d '.' -f 3 <<< "$(setuptools-git-versioning)") VERSION_BUILD=$(($(git rev-list --count $(git tag | sort -V -r | sed '1!d')..HEAD))) YEARS='2020-2023' envsubst < file_version_info_template.txt > build/file_version_info.txt"
+ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+ eval "$cmd"
+
+ # Run GUI from Docker
if $run_gui; then
cmd="docker run --rm -v "$(pwd)":/files:ro --device=\"/dev/video0:/dev/video0\" --env=\"DISPLAY\" -v /tmp/.X11-unix:/tmp/.X11-unix:ro extract_otp_secrets &"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
diff --git a/docker/.alias b/docker/.alias
new file mode 100644
index 0000000..e44949f
--- /dev/null
+++ b/docker/.alias
@@ -0,0 +1,4 @@
+alias ll='ls -lh'
+alias la='ls -lha'
+alias l='ls -alhF'
+alias ls-l='ls -lh'
diff --git a/docs/README_TOC.md b/docs/README_TOC.md
new file mode 100644
index 0000000..86e2a72
--- /dev/null
+++ b/docs/README_TOC.md
@@ -0,0 +1,56 @@
+Generate from file: README.md
+
+## Table of contents
+
+- [Table of contents](#table-of-contents)
+- [Usage](#usage)
+ - [Capture QR codes from camera (🆕 since version 2.0)](#capture-qr-codes-from-camera--since-version-20)
+ - [With builtin QR decoder from image files (🆕 since version 2.0)](#with-builtin-qr-decoder-from-image-files--since-version-20)
+ - [With external QR decoder app from text files](#with-external-qr-decoder-app-from-text-files)
+- [Installation](#installation)
+ - [Download binary executable (🆕 since v2.1)](#download-binary-executable--since-v21)
+ - [Run as script (recommend for developers or advanced users)](#run-as-script-recommend-for-developers-or-advanced-users)
+ - [Installation of shared system libraries](#installation-of-shared-system-libraries)
+- [Program help: arguments and options](#program-help-arguments-and-options)
+- [Examples](#examples)
+ - [Printing otp secrets form text file](#printing-otp-secrets-form-text-file)
+ - [Printing otp secrets from image file](#printing-otp-secrets-from-image-file)
+ - [Writing otp secrets to csv file](#writing-otp-secrets-to-csv-file)
+ - [Writing otp secrets to json file](#writing-otp-secrets-to-json-file)
+ - [Printing otp secrets multiple files](#printing-otp-secrets-multiple-files)
+ - [Printing otp secrets from stdin (text)](#printing-otp-secrets-from-stdin-text)
+ - [Printing otp secrets from stdin (image)](#printing-otp-secrets-from-stdin-image)
+ - [Printing otp secrets csv to stdout](#printing-otp-secrets-csv-to-stdout)
+ - [Printing otp secrets csv to stdout without header line](#printing-otp-secrets-csv-to-stdout-without-header-line)
+ - [Reading from stdin and printing to stdout](#reading-from-stdin-and-printing-to-stdout)
+- [Features](#features)
+- [KeePass](#keepass)
+- [How to export otp secrets from Google Authenticator app](#how-to-export-otp-secrets-from-google-authenticator-app)
+- [Glossary](#glossary)
+- [Alternative installation methods](#alternative-installation-methods)
+ - [pip using github](#pip-using-github)
+ - [local pip](#local-pip)
+ - [pipenv](#pipenv)
+ - [Visual Studio Code Remote - Containers / VSCode devcontainer](#visual-studio-code-remote---containers--vscode-devcontainer)
+ - [venv](#venv)
+ - [devbox](#devbox)
+ - [docker](#docker)
+ - [More docker examples](#more-docker-examples)
+- [Tests](#tests)
+ - [PyTest](#pytest)
+ - [unittest](#unittest)
+ - [VSCode Setup](#vscode-setup)
+- [Development](#development)
+ - [Build](#build)
+ - [Upgrade pip Packages](#upgrade-pip-packages)
+ - [Build docker images](#build-docker-images)
+ - [Create executables with pyinstaller](#create-executables-with-pyinstaller)
+ - [Full local build (bash)](#full-local-build-bash)
+- [Technical background](#technical-background)
+- [References](#references)
+- [Issues](#issues)
+- [Problems and Troubleshooting](#problems-and-troubleshooting)
+ - [Windows error message](#windows-error-message)
+- [Related projects](#related-projects)
+
+Table of contents generated.
diff --git a/cv2_capture_screenshot.png b/docs/cv2_capture_screenshot.png
similarity index 100%
rename from cv2_capture_screenshot.png
rename to docs/cv2_capture_screenshot.png
diff --git a/file_version_info_template.txt b/file_version_info_template.txt
new file mode 100644
index 0000000..cb4126c
--- /dev/null
+++ b/file_version_info_template.txt
@@ -0,0 +1,46 @@
+# UTF-8
+#
+# For more details about fixed file info 'ffi' see:
+# http://msdn.microsoft.com/en-us/library/ms646997.aspx
+VSVersionInfo(
+ ffi=FixedFileInfo(
+ # The elements of each tuple represent 16-bit values from most-significant to least-significant. For example the value (2, 0, 4, 0) resolves to 0002000000040000 in hex.
+ # filevers and prodvers should be always a tuple with four items: (1, 2, 3, 4)
+ # Set not needed items to zero 0.
+ filevers=($VERSION_MAJOR, $VERSION_MINOR, $VERSION_PATCH, $VERSION_BUILD),
+ prodvers=($VERSION_MAJOR, $VERSION_MINOR, $VERSION_PATCH, $VERSION_BUILD),
+ # Contains a bitmask that specifies the valid bits 'flags'r
+ mask=0x3f,
+ # Contains a bitmask that specifies the Boolean attributes of the file.
+ flags=0x0,
+ # The operating system for which this file was designed.
+ # 0x4 - NT and there is no need to change it.
+ OS=0x4,
+ # The general type of file.
+ # 0x1 - the file is an application.
+ fileType=0x1,
+ # The function of the file.
+ # 0x0 - the function is not defined for this fileType
+ subtype=0x0,
+ # Creation date and time stamp.
+ date=(0, 0)
+ ),
+ kids=[
+ StringFileInfo(
+ [
+ StringTable(
+ # 0x0409 (U.S. English) + 04B0 (1200 = Unicode), https://learn.microsoft.com/en-us/windows/win32/menurc/stringfileinfo-block
+ '040904B0',
+ [StringStruct('CompanyName', 'scito'),
+ StringStruct('FileDescription', 'extract_otp_secrets'),
+ StringStruct('FileVersion', '$VERSION_STR'),
+ StringStruct('InternalName', 'extract_otp_secrets'),
+ StringStruct('LegalCopyright', 'Copyright © $YEARS Scito.'),
+ StringStruct('OriginalFilename', 'extract_otp_secrets.exe'),
+ StringStruct('ProductName', 'extract_otp_secrets'),
+ StringStruct('ProductVersion', '$VERSION_STR')])
+ ]),
+ # 1033 (0x0409 = U.S. English), 1200 (Unicode)
+ VarFileInfo([VarStruct('Translation', [0, 1200])])
+ ]
+)
diff --git a/pyproject.toml b/pyproject.toml
index bcab23a..898a1d1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,6 +27,10 @@ classifiers = [
"Intended Audience :: Developers",
"Intended Audience :: System Administrators",
"Programming Language :: Python",
+ "Operating System :: POSIX :: Linux",
+ "Operating System :: Microsoft :: Windows :: Windows 10",
+ "Operating System :: Microsoft :: Windows :: Windows 11",
+ "Operating System :: MacOS",
"Natural Language :: English",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Typing :: Typed",
@@ -40,7 +44,9 @@ dependencies = [
"pyzbar",
"qrcode",
"qreader<2.0.0",
+ # workaround for PYTHON <= 3.7: compatibility
"typing_extensions; python_version<='3.7'",
+ "importlib_metadata; python_version<='3.7'",
]
description = "Extracts one time password (OTP) secrets from QR codes exported by two-factor authentication (2FA) apps such as 'Google Authenticator'"
dynamic = ["version"]
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 5e80330..2fb7df0 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,11 +1,14 @@
build
flake8
+gfm-toc
mypy
mypy-protobuf
+pyinstaller
pylint
pytest
pytest-cov
pytest-mock
setuptools
+setuptools-git-versioning
types-protobuf
wheel
diff --git a/requirements.txt b/requirements.txt
index c4e52cd..454f7d1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,3 +7,4 @@ pyzbar
qrcode
qreader<2.0.0
typing_extensions; python_version<='3.7'
+importlib_metadata; python_version<='3.7'
diff --git a/run_pytest.sh b/run_pytest.sh
index 08ccfdd..8c77ce5 100755
--- a/run_pytest.sh
+++ b/run_pytest.sh
@@ -1,3 +1,5 @@
#!/bin/sh
cd /extract
-pip install -U pytest pytest-mock && pip install --no-deps . && pytest "$@"
+mkdir -p tests
+ln -sf /extract/data tests/data
+pip install -U pytest pytest-mock && pytest "$@"
diff --git a/setup.cfg b/setup.cfg
index e867b90..a879cca 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -6,6 +6,10 @@ python_requires = >=3.7, <4
py_modules = extract_otp_secrets, protobuf_generated_python.google_auth_pb2
package_dir =
=src
+platforms =
+ Linux
+ Windows
+ MacOS
# packages=find:
# [options.packages.find]
diff --git a/src/extract_otp_secrets.py b/src/extract_otp_secrets.py
index 6b920c6..80c6165 100644
--- a/src/extract_otp_secrets.py
+++ b/src/extract_otp_secrets.py
@@ -38,11 +38,18 @@ import csv
import fileinput
import json
import os
+import platform
import re
import sys
import urllib.parse as urlparse
from enum import Enum, IntEnum
-from typing import Any, List, Optional, TextIO, Tuple, Union
+from typing import Any, List, Optional, Sequence, TextIO, Tuple, Union
+
+import colorama
+from pkg_resources import DistributionNotFound, get_distribution
+from qrcode import QRCode # type: ignore
+
+import protobuf_generated_python.google_auth_pb2 as pb
# workaround for PYTHON <= 3.7: compatibility
if sys.version_info >= (3, 8):
@@ -50,16 +57,17 @@ if sys.version_info >= (3, 8):
else:
from typing_extensions import Final, TypedDict
-from qrcode import QRCode # type: ignore
+# workaround for PYTHON <= 3.7: compatibility
+if sys.version_info >= (3, 8):
+ from importlib.metadata import PackageNotFoundError, version
+else:
+ from importlib_metadata import PackageNotFoundError, version
-import protobuf_generated_python.google_auth_pb2 as pb
-import colorama
debug_mode = '-d' in sys.argv[1:] or '--debug' in sys.argv[1:]
try:
import cv2 # type: ignore # TODO use cv2 types if available
-
import numpy as np # TODO use numpy types if available
try:
@@ -133,6 +141,8 @@ CAMERA: Final[str] = 'camera'
verbose: IntEnum = LogLevel.NORMAL
quiet: bool = False
colored: bool = True
+executable: bool = False
+__version__: str
def sys_main() -> None:
@@ -140,6 +150,7 @@ def sys_main() -> None:
def main(sys_args: list[str]) -> None:
+ global executable
# allow to use sys.stdout with with (avoid closing)
sys.stdout.close = lambda: None # type: ignore
# set encoding to utf-8, needed for Windows
@@ -150,11 +161,15 @@ def main(sys_args: list[str]) -> None:
# StringIO in tests do not have all attributes, ignore it
pass
+ # https://pyinstaller.org/en/stable/runtime-information.html#run-time-information
+ executable = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
+
args = parse_args(sys_args)
if colored:
colorama.just_fix_windows_console()
-
+ if verbose >= LogLevel.DEBUG:
+ print(f"Version: {get_full_version()}\n")
if args.debug:
sys.exit(0 if do_debug_checks() else 1)
@@ -237,15 +252,19 @@ def extract_otp_from_otp_url(otpauth_migration_url: str, otps: Otps, urls_count:
def parse_args(sys_args: list[str]) -> Args:
global verbose, quiet, colored
+
+ # For PYTHON <= 3.7: Use :=
+ name = os.path.basename(sys.argv[0])
+ cmd = f"python {name}" if name.endswith('.py') else f"{name}"
description_text = "Extracts one time password (OTP) secrets from QR codes exported by two-factor authentication (2FA) apps"
if qreader_available:
description_text += "\nIf no infiles are provided, a GUI window starts and QR codes are captured from the camera."
- example_text = """examples:
-python extract_otp_secrets.py
-python extract_otp_secrets.py example_*.txt
-python extract_otp_secrets.py - < example_export.txt
-python extract_otp_secrets.py --csv - example_*.png | tail -n+2
-python extract_otp_secrets.py = < example_export.png"""
+ example_text = f"""examples:
+{cmd}
+{cmd} example_*.txt
+{cmd} - < example_export.txt
+{cmd} --csv - example_*.png | tail -n+2
+{cmd} = < example_export.png"""
arg_parser = argparse.ArgumentParser(formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=32),
description=description_text,
@@ -262,6 +281,7 @@ b) image file containing a QR code or = for stdin for an image containing a QR c
arg_parser.add_argument('--qr', '-Q', help=f'QR reader (default: {QRMode.ZBAR.name})', type=str, choices=[mode.name for mode in QRMode], default=QRMode.ZBAR.name)
arg_parser.add_argument('-i', '--ignore', help='ignore duplicate otps', action='store_true')
arg_parser.add_argument('--no-color', '-n', help='do not use ANSI colors in console output', action='store_true')
+ arg_parser.add_argument('--version', '-V', help='print version and quit', action=PrintVersionAction)
output_group = arg_parser.add_mutually_exclusive_group()
output_group.add_argument('-d', '--debug', help='enter debug mode, do checks and quit', action='count')
output_group.add_argument('-v', '--verbose', help='verbose output', action='count')
@@ -731,6 +751,57 @@ def do_debug_checks() -> bool:
return True
+class PrintVersionAction(argparse.Action):
+ def __init__(self, option_strings: Sequence[str], dest: str, nargs: int = 0, **kwargs: Any) -> None:
+ super().__init__(option_strings, dest, nargs, **kwargs)
+
+ def __call__(self, parser: argparse.ArgumentParser, namespace: Args, values: Union[str, Sequence[Any], None], option_string: Optional[str] = None) -> None:
+ print_version()
+ parser.exit()
+
+
+def print_version() -> None:
+ print(get_full_version())
+
+
+def get_full_version() -> str:
+ version = get_raw_version()
+ meta = [
+ platform.python_implementation()
+ ]
+ if executable: meta.append('exe')
+ meta.append(f"called as {'package' if __package__ else 'script'}")
+ return (
+ f"extract_otp_secrets {version} {platform.system()} {platform.machine()}"
+ f" Python {platform.python_version()}"
+ f" ({'/'.join(meta)})"
+ )
+
+
+# https://setuptools-git-versioning.readthedocs.io/en/stable/runtime_version.html
+def get_raw_version() -> str:
+ global __version__
+
+ try:
+ __version__ = version("extract_otp_secrets")
+ return __version__
+ except PackageNotFoundError:
+ # package is not installed
+ pass
+
+ # In some cases importlib cannot properly detect package version, for example it was compiled into executable file, so it uses some custom import mechanism.
+ # Instead, use pkg_resources which is included in setuptools (but has a significant runtime cost)
+
+ try:
+ __version__ = get_distribution("package-name").version
+ return __version__
+ except DistributionNotFound:
+ # package is not installed
+ pass
+
+ return ''
+
+
# workaround for PYTHON <= 3.9 use: BaseException | None
def log_debug(*values: object, sep: Optional[str] = ' ') -> None:
if colored:
diff --git a/tests/data/print_verbose_output-n-vvv.txt b/tests/data/print_verbose_output-n-vvv.txt
index 4013a77..a12e0f4 100644
--- a/tests/data/print_verbose_output-n-vvv.txt
+++ b/tests/data/print_verbose_output-n-vvv.txt
@@ -2,6 +2,8 @@ QReader installed: True
CV2 version: 4.7.0
QR reading mode: ZBAR
+Version: extract_otp_secrets 2.0.2.post50+git.158245dd.dirty Linux x86_64 Python 3.11.1 (CPython/called as script)
+
Input files: ['example_export.txt']
Processing infile example_export.txt
Reading lines of example_export.txt
diff --git a/tests/data/print_verbose_output-vvv.txt b/tests/data/print_verbose_output-vvv.txt
index e81eb24..5f53095 100644
--- a/tests/data/print_verbose_output-vvv.txt
+++ b/tests/data/print_verbose_output-vvv.txt
@@ -2,6 +2,8 @@ QReader installed: True
CV2 version: 4.7.0
QR reading mode: ZBAR
+Version: extract_otp_secrets 2.0.2.post50+git.158245dd.dirty Linux x86_64 Python 3.11.1 (CPython/called as script)
+
Input files: ['example_export.txt']
[36mProcessing infile example_export.txt[39m
Reading lines of example_export.txt
diff --git a/tests/extract_otp_secrets_test.py b/tests/extract_otp_secrets_test.py
index 9c56021..e77c648 100644
--- a/tests/extract_otp_secrets_test.py
+++ b/tests/extract_otp_secrets_test.py
@@ -509,7 +509,7 @@ def test_extract_verbose(verbose_level: str, color: str, capsys: pytest.CaptureF
def normalize_verbose_text(text: str, relaxed: bool) -> str:
- normalized = re.sub('^.+ version: .+$', '', text, flags=re.MULTILINE | re.IGNORECASE)
+ normalized = re.sub('^.*version: .+$', '', text, flags=re.MULTILINE | re.IGNORECASE)
if not qreader_available:
normalized = normalized \
.replace('QReader installed: True', 'QReader installed: False') \
@@ -549,6 +549,20 @@ def test_extract_help(capsys: pytest.CaptureFixture[str]) -> None:
assert e.value.code == 0
+def test_extract_version(capsys: pytest.CaptureFixture[str]) -> None:
+ with pytest.raises(SystemExit) as e:
+ # Act
+ extract_otp_secrets.main(['--version'])
+
+ # Assert
+ captured = capsys.readouterr()
+
+ assert captured.out.startswith('extract_otp_secrets ')
+ assert captured.err == ''
+ assert e.type == SystemExit
+ assert e.value.code == 0
+
+
def test_extract_no_arguments(capsys: pytest.CaptureFixture[str], mocker: MockerFixture) -> None:
if qreader_available:
# Arrange