From 851cb6532cd81f88975879be609e80939cc73038 Mon Sep 17 00:00:00 2001 From: scito Date: Tue, 3 Jan 2023 17:25:13 +0100 Subject: [PATCH] improve build and README - clean pip - do not use sudo anymore - add missing mypy-protobuf package - sort package dependencies - fix order of build calls - add frame color docu to README --- Pipfile | 22 ++++++------ Pipfile.lock | 46 +++++++++++++++++++++++- README.md | 52 +++++++++++++++++---------- build.sh | 83 +++++++++++++++++++++++--------------------- pyproject.toml | 10 +++--- requirements-dev.txt | 7 ++-- requirements.txt | 10 +++--- 7 files changed, 149 insertions(+), 81 deletions(-) diff --git a/Pipfile b/Pipfile index 90e9ecd..ceec799 100644 --- a/Pipfile +++ b/Pipfile @@ -4,24 +4,26 @@ verify_ssl = true name = "pypi" [packages] -protobuf = "*" -qrcode = "*" -pillow = "*" -qreader = "*" -opencv-contrib-python = "*" colorama = ">=0.4.6" +opencv-contrib-python = "*" # for macOS: opencv-contrib-python = "<=4.7.0" # for PYTHON <= 3.7: typing_extensions = "*" +pillow = "*" +protobuf = "*" +qrcode = "*" +qreader = "*" [dev-packages] -pytest = "*" -pytest-mock = "*" -pytest-cov = "*" -wheel = "*" +build = "*" flake8 = "*" -pylint = "*" mypy = "*" +mypy-protobuf = "*" +pylint = "*" +pytest = "*" +pytest-cov = "*" +pytest-mock = "*" types-protobuf = "*" +wheel = "*" [requires] python_version = "3.11" diff --git a/Pipfile.lock b/Pipfile.lock index 7125a63..132d596 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "25b244c44cb891ac15ef20c4011eb043b87fb1f112396d68f470d0bb362e97f7" + "sha256": "13e2cc849fbc56593a2179a51a091717bcd4baeb9235b0843bb3abd8a9d1c698" }, "pipfile-spec": 6, "requires": { @@ -220,6 +220,14 @@ "markers": "python_version >= '3.6'", "version": "==22.2.0" }, + "build": { + "hashes": [ + "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c", + "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69" + ], + "index": "pypi", + "version": "==0.9.0" + }, "coverage": { "extras": [ "toml" @@ -387,6 +395,14 @@ ], "version": "==0.4.3" }, + "mypy-protobuf": { + "hashes": [ + "sha256:7d75a079651b105076776a35a5405e3fa773b8a167118f1b712e443e9a6c18a2", + "sha256:da33dfde7547ff57e5ba5564126cbfa114f14413b2fa50759b1fa5de1e4ab511" + ], + "index": "pypi", + "version": "==3.4.0" + }, "packaging": { "hashes": [ "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3", @@ -395,6 +411,14 @@ "markers": "python_version >= '3.7'", "version": "==22.0" }, + "pep517": { + "hashes": [ + "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b", + "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59" + ], + "markers": "python_version >= '3.6'", + "version": "==0.13.0" + }, "platformdirs": { "hashes": [ "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490", @@ -411,6 +435,26 @@ "markers": "python_version >= '3.6'", "version": "==1.0.0" }, + "protobuf": { + "hashes": [ + "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30", + "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b", + "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc", + "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791", + "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717", + "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec", + "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7", + "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab", + "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2", + "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5", + "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1", + "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462", + "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97", + "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574" + ], + "index": "pypi", + "version": "==4.21.12" + }, "pycodestyle": { "hashes": [ "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", diff --git a/README.md b/README.md index 1828817..42ffc87 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,13 @@ The Python script `extract_otp_secrets.py` extracts one time password (OTP) secrets from QR codes exported by two-factor authentication (2FA) apps such as "Google Authenticator". The exported QR codes from authentication apps can be read in three ways: -1. Capture from the system camera using a GUI, _(new!)_ -2. Read image files containing the QR codes, and _(new!)_ +1. Capture from the system camera using a GUI, 🆕 +2. Read image files containing the QR codes, and 🆕 3. Read text files containing the QR code data generated by third-party QR readers. The secret and otp values can be exported to json or csv files, as well as printed or saved to PNG images. -**The project and the script were renamed from extract_otp_secret_keys to extract_otp_secrets in version 2.0.0.** +⚡ **The project and the script were renamed from `extract_otp_secret_keys` to `extract_otp_secrets` in version 2.0.** ⚡ ## Installation @@ -70,7 +70,7 @@ OpenCV requires [Visual C++ redistributable 2015](https://www.microsoft.com/en-u ## Usage -### Capture QR codes from camera (since version 2.0.0) +### Capture QR codes from camera (🆕 since version 2.0) 1. Open "Google Authenticator" app on the mobile phone 2. Export the QR codes from "Google Authenticator" app @@ -81,7 +81,13 @@ OpenCV requires [Visual C++ redistributable 2015](https://www.microsoft.com/en-u ![CV2 Capture from camera screenshot](cv2_capture_screenshot.png) -### With builtin QR decoder from image files (since version 2.0.0) +Detected QR codes are surrounded with a frame. The color of the frame indicates the extracting result: + +* Green: The QR code is detected, decoded and the OTP secret was successfully extracted. +* Red: The QR code is detected and decoded, but could not be successfully extracted. This is the case if a QR code not containing OTP data is captured. +* Magenta: The QR code is detected, but could not be decoded. The QR code should be presented better to the camera or another QR reader could be used. + +### With builtin QR decoder from image files (🆕 since version 2.0) 1. Open "Google Authenticator" app on the mobile phone 2. Export the QR codes from "Google Authenticator" app @@ -177,13 +183,13 @@ python extract_otp_secrets.py = < example_export.png * Free and open source * Supports Google Authenticator exports (and compatible apps like Aegis Authenticator) -* Captures the the QR codes directly from the camera using different QR code libraries (based on OpenCV) +* Captures the the QR codes directly from the camera using different QR code libraries (based on OpenCV) (🆕 since v2.0) * ZBAR: [pyzbar](https://github.com/NaturalHistoryMuseum/pyzbar) - fast and reliable, good for images and video capture (default and recommended) * QREADER: [QReader](https://github.com/Eric-Canas/QReader) * QREADER_DEEP: [QReader](https://github.com/Eric-Canas/QReader) - very slow in GUI * CV2: [QRCodeDetector](https://docs.opencv.org/4.x/de/dc3/classcv_1_1QRCodeDetector.html) * CV2_WECHAT: [WeChatQRCode](https://docs.opencv.org/4.x/dd/d63/group__wechat__qrcode.html) -* Supports TOTP and HOTP standards +* Supports [TOTP](https://www.ietf.org/rfc/rfc6238.txt) and [HOTP](https://www.ietf.org/rfc/rfc4226.txt) standards * Generates QR codes * Exports to various formats: * CSV @@ -191,7 +197,8 @@ python extract_otp_secrets.py = < example_export.png * Dedicated CSV for KeePass * QR code images * Supports reading from stdin and writing to stdout, thus pipes can be used -* Reads QR codes images: (See [OpenCV docu](https://docs.opencv.org/4.x/d4/da8/group__imgcodecs.html#ga288b8b3da0892bd651fce07b3bbd3a56)) +* Handles multiple input files (🆕 since v2.0) +* Reads QR codes images: (See [OpenCV docu](https://docs.opencv.org/4.x/d4/da8/group__imgcodecs.html#ga288b8b3da0892bd651fce07b3bbd3a56)) (🆕 since v2.0) * Portable Network Graphics - *.png * WebP - *.webp * JPEG files - *.jpeg, *.jpg, *.jpe @@ -199,8 +206,8 @@ python extract_otp_secrets.py = < example_export.png * Windows bitmaps - *.bmp, *.dib * JPEG 2000 files - *.jp2 * Portable image format - *.pbm, *.pgm, *.ppm *.pxm, *.pnm -* Prints errors and warnings to stderr -* Prints colored output +* Prints errors and warnings to stderr (🆕 since v2.0) +* Prints colored output (🆕 since v2.0) * Many ways to run the script: * Native Python * pipenv @@ -209,7 +216,7 @@ python extract_otp_secrets.py = < example_export.png * Docker * VSCode devcontainer * devbox -* Prebuilt Docker images provided for amd64 and arm64 +* Prebuilt Docker images provided for amd64 and arm64 (🆕 since v2.0) * Compatible with major platforms: * Linux * macOS @@ -308,7 +315,7 @@ curl -s https://raw.githubusercontent.com/scito/extract_otp_secrets/master/examp ``` git clone https://github.com/scito/extract_otp_secrets.git pip install -U -e extract_otp_secrets -python -m extract_otp_secrets example_export.txt +python -m extract_otp_secrets extract_otp_secrets/example_export.txt ``` ### pipenv @@ -368,7 +375,7 @@ docker login -u USERNAME curl -s https://raw.githubusercontent.com/scito/extract_otp_secrets/master/example_export.png | docker run --pull always -i --rm -v "$(pwd)":/files:ro scit0/extract_otp_secrets = ``` -Capturing from camera in GUI (X Window system required on host): +Capturing from camera in GUI window (X Window system required on host): ``` docker run --pull always --rm -v "$(pwd)":/files:ro -i --device="/dev/video0:/dev/video0" --env="DISPLAY" -v /tmp/.X11-unix:/tmp/.X11-unix:ro scit0/extract_otp_secrets @@ -447,6 +454,7 @@ Setup for running the tests in VSCode. ### Build ``` +cd extract_otp_secrets/ pip install -U -e . python src/extract_otp_secrets.py @@ -463,29 +471,37 @@ pip install -U -r requirements.txt ### Build docker images +#### Debian (full functionality) + Build and run the app within the container: ```bash docker build . -t extract_otp_secrets --pull --build-arg RUN_TESTS=false ``` +Run tests in docker container: + ```bash -docker build . -t extract_otp_secrets_only_txt --pull -f Dockerfile_only_txt --build-arg RUN_TESTS=false +docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets ``` -Run tests in docker container: + +#### Alpine (only text file processing) ```bash -docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets +docker build . -t extract_otp_secrets_only_txt --pull -f Dockerfile_only_txt --build-arg RUN_TESTS=false ``` +Run tests in docker container: + ```bash -docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt extract_otp_secrets_test.py -k "not qreader" --relaxed +docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt tests/extract_otp_secrets_test.py -k "not qreader" --relaxed ``` ## Issues -* Known issue for macOS: https://github.com/opencv/opencv/issues/23072 +* Segmentation fault on macOS with CV2 4.7.0: https://github.com/opencv/opencv/issues/23072 +* CV2 window does not show icons: https://github.com/opencv/opencv-python/issues/585 ## Problems and Troubleshooting diff --git a/build.sh b/build.sh index dc3fe12..7b79cfc 100755 --- a/build.sh +++ b/build.sh @@ -138,6 +138,8 @@ PIPENV="$PYTHON -m pipenv" FLAKE8="$PYTHON -m flake8" MYPY="$PYTHON -m mypy" +# sudo ln -s /usr/bin/python3.11 /usr/bin/python + # Upgrade protoc DEST="protoc" @@ -147,6 +149,26 @@ echo -e "\nProtoc remote version $VERSION\n" echo -e "Protoc local version: $OLDVERSION\n" if $clean; then + cmd="docker image prune -f || echo 'No docker image pruned'" + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi + eval "$cmd" + + cmd="$PIP uninstall -y extract-otp-secrets || echo nothing done" + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi + eval "$cmd" + + cmd="$PIP freeze | grep -v -E '^-e|^#' | xargs sudo $PIP uninstall -y || echo nothing done" + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi + eval "$cmd" + + cmd="$PIP freeze --user | grep -v -E '^-e|^#' | xargs $PIP uninstall -y || echo nothing done" + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi + eval "$cmd" + + cmd="$PIP freeze | cut -d \"@\" -f1 | xargs pip uninstall -y || echo Nothing to do" + if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi + eval "$cmd" + cmd="rm -r dist/ build/ *.whl pytest.xml pytest-coverage.txt .coverage tests/reports || true; find . -name '*.pyc' -type f -delete; find . -name '__pycache__' -type d -exec rm -r {} \; || true; find . -name '*.egg-info' -type d -exec rm -r {} \; || true; find . -name '*_cache' -type d -exec rm -r {} \; || true; mkdir -p tests/reports;" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" @@ -160,6 +182,10 @@ if $clean; then eval "$cmd" fi +cmd="$PIP install --use-pep517 -U -r requirements-dev.txt" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + if [ "$OLDVERSION" != "$VERSION" ] || ! $ignore_version_check; then echo "Upgrade protoc from $OLDVERSION to $VERSION" @@ -216,7 +242,7 @@ fi # Upgrade pip requirements -cmd="sudo pip install -U pip" +cmd="pip install -U pip" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" @@ -226,10 +252,6 @@ cmd="$PIP install --use-pep517 -U -r requirements.txt" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -cmd="$PIP install --use-pep517 -U -r requirements-dev.txt" -if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi -eval "$cmd" - # Lint LINT_OUT_FILE="tests/reports/flake8_results.txt" @@ -254,64 +276,50 @@ cmd="$MYPY --strict src/*.py tests/*.py | tee $TYPE_CHECK_OUT_FILE" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -# Test +# pip -e install -cmd="$PYTHON src/extract_otp_secrets.py example_export.txt" -if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi -eval "$cmd" - -cmd="$PYTHON src/extract_otp_secrets.py - < example_export.txt" +cmd="$PIP install -U -e ." if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -COVERAGE_OUT_FILE="tests/reports/pytest-coverage.txt" -cmd="pytest --cov=extract_otp_secrets_test --junitxml=tests/reports/pytest.xml --cov-report html:tests/reports/html --cov-report=term-missing tests/ | tee $COVERAGE_OUT_FILE" +cmd="extract_otp_secrets example_export.txt" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -# Pipenv - -cmd="$PIP install -U pipenv" +cmd="extract_otp_secrets - < example_export.txt" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -$PIPENV --version +# Test (needs module) -cmd="$PIPENV update && $PIPENV --rm && $PIPENV install" +cmd="$PYTHON src/extract_otp_secrets.py example_export.txt" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -$PIPENV run python --version - -cmd="$PIPENV run pytest --cov=extract_otp_secrets_test tests/" +cmd="$PYTHON src/extract_otp_secrets.py - < example_export.txt" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -# sudo pip - -cmd="sudo $PIP install --use-pep517 -U -r requirements.txt" +COVERAGE_OUT_FILE="tests/reports/pytest-coverage.txt" +cmd="pytest --cov=extract_otp_secrets_test --junitxml=tests/reports/pytest.xml --cov-report html:tests/reports/html --cov-report=term-missing tests/ | tee $COVERAGE_OUT_FILE" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -cmd="sudo $PIP install --use-pep517 -U -r requirements-dev.txt" -if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi -eval "$cmd" +# Pipenv -cmd="sudo $PIP install -U pipenv" +cmd="$PIP install -U pipenv" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -# pip -e install (must be after other pip installs in order to have this environment for development) +$PIPENV --version -cmd="$PIP install -U -e ." +cmd="$PIPENV update && $PIPENV --rm && $PIPENV install" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -cmd="extract_otp_secrets example_export.txt" -if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi -eval "$cmd" +$PIPENV run python --version -cmd="extract_otp_secrets - < example_export.txt" +cmd="$PIPENV run pytest --cov=extract_otp_secrets_test tests/" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" @@ -378,10 +386,6 @@ if $build_docker; then if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" - cmd="docker image prune -f || echo 'No docker image pruned'" - if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi - eval "$cmd" - if $run_gui; then cmd="docker run --rm -v "$(pwd)":/files:ro --device=\"/dev/video0:/dev/video0\" --env=\"DISPLAY\" -v /tmp/.X11-unix:/tmp/.X11-unix:ro extract_otp_secrets &" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi @@ -402,6 +406,7 @@ cmd="cat $TYPE_CHECK_OUT_FILE $LINT_OUT_FILE $COVERAGE_OUT_FILE" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -echo -e "\n${greenBold}SUCCESS${reset}" +line=$(printf '#%.0s' $(eval echo {1..$(( ($COLUMNS - 10) / 2))})) +echo -e "\n${greenBold}$line SUCCESS $line${reset}" quit diff --git a/pyproject.toml b/pyproject.toml index cb86bbe..3eaf140 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,15 +29,15 @@ classifiers = [ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", ] dependencies = [ + "colorama>=0.4.6", + "opencv-contrib-python; sys_platform != 'darwin'", + "opencv-contrib-python<=4.7.0; sys_platform == 'darwin'", + "Pillow", "protobuf", + "pyzbar", "qrcode", - "Pillow", "qreader", - "pyzbar", - "opencv-contrib-python<=4.7.0; sys_platform == 'darwin'", - "opencv-contrib-python; sys_platform != 'darwin'", "typing_extensions; python_version<='3.7'", - "colorama>=0.4.6", ] description = "Extracts one time password (OTP) secrets from QR codes exported by two-factor authentication (2FA) apps such as 'Google Authenticator'" dynamic = ["version"] diff --git a/requirements-dev.txt b/requirements-dev.txt index fce22e5..5e80330 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,10 +1,11 @@ +build flake8 mypy -types-protobuf +mypy-protobuf pylint pytest -pytest-mock pytest-cov +pytest-mock setuptools +types-protobuf wheel -build diff --git a/requirements.txt b/requirements.txt index d456851..60001a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ +colorama>=0.4.6 +opencv-contrib-python; sys_platform != 'darwin' +opencv-contrib-python<=4.7.0; sys_platform == 'darwin' +Pillow protobuf +pyzbar qrcode -Pillow qreader -opencv-contrib-python<=4.7.0; sys_platform == 'darwin' -opencv-contrib-python; sys_platform != 'darwin' -pyzbar typing_extensions; python_version<='3.7' -colorama>=0.4.6