improve build and README

- clean pip - do not use sudo anymore - add missing mypy-protobuf package - sort package dependencies - fix order of build calls - add frame color docu to README
1 year ago · 851cb6532c
parent 2bef64e5f6
commit 851cb6532c
7 changed files with 149 additions and 81 deletions
--- a/22
+++ b/22
@ -4,24 +4,26 @@ verify_ssl = true
 name = "pypi"

 [packages]
-protobuf = "*"
-qrcode = "*"
-pillow = "*"
-qreader = "*"
-opencv-contrib-python = "*"
 colorama = ">=0.4.6"
+opencv-contrib-python = "*"
 # for macOS: opencv-contrib-python = "<=4.7.0"
 # for PYTHON <= 3.7: typing_extensions = "*"
+pillow = "*"
+protobuf = "*"
+qrcode = "*"
+qreader = "*"

 [dev-packages]
-pytest = "*"
-pytest-mock = "*"
-pytest-cov = "*"
-wheel = "*"
+build = "*"
 flake8 = "*"
-pylint = "*"
 mypy = "*"
+mypy-protobuf = "*"
+pylint = "*"
+pytest = "*"
+pytest-cov = "*"
+pytest-mock = "*"
 types-protobuf = "*"
+wheel = "*"

 [requires]
 python_version = "3.11"
--- a/Pipfile.lock
+++ b/Pipfile.lock
@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "25b244c44cb891ac15ef20c4011eb043b87fb1f112396d68f470d0bb362e97f7"
+            "sha256": "13e2cc849fbc56593a2179a51a091717bcd4baeb9235b0843bb3abd8a9d1c698"
        },
        "pipfile-spec": 6,
        "requires": {
@ -220,6 +220,14 @@
            "markers": "python_version >= '3.6'",
            "version": "==22.2.0"
        },
+        "build": {
+            "hashes": [
+                "sha256:1a07724e891cbd898923145eb7752ee7653674c511378eb9c7691aab1612bc3c",
+                "sha256:38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69"
+            ],
+            "index": "pypi",
+            "version": "==0.9.0"
+        },
        "coverage": {
            "extras": [
                "toml"
@ -387,6 +395,14 @@
            ],
            "version": "==0.4.3"
        },
+        "mypy-protobuf": {
+            "hashes": [
+                "sha256:7d75a079651b105076776a35a5405e3fa773b8a167118f1b712e443e9a6c18a2",
+                "sha256:da33dfde7547ff57e5ba5564126cbfa114f14413b2fa50759b1fa5de1e4ab511"
+            ],
+            "index": "pypi",
+            "version": "==3.4.0"
+        },
        "packaging": {
            "hashes": [
                "sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3",
@ -395,6 +411,14 @@
            "markers": "python_version >= '3.7'",
            "version": "==22.0"
        },
+        "pep517": {
+            "hashes": [
+                "sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b",
+                "sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.13.0"
+        },
        "platformdirs": {
            "hashes": [
                "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490",
@ -411,6 +435,26 @@
            "markers": "python_version >= '3.6'",
            "version": "==1.0.0"
        },
+        "protobuf": {
+            "hashes": [
+                "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30",
+                "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b",
+                "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc",
+                "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791",
+                "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717",
+                "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec",
+                "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7",
+                "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab",
+                "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2",
+                "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5",
+                "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1",
+                "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462",
+                "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97",
+                "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"
+            ],
+            "index": "pypi",
+            "version": "==4.21.12"
+        },
        "pycodestyle": {
            "hashes": [
                "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053",
--- a/README.md
+++ b/README.md
@ -15,13 +15,13 @@
 The Python script `extract_otp_secrets.py` extracts one time password (OTP) secrets from QR codes exported by two-factor authentication (2FA) apps such as "Google Authenticator".
 The exported QR codes from authentication apps can be read in three ways:

-1. Capture from the system camera using a GUI, _(new!)_
-2. Read image files containing the QR codes, and _(new!)_
+1. Capture from the system camera using a GUI, 🆕
+2. Read image files containing the QR codes, and 🆕
 3. Read text files containing the QR code data generated by third-party QR readers.

 The secret and otp values can be exported to json or csv files, as well as printed or saved to PNG images.

-**The project and the script were renamed from extract_otp_secret_keys to extract_otp_secrets in version 2.0.0.**
+⚡ **The project and the script were renamed from `extract_otp_secret_keys` to `extract_otp_secrets` in version 2.0.** ⚡

 ## Installation

@ -70,7 +70,7 @@ OpenCV requires [Visual C++ redistributable 2015](https://www.microsoft.com/en-u

 ## Usage

-### Capture QR codes from camera (since version 2.0.0)
+### Capture QR codes from camera (🆕 since version 2.0)

 1. Open "Google Authenticator" app on the mobile phone
 2. Export the QR codes from "Google Authenticator" app
@ -81,7 +81,13 @@ OpenCV requires [Visual C++ redistributable 2015](https://www.microsoft.com/en-u

 ![CV2 Capture from camera screenshot](cv2_capture_screenshot.png)

-### With builtin QR decoder from image files (since version 2.0.0)
+Detected QR codes are surrounded with a frame. The color of the frame indicates the extracting result:
+
+* Green: The QR code is detected, decoded and the OTP secret was successfully extracted.
+* Red: The QR code is detected and decoded, but could not be successfully extracted. This is the case if a QR code not containing OTP data is captured.
+* Magenta: The QR code is detected, but could not be decoded. The QR code should be presented better to the camera or another QR reader could be used.
+
+### With builtin QR decoder from image files (🆕 since version 2.0)

 1. Open "Google Authenticator" app on the mobile phone
 2. Export the QR codes from "Google Authenticator" app
@ -177,13 +183,13 @@ python extract_otp_secrets.py = < example_export.png</pre>

 * Free and open source
 * Supports Google Authenticator exports (and compatible apps like Aegis Authenticator)
-* Captures the the QR codes directly from the camera using different QR code libraries (based on OpenCV)
+* Captures the the QR codes directly from the camera using different QR code libraries (based on OpenCV) (🆕 since v2.0)
    * ZBAR: [pyzbar](https://github.com/NaturalHistoryMuseum/pyzbar) - fast and reliable, good for images and video capture (default and recommended)
    * QREADER: [QReader](https://github.com/Eric-Canas/QReader)
    * QREADER_DEEP: [QReader](https://github.com/Eric-Canas/QReader) - very slow in GUI
    * CV2: [QRCodeDetector](https://docs.opencv.org/4.x/de/dc3/classcv_1_1QRCodeDetector.html)
    * CV2_WECHAT: [WeChatQRCode](https://docs.opencv.org/4.x/dd/d63/group__wechat__qrcode.html)
-* Supports TOTP and HOTP standards
+* Supports [TOTP](https://www.ietf.org/rfc/rfc6238.txt) and [HOTP](https://www.ietf.org/rfc/rfc4226.txt) standards
 * Generates QR codes
 * Exports to various formats:
    * CSV
@ -191,7 +197,8 @@ python extract_otp_secrets.py = < example_export.png</pre>
    * Dedicated CSV for KeePass
    * QR code images
 * Supports reading from stdin and writing to stdout, thus pipes can be used
-* Reads QR codes images: (See [OpenCV docu](https://docs.opencv.org/4.x/d4/da8/group__imgcodecs.html#ga288b8b3da0892bd651fce07b3bbd3a56))
+* Handles multiple input files (🆕 since v2.0)
+* Reads QR codes images: (See [OpenCV docu](https://docs.opencv.org/4.x/d4/da8/group__imgcodecs.html#ga288b8b3da0892bd651fce07b3bbd3a56)) (🆕 since v2.0)
    * Portable Network Graphics - *.png
    * WebP - *.webp
    * JPEG files - *.jpeg, *.jpg, *.jpe
@ -199,8 +206,8 @@ python extract_otp_secrets.py = < example_export.png</pre>
    * Windows bitmaps - *.bmp, *.dib
    * JPEG 2000 files - *.jp2
    * Portable image format - *.pbm, *.pgm, *.ppm *.pxm, *.pnm
-* Prints errors and warnings to stderr
-* Prints colored output
+* Prints errors and warnings to stderr (🆕 since v2.0)
+* Prints colored output (🆕 since v2.0)
 * Many ways to run the script:
    * Native Python
    * pipenv
@ -209,7 +216,7 @@ python extract_otp_secrets.py = < example_export.png</pre>
    * Docker
    * VSCode devcontainer
    * devbox
-* Prebuilt Docker images provided for amd64 and arm64
+* Prebuilt Docker images provided for amd64 and arm64 (🆕 since v2.0)
 * Compatible with major platforms:
    * Linux
    * macOS
@ -308,7 +315,7 @@ curl -s https://raw.githubusercontent.com/scito/extract_otp_secrets/master/examp
 ```
 git clone https://github.com/scito/extract_otp_secrets.git
 pip install -U -e extract_otp_secrets
-python -m extract_otp_secrets example_export.txt
+python -m extract_otp_secrets extract_otp_secrets/example_export.txt
 ```

 ### pipenv
@ -368,7 +375,7 @@ docker login -u USERNAME
 curl -s https://raw.githubusercontent.com/scito/extract_otp_secrets/master/example_export.png | docker run --pull always -i --rm -v "$(pwd)":/files:ro scit0/extract_otp_secrets =
 ```

-Capturing from camera in GUI (X Window system required on host):
+Capturing from camera in GUI window (X Window system required on host):

 ```
 docker run --pull always --rm -v "$(pwd)":/files:ro -i --device="/dev/video0:/dev/video0" --env="DISPLAY" -v /tmp/.X11-unix:/tmp/.X11-unix:ro scit0/extract_otp_secrets
@ -447,6 +454,7 @@ Setup for running the tests in VSCode.
 ### Build

 ```
+cd extract_otp_secrets/
 pip install -U -e .
 python src/extract_otp_secrets.py

@ -463,29 +471,37 @@ pip install -U -r requirements.txt

 ### Build docker images

+#### Debian (full functionality)
+
 Build and run the app within the container:

 ```bash
 docker build . -t extract_otp_secrets --pull --build-arg RUN_TESTS=false
 ```

+Run tests in docker container:
+
 ```bash
-docker build . -t extract_otp_secrets_only_txt --pull -f Dockerfile_only_txt --build-arg RUN_TESTS=false
+docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets
 ```

-Run tests in docker container:
+
+#### Alpine (only text file processing)

 ```bash
-docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets
+docker build . -t extract_otp_secrets_only_txt --pull -f Dockerfile_only_txt --build-arg RUN_TESTS=false
 ```

+Run tests in docker container:
+
 ```bash
-docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt extract_otp_secrets_test.py -k "not qreader" --relaxed
+docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secrets_only_txt tests/extract_otp_secrets_test.py -k "not qreader" --relaxed
 ```

 ## Issues

-* Known issue for macOS: https://github.com/opencv/opencv/issues/23072
+* Segmentation fault on macOS with CV2 4.7.0: https://github.com/opencv/opencv/issues/23072
+* CV2 window does not show icons: https://github.com/opencv/opencv-python/issues/585

 ## Problems and Troubleshooting

--- a/build.sh
+++ b/build.sh
@ -138,6 +138,8 @@ PIPENV="$PYTHON -m pipenv"
 FLAKE8="$PYTHON -m flake8"
 MYPY="$PYTHON -m mypy"

+# sudo ln -s /usr/bin/python3.11 /usr/bin/python
+
 # Upgrade protoc

 DEST="protoc"
@ -147,6 +149,26 @@ echo -e "\nProtoc remote version $VERSION\n"
 echo -e "Protoc local version: $OLDVERSION\n"

 if $clean; then
+    cmd="docker image prune -f || echo 'No docker image pruned'"
+    if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+    eval "$cmd"
+
+    cmd="$PIP uninstall -y extract-otp-secrets || echo nothing done"
+    if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+    eval "$cmd"
+
+    cmd="$PIP freeze | grep -v -E '^-e|^#' | xargs sudo $PIP uninstall -y || echo nothing done"
+    if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+    eval "$cmd"
+
+    cmd="$PIP freeze --user | grep -v -E '^-e|^#' | xargs $PIP uninstall -y || echo nothing done"
+    if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+    eval "$cmd"
+
+    cmd="$PIP freeze | cut -d \"@\" -f1 | xargs pip uninstall -y || echo Nothing to do"
+    if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+    eval "$cmd"
+
    cmd="rm -r dist/ build/ *.whl pytest.xml pytest-coverage.txt .coverage tests/reports || true; find . -name '*.pyc' -type f -delete; find . -name '__pycache__' -type d -exec rm -r {} \; || true; find . -name '*.egg-info' -type d -exec rm -r {} \; || true; find . -name '*_cache' -type d -exec rm -r {} \; || true; mkdir -p tests/reports;"
    if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
    eval "$cmd"
@ -160,6 +182,10 @@ if $clean; then
    eval "$cmd"
 fi

+cmd="$PIP install --use-pep517 -U -r requirements-dev.txt"
+if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
+eval "$cmd"
+
 if [ "$OLDVERSION" != "$VERSION" ] || ! $ignore_version_check; then
    echo "Upgrade protoc from $OLDVERSION to $VERSION"

@ -216,7 +242,7 @@ fi

 # Upgrade pip requirements

-cmd="sudo pip install -U pip"
+cmd="pip install -U pip"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

@ -226,10 +252,6 @@ cmd="$PIP install --use-pep517 -U -r requirements.txt"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-cmd="$PIP install --use-pep517 -U -r requirements-dev.txt"
-if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
-eval "$cmd"
-
 # Lint

 LINT_OUT_FILE="tests/reports/flake8_results.txt"
@ -254,64 +276,50 @@ cmd="$MYPY --strict src/*.py tests/*.py | tee $TYPE_CHECK_OUT_FILE"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-# Test
+# pip -e install

-cmd="$PYTHON src/extract_otp_secrets.py example_export.txt"
-if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
-eval "$cmd"
-
-cmd="$PYTHON src/extract_otp_secrets.py - < example_export.txt"
+cmd="$PIP install -U -e ."
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-COVERAGE_OUT_FILE="tests/reports/pytest-coverage.txt"
-cmd="pytest --cov=extract_otp_secrets_test --junitxml=tests/reports/pytest.xml --cov-report html:tests/reports/html --cov-report=term-missing tests/ | tee $COVERAGE_OUT_FILE"
+cmd="extract_otp_secrets example_export.txt"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-# Pipenv
-
-cmd="$PIP install -U pipenv"
+cmd="extract_otp_secrets - < example_export.txt"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-$PIPENV --version
+# Test (needs module)

-cmd="$PIPENV update && $PIPENV --rm && $PIPENV install"
+cmd="$PYTHON src/extract_otp_secrets.py example_export.txt"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-$PIPENV run python --version
-
-cmd="$PIPENV run pytest --cov=extract_otp_secrets_test tests/"
+cmd="$PYTHON src/extract_otp_secrets.py - < example_export.txt"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-# sudo pip
-
-cmd="sudo $PIP install --use-pep517 -U -r requirements.txt"
+COVERAGE_OUT_FILE="tests/reports/pytest-coverage.txt"
+cmd="pytest --cov=extract_otp_secrets_test --junitxml=tests/reports/pytest.xml --cov-report html:tests/reports/html --cov-report=term-missing tests/ | tee $COVERAGE_OUT_FILE"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-cmd="sudo $PIP install --use-pep517 -U -r requirements-dev.txt"
-if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
-eval "$cmd"
+# Pipenv

-cmd="sudo $PIP install -U pipenv"
+cmd="$PIP install -U pipenv"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-# pip -e install (must be after other pip installs in order to have this environment for development)
+$PIPENV --version

-cmd="$PIP install -U -e ."
+cmd="$PIPENV update && $PIPENV --rm && $PIPENV install"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-cmd="extract_otp_secrets example_export.txt"
-if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
-eval "$cmd"
+$PIPENV run python --version

-cmd="extract_otp_secrets - < example_export.txt"
+cmd="$PIPENV run pytest --cov=extract_otp_secrets_test tests/"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

@ -378,10 +386,6 @@ if $build_docker; then
    if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
    eval "$cmd"

-    cmd="docker image prune -f || echo 'No docker image pruned'"
-    if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
-    eval "$cmd"
-
    if $run_gui; then
        cmd="docker run --rm -v "$(pwd)":/files:ro --device=\"/dev/video0:/dev/video0\" --env=\"DISPLAY\" -v /tmp/.X11-unix:/tmp/.X11-unix:ro extract_otp_secrets &"
        if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
@ -402,6 +406,7 @@ cmd="cat $TYPE_CHECK_OUT_FILE $LINT_OUT_FILE $COVERAGE_OUT_FILE"
 if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
 eval "$cmd"

-echo -e "\n${greenBold}SUCCESS${reset}"
+line=$(printf '#%.0s' $(eval echo {1..$(( ($COLUMNS - 10) / 2))}))
+echo -e "\n${greenBold}$line SUCCESS $line${reset}"

 quit
--- a/pyproject.toml
+++ b/pyproject.toml
@ -29,15 +29,15 @@ classifiers = [
    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
 ]
 dependencies = [
+  "colorama>=0.4.6",
+  "opencv-contrib-python; sys_platform != 'darwin'",
+  "opencv-contrib-python<=4.7.0; sys_platform == 'darwin'",
+  "Pillow",
  "protobuf",
+  "pyzbar",
  "qrcode",
-  "Pillow",
  "qreader",
-  "pyzbar",
-  "opencv-contrib-python<=4.7.0; sys_platform == 'darwin'",
-  "opencv-contrib-python; sys_platform != 'darwin'",
  "typing_extensions; python_version<='3.7'",
-  "colorama>=0.4.6",
 ]
 description = "Extracts one time password (OTP) secrets from QR codes exported by two-factor authentication (2FA) apps such as 'Google Authenticator'"
 dynamic = ["version"]
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@ -1,10 +1,11 @@
+build
 flake8
 mypy
-types-protobuf
+mypy-protobuf
 pylint
 pytest
-pytest-mock
 pytest-cov
+pytest-mock
 setuptools
+types-protobuf
 wheel
-build
--- a/requirements.txt
+++ b/requirements.txt
@ -1,9 +1,9 @@
+colorama>=0.4.6
+opencv-contrib-python; sys_platform != 'darwin'
+opencv-contrib-python<=4.7.0; sys_platform == 'darwin'
+Pillow
 protobuf
+pyzbar
 qrcode
-Pillow
 qreader
-opencv-contrib-python<=4.7.0; sys_platform == 'darwin'
-opencv-contrib-python; sys_platform != 'darwin'
-pyzbar
 typing_extensions; python_version<='3.7'
-colorama>=0.4.6