build docker images, run tests in docker build

- qreader_available flag
- echo commands in upgrade_deps.sh
cv2_1
scito 2 years ago
parent 0490e227e1
commit 672d18a5ca

@ -0,0 +1,56 @@
name: "Docker: build and publish"
# How to setup: https://event-driven.io/en/how_to_buid_and_push_docker_image_with_github_actions/
# How to run: https://aschmelyun.com/blog/using-docker-run-inside-of-github-actions/
on:
# run it on push to the default repository branch
push:
# branches: [master]
# run it during pull request
# pull_request:
jobs:
# define job to build and publish docker image
build-and-push-docker-image:
name: Build Docker image and push to repositories
# run only when code is compiling and tests are passing
runs-on: ubuntu-latest
# steps to perform in job
steps:
- name: Checkout code
uses: actions/checkout@v3
# setup Docker build action
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Github Packages
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- name: Build image and push to Docker Hub and GitHub Container Registry
uses: docker/build-push-action@v2
with:
# relative path to the place where source code with Dockerfile is located
context: .
# Note: tags has to be all lower-case
tags: |
scito/extract_otp_secret_keys:latest
ghcr.io/scito/extract_otp_secret_keys:latest
# build on feature branches, push only on master branch
push: ${{ github.ref == 'refs/heads/master' }}
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}

@ -1,46 +0,0 @@
name: nightly tests
on:
schedule:
- cron: '47 3 * * *'
jobs:
build:
strategy:
matrix:
python-version: ["3.x", "3.11", "3.10", "3.9", "pypy-3.9", "3.8", "pypy-3.8", "3.7", "pypy-3.7"]
platform: [ubuntu-latest, macos-latest, windows-latest]
exclude:
- platform: windows-latest
- python-version: [pypy-3.9]
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install zbar shared lib for QReader (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get install -y libzbar0
- name: Install zbar shared lib for QReader (macOS)
if: runner.os == 'macOS'
run: |
brew install zbar
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install flake8 pytest
pip install --use-pep517 -r requirements.txt
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=200 --statistics
- name: Test with pytest
run: pytest

@ -4,9 +4,10 @@ WORKDIR /extract
COPY . .
RUN apt-get update && apt-get install -y libzbar0 python3-opencv \
&& pip install -r requirements.txt
RUN apt-get update && apt-get install -y libzbar0 python3-opencv nano \
&& pip install -r requirements.txt \
&& /extract/run_pytest.sh
WORKDIR /files
ENTRYPOINT [ "python", "/extract/extract_otp_secret_keys.py" ]
ENTRYPOINT ["python", "/extract/extract_otp_secret_keys.py"]

@ -4,8 +4,9 @@ WORKDIR /extract
COPY . .
RUN pip install protobuf qrcode Pillow
RUN pip install protobuf qrcode Pillow \
&& /extract/run_pytest.sh test_extract_otp_secret_keys_pytest.py -k "not qreader" --relaxed
WORKDIR /files
ENTRYPOINT [ "python", "/extract/extract_otp_secret_keys.py" ]
ENTRYPOINT ["python", "/extract/extract_otp_secret_keys.py"]

58
Pipfile.lock generated

@ -18,37 +18,37 @@
"default": {
"numpy": {
"hashes": [
"sha256:0104d8adaa3a4cc60c2777cab5196593bf8a7f416eda133be1f3803dd0838886",
"sha256:0885d9a7666cafe5f9876c57bfee34226e2b2847bfb94c9505e18d81011e5401",
"sha256:12bba5561d8118981f2f1ff069ecae200c05d7b6c78a5cdac0911f74bc71cbd1",
"sha256:2f8e0df2ecc1928ef7256f18e309c9d6229b08b5be859163f5caa59c93d53646",
"sha256:4445f472b246cad6514cc09fbb5ecb7aab09ca2acc3c16f29f8dca6c468af501",
"sha256:4d01f7832fa319a36fd75ba10ea4027c9338ede875792f7bf617f4b45056fc3a",
"sha256:4f5e78b8b710cd7cd1a8145994cfffc6ddd5911669a437777d8cedfce6c83a98",
"sha256:667b5b1f6a352419e340f6475ef9930348ae5cb7fca15f2cc3afcb530823715e",
"sha256:6e73a1f4f5b74a42abb55bc2b3d869f1b38cbc8776da5f8b66bf110284f7a437",
"sha256:73cf2c5b5a07450f20a0c8e04d9955491970177dce8df8d6903bf253e53268e0",
"sha256:7ad6a024a32ee61d18f5b402cd02e9c0e22c0fb9dc23751991b3a16d209d972e",
"sha256:8b1ddfac6a82d4f3c8e99436c90b9c2c68c0bb14658d1684cdd00f05fab241f5",
"sha256:90075ef2c6ac6397d0035bcd8b298b26e481a7035f7a3f382c047eb9c3414db0",
"sha256:9387c7d6d50e8f8c31e7bfc034241e9c6f4b3eb5db8d118d6487047b922f82af",
"sha256:9af91f794d2d3007d91d749ebc955302889261db514eb24caef30e03e8ec1e41",
"sha256:ab11f6a7602cf8ea4c093e091938207de3068c5693a0520168ecf4395750f7ea",
"sha256:ac4fe68f1a5a18136acebd4eff91aab8bed00d1ef2fdb34b5d9192297ffbbdfc",
"sha256:ada6c1e9608ceadaf7020e1deea508b73ace85560a16f51bef26aecb93626a72",
"sha256:c4ab7c9711fe6b235e86487ca74c1b092a6dd59a3cb45b63241ea0a148501853",
"sha256:cec79ff3984b2d1d103183fc4a3361f5b55bbb66cb395cbf5a920a4bb1fd588d",
"sha256:cf8960f72997e56781eb1c2ea256a70124f92a543b384f89e5fb3503a308b1d3",
"sha256:d7f223554aba7280e6057727333ed357b71b7da7422d02ff5e91b857888c25d1",
"sha256:dbb0490f0a880700a6cc4d000384baf19c1f4df59fff158d9482d4dbbca2b239",
"sha256:e63d2157f9fc98cc178870db83b0e0c85acdadd598b134b00ebec9e0db57a01f",
"sha256:ec3e5e8172a0a6a4f3c2e7423d4a8434c41349141b04744b11a90e017a95bad5",
"sha256:f3c4a9a9f92734a4728ddbd331e0124eabbc968a0359a506e8e74a9b0d2d419b",
"sha256:f9168790149f917ad8e3cf5047b353fefef753bd50b07c547da0bdf30bc15d91",
"sha256:fe44e925c68fb5e8db1334bf30ac1a1b6b963b932a19cf41d2e899cf02f36aab"
"sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9",
"sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398",
"sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7",
"sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2",
"sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954",
"sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6",
"sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032",
"sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36",
"sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c",
"sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8",
"sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7",
"sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f",
"sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700",
"sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086",
"sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7",
"sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2",
"sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9",
"sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407",
"sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2",
"sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36",
"sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566",
"sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1",
"sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e",
"sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51",
"sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1",
"sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d",
"sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf",
"sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"
],
"markers": "python_version >= '3.10'",
"version": "==1.24.0"
"version": "==1.24.1"
},
"opencv-python": {
"hashes": [

@ -45,6 +45,7 @@ import argparse
import base64
import csv
import fileinput
import importlib
import json
import os
import re
@ -63,12 +64,13 @@ def sys_main():
def main(sys_args):
global verbose, quiet
global verbose, quiet, qreader_available
# allow to use sys.stdout with with (avoid closing)
sys.stdout.close = lambda: None
# sys.stdout.reconfigure(encoding='utf-8')
args = parse_args(sys_args)
verbose = args.verbose if args.verbose else 0
quiet = args.quiet
@ -80,8 +82,15 @@ def main(sys_args):
def parse_args(sys_args):
formatter = lambda prog: argparse.HelpFormatter(prog, max_help_position=52)
arg_parser = argparse.ArgumentParser(formatter_class=formatter)
formatter = lambda prog: argparse.RawDescriptionHelpFormatter(prog, max_help_position=52)
example_text = '''examples:
python extract_otp_secret_keys.py example_*.txt
python extract_otp_secret_keys.py - < example_export.txt
python extract_otp_secret_keys.py --csv - example_*.png | tail -n+2
python extract_otp_secret_keys.py = < example_export.png'''
arg_parser = argparse.ArgumentParser(formatter_class=formatter,
epilog=example_text)
arg_parser.add_argument('infile', help='1) file or - for stdin with "otpauth-migration://..." URLs separated by newlines, lines starting with # are ignored; or 2) image file containing a QR code or = for stdin for an image containing a QR code', nargs='+')
arg_parser.add_argument('--json', '-j', help='export json file or - for stdout', metavar=('FILE'))
arg_parser.add_argument('--csv', '-c', help='export csv file or - for stdout', metavar=('FILE'))
@ -145,6 +154,7 @@ def extract_otps(args):
def get_lines_from_file(filename):
global qreader_available
# stdin stream cannot be rewinded, thus distinguish, use - for utf-8 stdin and = for binary image stdin
if filename != '=':
check_file_exists(filename)
@ -422,6 +432,11 @@ def is_binary(line):
return True
def check_module_available(module_name):
module_spec = importlib.util.find_spec(module_name)
return module_spec is not None
def eprint(*args, **kwargs):
'''Print to stderr.'''
print(*args, file=sys.stderr, **kwargs)

@ -28,6 +28,8 @@ import pytest
import extract_otp_secret_keys
from utils import *
qreader_available = extract_otp_secret_keys.check_module_available('cv2')
def test_extract_stdout(capsys):
# Act
@ -100,20 +102,31 @@ def test_extract_stdin_empty(capsys, monkeypatch):
assert captured.err == 'WARN: stdin is empty\n'
def test_extract_empty_file(capsys):
# Act
with pytest.raises(SystemExit) as e:
extract_otp_secret_keys.main(['test/empty_file.txt'])
# @pytest.mark.skipif(not qreader_available, reason='Test if cv2 and qreader are not available.')
def test_extract_empty_file_no_qreader(capsys):
if qreader_available:
# Act
with pytest.raises(SystemExit) as e:
extract_otp_secret_keys.main(['test/empty_file.txt'])
# Assert
captured = capsys.readouterr()
# Assert
captured = capsys.readouterr()
expected_stderr = 'WARN: test/empty_file.txt is empty\n\nERROR: Unable to open file for reading.\ninput file: test/empty_file.txt\n'
expected_stderr = 'WARN: test/empty_file.txt is empty\n\nERROR: Unable to open file for reading.\ninput file: test/empty_file.txt\n'
assert captured.err == expected_stderr
assert captured.out == ''
assert e.value.code == 1
assert e.type == SystemExit
assert captured.err == expected_stderr
assert captured.out == ''
assert e.value.code == 1
assert e.type == SystemExit
else:
# Act
extract_otp_secret_keys.main(['test/empty_file.txt'])
# Assert
captured = capsys.readouterr()
assert captured.err == ''
assert captured.out == ''
@pytest.mark.qreader

@ -128,45 +128,45 @@ if [ "$OLDVERSION" != "$VERSION" ]; then
mkdir -p $DOWNLOADS
# https://github.com/protocolbuffers/protobuf/releases/download/v21.6/protoc-21.6-linux-x86_64.zip
cmd="wget --trust-server-names https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-linux-x86_64.zip -O $DOWNLOADS/$ARCHIVE"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="echo -e '\nSize [Byte]'; stat --printf='%s\n' $DOWNLOADS/$ARCHIVE; echo -e '\nMD5'; md5sum $DOWNLOADS/$ARCHIVE; echo -e '\nSHA256'; sha256sum $DOWNLOADS/$ARCHIVE;"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="mkdir -p $BIN/$NAME; unzip $DOWNLOADS/$ARCHIVE -d $BIN/$NAME"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="echo $VERSION > $BIN/$NAME/.VERSION.txt; echo $VERSION > $BIN/$NAME/.VERSION_$VERSION.txt"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="[ -d $BIN/$DEST.old ] && rm -rf $BIN/$DEST.old || echo 'No old dir to delete'"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="[ -d $BIN/$DEST ] && mv -iT $BIN/$DEST $BIN/$DEST.old || echo 'No previous dir to keep'"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="mv -iT $BIN/$NAME $BIN/$DEST"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="rm $DOWNLOADS/$ARCHIVE"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="$BIN/$DEST/bin/protoc --python_out=protobuf_generated_python google_auth.proto"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
# Update README.md
cmd="perl -i -pe 's%proto(buf|c)([- ])(\d\.)?$OLDVERSION%proto\$1\$2\${3}$VERSION%g' README.md && perl -i -pe 's%(protobuf/releases/tag/v)$OLDVERSION%\${1}$VERSION%g' README.md"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
else
echo -e "\nVersion has not changed. Quit"
@ -176,27 +176,27 @@ fi
# Upgrade pip requirements
cmd="sudo pip install --upgrade pip"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
$PIP --version
cmd="$PIP install --use-pep517 -U -r requirements.txt"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="$PIP install --use-pep517 -U -r requirements-dev.txt"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="$PIP install -U pipenv"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
$PIPENV --version
cmd="$PIPENV update && $PIPENV --rm && $PIPENV install"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
$PIPENV run python --version
@ -204,11 +204,33 @@ $PIPENV run python --version
# Test
cmd="pytest"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="$PIPENV run pytest"
if $interactive ; then askContinueYn "$cmd"; fi
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
# Build docker
cmd="docker build . -t extract_otp_secret_keys_no_qr_reader -f Dockerfile_no_qr_reader --pull"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys_no_qr_reader test_extract_otp_secret_keys_pytest.py -k 'not qreader' -vvv --relaxed"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="docker build . -t extract_otp_secret_keys --pull"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v "$(pwd)":/files:ro extract_otp_secret_keys"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
cmd="docker image prune"
if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi
eval "$cmd"
quit

Loading…
Cancel
Save