From 144c9e6320f55f77d3481f79f00b9e13a8931a2b Mon Sep 17 00:00:00 2001 From: scito Date: Fri, 30 Dec 2022 15:31:41 +0100 Subject: [PATCH] fixes after change to src-layout --- .github/workflows/ci.yml | 8 +-- .gitignore | 4 +- Dockerfile | 2 +- Dockerfile_no_qr_reader | 4 +- pyproject.toml | 4 +- run_pytest.sh | 2 +- google_auth.proto => src/google_auth.proto | 0 conftest.py => tests/conftest.py | 0 tests/utils.py | 2 +- upgrade_deps.sh | 58 +++++++++++++++++++--- 10 files changed, 63 insertions(+), 21 deletions(-) rename google_auth.proto => src/google_auth.proto (100%) rename conftest.py => tests/conftest.py (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76d8049..4e51b3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,8 +36,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest pytest-mock pytest-cov mypy types-protobuf - pip install --use-pep517 -r requirements.txt + pip install -r requirements-dev.txt + pip install . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -46,8 +46,8 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=200 --statistics - name: Type checking with mypy run: | - mypy --install-types --non-interactive *.py - mypy --strict *.py + mypy --install-types --non-interactive src tests + mypy --strict src tests if: matrix.python-version == '3.x' - name: Test with pytest run: pytest diff --git a/.gitignore b/.gitignore index bf72bec..cb80958 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,7 @@ venv/ !example_output.csv !.github/ !.flake8 -.vscode -!.vscode/settings.json +!.vscode/ !.devcontainer/ !.devcontainer/*.json *.whl @@ -19,3 +18,4 @@ dist/ extract_otp_secret_keys.egg-info/ *.xml pytest-coverage.txt +tests/reports/ diff --git a/Dockerfile b/Dockerfile index e390a54..2093f99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,6 @@ RUN apt-get update && apt-get install -y libzbar0 libsm6 python3-opencv nano \ WORKDIR /files -ENTRYPOINT ["python", "/extract/extract_otp_secret_keys.py"] +ENTRYPOINT ["python", "/extract/src/extract_otp_secret_keys.py"] LABEL org.opencontainers.image.source https://github.com/scito/extract_otp_secret_keys diff --git a/Dockerfile_no_qr_reader b/Dockerfile_no_qr_reader index 7136d8f..65672e7 100644 --- a/Dockerfile_no_qr_reader +++ b/Dockerfile_no_qr_reader @@ -15,10 +15,10 @@ RUN apk add --no-cache nano zlib jpeg \ && if [[ "$(apk --print-arch)" == "aarch64" ]]; then apk add --no-cache --virtual .build-deps gcc libc-dev python3-dev py3-setuptools zlib-dev jpeg-dev; fi \ && pip install --no-cache-dir protobuf qrcode Pillow \ && if [[ "$(apk --print-arch)" == "aarch64" ]]; then apk del .build-deps; fi \ - && if [[ "$RUN_TESTS" == "true" ]]; then /extract/run_pytest.sh test_extract_otp_secret_keys_pytest.py -k "not qreader" --relaxed; else echo "Not running tests..."; fi + && if [[ "$RUN_TESTS" == "true" ]]; then /extract/run_pytest.sh tests/test_extract_otp_secret_keys_pytest.py -k "not qreader" --relaxed; else echo "Not running tests..."; fi WORKDIR /files -ENTRYPOINT ["python", "/extract/extract_otp_secret_keys.py"] +ENTRYPOINT ["python", "/extract/src/extract_otp_secret_keys.py"] LABEL org.opencontainers.image.source https://github.com/scito/extract_otp_secret_keys diff --git a/pyproject.toml b/pyproject.toml index e5b2522..709d87f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,13 +48,13 @@ scripts = {extract_otp_secret_keys = "extract_otp_secret_keys:sys_main"} urls = {Project-URL = "https://github.com/scito/extract_otp_secret_keys", Bug-Reports = "https://github.com/scito/extract_otp_secret_keys/issues", Source = "https://github.com/scito/extract_otp_secret_keys" } [tool.setuptools] -py-modules = ["extract_otp_secret_keys", "protobuf_generated_python.google_auth_pb2"] +py-modules = ["extract_otp_secret_keys"] [tool.setuptools.dynamic] # TODO version = {attr = "extract_otp_secret_keys.VERSION"} [tool.setuptools.package-data] -"." = ["*.txt", "*.json", "*.csv", "*.png", "*.md"] +"extract_otp_secret_keys" = ["*.txt", "*.json", "*.csv", "*.png", "*.md"] # https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-structure%3E # https://docs.pytest.org/en/7.1.x/explanation/goodpractices.html#which-import-mode diff --git a/run_pytest.sh b/run_pytest.sh index 8b93975..08ccfdd 100755 --- a/run_pytest.sh +++ b/run_pytest.sh @@ -1,3 +1,3 @@ #!/bin/sh cd /extract -pip install -U pytest pytest-mock && pytest "$@" +pip install -U pytest pytest-mock && pip install --no-deps . && pytest "$@" diff --git a/google_auth.proto b/src/google_auth.proto similarity index 100% rename from google_auth.proto rename to src/google_auth.proto diff --git a/conftest.py b/tests/conftest.py similarity index 100% rename from conftest.py rename to tests/conftest.py diff --git a/tests/utils.py b/tests/utils.py index f1372f4..9504904 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -39,7 +39,7 @@ Usage: with Capturing() as output: print("Output") ''' - # TODO remove type ignore when fixed, see https://github.com/python/mypy/issues/11871, https://stackoverflow.com/questions/72174409/type-hinting-the-return-value-of-a-class-method-that-returns-self + # TODO remove type ignore if fixed, see https://github.com/python/mypy/issues/11871, https://stackoverflow.com/questions/72174409/type-hinting-the-return-value-of-a-class-method-that-returns-self def __enter__(self): # type: ignore self._stdout = sys.stdout sys.stdout = self._stringio_std = io.StringIO() diff --git a/upgrade_deps.sh b/upgrade_deps.sh index 4e4e9bc..4c7019e 100755 --- a/upgrade_deps.sh +++ b/upgrade_deps.sh @@ -162,7 +162,7 @@ if [ "$OLDVERSION" != "$VERSION" ] || ! $ignore_version_check; then if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" - cmd="$BIN/$DEST/bin/protoc --plugin=protoc-gen-mypy=/home/rkurmann/.local/bin/protoc-gen-mypy --python_out=protobuf_generated_python --mypy_out=protobuf_generated_python google_auth.proto" + cmd="$BIN/$DEST/bin/protoc --plugin=protoc-gen-mypy=$HOME/.local/bin/protoc-gen-mypy --python_out=protobuf_generated_python --mypy_out=protobuf_generated_python src/google_auth.proto" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" @@ -232,17 +232,42 @@ cmd="$MYPY --install-types --non-interactive *.py" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -cmd="$MYPY --strict *.py" +# change to src as python -m mypy adds the current dir Python sys.path +# execute in a subshell in order not to loose the exit code and not to change the dir in the currrent shell +cmd="(cd src && $MYPY --strict *.py ../tests)" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +# pip install + +cmd="$PIP install -e ." +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="extract_otp_secret_keys example_export.txt" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="extract_otp_secret_keys - < example_export.txt" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" # Test -cmd="pytest --cov=test_extract_otp_secret_keys_pytest --junitxml=pytest.xml --cov-report=term-missing | tee pytest-coverage.txt" +cmd="$PYTHON src/extract_otp_secret_keys.py example_export.txt" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="$PYTHON src/extract_otp_secret_keys.py - < example_export.txt" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +COVERAGE_OUT="tests/reports/pytest-coverage.txt" +cmd="pytest --cov=test_extract_otp_secret_keys_pytest --junitxml=tests/reports/pytest.xml --cov-report html:tests/reports/html --cov-report=term-missing tests/ | tee $COVERAGE_OUT" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -cmd="$PIPENV run pytest --cov=test_extract_otp_secret_keys_pytest" +cmd="$PIPENV run pytest --cov=test_extract_otp_secret_keys_pytest tests/" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" @@ -250,7 +275,8 @@ eval "$cmd" # https://github.com/marketplace/actions/pytest-coverage-comment # Coverage-95%25-yellowgreen -TOTAL_COVERAGE=$(cat pytest-coverage.txt | grep 'TOTAL' | perl -ne 'print "$&" if /\b(\d{1,3})%/') && perl -i -pe "s/coverage-(\d{1,3}%)25-/coverage-${TOTAL_COVERAGE}25-/" README.md +echo -e "Upgrade code coverage in README.md" +TOTAL_COVERAGE=$(cat $COVERAGE_OUT | grep 'TOTAL' | perl -ne 'print "$&" if /\b(\d{1,3})%/') && perl -i -pe "s/coverage-(\d{1,3}%)25-/coverage-${TOTAL_COVERAGE}25-/" README.md # Build docker @@ -258,7 +284,15 @@ cmd="docker build . -t extract_otp_secret_keys_no_qr_reader -f Dockerfile_no_qr_ if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v \"$(pwd)\":/files:ro extract_otp_secret_keys_no_qr_reader test_extract_otp_secret_keys_pytest.py -k 'not qreader' -vvv --relaxed" +cmd="docker run --rm -v \"$(pwd)\":/files:ro extract_otp_secret_keys_no_qr_reader example_export.txt" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="docker run --rm -i -v \"$(pwd)\":/files:ro extract_otp_secret_keys_no_qr_reader - < example_export.txt" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v \"$(pwd)\":/files:ro extract_otp_secret_keys_no_qr_reader tests/test_extract_otp_secret_keys_pytest.py -k 'not qreader' -vvv --relaxed" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" @@ -266,15 +300,23 @@ cmd="docker build . -t extract_otp_secret_keys --pull" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" +cmd="docker run --rm -v \"$(pwd)\":/files:ro extract_otp_secret_keys example_export.txt" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + +cmd="docker run --rm -i -v \"$(pwd)\":/files:ro extract_otp_secret_keys - < example_export.txt" +if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi +eval "$cmd" + cmd="docker run --entrypoint /extract/run_pytest.sh --rm -v \"$(pwd)\":/files:ro extract_otp_secret_keys" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -cmd="docker image prune" +cmd="docker image prune || echo 'No docker image pruning'" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd" -cmd="$PYTHON extract_otp_secret_keys.py &" +cmd="$PYTHON src/extract_otp_secret_keys.py &" if $interactive ; then askContinueYn "$cmd"; else echo -e "${cyan}$cmd${reset}";fi eval "$cmd"