Travis: speed up by caching base and running luacheck earlier (#2741)

* Travis: speed up by caching base and running luacheck earlier * ignore bin and install for git status change detection * skip coverage except on official master branch. It adds 3 whole minutes and does nothing to prevent regressions * also cache ~/.luarocks. It evens out but would generally prevent remote timeout shenenigans * remove base cache dir before caching with verbose remove to see what's going on * more inclusive shell code quality analysis * fixed more shellcheck issues * better shellcheck/shfmt debugging info
7 years ago · 9d39f11f59
parent 5bc19fa084
commit 9d39f11f59
9 changed files with 114 additions and 40 deletions
--- a/.ci/after_success.sh
+++ b/.ci/after_success.sh
@ -33,11 +33,12 @@ if [ "${TRAVIS_PULL_REQUEST}" = false ] && [ "${TRAVIS_BRANCH}" = 'master' ]; th
        commit -a --amend -m 'Automated documentation build from travis-ci.'
    git push -f --quiet origin gh-pages > /dev/null
    echo -e "\n${ANSI_GREEN}Documentation update pushed."
+    popd
+
+    travis_retry make coverage
+    pushd koreader-*/koreader
+        luajit "$(which luacov-coveralls)"
+    popd
 else
-    echo -e "\n${ANSI_GREEN}Not on official master branch, skip documentation update."
+    echo -e "\n${ANSI_GREEN}Not on official master branch, skip documentation update and coverage."
 fi
-
-travis_retry make coverage
-pushd koreader-*/koreader
-    luajit "$(which luacov-coveralls)"
-popd
--- a/.ci/helper_luarocks.sh
+++ b/.ci/helper_luarocks.sh
@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${CI_DIR}/common.sh"
+
+rm -rf "${HOME}/.luarocks"
+mkdir "${HOME}/.luarocks"
+cp "${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua" "${HOME}/.luarocks/config.lua"
+echo "wrap_bin_scripts = false" >> "$HOME/.luarocks/config.lua"
+travis_retry luarocks --local install luafilesystem
+# for verbose_print module
+travis_retry luarocks --local install ansicolors
+travis_retry luarocks --local install busted 2.0.rc12-1
+#- mv -f $HOME/.luarocks/bin/busted_bootstrap $HOME/.luarocks/bin/busted
+travis_retry luarocks --local install luacov
+# luasec doesn't automatically detect 64-bit libs
+travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linux-gnu
+travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev
+travis_retry luarocks --local install luacheck
+travis_retry luarocks --local install lanes  # for parallel luacheck
--- a/.ci/helper_shellchecks.sh
+++ b/.ci/helper_shellchecks.sh
@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "${CI_DIR}/common.sh"
+
+# shellcheck disable=2016
+mapfile -t shellscript_locations < <( { git grep -lE '^#!(/usr)?/bin/(env )?(bash|sh)' && git submodule --quiet foreach '[ "$path" = "base" ] || git grep -lE "^#!(/usr)?/bin/(env )?(bash|sh)" | sed "s|^|$path/|"' && git ls-files ./*.sh ; } | sort | uniq )
+
+SHELLSCRIPT_ERROR=0
+
+for shellscript in "${shellscript_locations[@]}"; do
+    echo -e "${ANSI_GREEN}Running shellcheck on ${shellscript}"
+    shellcheck "${shellscript}" || SHELLSCRIPT_ERROR=1
+    echo -e "${ANSI_GREEN}Running shfmt on ${shellscript}"
+    [ "$(cat "${shellscript}" )" != "$(shfmt -i 4 "${shellscript}")" ] && echo  -e "${ANSI_RED}Warning: ${shellscript} does not abide by coding style"
+    # @TODO add error handling with something like && shfmt -i 4 "${shellscript}" | diff "${shellscript}"
+done
+
+exit "${SHELLSCRIPT_ERROR}"
--- a/.ci/install.sh
+++ b/.ci/install.sh
@ -4,6 +4,22 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"

+# print some useful info
+echo "TRAVIS_BUILD_DIR: ${TRAVIS_BUILD_DIR}"
+echo "pwd: $(pwd)"
+ls
+
+# toss submodules if there are any changes
+if [ "$(git status --ignore-submodules=dirty --porcelain)" ]; then
+    # what changed?
+    git status
+    # purge and reinit submodules
+    git submodule deinit -f .
+    git submodule update --init
+else
+    echo -e "${ANSI_GREEN}Using cached submodules."
+fi
+
 # install our own updated luarocks
 if [ ! -f "${TRAVIS_BUILD_DIR}/install/bin/luarocks" ]; then
    git clone https://github.com/torch/luajit-rocks.git
@ -12,22 +28,17 @@ if [ ! -f "${TRAVIS_BUILD_DIR}/install/bin/luarocks" ]; then
        cmake . -DWITH_LUAJIT21=ON -DCMAKE_INSTALL_PREFIX="${TRAVIS_BUILD_DIR}/install"
        make install
    popd
+else
+    echo -e "${ANSI_GREEN}Using cached luarocks."
 fi

-mkdir "${HOME}/.luarocks"
-cp "${TRAVIS_BUILD_DIR}/install/etc/luarocks/config.lua" "$HOME/.luarocks/config.lua"
-echo "wrap_bin_scripts = false" >> "$HOME/.luarocks/config.lua"
-travis_retry luarocks --local install luafilesystem
-# for verbose_print module
-travis_retry luarocks --local install ansicolors
-travis_retry luarocks --local install busted 2.0.rc12-1
-#- mv -f $HOME/.luarocks/bin/busted_bootstrap $HOME/.luarocks/bin/busted
-travis_retry luarocks --local install luacov
-# luasec doesn't automatically detect 64-bit libs
-travis_retry luarocks --local install luasec OPENSSL_LIBDIR=/usr/lib/x86_64-linux-gnu
-travis_retry luarocks --local install luacov-coveralls --server=http://rocks.moonscript.org/dev
-travis_retry luarocks --local install luacheck
-travis_retry luarocks --local install lanes  # for parallel luacheck
+if [ ! -d "${HOME}/.luarocks" ] || [ ! -f "${HOME}/.luarocks/$(md5sum < "${CI_DIR}/helper_luarocks.sh")" ] ; then
+    echo -e "${ANSI_GREEN}Grabbing new .luarocks."
+    "${CI_DIR}/helper_luarocks.sh"
+    touch "${HOME}/.luarocks/$(md5sum < "${CI_DIR}/helper_luarocks.sh")"
+else
+    echo -e "${ANSI_GREEN}Using cached .luarocks."
+fi

 #install our own updated shellcheck
 SHELLCHECK_URL="https://s3.amazonaws.com/travis-blue-public/binaries/ubuntu/14.04/x86_64/shellcheck-0.4.5.tar.bz2"
@ -35,6 +46,8 @@ if ! command -v shellcheck ; then
    curl -sSL "${SHELLCHECK_URL}" | tar --exclude 'SHA256SUMS' --strip-components=1 -C "${HOME}/bin" -xjf -;
    chmod +x "${HOME}/bin/shellcheck"
    shellcheck --version
+else
+    echo -e "${ANSI_GREEN}Using cached shellcheck."
 fi

 # install shfmt
@ -42,4 +55,6 @@ SHFMT_URL="https://github.com/mvdan/sh/releases/download/v1.2.0/shfmt_v1.2.0_lin
 if ! command -v shfmt ; then
    curl -sSL "${SHFMT_URL}" -o "${HOME}/bin/shfmt"
    chmod +x "${HOME}/bin/shfmt"
+else
+    echo -e "${ANSI_GREEN}Using cached shfmt."
 fi
--- a/.ci/script.sh
+++ b/.ci/script.sh
@ -4,9 +4,15 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"

+echo -e "\n${ANSI_GREEN}make fetchthirdparty"
 travis_retry make fetchthirdparty
-find . -type f -name '*.sh' -not -path "./base/*" -not -path "./luajit-rocks/*" -print0 | xargs --null shellcheck
-find . -type f -name '*.sh' -not -path "./base/*" -not -path "./luajit-rocks/*" -print0 | xargs shfmt -i 0 -w
+
+"${CI_DIR}/helper_shellchecks.sh"
+
+echo -e "\n${ANSI_GREEN}Luacheck results"
+luajit "$(which luacheck)" --no-color -q {reader,setupkoenv,datastorage}.lua frontend plugins
+
+echo -e "\n${ANSI_GREEN}make all"
 make all
+echo -e "\n${ANSI_GREEN}make testfront"
 make testfront
-luajit "$(which luacheck)" --no-color -q {reader,setupkoenv,datastorage}.lua frontend plugins
--- a/.gitignore
+++ b/.gitignore
@ -12,6 +12,8 @@ tags
 test/*
 *.tar
 *.log
+bin/
+install/
 spec/unit/data
 doc/html
 git-rev
--- a/.travis.yml
+++ b/.travis.yml
@ -18,8 +18,14 @@ cache:
  directories:
    - "${HOME}/bin"
    # compiled luarocks binaries
-    - "${HOME}/build/koreader/koreader/install"
+    - "${TRAVIS_BUILD_DIR}/install"
+    # base build
+    - "${TRAVIS_BUILD_DIR}/base"
    - "${HOME}/.ccache"
+    - "${HOME}/.luarocks"
+before_cache:
+  # don't quote like you normally would or it won't expand
+  - rm -frv ${TRAVIS_BUILD_DIR}/base/build/*/cache/*

 addons:
  apt:
@ -28,7 +34,7 @@ addons:
    packages:
      - g++-4.8
      - libsdl1.2-dev
-      # for luasec
+      # luasec dependencies
      - libssl1.0.0
      - nasm
      # OpenSSL likes this (package contains makedepend)
--- a/platform/kindle/kotar_cpoint
+++ b/platform/kindle/kotar_cpoint
@ -8,21 +8,21 @@ export EIPS_NO_SLEEP="true"

 # Load our helper functions...
 if [ -f "${KOREADER_DIR}/libkohelper.sh" ] ; then
-	source "${KOREADER_DIR}/libkohelper.sh"
+	# shellcheck source=/dev/null
+	. "${KOREADER_DIR}/libkohelper.sh"
 else
 	echo "Can't source helper functions, aborting!"
 	exit 1
 fi

-
 ## First arg is the chekpoint number, and we get one every 200 checkpoints.
 CHECKPOINT_NUM="${1}"
 CHECKPOINT_GRANULARITY="200"

 # Use that to build a poor man's progress bar, with dots.
-PROGRESS_AMOUNT="$(( ${CHECKPOINT_NUM} / ${CHECKPOINT_GRANULARITY} ))"
+PROGRESS_AMOUNT="$(( CHECKPOINT_NUM / CHECKPOINT_GRANULARITY ))"
 PROGRESS_STRING="Updating koreader "
-for foo in $( seq 1 ${PROGRESS_AMOUNT} ) ; do
+for _ in $( seq 1 ${PROGRESS_AMOUNT} ) ; do
 	# Append a dot until we hit the needed amount
 	PROGRESS_STRING="${PROGRESS_STRING}."
 done
--- a/platform/pocketbook/koreader.app
+++ b/platform/pocketbook/koreader.app
@ -7,16 +7,16 @@ KOREADER_DIR=/mnt/ext1/applications/koreader
 # update to new version from OTA directory
 NEWUPDATE="${KOREADER_DIR}/ota/koreader.updated.tar"
 INSTALLED="${KOREADER_DIR}/ota/koreader.installed.tar"
-if [ -f $NEWUPDATE ]; then
+if [ -f ${NEWUPDATE} ]; then
    # TODO: any graphic indication for the updating progress?
-    cd /mnt/ext1/ && tar xf $NEWUPDATE && mv $NEWUPDATE $INSTALLED
+    cd /mnt/ext1/ && tar xf ${NEWUPDATE} && mv ${NEWUPDATE} ${INSTALLED}
 fi

 # we're always starting from our working directory
-cd $KOREADER_DIR
+cd ${KOREADER_DIR} || exit

 # export load library path for some old firmware
-export LD_LIBRARY_PATH=${KOREADER_DIR}/libs:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${KOREADER_DIR}/libs:${LD_LIBRARY_PATH}

 # export trained OCR data directory
 export TESSDATA_PREFIX="data"
@ -27,18 +27,21 @@ export STARDICT_DATA_DIR="data/dict"
 # export external font directory
 export EXT_FONT_DIR="/mnt/ext1/fonts"

-if [ `echo $@ | wc -c` -eq 1 ]; then
+# shellcheck disable=2000
+if [ "$(echo "$@" | wc -c)" -eq 1 ]; then
    args="/mnt/ext1/"
 else
-    args="$@"
+    args="$*"
 fi

 # we keep maximum 500K worth of crash log
-cat crash.log 2> /dev/null | tail -c 500000 > crash.log.new
-mv -f crash.log.new crash.log
-./reader.lua "$args" >> crash.log 2>&1
+if [ -e crash.log ]; then
+    tail -c 500000 crash.log > crash.log.new
+    mv -f crash.log.new crash.log
+fi
+
+./reader.lua "${args}" >> crash.log 2>&1

 if pidof reader.lua > /dev/null 2>&1 ; then
-	killall -TERM reader.lua
+    killall -TERM reader.lua
 fi
-