ci/circle: improve caching

Don't cache the whole base directory; instead independently cache:
- the build output directory
- the ccache directory

This allow completely avoiding the need for building base if a
cache (keyed on the relevant parts of the git tree for base) is
successfully restored, as well as reducing the amount of cached
data.

Additionally, ensure caches are isolated by branch name (but allow
falling back to a "master" cache for restore): we don't want PRs
using each others' caches.

.ci/after_success.sh

@@ -15,7 +15,7 @@ if [ -z "${CIRCLE_PULL_REQUEST}" ] && [ "${CIRCLE_BRANCH}" = 'master' ]; then
 
     if [ "${CIRCLE_NODE_INDEX}" = 0 ]; then
         travis_retry make coverage
-        pushd koreader-*/koreader && {
+        pushd install/koreader && {
             # see https://github.com/codecov/example-lua
             bash <(curl -s https://codecov.io/bash)
         } && popd || exit

.ci/build.sh

@@ -4,4 +4,11 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"
 
-make all
+# Build.
+cmd=(make all)
+if [[ -d base/build ]]; then
+    cmd+=(--assume-old=base)
+fi
+"${cmd[@]}"
+
+# vim: sw=4

.ci/test.sh

@@ -4,9 +4,11 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=/dev/null
 source "${CI_DIR}/common.sh"
 
-pushd koreader-emulator-x86_64-linux-gnu/koreader && {
+pushd install/koreader && {
     # the circleci command spits out newlines; we want spaces instead
     BUSTED_SPEC_FILE="$(circleci tests glob "spec/front/unit/*_spec.lua" | circleci tests split --split-by=timings --timings-type=filename | tr '\n' ' ')"
 } && popd || exit
 
 make testfront BUSTED_SPEC_FILE="${BUSTED_SPEC_FILE}"
+
+# vim: sw=4

.circleci/config.yml

@@ -1,80 +1,125 @@
-version: 2
+version: "2.1"
 
-workflows:
-  version: 2
-  build:
-    jobs:
-      - build
-      - docs:
-          context: koreader-vars
-          filters:
-            branches:
-              only: master
-          requires:
-            - build
+# Parameters. {{{
 
-jobs:
-  build:
+parameters:
+
+    # Bump this to reset all caches.
+    cache_epoch:
+      type: integer
+      default: 0
+
+# }}}
+
+# Executors. {{{
+
+executors:
+
+  base:
     docker:
       - image: koreader/kobase:0.3.2-20.04
         auth:
           username: $DOCKER_USERNAME
           password: $DOCKER_PASSWORD
-        environment:
-          EMULATE_READER: 1
+
+# }}}
+
+# Jobs. {{{
+
+jobs:
+
+  # Build. {{{
+
+  build:
+    executor: base
     environment:
       BASH_ENV: "~/.bashrc"
+      CCACHE_MAXSIZE: "256M"
+      EMULATE_READER: "1"
+      MAKEFLAGS: "OUTPUT_DIR=build INSTALL_DIR=install"
     parallelism: 2
     steps:
+      # Checkout / fetch. {{{
       - checkout
-      # need to init some stuff first or git will complain when sticking in base cache
-      - run: git submodule update --init base && git -C base submodule update --depth 1 --init --jobs 3 --recursive
-      # we can't use command output directly for cache check so we write it to git-rev-base
-      - run: pushd base && git_rev_base=$(git describe HEAD) && popd && echo $git_rev_base && echo $git_rev_base >git-rev-base
-      - restore_cache:
-          keys:
-            - build-{{ arch }}-{{ checksum "git-rev-base" }}
-
-      # installs everything and caches base
       - run:
-          name: fetch
+          name: Fetch
           command: .ci/fetch.sh
+      # }}}
+      # Check.
       - run:
-          name: check
+          name: Check
           command: .ci/check.sh
+      # Restore / setup caches. {{{
       - run:
-          name: build
+          name: Generate cache key
+          command: make -C base TARGET= cache-key
+      - restore_cache:
+          name: Restore build directory
+          keys:
+            - &CACHE_KEY_BUILD_DIR '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-build-{{ arch }}-{{ checksum "base/cache-key" }}'
+      - restore_cache:
+          name: Restore build cache
+          keys:
+            - &CACHE_KEY_BUILD_CACHE '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-ccache-{{ arch }}-{{ checksum "base/cache-key" }}'
+            - '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-ccache-{{ arch }}-'
+      - run:
+          name: Setup build cache
+          command: |
+            set -x
+            which ccache
+            ccache --version
+            ccache --zero-stats
+            ccache --show-config
+      # }}}
+      # Build.
+      - run:
+          name: Build
           command: .ci/build.sh
-      # we want to save cache prior to testing so we don't have to clean it up
+      # Clean / save caches. {{{
+      # We want to save cache prior to testing so we don't have to clean it up.
+      - run:
+          name: Clean caches
+          when: always
+          command: |
+            set -x
+            # Trim the build directory.
+            rm -rf base/build/thirdparty
+            ccache --cleanup >/dev/null
+            ccache --show-stats
       - save_cache:
-          key: build-{{ arch }}-{{ checksum "git-rev-base" }}
+          name: Save build cache
+          key: *CACHE_KEY_BUILD_CACHE
           paths:
-            - "/home/ko/.ccache"
-            - "base"
-
-      # our lovely unit tests
+            - /home/ko/.ccache
+      - save_cache:
+          name: Save build directory
+          key: *CACHE_KEY_BUILD_DIR
+          paths:
+            - base/build
+      # }}}
+      # Tests / coverage. {{{
+      # Our lovely unit tests.
       - run:
-          name: test
+          name: Test
           command: .ci/test.sh
-
-      # docs, coverage, and test timing (can we use two outputs at once?); master branch only
+      # Docs, coverage, and test timing (can we use two outputs at once?); master branch only.
       - run:
-          name: coverage
+          name: Coverage
           command: .ci/after_success.sh
-      # by storing the test results CircleCI automatically distributes tests based on execution time
+      # By storing the test results CircleCI automatically distributes tests based on execution time.
       - store_test_results:
-          path: koreader-emulator-x86_64-linux-gnu/koreader
-      # CircleCI doesn't make the test results available as artifacts (October 2017)
+          path: &TESTS_XML install/koreader/junit-test-results.xml
+      # CircleCI doesn't make the test results available as artifacts (October 2017).
       - store_artifacts:
-          path: koreader-emulator-x86_64-linux-gnu/koreader/junit-test-results.xml
+          path: *TESTS_XML
+      # }}}
+
+  # }}}
+
+  # Docs. {{{
+
   docs:
-    docker:
-      - image: koreader/kobase:0.3.2-20.04
-        auth:
-          username: $DOCKER_USERNAME
-          password: $DOCKER_PASSWORD
-        environment:
-          EMULATE_READER: 1
+    executor: base
     environment:
       BASH_ENV: "~/.bashrc"
     parallelism: 1
@@ -83,8 +128,35 @@ jobs:
       - run:
           name: fetch
           command: .ci/fetch.sh
-
       # docs, coverage, and test timing (can we use two outputs at once?); master branch only
       - run:
           name: docs-and-translation
           command: .ci/after_success_docs_translation.sh
+
+  # }}}
+
+# }}}
+
+# Workflows. {{{
+
+workflows:
+
+  version: 2
+
+  build:
+
+    jobs:
+
+      - build
+
+      - docs:
+          context: koreader-vars
+          filters:
+            branches:
+              only: master
+          requires:
+            - build
+
+# }}}
+
+# vim: foldmethod=marker foldlevel=0