2
0
mirror of https://github.com/koreader/koreader synced 2024-10-31 21:20:20 +00:00

ci/circle: improve caching

Don't cache the whole base directory; instead independently cache:
- the build output directory
- the ccache directory

This allow completely avoiding the need for building base if a
cache (keyed on the relevant parts of the git tree for base) is
successfully restored, as well as reducing the amount of cached
data.

Additionally, ensure caches are isolated by branch name (but allow
falling back to a "master" cache for restore): we don't want PRs
using each others' caches.
This commit is contained in:
Benoit Pierre 2024-05-13 21:33:53 +02:00 committed by Frans de Jonge
parent 8a316f928a
commit 76bdb7e65c
4 changed files with 138 additions and 57 deletions

View File

@ -15,7 +15,7 @@ if [ -z "${CIRCLE_PULL_REQUEST}" ] && [ "${CIRCLE_BRANCH}" = 'master' ]; then
if [ "${CIRCLE_NODE_INDEX}" = 0 ]; then
travis_retry make coverage
pushd koreader-*/koreader && {
pushd install/koreader && {
# see https://github.com/codecov/example-lua
bash <(curl -s https://codecov.io/bash)
} && popd || exit

View File

@ -4,4 +4,11 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${CI_DIR}/common.sh"
make all
# Build.
cmd=(make all)
if [[ -d base/build ]]; then
cmd+=(--assume-old=base)
fi
"${cmd[@]}"
# vim: sw=4

View File

@ -4,9 +4,11 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${CI_DIR}/common.sh"
pushd koreader-emulator-x86_64-linux-gnu/koreader && {
pushd install/koreader && {
# the circleci command spits out newlines; we want spaces instead
BUSTED_SPEC_FILE="$(circleci tests glob "spec/front/unit/*_spec.lua" | circleci tests split --split-by=timings --timings-type=filename | tr '\n' ' ')"
} && popd || exit
make testfront BUSTED_SPEC_FILE="${BUSTED_SPEC_FILE}"
# vim: sw=4

View File

@ -1,80 +1,125 @@
version: 2
version: "2.1"
workflows:
version: 2
build:
jobs:
- build
- docs:
context: koreader-vars
filters:
branches:
only: master
requires:
- build
# Parameters. {{{
parameters:
# Bump this to reset all caches.
cache_epoch:
type: integer
default: 0
# }}}
# Executors. {{{
executors:
base:
docker:
- image: koreader/kobase:0.3.2-20.04
auth:
username: $DOCKER_USERNAME
password: $DOCKER_PASSWORD
# }}}
# Jobs. {{{
jobs:
# Build. {{{
build:
docker:
- image: koreader/kobase:0.3.2-20.04
auth:
username: $DOCKER_USERNAME
password: $DOCKER_PASSWORD
environment:
EMULATE_READER: 1
executor: base
environment:
BASH_ENV: "~/.bashrc"
CCACHE_MAXSIZE: "256M"
EMULATE_READER: "1"
MAKEFLAGS: "OUTPUT_DIR=build INSTALL_DIR=install"
parallelism: 2
steps:
# Checkout / fetch. {{{
- checkout
# need to init some stuff first or git will complain when sticking in base cache
- run: git submodule update --init base && git -C base submodule update --depth 1 --init --jobs 3 --recursive
# we can't use command output directly for cache check so we write it to git-rev-base
- run: pushd base && git_rev_base=$(git describe HEAD) && popd && echo $git_rev_base && echo $git_rev_base >git-rev-base
- restore_cache:
keys:
- build-{{ arch }}-{{ checksum "git-rev-base" }}
# installs everything and caches base
- run:
name: fetch
name: Fetch
command: .ci/fetch.sh
# }}}
# Check.
- run:
name: check
name: Check
command: .ci/check.sh
# Restore / setup caches. {{{
- run:
name: build
name: Generate cache key
command: make -C base TARGET= cache-key
- restore_cache:
name: Restore build directory
keys:
- &CACHE_KEY_BUILD_DIR '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-build-{{ arch }}-{{ checksum "base/cache-key" }}'
- restore_cache:
name: Restore build cache
keys:
- &CACHE_KEY_BUILD_CACHE '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-ccache-{{ arch }}-{{ checksum "base/cache-key" }}'
- '<< pipeline.parameters.cache_epoch >>-{{ .Environment.CIRCLE_JOB }}-ccache-{{ arch }}-'
- run:
name: Setup build cache
command: |
set -x
which ccache
ccache --version
ccache --zero-stats
ccache --show-config
# }}}
# Build.
- run:
name: Build
command: .ci/build.sh
# we want to save cache prior to testing so we don't have to clean it up
# Clean / save caches. {{{
# We want to save cache prior to testing so we don't have to clean it up.
- run:
name: Clean caches
when: always
command: |
set -x
# Trim the build directory.
rm -rf base/build/thirdparty
ccache --cleanup >/dev/null
ccache --show-stats
- save_cache:
key: build-{{ arch }}-{{ checksum "git-rev-base" }}
name: Save build cache
key: *CACHE_KEY_BUILD_CACHE
paths:
- "/home/ko/.ccache"
- "base"
# our lovely unit tests
- /home/ko/.ccache
- save_cache:
name: Save build directory
key: *CACHE_KEY_BUILD_DIR
paths:
- base/build
# }}}
# Tests / coverage. {{{
# Our lovely unit tests.
- run:
name: test
name: Test
command: .ci/test.sh
# docs, coverage, and test timing (can we use two outputs at once?); master branch only
# Docs, coverage, and test timing (can we use two outputs at once?); master branch only.
- run:
name: coverage
name: Coverage
command: .ci/after_success.sh
# by storing the test results CircleCI automatically distributes tests based on execution time
# By storing the test results CircleCI automatically distributes tests based on execution time.
- store_test_results:
path: koreader-emulator-x86_64-linux-gnu/koreader
# CircleCI doesn't make the test results available as artifacts (October 2017)
path: &TESTS_XML install/koreader/junit-test-results.xml
# CircleCI doesn't make the test results available as artifacts (October 2017).
- store_artifacts:
path: koreader-emulator-x86_64-linux-gnu/koreader/junit-test-results.xml
path: *TESTS_XML
# }}}
# }}}
# Docs. {{{
docs:
docker:
- image: koreader/kobase:0.3.2-20.04
auth:
username: $DOCKER_USERNAME
password: $DOCKER_PASSWORD
environment:
EMULATE_READER: 1
executor: base
environment:
BASH_ENV: "~/.bashrc"
parallelism: 1
@ -83,8 +128,35 @@ jobs:
- run:
name: fetch
command: .ci/fetch.sh
# docs, coverage, and test timing (can we use two outputs at once?); master branch only
- run:
name: docs-and-translation
command: .ci/after_success_docs_translation.sh
# }}}
# }}}
# Workflows. {{{
workflows:
version: 2
build:
jobs:
- build
- docs:
context: koreader-vars
filters:
branches:
only: master
requires:
- build
# }}}
# vim: foldmethod=marker foldlevel=0