Browse Source

ci: Travis: improve/revisit caching (#10358)

- use CACHE_NVIM_DEPS_DIR

- do not cache pip
  This is handled through http caches in general/better, and it is not
  used much anyway.

- do not cache DEPS_DOWNLOAD_DIR
  Built deps are cached, downloads are not needed then.

- display ccache stats before clearing

- do not cache ccache stats

- improve output of `du` (do not list pages of output for "/home/travis/.cache/go-build")
tags/v0.4.0
Daniel Hahler GitHub 8 months ago
parent
commit
208f56d3b9
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 15 additions and 21 deletions
  1. +5
    -8
      .travis.yml
  2. +7
    -10
      ci/before_cache.sh
  3. +2
    -0
      ci/before_script.sh
  4. +1
    -3
      ci/common/build.sh

+ 5
- 8
.travis.yml View File

@@ -10,8 +10,6 @@ env:
- BUILD_DIR="$TRAVIS_BUILD_DIR/build"
# Build directory for third-party dependencies.
- DEPS_BUILD_DIR="$HOME/nvim-deps"
# Directory where third-party dependency sources are downloaded to.
- DEPS_DOWNLOAD_DIR="$TRAVIS_BUILD_DIR/deps-downloads"
# Install directory for Neovim.
- INSTALL_PREFIX="$HOME/nvim-install"
# Log directory for Clang sanitizers and Valgrind.
@@ -25,7 +23,7 @@ env:
-DBUSTED_OUTPUT_TYPE=nvim
-DDEPS_PREFIX=$DEPS_BUILD_DIR/usr
-DMIN_LOG_LEVEL=3"
- DEPS_CMAKE_FLAGS="-DDEPS_DOWNLOAD_DIR:PATH=$DEPS_DOWNLOAD_DIR -DUSE_BUNDLED_GPERF=OFF"
- DEPS_CMAKE_FLAGS="-DUSE_BUNDLED_GPERF=OFF"
# Additional CMake flags for 32-bit builds.
- CMAKE_FLAGS_32BIT="-DCMAKE_SYSTEM_LIBRARY_PATH=/lib32:/usr/lib32:/usr/local/lib32
-DCMAKE_IGNORE_PATH=/lib:/usr/lib:/usr/local/lib
@@ -36,8 +34,9 @@ env:
- UBSAN_OPTIONS="print_stacktrace=1 log_path=$LOG_DIR/ubsan"
# Environment variables for Valgrind.
- VALGRIND_LOG="$LOG_DIR/valgrind-%p.log"
- CACHE_NVIM_DEPS_DIR="$HOME/.cache/nvim-deps"
# If this file exists, the cache is valid (compile was successful).
- CACHE_MARKER="$HOME/.cache/nvim-deps/.travis_cache_marker"
- CACHE_MARKER="$CACHE_NVIM_DEPS_DIR/.travis_cache_marker"
# default target name for functional tests
- FUNCTIONALTEST=functionaltest
- CI_TARGET=tests
@@ -52,7 +51,7 @@ anchors:
envs: &common-job-env
# Do not fall back to cache for "master" for PR on "release" branch:
# adds the target branch to the cache key.
FOR_TRAVIS_CACHE=$TRAVIS_BRANCH
FOR_TRAVIS_CACHE=v1-$TRAVIS_BRANCH

addons:
apt:
@@ -165,9 +164,7 @@ cache:
apt: true
ccache: true
directories:
- "$HOME/.cache/pip"
- "$HOME/.cache/nvim-deps"
- "$HOME/.cache/nvim-deps-downloads"
- "$CACHE_NVIM_DEPS_DIR"

notifications:
webhooks:


+ 7
- 10
ci/before_cache.sh View File

@@ -7,23 +7,20 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${CI_DIR}/common/build.sh"
source "${CI_DIR}/common/suite.sh"

# Don't cache pip's log and selfcheck.
rm -rf "${HOME}/.cache/pip/log"
rm -f "${HOME}/.cache/pip/selfcheck.json"

echo "before_cache.sh: cache size"
du -d 2 "${HOME}/.cache" | sort -n
du -chd 1 "${HOME}/.cache" | sort -rh | head -20

echo "before_cache.sh: ccache stats"
ccache -s 2>/dev/null || true
# Do not keep ccache stats (uploaded to cache otherwise; reset initially anyway).
find "${HOME}/.ccache" -name stats -delete

# Update the third-party dependency cache only if the build was successful.
if ended_successfully; then
rm -rf "${HOME}/.cache/nvim-deps"
mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps"

rm -rf "${HOME}/.cache/nvim-deps-downloads"
mv "${DEPS_DOWNLOAD_DIR}" "${HOME}/.cache/nvim-deps-downloads"
# Do not cache downloads. They should not be needed with up-to-date deps.
rm -rf "${DEPS_BUILD_DIR}/build/downloads"
rm -rf "${CACHE_NVIM_DEPS_DIR}"
mv "${DEPS_BUILD_DIR}" "${CACHE_NVIM_DEPS_DIR}"

touch "${CACHE_MARKER}"
echo "Updated third-party dependencies (timestamp: $(_stat "${CACHE_MARKER}"))."


+ 2
- 0
ci/before_script.sh View File

@@ -20,6 +20,8 @@ if [[ -n "${LLVM_SYMBOLIZER}" ]] && [[ ! $(type -P "${LLVM_SYMBOLIZER}") ]]; the
exit 1
fi

echo "before_script.sh: ccache stats (will be cleared)"
ccache -s
# Reset ccache stats for real results in before_cache.
ccache --zero-stats



+ 1
- 3
ci/common/build.sh View File

@@ -27,15 +27,13 @@ build_deps() {
fi

mkdir -p "${DEPS_BUILD_DIR}"
mkdir -p "${DEPS_DOWNLOAD_DIR}"

# Use cached dependencies if $CACHE_MARKER exists.
if test "${CACHE_ENABLE}" = "false" ; then
export CCACHE_RECACHE=1
elif test -f "${CACHE_MARKER}" ; then
echo "Using third-party dependencies from Travis cache (last update: $(_stat "${CACHE_MARKER}"))."
cp -a "${HOME}/.cache/nvim-deps"/. "${DEPS_BUILD_DIR}"
cp -a "${HOME}/.cache/nvim-deps-downloads"/. "${DEPS_DOWNLOAD_DIR}"
cp -a "${CACHE_NVIM_DEPS_DIR}"/. "${DEPS_BUILD_DIR}"
fi

# Even if we're using cached dependencies, run CMake and make to


Loading…
Cancel
Save