ci: Travis: improve/revisit caching (#10358)

- use CACHE_NVIM_DEPS_DIR

- do not cache pip
  This is handled through http caches in general/better, and it is not
  used much anyway.

- do not cache DEPS_DOWNLOAD_DIR
  Built deps are cached, downloads are not needed then.

- display ccache stats before clearing

- do not cache ccache stats

- improve output of `du` (do not list pages of output for "/home/travis/.cache/go-build")

.travis.yml

@@ -10,8 +10,6 @@ env:
     - BUILD_DIR="$TRAVIS_BUILD_DIR/build"
     # Build directory for third-party dependencies.
     - DEPS_BUILD_DIR="$HOME/nvim-deps"
-    # Directory where third-party dependency sources are downloaded to.
-    - DEPS_DOWNLOAD_DIR="$TRAVIS_BUILD_DIR/deps-downloads"
     # Install directory for Neovim.
     - INSTALL_PREFIX="$HOME/nvim-install"
     # Log directory for Clang sanitizers and Valgrind.
@@ -25,7 +23,7 @@ env:
                    -DBUSTED_OUTPUT_TYPE=nvim
                    -DDEPS_PREFIX=$DEPS_BUILD_DIR/usr
                    -DMIN_LOG_LEVEL=3"
-    - DEPS_CMAKE_FLAGS="-DDEPS_DOWNLOAD_DIR:PATH=$DEPS_DOWNLOAD_DIR -DUSE_BUNDLED_GPERF=OFF"
+    - DEPS_CMAKE_FLAGS="-DUSE_BUNDLED_GPERF=OFF"
     # Additional CMake flags for 32-bit builds.
     - CMAKE_FLAGS_32BIT="-DCMAKE_SYSTEM_LIBRARY_PATH=/lib32:/usr/lib32:/usr/local/lib32
                          -DCMAKE_IGNORE_PATH=/lib:/usr/lib:/usr/local/lib
@@ -36,8 +34,9 @@ env:
     - UBSAN_OPTIONS="print_stacktrace=1 log_path=$LOG_DIR/ubsan"
     # Environment variables for Valgrind.
     - VALGRIND_LOG="$LOG_DIR/valgrind-%p.log"
+    - CACHE_NVIM_DEPS_DIR="$HOME/.cache/nvim-deps"
     # If this file exists, the cache is valid (compile was successful).
-    - CACHE_MARKER="$HOME/.cache/nvim-deps/.travis_cache_marker"
+    - CACHE_MARKER="$CACHE_NVIM_DEPS_DIR/.travis_cache_marker"
     # default target name for functional tests
     - FUNCTIONALTEST=functionaltest
     - CI_TARGET=tests
@@ -52,7 +51,7 @@ anchors:
   envs: &common-job-env
     # Do not fall back to cache for "master" for PR on "release" branch:
     # adds the target branch to the cache key.
-    FOR_TRAVIS_CACHE=$TRAVIS_BRANCH
+    FOR_TRAVIS_CACHE=v1-$TRAVIS_BRANCH
 
 addons:
   apt:
@@ -165,9 +164,7 @@ cache:
   apt: true
   ccache: true
   directories:
-    - "$HOME/.cache/pip"
-    - "$HOME/.cache/nvim-deps"
-    - "$HOME/.cache/nvim-deps-downloads"
+    - "$CACHE_NVIM_DEPS_DIR"
 
 notifications:
   webhooks:

ci/before_cache.sh

@@ -7,23 +7,20 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/suite.sh"
 
-# Don't cache pip's log and selfcheck.
-rm -rf "${HOME}/.cache/pip/log"
-rm -f "${HOME}/.cache/pip/selfcheck.json"
-
 echo "before_cache.sh: cache size"
-du -d 2 "${HOME}/.cache" | sort -n
+du -chd 1 "${HOME}/.cache" | sort -rh | head -20
 
 echo "before_cache.sh: ccache stats"
 ccache -s 2>/dev/null || true
+# Do not keep ccache stats (uploaded to cache otherwise; reset initially anyway).
+find "${HOME}/.ccache" -name stats -delete
 
 # Update the third-party dependency cache only if the build was successful.
 if ended_successfully; then
-  rm -rf "${HOME}/.cache/nvim-deps"
-  mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps"
-
-  rm -rf "${HOME}/.cache/nvim-deps-downloads"
-  mv "${DEPS_DOWNLOAD_DIR}" "${HOME}/.cache/nvim-deps-downloads"
+  # Do not cache downloads.  They should not be needed with up-to-date deps.
+  rm -rf "${DEPS_BUILD_DIR}/build/downloads"
+  rm -rf "${CACHE_NVIM_DEPS_DIR}"
+  mv "${DEPS_BUILD_DIR}" "${CACHE_NVIM_DEPS_DIR}"
 
   touch "${CACHE_MARKER}"
   echo "Updated third-party dependencies (timestamp: $(_stat "${CACHE_MARKER}"))."

ci/before_script.sh

@@ -20,6 +20,8 @@ if [[ -n "${LLVM_SYMBOLIZER}" ]] && [[ ! $(type -P "${LLVM_SYMBOLIZER}") ]]; the
   exit 1
 fi
 
+echo "before_script.sh: ccache stats (will be cleared)"
+ccache -s
 # Reset ccache stats for real results in before_cache.
 ccache --zero-stats
 

ci/common/build.sh

@@ -27,15 +27,13 @@ build_deps() {
   fi
 
   mkdir -p "${DEPS_BUILD_DIR}"
-  mkdir -p "${DEPS_DOWNLOAD_DIR}"
 
   # Use cached dependencies if $CACHE_MARKER exists.
   if test "${CACHE_ENABLE}" = "false" ; then
     export CCACHE_RECACHE=1
   elif test -f "${CACHE_MARKER}" ; then
     echo "Using third-party dependencies from Travis cache (last update: $(_stat "${CACHE_MARKER}"))."
-    cp -a "${HOME}/.cache/nvim-deps"/. "${DEPS_BUILD_DIR}"
-    cp -a "${HOME}/.cache/nvim-deps-downloads"/. "${DEPS_DOWNLOAD_DIR}"
+    cp -a "${CACHE_NVIM_DEPS_DIR}"/. "${DEPS_BUILD_DIR}"
   fi
 
   # Even if we're using cached dependencies, run CMake and make to