ci: Really continue tests on failure, print global summary

.travis.yml

@@ -43,11 +43,6 @@ env:
     # If this file exists, we know that the cache contains compiled
     # dependencies and we can use it.
     - CACHE_MARKER="$HOME/.cache/nvim-deps/.travis_cache_marker"
-    # Test success marker. If this file exists, we know that all tests
-    # were successful. Required because we only want to update the cache
-    # if the tests were successful, but don't have this information
-    # available in before_cache (which is run before after_success).
-    - SUCCESS_MARKER="$BUILD_DIR/.tests_successful"
     # default target name for functional tests
     - FUNCTIONALTEST=functionaltest
     - CI_TARGET=tests

ci/before_cache.sh

@@ -3,12 +3,15 @@
 set -e
 set -o pipefail
 
+CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${CI_DIR}/common/suite.sh"
+
 # Don't cache pip's log and selfcheck.
 rm -rf "${HOME}/.cache/pip/log"
 rm -f "${HOME}/.cache/pip/selfcheck.json"
 
 # Update the third-party dependency cache only if the build was successful.
-if [[ -f "${SUCCESS_MARKER}" ]]; then
+if ended_successfully; then
   rm -rf "${HOME}/.cache/nvim-deps"
   mv "${DEPS_BUILD_DIR}" "${HOME}/.cache/nvim-deps"
   touch "${CACHE_MARKER}"

ci/common/suite.sh

@@ -2,11 +2,18 @@
 NL="$(printf '\nE')"
 NL="${NL%E}"
 
-FAILED=0
-
 FAIL_SUMMARY=""
 
+# Test success marker. If END_MARKER file exists, we know that all tests 
+# finished. If FAIL_SUMMARY_FILE exists we know that some tests failed, this 
+# file will contain information about failed tests. Build is considered 
+# successful if tests ended without any of them failing.
+END_MARKER="$BUILD_DIR/.tests_finished"
+FAIL_SUMMARY_FILE="$BUILD_DIR/.test_errors"
+
 enter_suite() {
+  FAILED=0
+  rm -f "${END_MARKER}"
   local suite_name="$1"
   export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name"
 }
@@ -19,17 +26,16 @@ exit_suite() {
   export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}"
   if test "x$1" != "x--continue" ; then
     exit $FAILED
+  else
+    local saved_failed=$FAILED
+    FAILED=0
+    return $saved_failed
   fi
 }
 
 fail() {
-  local allow_failure=
-  if test "x$1" = "x--allow-failure" ; then
-    shift
-    allow_failure=A
-  fi
   local test_name="$1"
-  local fail_char="$allow_failure$2"
+  local fail_char="$2"
   local message="$3"
 
   : ${fail_char:=F}
@@ -37,10 +43,9 @@ fail() {
 
   local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message"
   FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}"
+  echo "${full_msg}" >> "${FAIL_SUMMARY_FILE}"
   echo "Failed: $full_msg"
-  if test "x$allow_failure" = "x" ; then
-    FAILED=1
-  fi
+  FAILED=1
 }
 
 run_test() {
@@ -77,14 +82,13 @@ run_test_wd() {
   while test $restarts -gt 0 ; do
     : > "${status_file}"
     (
-      FAILED=0
-      if ! (
-        set -o pipefail
-        eval "$cmd" 2>&1 | tee -a "$output_file"
-      ) ; then
-        fail "${test_name}" "$@"
+      set -o pipefail
+      ret=0
+      if ! eval "$cmd" 2>&1 | tee -a "$output_file" ; then
+        ret=1
       fi
-      echo "$FAILED" > "$status_file"
+      echo "$ret" > "$status_file"
+      exit $ret
     ) &
     local pid=$!
     while test "$(stat -c "%s" "$status_file")" -eq 0 ; do
@@ -116,6 +120,20 @@ run_test_wd() {
   done
 }
 
-succeeded() {
-  return $FAILED
+ended_successfully() {
+  if [[ -f "${FAIL_SUMMARY_FILE}" ]]; then
+    echo 'Test failed, complete summary:'
+    cat "${FAIL_SUMMARY_FILE}"
+    return 1
+  fi
+  if ! [[ -f "${END_MARKER}" ]] ; then
+    echo 'ended_successfully called before end marker was touched'
+    return 1
+  fi
+  return 0
+}
+
+end_tests() {
+  touch "${END_MARKER}"
+  ended_successfully
 }

ci/common/test.sh

@@ -1,4 +1,5 @@
 source "${CI_DIR}/common/build.sh"
+source "${CI_DIR}/common/suite.sh"
 
 print_core() {
   local app="$1"
@@ -40,10 +41,9 @@ check_core_dumps() {
       print_core "$app" "$core"
     fi
   done
-  if test "$app" = quiet ; then
-    return 0
+  if test "$app" != quiet ; then
+    fail 'cores' E 'Core dumps found'
   fi
-  exit 1
 }
 
 check_logs() {
@@ -62,8 +62,7 @@ check_logs() {
     err=1
   done
   if [[ -n "${err}" ]]; then
-    echo "Runtime errors detected."
-    exit 1
+    fail 'logs' E 'Runtime errors detected.'
   fi
 }
 
@@ -75,50 +74,53 @@ asan_check() {
   check_logs "${1}" "*san.*"
 }
 
-run_unittests() {
+run_unittests() {(
+  enter_suite unittests
   ulimit -c unlimited
   if ! build_make unittest ; then
-    check_core_dumps "$(which luajit)"
-    exit 1
+    fail 'unittests' F 'Unit tests failed'
   fi
   check_core_dumps "$(which luajit)"
-}
+  exit_suite
+)}
 
-run_functionaltests() {
+run_functionaltests() {(
+  enter_suite functionaltests
   ulimit -c unlimited
   if ! build_make ${FUNCTIONALTEST}; then
-    asan_check "${LOG_DIR}"
-    valgrind_check "${LOG_DIR}"
-    check_core_dumps
-    exit 1
+    fail 'functionaltests' F 'Functional tests failed'
   fi
   asan_check "${LOG_DIR}"
   valgrind_check "${LOG_DIR}"
   check_core_dumps
-}
+  exit_suite
+)}
 
-run_oldtests() {
+run_oldtests() {(
+  enter_suite oldtests
   ulimit -c unlimited
   if ! make -C "${TRAVIS_BUILD_DIR}/src/nvim/testdir"; then
     reset
-    asan_check "${LOG_DIR}"
-    valgrind_check "${LOG_DIR}"
-    check_core_dumps
-    exit 1
+    fail 'oldtests' F 'Legacy tests failed'
   fi
   asan_check "${LOG_DIR}"
   valgrind_check "${LOG_DIR}"
   check_core_dumps
-}
+  exit_suite
+)}
 
-install_nvim() {
-  build_make install
+install_nvim() {(
+  enter_suite 'install_nvim'
+  if ! build_make install ; then
+    fail 'install' E 'make install failed'
+    exit_suite
+  fi
 
   "${INSTALL_PREFIX}/bin/nvim" --version
   "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' || {
     echo "Running ':help' in the installed nvim failed."
     echo "Maybe the helptags have not been generated properly."
-    exit 1
+    fail 'help' F 'Failed running :help'
   }
 
   local genvimsynf=syntax/vim/generated.vim
@@ -127,24 +129,22 @@ install_nvim() {
     cd runtime ; git ls-files | grep -e '.vim$' -e '.ps$' -e '.dict$' -e '.py$' -e '.tutor$'
   ) ; do
     if ! test -e "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
-      echo "It appears that $file is not installed."
-      exit 1
+      fail 'runtime-install' F "It appears that $file is not installed."
     fi
   done
 
   # Check that generated syntax file has function names, #5060.
   local gpat='syn keyword vimFuncName .*eval'
   if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf"; then
-    echo "It appears that $genvimsynf does not contain $gpat."
-    exit 1
+    fail 'funcnames' F "It appears that $genvimsynf does not contain $gpat."
   fi
 
   for file in $(
     cd runtime ; git ls-files | grep -e '.awk$' -e '.sh$' -e '.bat$'
   ) ; do
     if ! test -x "${INSTALL_PREFIX}/share/nvim/runtime/$file" ; then
-      echo "It appears that $file is not installed or is not executable."
-      exit 1
+      fail 'not-exe' F "It appears that $file is not installed or is not executable."
     fi
   done
-}
+  exit_suite
+)}

ci/run_lint.sh

@@ -25,4 +25,4 @@ CLICOLOR_FORCE=1 run_test_wd \
   'csi_clean' \
   single-includes
 
-exit_suite
+end_tests

ci/run_tests.sh

@@ -27,8 +27,4 @@ run_test run_oldtests
 
 run_test install_nvim
 
-if succeeded ; then
-  touch "${SUCCESS_MARKER}"
-fi
-
-exit_suite
+end_tests