build: move luarocks and rocks installation to main build

This will ensure luacheck and busted are only installed when they're actually needed. This cuts total build time by over 50%. Closes https://github.com/neovim/neovim/issues/22797.
2023-05-21 20:57:39 +02:00 · 2023-05-21 20:57:39 +02:00 · 8b8e607284
parent 384a3bc308
commit 8b8e607284
10 changed files with 91 additions and 125 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -35,6 +35,9 @@ include(InstallHelpers)
 include(PreventInTreeBuilds)
 include(Util)

+set_directory_properties(PROPERTIES
+  EP_PREFIX "${DEPS_BUILD_DIR}")
+
 set(TOUCHES_DIR ${PROJECT_BINARY_DIR}/touches)

 find_program(CCACHE_PRG ccache)
@ -229,18 +232,18 @@ endif()
 #
 # Lint
 #
-find_program(LUACHECK_PRG luacheck)
 find_program(SHELLCHECK_PRG shellcheck)
 find_program(STYLUA_PRG stylua)

 add_glob_target(
  REQUIRED
  TARGET lintlua-luacheck
-  COMMAND ${LUACHECK_PRG}
+  COMMAND ${DEPS_BIN_DIR}/luacheck
  FLAGS -q
  GLOB_DIRS runtime/ scripts/ src/ test/
  GLOB_PAT *.lua
  TOUCH_STRATEGY SINGLE)
+add_dependencies(lintlua-luacheck luacheck)

 add_glob_target(
  TARGET lintlua-stylua
@ -313,7 +316,6 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
 endif()

 ExternalProject_Add(uncrustify
-  PREFIX ${DEPS_BUILD_DIR}
  URL https://github.com/uncrustify/uncrustify/archive/uncrustify-0.77.1.tar.gz
  URL_HASH SHA256=414bbc9f7860eb18a53074f9af14ed04638a633b2216a73f2629291300d37c1b
  DOWNLOAD_NO_PROGRESS TRUE
@ -321,3 +323,5 @@ ExternalProject_Add(uncrustify
  CMAKE_ARGS ${DEPS_CMAKE_ARGS}
  CMAKE_CACHE_ARGS ${DEPS_CMAKE_CACHE_ARGS}
  EXCLUDE_FROM_ALL TRUE)
+
+include(BuildLuarocks)
--- a/cmake.deps/CMakeLists.txt
+++ b/cmake.deps/CMakeLists.txt
@ -38,7 +38,6 @@ option(USE_BUNDLED_LIBVTERM "Use the bundled libvterm." ${USE_BUNDLED})
 option(USE_BUNDLED_LIBUV "Use the bundled libuv." ${USE_BUNDLED})
 option(USE_BUNDLED_MSGPACK "Use the bundled msgpack." ${USE_BUNDLED})
 option(USE_BUNDLED_LUAJIT "Use the bundled version of luajit." ${USE_BUNDLED})
-option(USE_BUNDLED_LUAROCKS "Use the bundled version of luarocks." ${USE_BUNDLED})
 option(USE_BUNDLED_LUV "Use the bundled version of luv." ${USE_BUNDLED})
 option(USE_BUNDLED_LPEG "Use the bundled lpeg." ${USE_BUNDLED})
 #XXX(tarruda): Lua is only used for debugging the functional test client, don't
@ -66,45 +65,6 @@ if(NOT Git_FOUND)
  message(FATAL_ERROR "Git is required to apply patches.")
 endif()

-if(UNIX)
-  find_program(MAKE_PRG NAMES gmake make)
-  if(NOT MAKE_PRG)
-    message(FATAL_ERROR "GNU Make is required to build the dependencies.")
-  else()
-    message(STATUS "Found GNU Make at ${MAKE_PRG}")
-  endif()
-endif()
-
-# When using make, use the $(MAKE) variable to avoid warning about the job
-# server.
-if(CMAKE_GENERATOR MATCHES "Makefiles")
-  set(MAKE_PRG "$(MAKE)")
-endif()
-
-if(MINGW AND CMAKE_GENERATOR MATCHES "Ninja")
-  find_program(MAKE_PRG NAMES mingw32-make)
-  if(NOT MAKE_PRG)
-    message(FATAL_ERROR "GNU Make for mingw32 is required to build the dependencies.")
-  else()
-    message(STATUS "Found GNU Make for mingw32: ${MAKE_PRG}")
-  endif()
-endif()
-
-set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}")
-
-if(CMAKE_OSX_SYSROOT)
-  set(DEPS_C_COMPILER "${DEPS_C_COMPILER} -isysroot${CMAKE_OSX_SYSROOT}")
-endif()
-
-if(CMAKE_OSX_ARCHITECTURES)
-  # The LuaJIT build does not like being passed multiple `-arch` flags
-  # so we handle a universal build the old-fashioned way.
-  set(LUAJIT_C_COMPILER "${DEPS_C_COMPILER}")
-  foreach(ARCH IN LISTS CMAKE_OSX_ARCHITECTURES)
-    set(DEPS_C_COMPILER "${DEPS_C_COMPILER} -arch ${ARCH}")
-  endforeach()
-endif()
-
 # If the macOS deployment target is not set manually (via $MACOSX_DEPLOYMENT_TARGET),
 # fall back to local system version. Needs to be done here and in top-level CMakeLists.txt.
 if(APPLE)
@ -163,10 +123,6 @@ if(USE_BUNDLED_LUA)
  include(BuildLua)
 endif()

-if(USE_BUNDLED_LUAROCKS)
-  include(BuildLuarocks)
-endif()
-
 if(USE_BUNDLED_LUV)
  include(BuildLuv)
 endif()
--- a/cmake.deps/cmake/BuildLua.cmake
+++ b/cmake.deps/cmake/BuildLua.cmake
@ -51,12 +51,3 @@ ExternalProject_Add(lua
  BUILD_IN_SOURCE 1
  BUILD_COMMAND ${MAKE_PRG} ${LUA_INSTALL_TOP_ARG} ${LUA_TARGET}
  INSTALL_COMMAND ${MAKE_PRG} ${LUA_INSTALL_TOP_ARG} install)
-
-set(BUSTED ${DEPS_BIN_DIR}/busted)
-set(BUSTED_LUA ${BUSTED}-lua)
-
-add_custom_command(OUTPUT ${BUSTED_LUA}
-  COMMAND sed -e 's/^exec/exec $$LUA_DEBUGGER/' -e 's/jit//g' < ${BUSTED} > ${BUSTED_LUA} && chmod +x ${BUSTED_LUA}
-  DEPENDS lua busted ${BUSTED})
-add_custom_target(busted-lua ALL
-  DEPENDS ${DEPS_BIN_DIR}/busted-lua)
--- a/cmake.deps/cmake/BuildLuajit.cmake
+++ b/cmake.deps/cmake/BuildLuajit.cmake
@ -66,6 +66,11 @@ if((UNIX AND NOT APPLE) OR (APPLE AND NOT CMAKE_OSX_ARCHITECTURES))

 elseif(CMAKE_OSX_ARCHITECTURES AND APPLE)

+  set(LUAJIT_C_COMPILER "${CMAKE_C_COMPILER}")
+  if(CMAKE_OSX_SYSROOT)
+    set(LUAJIT_C_COMPILER "${LUAJIT_C_COMPILER} -isysroot${CMAKE_OSX_SYSROOT}")
+  endif()
+
  # Passing multiple `-arch` flags to the LuaJIT build will cause it to fail.
  # To get a working universal build, we build each requested architecture slice
  # individually then `lipo` them all up.
--- a/cmake.deps/deps.txt
+++ b/cmake.deps/deps.txt
@ -10,9 +10,6 @@ LUAJIT_SHA256 a9bcd9e646e2b188e1d7e3fb594e04c61dda3b332dfd0378d41be19c1eae9d09
 LUA_URL https://www.lua.org/ftp/lua-5.1.5.tar.gz
 LUA_SHA256 2640fc56a795f29d28ef15e13c34a47e223960b0240e8cb0a82d9b0738695333

-LUAROCKS_URL https://github.com/luarocks/luarocks/archive/v3.9.2.tar.gz
-LUAROCKS_SHA256 a0b36cd68586cd79966d0106bb2e5a4f5523327867995fd66bee4237062b3e3b
-
 UNIBILIUM_URL https://github.com/neovim/unibilium/archive/d72c3598e7ac5d1ebf86ee268b8b4ed95c0fa628.tar.gz
 UNIBILIUM_SHA256 9c4747c862ab5e3076dcf8fa8f0ea7a6b50f20ec5905618b9536655596797487

--- a/cmake.deps/cmake/BuildLuarocks.cmake
+++ b/cmake.deps/cmake/BuildLuarocks.cmake
@ -13,66 +13,41 @@ if(NOT MSVC)
  set(LUAROCKS_BUILDARGS CC=${DEPS_C_COMPILER} LD=${DEPS_C_COMPILER})
 endif()

-# Lua version, used with rocks directories.
-# Defaults to 5.1 for bundled LuaJIT/Lua.
-set(LUA_VERSION "5.1")
-
 if(UNIX)
-
-  if(USE_BUNDLED_LUAJIT)
+  if(PREFER_LUA)
+    find_package(Lua 5.1 EXACT REQUIRED)
+    get_filename_component(LUA_ROOT ${LUA_INCLUDE_DIR} DIRECTORY)
    list(APPEND LUAROCKS_OPTS
-      --with-lua=${DEPS_INSTALL_DIR}
-      --with-lua-include=${DEPS_INSTALL_DIR}/include/luajit-2.1
-      --with-lua-interpreter=luajit)
-  elseif(USE_BUNDLED_LUA)
-    list(APPEND LUAROCKS_OPTS
-      --with-lua=${DEPS_INSTALL_DIR})
+      --with-lua=${LUA_ROOT})
  else()
-    find_package(Luajit)
-    if(LUAJIT_FOUND)
-      list(APPEND LUAROCKS_OPTS
-        --with-lua-include=${LUAJIT_INCLUDE_DIR}
-        --with-lua-interpreter=luajit)
-    endif()
-
-    # Get LUA_VERSION used with rocks output.
-    if(LUAJIT_FOUND)
-      set(LUA_EXE "luajit")
-    else()
-      set(LUA_EXE "lua")
-    endif()
-    execute_process(
-      COMMAND ${LUA_EXE} -e "print(string.sub(_VERSION, 5))"
-      OUTPUT_VARIABLE LUA_VERSION
-      ERROR_VARIABLE ERR
-      RESULT_VARIABLE RES)
-    if(RES)
-      message(FATAL_ERROR "Could not get LUA_VERSION with ${LUA_EXE}: ${ERR}")
-    endif()
+    find_package(Luajit REQUIRED)
+    get_filename_component(LUA_ROOT ${LUAJIT_INCLUDE_DIR} DIRECTORY)
+    get_filename_component(LUA_ROOT ${LUA_ROOT} DIRECTORY)
+    list(APPEND LUAROCKS_OPTS
+      --with-lua=${LUA_ROOT}
+      --with-lua-include=${LUAJIT_INCLUDE_DIR}
+      --with-lua-interpreter=luajit)
  endif()

  set(LUAROCKS_CONFIGURE_COMMAND ${DEPS_BUILD_DIR}/src/luarocks/configure
      --prefix=${DEPS_INSTALL_DIR} --force-config ${LUAROCKS_OPTS})
  set(LUAROCKS_INSTALL_COMMAND ${MAKE_PRG} -j1 bootstrap)
 elseif(MSVC OR MINGW)
-
  if(MINGW)
    set(COMPILER_FLAG /MW)
  elseif(MSVC)
    set(COMPILER_FLAG /MSVC)
  endif()

-  # Ignore USE_BUNDLED_LUAJIT - always ON for native Win32
+  find_package(Luajit REQUIRED)
+  # Always assume bundled luajit for native Win32
  set(LUAROCKS_INSTALL_COMMAND install.bat /FORCECONFIG /NOREG /NOADMIN /Q /F
-    /LUA ${DEPS_INSTALL_DIR}
-    /LIB ${DEPS_LIB_DIR}
-    /BIN ${DEPS_BIN_DIR}
-    /INC ${DEPS_INSTALL_DIR}/include/luajit-2.1
-    /P ${DEPS_INSTALL_DIR}/luarocks /TREE ${DEPS_INSTALL_DIR}
+    /LUA ${DEPS_PREFIX}
+    /INC ${LUAJIT_INCLUDE_DIR}
+    /P ${DEPS_INSTALL_DIR}/luarocks
+    /TREE ${DEPS_INSTALL_DIR}
    /SCRIPTS ${DEPS_BIN_DIR}
-    /CMOD ${DEPS_BIN_DIR}
-    ${COMPILER_FLAG}
-    /LUAMOD ${DEPS_BIN_DIR}/lua)
+    ${COMPILER_FLAG})

  set(LUAROCKS_BINARY ${DEPS_INSTALL_DIR}/luarocks/luarocks.bat)
 else()
@ -80,21 +55,17 @@ else()
 endif()

 ExternalProject_Add(luarocks
-  URL ${LUAROCKS_URL}
-  URL_HASH SHA256=${LUAROCKS_SHA256}
+  URL https://github.com/luarocks/luarocks/archive/v3.9.2.tar.gz
+  URL_HASH SHA256=a0b36cd68586cd79966d0106bb2e5a4f5523327867995fd66bee4237062b3e3b
  DOWNLOAD_NO_PROGRESS TRUE
  DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/luarocks
  BUILD_IN_SOURCE 1
  CONFIGURE_COMMAND "${LUAROCKS_CONFIGURE_COMMAND}"
  BUILD_COMMAND ""
-  INSTALL_COMMAND "${LUAROCKS_INSTALL_COMMAND}")
+  INSTALL_COMMAND "${LUAROCKS_INSTALL_COMMAND}"
+  EXCLUDE_FROM_ALL TRUE)

-if(USE_BUNDLED_LUAJIT)
-  add_dependencies(luarocks luajit)
-elseif(USE_BUNDLED_LUA)
-  add_dependencies(luarocks lua)
-endif()
-set(ROCKS_DIR ${DEPS_LIB_DIR}/luarocks/rocks-${LUA_VERSION})
+set(ROCKS_DIR ${DEPS_LIB_DIR}/luarocks/rocks)

 if(MSVC)
  # Workaround for luarocks failing to find the md5sum.exe it is shipped with.
@ -113,7 +84,7 @@ function(Download ROCK VER)
  add_custom_command(OUTPUT ${OUTPUT}
    COMMAND ${CMAKE_COMMAND} -E env "${PATH}" ${LUAROCKS_BINARY} build ${ROCK} ${VER} ${LUAROCKS_BUILDARGS}
    DEPENDS ${CURRENT_DEP})
-  add_custom_target(${ROCK} ALL DEPENDS ${OUTPUT})
+  add_custom_target(${ROCK} DEPENDS ${OUTPUT})
  set(CURRENT_DEP ${ROCK} PARENT_SCOPE)
 endfunction()

@ -125,9 +96,14 @@ else()
  set(LUACHECK_EXE "${DEPS_BIN_DIR}/luacheck")
 endif()

-Download(busted 2.1.1 ${BUSTED_EXE})
+add_custom_target(test_deps)
+
 Download(luacheck 1.1.0-1 ${LUACHECK_EXE})

-if (USE_BUNDLED_LUA OR NOT USE_BUNDLED_LUAJIT)
+Download(busted 2.1.1 ${BUSTED_EXE})
+add_dependencies(test_deps busted)
+
+if(PREFER_LUA)
  Download(coxpcall 1.17.0-1)
+  add_dependencies(test_deps coxpcall)
 endif()
--- a/cmake/Deps.cmake
+++ b/cmake/Deps.cmake
@ -18,3 +18,37 @@ if(APPLE)
 endif()

 set(DEPS_CMAKE_CACHE_ARGS -DCMAKE_OSX_ARCHITECTURES:STRING=${CMAKE_OSX_ARCHITECTURES})
+
+# MAKE_PRG
+if(UNIX)
+  find_program(MAKE_PRG NAMES gmake make)
+  if(NOT MAKE_PRG)
+    message(FATAL_ERROR "GNU Make is required to build the dependencies.")
+  else()
+    message(STATUS "Found GNU Make at ${MAKE_PRG}")
+  endif()
+endif()
+# When using make, use the $(MAKE) variable to avoid warning about the job
+# server.
+if(CMAKE_GENERATOR MATCHES "Makefiles")
+  set(MAKE_PRG "$(MAKE)")
+endif()
+if(MINGW AND CMAKE_GENERATOR MATCHES "Ninja")
+  find_program(MAKE_PRG NAMES mingw32-make)
+  if(NOT MAKE_PRG)
+    message(FATAL_ERROR "GNU Make for mingw32 is required to build the dependencies.")
+  else()
+    message(STATUS "Found GNU Make for mingw32: ${MAKE_PRG}")
+  endif()
+endif()
+
+# DEPS_C_COMPILER
+set(DEPS_C_COMPILER "${CMAKE_C_COMPILER}")
+if(CMAKE_OSX_SYSROOT)
+  set(DEPS_C_COMPILER "${DEPS_C_COMPILER} -isysroot${CMAKE_OSX_SYSROOT}")
+endif()
+if(CMAKE_OSX_ARCHITECTURES)
+  foreach(ARCH IN LISTS CMAKE_OSX_ARCHITECTURES)
+    set(DEPS_C_COMPILER "${DEPS_C_COMPILER} -arch ${ARCH}")
+  endforeach()
+endif()
--- a/cmake/RunTests.cmake
+++ b/cmake/RunTests.cmake
@ -63,7 +63,7 @@ if(NOT DEFINED ENV{TEST_TIMEOUT} OR "$ENV{TEST_TIMEOUT}" STREQUAL "")
 endif()

 set(ENV{SYSTEM_NAME} ${CMAKE_HOST_SYSTEM_NAME})  # used by test/helpers.lua.
-set(ENV{DEPS_PREFIX} ${DEPS_PREFIX})  # used by test/busted_runner.lua on windows
+set(ENV{DEPS_INSTALL_DIR} ${DEPS_INSTALL_DIR})  # used by test/busted_runner.lua

 execute_process(
  COMMAND ${NVIM_PRG} -ll ${WORKING_DIR}/test/busted_runner.lua -v -o test.busted.outputHandlers.${BUSTED_OUTPUT_TYPE}
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@ -20,12 +20,14 @@ if(LUA_HAS_FFI)
      -D BUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
      -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
      -D BUILD_DIR=${CMAKE_BINARY_DIR}
+      -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
      -D TEST_TYPE=unit
      -D CIRRUS_CI=$ENV{CIRRUS_CI}
      -D CI_BUILD=${CI_BUILD}
      -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake
    DEPENDS ${UNITTEST_PREREQS}
    USES_TERMINAL)
+  add_dependencies(unittest test_deps)
 else()
  message(WARNING "disabling unit tests: no Luajit FFI in ${LUA_PRG}")
 endif()
@ -41,7 +43,7 @@ add_custom_target(functionaltest
    -D BUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
    -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
    -D BUILD_DIR=${CMAKE_BINARY_DIR}
-    -D DEPS_PREFIX=${DEPS_PREFIX}
+    -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
    -D TEST_TYPE=functional
    -D CIRRUS_CI=$ENV{CIRRUS_CI}
    -D CI_BUILD=${CI_BUILD}
@ -56,6 +58,7 @@ add_custom_target(benchmark
    -D BUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
    -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
    -D BUILD_DIR=${CMAKE_BINARY_DIR}
+    -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
    -D TEST_TYPE=benchmark
    -D CIRRUS_CI=$ENV{CIRRUS_CI}
    -D CI_BUILD=${CI_BUILD}
@ -70,9 +73,14 @@ add_custom_target(functionaltest-lua
    -D BUSTED_OUTPUT_TYPE=${BUSTED_OUTPUT_TYPE}
    -D TEST_DIR=${CMAKE_CURRENT_SOURCE_DIR}
    -D BUILD_DIR=${CMAKE_BINARY_DIR}
+    -D DEPS_INSTALL_DIR=${DEPS_INSTALL_DIR}
    -D TEST_TYPE=functional
    -D CIRRUS_CI=$ENV{CIRRUS_CI}
    -D CI_BUILD=${CI_BUILD}
    -P ${PROJECT_SOURCE_DIR}/cmake/RunTests.cmake
  DEPENDS ${FUNCTIONALTEST_PREREQS}
  USES_TERMINAL)
+
+add_dependencies(functionaltest test_deps)
+add_dependencies(benchmark test_deps)
+add_dependencies(functionaltest-lua test_deps)
--- a/test/busted_runner.lua
+++ b/test/busted_runner.lua
@ -1,12 +1,7 @@
 local platform = vim.loop.os_uname()
-if platform and platform.sysname:lower():find'windows' then
-  local deps_prefix = os.getenv 'DEPS_PREFIX'
-  if deps_prefix ~= nil and deps_prefix ~= "" then
-    package.path = deps_prefix.."/share/lua/5.1/?.lua;"..deps_prefix.."/share/lua/5.1/?/init.lua;"..package.path
-    package.path = deps_prefix.."/bin/lua/?.lua;"..deps_prefix.."/bin/lua/?/init.lua;"..package.path
-    package.cpath = deps_prefix.."/lib/lua/5.1/?.dll;"..package.cpath;
-    package.cpath = deps_prefix.."/bin/?.dll;"..deps_prefix.."/bin/loadall.dll;"..package.cpath;
-  end
-end
+local deps_install_dir = os.getenv 'DEPS_INSTALL_DIR'
+local suffix = (platform and platform.sysname:lower():find'windows') and '.dll' or '.so'
+package.path = deps_install_dir.."/share/lua/5.1/?.lua;"..deps_install_dir.."/share/lua/5.1/?/init.lua;"..package.path
+package.cpath = deps_install_dir.."/lib/lua/5.1/?"..suffix..";"..package.cpath;

 require 'busted.runner'({ standalone = false })