Skip Go code in CI if it hasn’t changed. (#17077)

* Skip building Go components for Docker CI if they have not changed. * Properly handle Go code in general checks PR. * Skip Go code in build checks if it hasn’t changed. * Fix linting issues. * Fix propagation of installer flags. * Fix propagation of environment variables through static build process. * Fix handling of extra install options in static builds. * Skip starting the agent in updater checks. * Fix actionlint warning.
2024-04-10 09:38:44 -04:00 · 2024-04-10 09:38:44 -04:00 · 5ce422daf0
parent 0f5b137471
commit 5ce422daf0
10 changed files with 138 additions and 76 deletions
--- a/.github/scripts/build-static.sh
+++ b/.github/scripts/build-static.sh
@ -22,7 +22,7 @@ prepare_build() {
 build_static() {
  progress "Building static ${BUILDARCH}"
  (
-    USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}"
+    EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}" USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}"
  ) >&2
 }

--- a/.github/scripts/run-updater-check.sh
+++ b/.github/scripts/run-updater-check.sh
@ -4,7 +4,7 @@ echo ">>> Installing CI support packages..."
 /netdata/.github/scripts/ci-support-pkgs.sh
 mkdir -p /etc/cron.daily # Needed to make auto-update checking work correctly on some platforms.
 echo ">>> Installing Netdata..."
-/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1
+/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --dont-start-it --disable-telemetry "${EXTRA_INSTALL_FLAGS:+--local-build-options "${EXTRA_INSTALL_FLAGS}"}" || exit 1
 echo "::group::>>> Pre-Update Environment File Contents"
 cat /etc/netdata/.environment
 echo "::endgroup::"
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -25,6 +25,7 @@ jobs:
    runs-on: ubuntu-latest
    outputs:
      run: ${{ steps.check-run.outputs.run }}
+      skip-go: ${{ steps.check-go.outputs.skip-go }}
    steps:
      - name: Checkout
        id: checkout
@ -32,8 +33,8 @@ jobs:
        with:
          fetch-depth: 0
          submodules: recursive
-      - name: Check files
-        id: check-files
+      - name: Check source files
+        id: check-source-files
        uses: tj-actions/changed-files@v44
        with:
          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
@ -44,6 +45,19 @@ jobs:
            **/*.hh
            **/*.in
            **/*.patch
+            src/aclk/aclk-schemas/
+            src/ml/dlib/
+            src/fluent-bit/
+            src/web/server/h2o/libh2o/
+          files_ignore: |
+            netdata.spec.in
+            **/*.md
+      - name: Check build files
+        id: check-build-files
+        uses: tj-actions/changed-files@v43
+        with:
+          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
+          files: |
            **/*.cmake
            CMakeLists.txt
            netdata-installer.sh
@ -59,29 +73,39 @@ jobs:
            packaging/*.sh
            packaging/*.version
            packaging/*.checksums
-            src/aclk/aclk-schemas/
-            src/ml/dlib/
-            src/fluent-bit/
-            src/web/server/h2o/libh2o/
          files_ignore: |
-            netdata.spec.in
            **/*.md
      - name: List all changed files in pattern
        continue-on-error: true
        env:
-          ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
+          CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
+          CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
        run: |
-          for file in ${ALL_CHANGED_FILES}; do
+          for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
            echo "$file was changed"
          done
      - name: Check Run
        id: check-run
        run: |
-          if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            echo 'run=true' >> "${GITHUB_OUTPUT}"
          else
            echo 'run=false' >> "${GITHUB_OUTPUT}"
          fi
+      - name: Check Go
+        id: check-go
+        env:
+          OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
+        run: |
+          if [ '${{ github.event_name }}' == 'pull_request' ]; then
+            if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
+              echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+            else
+              echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
+            fi
+          else
+            echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+          fi

  build-dist: # Build the distribution tarball and store it as an artifact.
    name: Build Distribution Tarball
@ -204,7 +228,9 @@ jobs:
          key: ${{ steps.cache-key.outputs.key }}
      - name: Build
        if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs.
-        run: .github/scripts/build-static.sh ${{ matrix.arch }}
+        run: |
+          export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
+          .github/scripts/build-static.sh ${{ matrix.arch }}
      - name: Build
        if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true'
        id: build
@ -212,7 +238,9 @@ jobs:
        with:
          timeout_minutes: 180
          max_attempts: 3
-          command: .github/scripts/build-static.sh ${{ matrix.arch }}
+          command: |
+            export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
+            .github/scripts/build-static.sh ${{ matrix.arch }}
      - name: Store
        id: store
        if: needs.file-check.outputs.run == 'true'
@ -432,19 +460,19 @@ jobs:
        if: needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
-              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build'
+              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
      - name: netdata-installer on ${{ matrix.distro }}, require cloud
        id: build-cloud
        if: needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
-              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build'
+              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
      - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
        id: build-no-jsonc
        if: matrix.jsonc_removal != '' && needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
-              /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build'
+              /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
@ -545,8 +573,9 @@ jobs:
        id: updater-check
        if: needs.file-check.outputs.run == 'true'
        run: |
-          docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata test:${{ matrix.artifact_key }} \
-                /netdata/.github/scripts/run-updater-check.sh
+          docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata \
+                 -e EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} \
+                 test:${{ matrix.artifact_key }} /netdata/.github/scripts/run-updater-check.sh
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@ -16,6 +16,7 @@ jobs:
    runs-on: ubuntu-latest
    outputs:
      run: ${{ steps.check-run.outputs.run }}
+      skip-go: ${{ steps.check-go.outputs.skip-go }}
    steps:
      - name: Checkout
        id: checkout
@ -23,8 +24,8 @@ jobs:
        with:
          fetch-depth: 0
          submodules: recursive
-      - name: Check files
-        id: check-files
+      - name: Check source files
+        id: check-source-files
        uses: tj-actions/changed-files@v44
        with:
          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
@ -35,18 +36,6 @@ jobs:
            **/*.hh
            **/*.in
            **/*.patch
-            **/*.cmake
-            CMakeLists.txt
-            .gitignore
-            .github/data/distros.yml
-            .github/workflows/build.yml
-            .github/scripts/build-static.sh
-            .github/scripts/get-static-cache-key.sh
-            .github/scripts/gen-matrix-build.py
-            .github/scripts/run-updater-check.sh
-            packaging/cmake/
-            packaging/*.version
-            packaging/*.checksums
            src/aclk/aclk-schemas/
            src/ml/dlib/
            src/fluent-bit/
@ -54,22 +43,53 @@ jobs:
          files_ignore: |
            netdata.spec.in
            **/*.md
+      - name: Check build files
+        id: check-build-files
+        uses: tj-actions/changed-files@v43
+        with:
+          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
+          files: |
+            **/*.cmake
+            CMakeLists.txt
+            .gitignore
+            .github/data/distros.yml
+            .github/workflows/build.yml
+            packaging/cmake/
+            packaging/*.version
+            packaging/*.checksums
+          files_ignore: |
+            **/*.md
      - name: List all changed files in pattern
        continue-on-error: true
        env:
-          ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
+          CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
+          CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
        run: |
-          for file in ${ALL_CHANGED_FILES}; do
+          for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
            echo "$file was changed"
          done
      - name: Check Run
        id: check-run
        run: |
-          if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            echo 'run=true' >> "${GITHUB_OUTPUT}"
          else
            echo 'run=false' >> "${GITHUB_OUTPUT}"
          fi
+      - name: Check Go
+        id: check-go
+        env:
+          OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
+        run: |
+          if [ '${{ github.event_name }}' == 'pull_request' ]; then
+            if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
+              echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+            else
+              echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
+            fi
+          else
+            echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+          fi

  libressl-checks:
    name: LibreSSL
@ -94,7 +114,8 @@ jobs:
          ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata;
          apk del openssl openssl-dev;
          apk add libressl libressl-dev protobuf-dev;
-          ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build;'
+          ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build --disable-go;'
+
  clang-checks:
    name: Clang
    needs:
@ -134,7 +155,7 @@ jobs:
        run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
      - name: Build netdata
        if: needs.file-check.outputs.run == 'true'
-        run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build
+        run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build ${{ needs.file-check.outputs.skip-go }}
      - name: Check that repo is clean
        if: needs.file-check.outputs.run == 'true'
        run: |
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@ -31,6 +31,7 @@ jobs:
    runs-on: ubuntu-latest
    outputs:
      run: ${{ steps.check-run.outputs.run }}
+      skip-go: ${{ steps.check-go.outputs.skip-go }}
    steps:
      - name: Checkout
        id: checkout
@ -39,8 +40,8 @@ jobs:
        with:
          fetch-depth: 0
          submodules: recursive
-      - name: Check files
-        id: check-files
+      - name: Check source files
+        id: check-source-files
        if: github.event_name != 'workflow_dispatch'
        uses: tj-actions/changed-files@v44
        with:
@ -52,7 +53,20 @@ jobs:
            **/*.hh
            **/*.in
            **/*.patch
-            **/*.cmake
+            src/aclk/aclk-schemas/
+            src/ml/dlib/
+            src/fluent-bit/
+            src/web/server/h2o/libh2o/
+          files_ignore: |
+            netdata.spec.in
+            **/*.md
+      - name: Check build system files
+        id: check-build-files
+        if: github.event_name != 'workflow_dispatch'
+        uses: tj-actions/changed-files@v42
+        with:
+          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
+          files: |
            .dockerignore
            CMakeLists.txt
            netdata-installer.sh
@ -66,30 +80,40 @@ jobs:
            packaging/runtime-check.sh
            packaging/*.version
            packaging/*.checksums
-            src/aclk/aclk-schemas/
-            src/ml/dlib/
-            src/fluent-bit/
-            src/web/server/h2o/libh2o/
          files_ignore: |
-            netdata.spec.in
            **/*.md
      - name: List all changed files in pattern
        continue-on-error: true
        if: github.event_name != 'workflow_dispatch'
        env:
-          ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }}
+          CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
+          CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
        run: |
-          for file in ${ALL_CHANGED_FILES}; do
+          for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
            echo "$file was changed"
          done
      - name: Check Run
        id: check-run
        run: |
-          if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            echo 'run=true' >> "${GITHUB_OUTPUT}"
          else
            echo 'run=false' >> "${GITHUB_OUTPUT}"
          fi
+      - name: Check Go
+        id: check-go
+        env:
+          OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
+        run: |
+          if [ '${{ github.event_name }}' == 'pull_request' ]; then
+            if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
+              echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+            else
+              echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
+            fi
+          else
+            echo 'skip-go=' >> "${GITHUB_OUTPUT}"
+          fi

  build-images:
    name: Build Docker Images
@ -143,7 +167,9 @@ jobs:
          tags: netdata/netdata:test
          load: true
          cache-to: type=local,dest=/tmp/build-cache,mode=max
-          build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
+          build-args: |
+            OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
+            EXTRA_INSTALL_OPTS=${{ needs.file-check.outputs.skip-go }}
      - name: Test Image
        id: test
        if: needs.file-check.outputs.run == 'true' && matrix.platform == 'linux/amd64'
@ -257,24 +283,8 @@ jobs:
        with:
          platforms: ${{ matrix.platform }}
          cache-from: type=local,src=/tmp/build-cache
-          build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
          outputs: type=image,name=netdata/netdata,push-by-digest=true,name-canonical=true,push=true
-      - name: Export Digest
-        id: export-digest
-        if: github.repository == 'netdata/netdata'
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      - name: Upload digest
-        id: upload-digest
-        if: github.repository == 'netdata/netdata'
-        uses: actions/upload-artifact@v4
-        with:
-          name: docker-digests-${{ steps.artifact-name.outputs.platform }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
+          build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
--- a/netdata-installer.sh
+++ b/netdata-installer.sh
@ -211,7 +211,7 @@ USAGE: ${PROGRAM} [options]
  --disable-dbengine         Explicitly disable DB engine support.
  --enable-plugin-go         Enable the Go plugin. Default: Enabled when possible.
  --disable-plugin-go        Disable the Go plugin.
-  --disable-go               Equivalent to --disable-go-plugin
+  --disable-go               Disable all Go components.
  --enable-plugin-nfacct     Enable nfacct plugin. Default: enable it when libmnl and libnetfilter_acct are available.
  --disable-plugin-nfacct    Explicitly disable the nfacct plugin.
  --enable-plugin-xenstat    Enable the xenstat plugin. Default: enable it when libxenstat and libyajl are available.
--- a/packaging/makeself/build-static.sh
+++ b/packaging/makeself/build-static.sh
@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash

 # SPDX-License-Identifier: GPL-3.0-or-later

@ -54,10 +54,11 @@ fi
 # Run the build script inside the container
 if [ -t 1 ]; then
  run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
-    --platform "${platform}" "${DOCKER_IMAGE_NAME}" \
-    /bin/sh /netdata/packaging/makeself/build.sh "${@}"
+    --platform "${platform}" ${EXTRA_INSTALL_FLAGS:+-e EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}"} \
+    "${DOCKER_IMAGE_NAME}" /bin/sh /netdata/packaging/makeself/build.sh "${@}"
 else
  run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
-    -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" "${DOCKER_IMAGE_NAME}" \
-    /bin/sh /netdata/packaging/makeself/build.sh "${@}"
+    -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" \
+    ${EXTRA_INSTALL_FLAGS:+-e EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}"} \
+    "${DOCKER_IMAGE_NAME}" /bin/sh /netdata/packaging/makeself/build.sh "${@}"
 fi
--- a/packaging/makeself/build.sh
+++ b/packaging/makeself/build.sh
@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/bin/bash
 # SPDX-License-Identifier: GPL-3.0-or-later

 # -----------------------------------------------------------------------------
--- a/packaging/makeself/jobs/70-netdata-git.install.sh
+++ b/packaging/makeself/jobs/70-netdata-git.install.sh
@ -37,7 +37,8 @@ run ./netdata-installer.sh \
  --dont-scrub-cflags-even-though-it-may-break-things \
  --one-time-build \
  --disable-logsmanagement \
-  --enable-lto
+  --enable-lto \
+  ${EXTRA_INSTALL_FLAGS:+${EXTRA_INSTALL_FLAGS}} \

 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Finishing netdata install" || true
--- a/packaging/makeself/run-all-jobs.sh
+++ b/packaging/makeself/run-all-jobs.sh
@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # SPDX-License-Identifier: GPL-3.0-or-later

 set -e