Upload packages to new self-hosted repository infrastructure. (#13240)

* Match packaging arches to distro architecture names.

This will simplify handling of uploads to our new self-hosted packaging
infrastructure.

* Add package uploads for RPM packages on new infrastructure.

* Add package upload support for DEB packages.

* Saner tmpdir handling.

* Upload fixes.

* Differentiate different DEB distros in package names.

* Workflow fixes.

.github/data/distros.yml

@@ -1,10 +1,21 @@
 # This defines the full set of distros we run CI on.
 ---
 platform_map: # map packaging architectures to docker platforms
-  i386: linux/i386
+  aarch64: linux/arm64/v8
   amd64: linux/amd64
-  armhf: linux/arm/v7
   arm64: linux/arm64/v8
+  armhf: linux/arm/v7
+  armhfp: linux/arm/v7
+  i386: linux/i386
+  x86_64: linux/amd64
+arch_order: # sort order for per-architecture jobs in CI
+  - amd64
+  - x86_64
+  - i386
+  - armhf
+  - armhfp
+  - arm64
+  - aarch64
 include:
   - &alpine
     distro: alpine
@@ -37,8 +48,8 @@ include:
       type: rpm
       repo_distro: el/9
       arches:
-        - amd64
-        - arm64
+        - x86_64
+        - aarch64
   - <<: *alma
     version: "8"
     packages:
@@ -51,7 +62,7 @@ include:
       type: rpm
       repo_distro: el/7
       arches:
-        - amd64
+        - x86_64
 
   - &debian
     distro: debian
@@ -88,9 +99,9 @@ include:
       type: rpm
       repo_distro: fedora/36
       arches:
-        - amd64
-        - armhf
-        - arm64
+        - x86_64
+        - armhfp
+        - aarch64
   - <<: *fedora
     version: "35"
     packages:
@@ -107,8 +118,8 @@ include:
       type: rpm
       repo_distro: opensuse/15.4
       arches:
-        - amd64
-        - arm64
+        - x86_64
+        - aarch64
   - <<: *opensuse
     version: "15.3"
     packages:
@@ -123,8 +134,8 @@ include:
       type: rpm
       repo_distro: ol/8
       arches:
-        - amd64
-        - arm64
+        - x86_64
+        - aarch64
 
   - &ubuntu
     distro: ubuntu

.github/scripts/package-upload.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -e
+
+host="packages.netdata.cloud"
+user="netdatabot"
+
+distro="${1}"
+arch="${2}"
+format="${3}"
+repo="${4}"
+
+staging="${TMPDIR:-/tmp}/package-staging"
+prefix="/var/www/html/repos/${repo}/"
+
+packages="$(find artifacts -name "*.${format}")"
+
+mkdir -p "${staging}"
+
+case "${format}" in
+    deb)
+        src="${staging}/$(echo "${distro}" | cut -f 1 -d '/')/pool/"
+        mkdir -p "${src}"
+
+        for pkg in ${packages}; do
+            cp "${pkg}" "${src}"
+        done
+        ;;
+    rpm)
+        src="${staging}/${distro}/${arch}/"
+        mkdir -p "${src}"
+
+        for pkg in ${packages}; do
+            cp "${pkg}" "${src}"
+        done
+        ;;
+    *)
+        echo "Unrecognized package format ${format}."
+        exit 1
+        ;;
+esac
+
+rsync -vrptO "${staging}/" "${user}@${host}:${prefix}"

.github/workflows/packaging.yml

@@ -65,14 +65,14 @@ jobs:
                           entries.append({
                               'distro': data['include'][i]['distro'],
                               'version': data['include'][i]['version'],
-                              'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
+                              'repo_distro': data['include'][i]['packages']['repo_distro'],
                               'format': data['include'][i]['packages']['type'],
                               'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
                               'platform': data['platform_map'][arch],
                               'arch': arch
                           })
 
-          entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
+          entries.sort(key=lambda k: (data['arch_order'].index(k['arch']), k['distro'], k['version']))
           matrix = json.dumps({'include': entries}, sort_keys=True)
           print('Generated Matrix: ' + matrix)
           print('::set-output name=matrix::' + matrix)
@@ -223,10 +223,27 @@ jobs:
         run: |
           printf "Packages to upload:\n%s" "$(ls artifacts/*.${{ matrix.format }})"
           for pkgfile in artifacts/*.${{ matrix.format }} ; do
-            .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} \
+            .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} \
             "$(basename "${pkgfile}")" || true
-            .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} "${pkgfile}"
+            .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} "${pkgfile}"
           done
+      - name: SSH setup
+        id: ssh-setup
+        if: github.event_name == 'workflow_dispatch'
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
+          name: id_ecdsa
+          known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
+      - name: Upload to packages.netdata.cloud
+        id: package-upload
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          .github/scripts/package-upload.sh \
+          ${{ matrix.repo_distro }} \
+          ${{ matrix.arch }} \
+          ${{ matrix.format }} \
+          ${{ needs.version-check.outputs.repo }}
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:
@@ -235,7 +252,7 @@ jobs:
           SLACK_TITLE: 'Package Build failed:'
           SLACK_USERNAME: 'GitHub Actions'
           SLACK_MESSAGE: |-
-              ${{ github.repository }}: ${{ matrix.pkgclouddistro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed.
+              ${{ github.repository }}: ${{ matrix.repo_distro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed.
               Checkout: ${{ steps.checkout.outcome }}
               Setup QEMU: ${{ steps.qemu.outcome }}
               Setup Docker: ${{ steps.docker-config.outcome }}
@@ -243,6 +260,8 @@ jobs:
               Build: ${{ steps.build.outcome }}
               Test: ${{ steps.test.outcome }}
               Publish: ${{ steps.upload.outcome }}
+              Import SSH Key: ${{ steps.ssh-setup.outcome }}
+              Publish: ${{ steps.package-upload.outcome }}
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
         if: >-
           ${{

.github/workflows/repoconfig-packages.yml

@@ -50,11 +50,10 @@ jobs:
                       'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
                       'format': data['include'][i]['packages']['type'],
                       'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
-                      'platform': data['platform_map']['amd64'],
-                      'arch': 'amd64'
+                      'platform': data['platform_map']['amd64']
                   })
 
-          entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
+          entries.sort(key=lambda k: (k['distro'], k['version']))
           matrix = json.dumps({'include': entries}, sort_keys=True)
           print('Generated Matrix: ' + matrix)
           print('::set-output name=matrix::' + matrix)