cockpit/test/image-prepare

#!/usr/bin/python3 -cimport os, sys; os.execv(os.path.dirname(sys.argv[1]) + "/common/pywrap", sys.argv)
# Build and run a bots/image-customize command to prepare a VM for testing Cockpit.

# This file is part of Cockpit.
#
# Copyright (C) 2022 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

import argparse
import os
import shlex
import shutil
import subprocess
import sys

from lib import testmap
from lib.constants import DEFAULT_IMAGE
from machine.machine_core import machine_virtual

BASE_DIR = os.path.realpath(f'{__file__}/../..')
TEST_DIR = f'{BASE_DIR}/test'
BOTS_DIR = f'{BASE_DIR}/bots'


def build_rpms(dist_tar, image, verbose, quick):
    '''build RPMs from a tarball in an image

    Return local rpm path list.
    '''
    subprocess.check_call([os.path.join(BOTS_DIR, "image-download"), image])
    machine = machine_virtual.VirtMachine(image=image)
    try:
        machine.start()
        machine.wait_boot()
        vm_tar = os.path.join("/var/tmp", os.path.basename(dist_tar))
        machine.upload([dist_tar], vm_tar)

        # build srpm
        machine.execute(fr'''
            su builder -c 'rpmbuild --define "_topdir /var/tmp/build" -ts "{vm_tar}"'
        ''')

        # build rpms
        mock_opts = ("--verbose" if verbose else "") + (" --nocheck" if quick else "")
        machine.execute(fr'''
            su builder -c 'mock --no-clean --disablerepo=* --offline \
                                --resultdir /var/tmp/build {mock_opts} \
                                --rebuild /var/tmp/build/SRPMS/*.src.rpm'
        ''', timeout=1800)

        # download rpms
        vm_rpms = machine.execute("find /var/tmp/build -name '*.rpm' -not -name '*.src.rpm'").strip().split()

        destdir = os.path.abspath("tmp/rpms")
        if os.path.exists(destdir):
            shutil.rmtree(destdir)
        os.makedirs(destdir)

        rpms = []
        for rpm in vm_rpms:
            machine.download(rpm, destdir)
            rpms.append(os.path.join(destdir, os.path.basename(rpm)))
        return rpms
    finally:
        machine.stop()


#
# Helper functions to build image-customize options for various steps
#

def build_install_package(dist_tar, image):
    '''Default rpm/deb/arch package build/install'''

    # our images have distro cockpit packages pre-installed, remove them
    args = ["--run-command"]
    if 'debian' in image or 'ubuntu' in image:
        args.append("dpkg --purge cockpit cockpit-ws cockpit-bridge cockpit-system")
    else:
        # subscription-manager-cockpit needs these, thus --nodeps
        args.append(r"""
            if rpm -q cockpit-ws >/dev/null 2>&1
                then rpm --erase --nodeps --verbose cockpit cockpit-ws cockpit-bridge cockpit-system
            fi
        """)

    args += ["--build", dist_tar]

    if 'debian' in image or 'ubuntu' in image:
        args.append("--run-command")
        suppress = {'initial-upload-closes-no-bugs', 'newer-standards-version'}
        # older lintian not yet complain about pkg/static/login.html and src/common/fail.html
        if image in ["debian-stable", "ubuntu-2204"]:
            suppress.add("mismatched-override")
        # Ubuntu 22.04 raises elf-error on *-dbgsym: "In program headers: Unable to find program interpreter name"
        if image == "ubuntu-2204":
            suppress.add("elf-error")

        args.append(fr"""
            cd /var/tmp/build;
            runuser -u admin -- \
                lintian --fail-on warning,error \
                --tag-display-limit 0 --display-info \
                --suppress-tags {','.join(suppress)} \
                cockpit*.changes >&2""")
    return args


def build_install_ostree(dist_tar, image, verbose, quick):
    '''Special treatment of build/install on CoreOS

    OSTree image can't build packages, build them on corresponding OSes and invoke
    test/ostree.install to install them into the OSTree and the cockpit/ws container.
    '''
    rpms = build_rpms(dist_tar, testmap.get_build_image(image), verbose, quick)
    args = []
    for rpm in rpms:
        args += ["--upload", f"{rpm}:/var/tmp/"]
    args += [
        "--upload", os.path.join(BASE_DIR, "containers") + ":/var/tmp/",
        "--script", os.path.join(TEST_DIR, "ostree.install")]
    return args


def build_install_rhel8(dist_tar, image, verbose, quick):
    '''Special treatment of build/install on RHEL/CentOS 8

    Here, cockpit is delivered as two mostly identical source packages: "cockpit" with
    build_basic=1, and "cockpit-appstream" with build_optional=1. The spec has proper build_*
    defaults depending on the Name:.
    '''
    vm_dist_tar = os.path.join("/var/tmp", os.path.basename(dist_tar))

    args = ["--upload", f"{dist_tar}:{vm_dist_tar}"]

    # in distropkg, keep basic OS packages, otherwise build/install cockpit
    if 'distropkg' not in image:
        args += [
            "--run-command",
            fr"""
                # remove already installed packages
                if rpm -q cockpit-ws >/dev/null 2>&1; then
                    rpm --erase --nodeps --verbose cockpit cockpit-ws cockpit-bridge cockpit-system
                fi

                # create cockpit.spec
                tar xf '{vm_dist_tar}' -O '*/tools/cockpit.spec' | sed '/%define build_all/d' > /var/tmp/cockpit.spec

                # create srpm
                su builder -c 'rpmbuild --define "_topdir /var/tmp/build" \
                                       --define "_sourcedir /var/tmp" \
                                       -bs /var/tmp/cockpit.spec'

                # build rpms in mock
                su builder -c '
                    mock --no-clean --no-cleanup-after \
                                    --disablerepo=* \
                                    --offline \
                                    --resultdir /var/tmp/build \
                                    {"--nocheck" if quick else ""} \
                                    {"--verbose" if verbose else ""} \
                                    --rebuild /var/tmp/build/SRPMS/*.src.rpm'

                # install rpms
                rpm -U --force --verbose $(find /var/tmp/build -name '*.rpm' -not -name '*.src.rpm')
            """
        ]

    # always build cockpit-appstream
    args += [
        "--run-command",
        fr"""
            # create cockpit-appstream.spec
            tar xf '{vm_dist_tar}' -O '*/tools/cockpit.spec' |
                sed '/^Name:/ s/$/-appstream/; /%define build_all/d' > /var/tmp/cockpit-appstream.spec

            # create srpm
            su builder -c 'rpmbuild --define "_topdir /var/tmp/appstream" \
                                    --define "_sourcedir /var/tmp" \
                                    -bs /var/tmp/cockpit-appstream.spec'

            # build rpms in mock
            su builder -c 'mock --no-clean --disablerepo=* --offline \
                                --resultdir /var/tmp/appstream \
                                --nocheck {"--verbose" if verbose else ""} \
                                --rebuild /var/tmp/appstream/SRPMS/*.src.rpm'

            # install rpms
            rpm -i --verbose $(find /var/tmp/appstream -name '*.rpm' -not -name '*.src.rpm')
        """
    ]
    return args


def validate_packages():
    '''Post-install package checks'''

    # check for files that are shipped by more than one RPM
    return ["--run-command",
            """set -eu
                fail=
                for f in $(find $(rpm -ql $(rpm -qa '*cockpit*') | sort | uniq -d) -maxdepth 0 -type f); do
                    # -debugsource overlap is legit
                    [ "${f#/usr/src/debug}" = "$f" ] || continue
                    echo "ERROR: $f is shipped by multiple packages: $(rpm -qf $f)" >&2
                    fail=1
                done
                [ -z "${fail}" ] || exit 1
            """]


def main():
    parser = argparse.ArgumentParser(
        description='Prepare testing environment, download images and build and install cockpit',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-e', '--env', metavar='NAME=VAL', action='append',
                        help="Add a line to /etc/environment")
    parser.add_argument('-d', '--debug', action='append_const', dest='env', const='COCKPIT_DEBUG=all',
                        help="Equivalent to --env=COCKPIT_DEBUG=all")
    parser.add_argument('-v', '--verbose', action='store_true', help='Display verbose progress details')
    parser.add_argument('-q', '--quick', action='store_true', help='Skip unit tests to build faster')
    parser.add_argument('-o', '--overlay', action='store_true',
                        help='Install into existing test/image/ overlay instead of from pristine base image')
    parser.add_argument('-p', '--python', action='store_true',
                        help='Install the python bridge into the image (experimental)')
    parser.add_argument('image', nargs='?', default=DEFAULT_IMAGE, help='The image to use')
    args = parser.parse_args()

    customize = [os.path.join(BOTS_DIR, "image-customize"), "--no-network"]
    if not args.overlay:
        customize.append("--fresh")
    if args.verbose:
        customize.append("--verbose")
    if args.quick:
        customize.append("--quick")

    makevars = ['HACK_SPEC_FOR_PYTHON=1'] if args.python else []
    dist_tar = subprocess.check_output([f'{BASE_DIR}/tools/make-dist', *makevars], text=True).strip()

    if args.image in ["fedora-coreos", "rhel4edge"]:
        customize += build_install_ostree(dist_tar, args.image, args.verbose, args.quick)
    elif args.image.startswith("rhel-8") or args.image.startswith("centos-8"):
        customize += build_install_rhel8(dist_tar, args.image, args.verbose, args.quick)
    else:
        customize += build_install_package(dist_tar, args.image)

    if not args.quick:
        customize += validate_packages()

    # post build/install test preparation
    customize += ["--script", os.path.join(TEST_DIR, "vm.install")]

    if args.env:
        customize += ['--run-command', r'printf "%s\n" >>/etc/environment ' + shlex.join(args.env)]

    customize.append(args.image)

    # show final command for easy copy&paste reproduction/debugging
    if args.verbose:
        print(' '.join([shlex.quote(arg) for arg in customize]))

    return subprocess.call(customize)


if __name__ == "__main__":
    sys.exit(main())