test: Add FMF test metadata and scripts

Copy and adjust the browser.sh and run-test.sh scripts from
Fedora's downstream dist-git gating tests [1], and translate the STI
yaml to an FMF metadata file test/verify.fmf. This replaces the previous
test/main.fmf, which was not functional and just a stub.

With this we can share the same test scripts upstream (for testing in
Packit) and downstream.

[1] 71ee0da592/f/tests
This commit is contained in:
Martin Pitt 2021-03-05 09:36:15 +01:00 committed by Martin Pitt
parent daeeaf861a
commit 4aa565f59d
7 changed files with 206 additions and 7 deletions

6
plans/all.fmf Normal file
View File

@ -0,0 +1,6 @@
summary:
Run all tests
discover:
how: fmf
execute:
how: tmt

View File

@ -1,6 +0,0 @@
duration: 2h
test: common/run-tests
environment:
TEST_DIR: verify
#TRACE: true
#SIT: True

116
test/run-verify-host-user.sh Executable file
View File

@ -0,0 +1,116 @@
#!/bin/sh
set -eux
cd "$SOURCE"
. /etc/os-release
test_optional=
test_basic=
if ls ../cockpit-appstream* 1> /dev/null 2>&1; then
test_optional=1
else
test_basic=1
fi
if [ "$ID" = "fedora" ]; then
test_basic=1
test_optional=1
fi
# tests need cockpit's bots/ libraries
git clone --depth=1 https://github.com/cockpit-project/bots
# support running from clean git tree
if [ ! -d node_modules/chrome-remote-interface ]; then
npm install chrome-remote-interface sizzle
fi
export TEST_OS="${ID}-${VERSION_ID/./-}"
# HACK: upstream does not yet know about rawhide
if [ "$TEST_OS" = "fedora-35" ]; then
export TEST_OS=fedora-33
fi
# HACK: CI hits this selinux denial. Unrelated to our tests.
export TEST_ALLOW_JOURNAL_MESSAGES=".*Permission denied:.*/var/cache/app-info/xmls.*"
# select tests
TESTS=""
EXCLUDES=""
RC=0
if [ -n "$test_optional" ]; then
# pre-download cirros image for Machines tests
bots/image-download cirros
# triggers SELinux violation
# See journal: SELinux is preventing /usr/libexec/qemu-kvm from open access on the file /var/lib/cockpittest/nfs_pool/nfs-volume-0.
EXCLUDES="$EXCLUDES TestMachinesDisks.testAddDiskNFS"
# not investigated yet
EXCLUDES="$EXCLUDES
TestAutoUpdates.testPrivilegeChange"
# TestUpdates: we can't run rebooting tests
TESTS="$TESTS
TestAutoUpdates
TestStorage
TestUpdates.testBasic
TestUpdates.testSecurityOnly"
# Fedora gating tests are running on infra without /dev/kvm; Machines tests are too darn slow there
if [ "$ID" = "fedora" ]; then
TESTS="$TESTS TestMachinesCreate.testCreateImportDisk"
else
TESTS="$TESTS TestMachines"
fi
fi
if [ -n "$test_basic" ]; then
# still too unstable
EXCLUDES="$EXCLUDES TestFirewall.testNetworkingPage"
# TODO: fix for CI environment
EXCLUDES="$EXCLUDES TestLogin.testTally"
EXCLUDES="$EXCLUDES TestAccounts.testBasic"
# PCI devices list is not predictable
EXCLUDES="$EXCLUDES TestSystemInfo.testHardwareInfo"
# Known issue #1008
EXCLUDES="$EXCLUDES TestTuned.testBasic"
TESTS="$TESTS
TestAccounts
TestBonding
TestBridge
TestFirewall
TestKdump
TestLogin
TestNetworking
TestServices
TestSOS
TestSystemInfo
TestTeam
TestTerminal
TestTuned
"
fi
exclude_options=""
for t in $EXCLUDES; do
exclude_options="$exclude_options --exclude $t"
done
# execute run-tests
test/common/run-tests --test-dir test/verify --nondestructive $exclude_options \
--machine localhost:22 --browser localhost:9090 $TESTS || RC=$?
# check-shell-menu is not @nondestructive yet, keep it last
if [ -n "$test_basic" ]; then
test/verify/check-shell-menu --machine localhost:22 --browser localhost:9090 || RC=$?
fi
echo $RC > "$LOGS/exitcode"
cp --verbose Test* "$LOGS" || true
# deliver test result via exitcode file
exit 0

60
test/run-verify-host.sh Executable file
View File

@ -0,0 +1,60 @@
#!/bin/sh
# This script is meant to be run on an ephemeral CI host, for packit/Fedora/RHEL gating.
set -eux
TESTS="$(realpath $(dirname "$0"))"
if [ -d source ]; then
# path for standard-test-source
SOURCE="$(pwd)/source"
else
SOURCE="$(realpath $TESTS/..)"
fi
LOGS="$(pwd)/logs"
mkdir -p "$LOGS"
chmod a+w "$LOGS"
# install browser; on RHEL, use chromium from epel
# HACK: chromium-headless ought to be enough, but version 88 has a crash: https://bugs.chromium.org/p/chromium/issues/detail?id=1170634
if ! rpm -q chromium; then
if grep -q 'ID=.*rhel' /etc/os-release; then
dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
dnf config-manager --enable epel
fi
dnf install -y chromium
fi
# make libpwquality less aggressive, so that our "foobar" password works
printf 'dictcheck = 0\nminlen = 6\n' >> /etc/security/pwquality.conf
# set root password for logging in
echo root:foobar | chpasswd
# create user account for logging in
if ! id admin 2>/dev/null; then
useradd -c Administrator -G wheel admin
echo admin:foobar | chpasswd
fi
# create user account for running the test
if ! id runtest 2>/dev/null; then
useradd -c 'Test runner' runtest
# allow test to set up things on the machine
mkdir -p /root/.ssh
curl https://raw.githubusercontent.com/cockpit-project/bots/master/machine/identity.pub >> /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
fi
chown -R runtest "$SOURCE"
# disable core dumps, we rather investigate them upstream where test VMs are accessible
echo core > /proc/sys/kernel/core_pattern
# make sure that we can access cockpit through the firewall
systemctl start firewalld
firewall-cmd --add-service=cockpit --permanent
firewall-cmd --add-service=cockpit
# Run tests as unprivileged user
su - -c "env SOURCE=$SOURCE LOGS=$LOGS $TESTS/run-verify-host-user.sh" runtest
RC=$(cat $LOGS/exitcode)
exit ${RC:-1}

23
test/verify.fmf Normal file
View File

@ -0,0 +1,23 @@
summary: Run browser integration tests on the host
require:
- cockpit
- cockpit-tests
- cockpit-machines
- cockpit-sosreport
- cockpit-storaged
- createrepo_c
- cryptsetup
- dnf-automatic
- firewalld
- git
- libvirt-daemon-config-network
- libvirt-python3
- make
- NetworkManager-team
- npm
- python3
- sssd-dbus
- targetcli
- tlog
test: ./run-verify-host.sh
duration: 2h

View File

@ -120,7 +120,7 @@ if [ -z "${PYTHON_STYLE_CHECKER-}" ]; then
echo "ok 7 pycodestyle test # SKIP pycodestyle not installed"
else
# FIXME: Fix code for the warnings and re-enable them
out=$(python3 -m "$PYTHON_STYLE_CHECKER" --ignore E501,E265,E261,W504,W605 test/* --exclude=test/verify/nested-kvm,test/README.md,test/run,test/main.fmf) || true
out=$(python3 -m "$PYTHON_STYLE_CHECKER" --ignore E501,E265,E261,W504,W605 test/* --exclude=test/verify/nested-kvm,test/README.md,test/run,test/verify.fmf,test/run-verify-host.sh,test/run-verify-host-user.sh) || true
if [ -n "$out" ]; then
echo "$out" >&2
echo "not ok 7 $PYTHON_STYLE_CHECKER test"