$ks
-install
-text
-shutdown
-lang en_US.UTF-8
-keyboard us
-network --bootproto dhcp
-rootpw foobar
-firewall --enabled --ssh
-selinux --enforcing
-timezone --utc America/New_York
-bootloader --location=mbr --append="console=ttyS0,115200 rd_NO_PLYMOUTH"
-zerombr
-clearpart --all --initlabel
-autopart
-
-%packages
-@core
-%end
-
-%post
-mkdir /root/.ssh
-chmod 700 /root/.ssh
-echo "$(cat $base/../../machine/identity.pub)" > /root/.ssh/authorized_keys
-chmod 600 /root/.ssh/authorized_keys
-mkdir -p /etc/sysconfig/network-scripts
-echo "$(cat $base/network-ifcfg-eth0)" > /etc/sysconfig/network-scripts/ifcfg-eth0
-echo "$(cat $base/network-ifcfg-eth1)" > /etc/sysconfig/network-scripts/ifcfg-eth1
-sed -i 's/GRUB_TIMEOUT.*/GRUB_TIMEOUT=0/; /GRUB_CMDLINE_LINUX=/ s/"$/ net.ifnames=0 biosdevname=0"/' /etc/default/grub
-grub2-mkconfig -o /boot/grub2/grub.cfg
-systemctl is-enabled sshd.socket || systemctl is-enabled sshd.service || systemctl enable sshd.socket
-%end
-EOF
-
-qemu-img create -f qcow2 "$out" 12G
-
-connect="--connect=qemu:///session"
-name=$(basename $1)-builder
-
-cleanup_builder_vm () {
- if virsh $connect list --state-running --name | grep -q $name; then
- virsh $connect destroy $name
- fi
- if virsh $connect list --all --name | grep -q $name; then
- virsh $connect undefine $name
- fi
-}
-
-# Using virt-install is fundamentally an interactive process. We
-# expect the installation to finish unattended, but it might drop into
-# an emergency shell upon unexpected errors, for example. Thus, we
-# run it under 'expect' and catch a few known scenarios that require
-# special handling.
-#
-# (Also, virt-install works best when it has a tty.)
-
-supervised_virt_install () {
-expect -- - virt-install "$@" <<'EOF'
-eval spawn $argv
-set timeout -1
-expect {
- "emergency mode" { send_user "\n\n\n\nABORT - emergency shell\n"
- exit 1
- }
- "Please make your choice" { send_user "\n\n\n\nABORT - Anaconda stopped\n"
- exit 1
- }
- "Please make a selection" { send_user "\n\n\n\nABORT - Anaconda stopped\n"
- exit 1
- }
-}
-EOF
-}
-
-cleanup_builder_vm
-
-# HACK: Due to #1686464
-ks_path="file:/$ks"
-ks_inject="--initrd-inject=$ks"
-if [[ "$url" == *30/Server* ]]
-then
- tmpdir=`mktemp -d`
- pushd .
- cp $ks $tmpdir
- cd $tmpdir
- python3 -m http.server &
- popd
- server_pid=$!
- ks_path="http://_gateway:8000/$ks"
- ks_inject=""
-fi
-
-supervised_virt_install $connect \
- $ks_inject \
- --extra-args="ks=$ks_path console=ttyS0,115200" \
- --name=$name \
- --disk "path=$out,format=qcow2" \
- --ram 4096 \
- --vcpus=1 \
- --os-type linux \
- --os-variant fedora21 \
- --location="$url" \
- --nographics \
- --noreboot
-
-if [ -n "$server_pid" ]
-then
- kill $server_pid
- rm -rf $tmpdir
-fi
-
-rm $ks
-
-cleanup_builder_vm
diff --git a/bots/images/scripts/windows-10.bootstrap b/bots/images/scripts/windows-10.bootstrap
deleted file mode 100755
index ac350e5d0..000000000
--- a/bots/images/scripts/windows-10.bootstrap
+++ /dev/null
@@ -1,71 +0,0 @@
-#! /bin/bash
-set -ex
-
-ORIGIN="$1"
-VM_NAME=win10
-
-# from https://developer.microsoft.com/en-us/microsoft-edge/tools/vms/
-wget --continue https://az792536.vo.msecnd.net/vms/VMBuild_20180425/VMWare/MSEdge/MSEdge.Win10.VMWare.zip -O MSEdge.Win10.VMWare.zip
-# from https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/
-wget --continue https://download.microsoft.com/download/F/8/A/F8AF50AB-3C3A-4BC4-8773-DC27B32988DD/MicrosoftWebDriver.exe -O MicrosoftWebDriver.exe
-wget --continue http://javadl.oracle.com/webapps/download/AutoDL?BundleId=233172_512cd62ec5174c3487ac17c61aaa89e8 -O java-installer.exe
-# from https://selenium-release.storage.googleapis.com/
-wget --continue https://selenium-release.storage.googleapis.com/3.13/selenium-server-standalone-3.13.0.jar -O selenium.jar
-wget --continue https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/archive-virtio/virtio-win-0.1.141-1/virtio-win.iso -O virtio-win.iso
-
-[ -e disk.vmdk ] || unzip -o MSEdge.Win10.VMWare.zip
-
-cat <<'EOF' > Autorun.inf
-[autorun]
-open=install.bat
-label=Selenium installer
-EOF
-
-cat <<'EOF' > selenium.bat
-powershell -Command "while (!(Get-NetIPAddress -AddressFamily ipv4 | Where-Object IPAddress -eq '10.111.112.10')) { sleep 2 }"
-timeout 10
-START "hub" java -jar C:\selenium\selenium.jar -role hub
-timeout 20
-java -Dwebdriver.edge.driver=c:\selenium\MicrosoftWebDriver.exe -jar C:\selenium\selenium.jar -role node -hub http://localhost:4444/grid/register -browser browserName="MicrosoftEdge",platform=WINDOWS
-timeout 50
-EOF
-
-cat <<'EOF' > install.bat
-net file 1>nul 2>nul && goto :run || powershell -ex unrestricted -Command "Start-Process -Verb RunAs -FilePath '%comspec%' -ArgumentList '/c %~fnx0 %*'"
-goto :eof
-:run
-netsh advfirewall firewall add rule name="Open Port 4444" dir=in action=allow protocol=TCP localport=4444
-netsh advfirewall firewall add rule name="Open Port 5555" dir=in action=allow protocol=TCP localport=5555
-netsh advfirewall set allprofiles state off
-pnputil.exe -i -a E:\qxldod\w10\amd64\qxldod.inf
-mkdir c:\selenium
-echo f | xcopy /f /y d:\selenium.jar c:\selenium\selenium.jar
-echo f | xcopy /f /y d:\selenium.bat c:\selenium\selenium.bat
-echo f | xcopy /f /y d:\MicrosoftWebDriver.exe c:\selenium\MicrosoftWebDriver.exe
-REG ADD "HKCU\SOFTWARE\Microsoft\Windows\CurrentVersion\Run" /V "Selenium Hub" /t REG_SZ /F /D "C:\selenium\selenium.bat"
-d:\java-installer.exe /s
-timeout 30
-shutdown -s -t 0
-EOF
-
-CR=$(printf '\r')
-sed -i "s/\$/$CR/" *.bat Autorun.inf
-genisoimage -output wininit.iso -volid cidata -joliet *.bat *.exe selenium.jar Autorun.inf
-echo "Converting vmdk to qcow2..."
-qemu-img convert -f vmdk -O qcow2 disk.vmdk "$ORIGIN"
-virsh destroy "$VM_NAME" || true
-virsh undefine --nvram "$VM_NAME" || true
-virt-install -n "$VM_NAME" -r 4000 --vcpus=4 --os-variant=win10 \
- --disk "$ORIGIN",device=disk --network user --boot uefi \
- --noautoconsole --wait=-1 --noreboot \
- --disk wininit.iso,device=cdrom \
- --disk virtio-win.iso,device=cdrom
-virsh start "$VM_NAME"
-
-echo "Manual part of installation, open graphics console and run D:\install.bat"
-set +x
-while [[ $(virsh domstate "$VM_NAME") =~ 'running' ]]; do
- sleep 2
-done
-set -x
-echo "Installation finished"
diff --git a/bots/images/selenium b/bots/images/selenium
deleted file mode 120000
index d6d011a37..000000000
--- a/bots/images/selenium
+++ /dev/null
@@ -1 +0,0 @@
-selenium-51bd1a8db6923e2bfd4eba0e36c53ca978f0a4109b31a9c1ffb6ff2ca23bf7d4.qcow2
\ No newline at end of file
diff --git a/bots/images/ubuntu-1804 b/bots/images/ubuntu-1804
deleted file mode 120000
index 3b4255a3d..000000000
--- a/bots/images/ubuntu-1804
+++ /dev/null
@@ -1 +0,0 @@
-ubuntu-1804-8f4086635b6afc6848dc329c60bb43e51999c900cc8d6509183c0e0cd77aeff0.qcow2
\ No newline at end of file
diff --git a/bots/images/ubuntu-stable b/bots/images/ubuntu-stable
deleted file mode 120000
index f85157319..000000000
--- a/bots/images/ubuntu-stable
+++ /dev/null
@@ -1 +0,0 @@
-ubuntu-stable-a8d26f908dee3428713ec34fda52164d45b15ad0b7081f650aca191698bbc6ce.qcow2
\ No newline at end of file
diff --git a/bots/images/windows-10 b/bots/images/windows-10
deleted file mode 120000
index 94f4a0793..000000000
--- a/bots/images/windows-10
+++ /dev/null
@@ -1 +0,0 @@
-windows-10-d62c352db36aa2cb37ec4a51d19c2236ad67a216f698dc3784862047aa762240.qcow2
\ No newline at end of file
diff --git a/bots/inspect-queue b/bots/inspect-queue
deleted file mode 100755
index 57ddfb664..000000000
--- a/bots/inspect-queue
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-
-# This file is part of Cockpit.
-#
-# Copyright (C) 2018 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-MAX_PRIORITY = 9
-
-import argparse
-import sys
-
-from task import distributed_queue
-
-def main():
- parser = argparse.ArgumentParser(description='Read and print messages from the queue without acknowleding them')
- parser.add_argument('--amqp', default='localhost:5671',
- help='The host:port of the AMQP server to consume from (default: %(default)s)')
- opts = parser.parse_args()
-
- with distributed_queue.DistributedQueue(opts.amqp, ['public', 'rhel'], passive=True) as q:
- def print_queue(queue):
- if q.declare_results[queue] is None:
- print("queue {} does not exist".format(queue))
- return
- message_count = q.declare_results[queue].method.message_count
- if message_count == 0:
- print("queue {} is empty".format(queue))
- return
- for i in range(message_count):
- method_frame, header_frame, body = q.channel.basic_get(queue=queue)
- if method_frame:
- print(body.decode())
-
- print('public queue:')
- print_queue('public')
- print('rhel queue:')
- print_queue('rhel')
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/bots/issue-scan b/bots/issue-scan
deleted file mode 100755
index 4c2727c0f..000000000
--- a/bots/issue-scan
+++ /dev/null
@@ -1,230 +0,0 @@
-#!/usr/bin/env python3
-
-# This file is part of Cockpit.
-#
-# Copyright (C) 2017 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-NAMES = [
- "example-task",
- "po-refresh",
- "image-refresh",
- "npm-update",
- "naughty-prune",
- "learn-tests",
- "tests-data",
- "flakes-refresh",
-]
-
-KVM_TASKS = [
- "image-refresh"
-]
-
-# RHEL tasks have to be done inside Red Hat network
-REDHAT_TASKS = [
- "rhel",
- "redhat"
-]
-
-# Windows tasks have to be done by a human
-WINDOWS_TASKS = [
- "windows"
-]
-
-KUBERNETES_TASKS = [
- ".svc.cluster.local"
-]
-
-# Credentials for working on above contexts
-REDHAT_CREDS = "~/.rhel/login"
-
-import argparse
-import pipes
-import os
-import sys
-import json
-
-sys.dont_write_bytecode = True
-
-from task import github, redhat_network, distributed_queue, labels_of_pull
-
-BOTS = os.path.normpath(os.path.join(os.path.dirname(__file__)))
-BASE = os.path.normpath(os.path.join(BOTS, ".."))
-
-no_amqp = False
-try:
- import pika
-except ImportError:
- no_amqp = True
-
-
-def main():
- parser = argparse.ArgumentParser(description="Scan issues for tasks")
- parser.add_argument("-v", "--human-readable", "--verbose", action="store_true", default=False,
- dest="verbose", help="Print verbose information")
- parser.add_argument('--amqp', default=None,
- help='The host:port of the AMQP server to publish to (format host:port)')
- parser.add_argument('--issues-data', default=None,
- help='issues or pull request event GitHub JSON data to evaluate')
- opts = parser.parse_args()
-
- if opts.amqp and no_amqp:
- parser.error("AMQP host:port specified but python-amqp not available")
-
- kvm = os.access("/dev/kvm", os.R_OK | os.W_OK)
- if not kvm:
- sys.stderr.write("issue-scan: No /dev/kvm access, not creating images here\n")
-
- # Figure if we're in a Kubernetes namespace. This file will always exist
- try:
- with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r") as f:
- namespace = f.read().strip()
- except IOError:
- namespace = None
-
- for result in scan(opts.issues_data, opts.verbose):
- if opts.amqp:
- with distributed_queue.DistributedQueue(opts.amqp, queues=['rhel', 'public']) as q:
- queue_task(q.channel, result)
- continue
-
- if not kvm and contains_any(result, KVM_TASKS):
- sys.stderr.write("issue-scan: skipping (no kvm): {0}\n".format(result))
- continue
- elif not redhat_network() and contains_any(result, REDHAT_TASKS):
- sys.stderr.write("issue-scan: skipping (outside redhat): {0}\n".format(result))
- continue
- elif contains_any(result, WINDOWS_TASKS):
- sys.stderr.write("issue-scan: skipping (windows task): {0}\n".format(result))
- continue
- elif contains_any(result, KUBERNETES_TASKS):
- if not namespace:
- sys.stderr.write("issue-scan: skipping (not in kubernetes): {0}\n".format(result))
- continue
- url = namespace + KUBERNETES_TASKS[0]
- if url not in result:
- sys.stderr.write("issue-scan: skipping (not same namespace): {0}\n".format(result))
- continue
- sys.stdout.write(result + "\n")
-
- return 0
-
-def contains_any(string, matches):
- for match in matches:
- if match in string:
- return True
- return False
-
-# Map all checkable work items to fixtures
-def tasks_for_issues(issues_data):
- results = [ ]
- issues = [ ]
-
- if issues_data:
- event = json.loads(issues_data)
- repo = event["repository"]["full_name"]
- issue = event.get("issue") or event.get("pull_request")
- labels = labels_of_pull(issue)
- if 'bot' in labels:
- issues.append(issue)
- api = github.GitHub(repo=repo)
- else:
- api = github.GitHub()
- issues = api.issues(state="open")
- whitelist = api.whitelist()
-
- for issue in issues:
- if issue["title"].strip().startswith("WIP"):
- continue
- login = issue.get("user", { }).get("login", { })
- if login not in whitelist:
- continue
-
- #
- # We only consider the first unchecked item per issue
- #
- # The bots think the list needs to be done in order.
- # If the first item in the checklist is not something
- # the bots can do, then the bots will ignore this issue
- # (below in output_task)
- #
- checklist = github.Checklist(issue["body"])
- for item, checked in checklist.items.items():
- if not checked:
- results.append((item, issue, api.repo))
- break
- return results
-
-def output_task(command, issue, repo, verbose):
- name, unused, context = command.partition(" ")
- if name not in NAMES:
- return None
- number = issue.get("number", None)
- if number is None:
- return None
-
- context = context.strip()
- checkout = "PRIORITY={priority:04d} "
- cmd = "bots/{name} --verbose --issue='{issue}' {context}"
-
- # `--issues-data` should also be able to receive pull_request events, in that
- # case pull_request won't be present in the object, but commits will be
- if "pull_request" in issue or "commits" in issue:
- checkout += "bots/make-checkout --verbose --repo {repo} pull/{issue}/head && "
- else:
- checkout += "bots/make-checkout --verbose --repo {repo} master && "
-
- if verbose:
- return "issue-{issue} {name} {context} {priority}".format(
- issue=int(number),
- priority=distributed_queue.MAX_PRIORITY,
- name=name,
- context=context
- )
- else:
- if context:
- context = pipes.quote(context)
- return (checkout + "cd bots/make-checkout-workdir && " + cmd + " ; cd ../..").format(
- issue=int(number),
- priority=distributed_queue.MAX_PRIORITY,
- name=name,
- context=context,
- repo=repo,
- )
-
-def queue_task(channel, result):
- body = {
- "command": result,
- "type": "issue",
- }
- queue = 'rhel' if contains_any(result, REDHAT_TASKS) else 'public'
- channel.basic_publish('', queue, json.dumps(body), properties=pika.BasicProperties(priority=distributed_queue.MAX_PRIORITY))
-
-# Default scan behavior run for each task
-def scan(issues_data, verbose):
- global issues
-
- results = [ ]
-
- # Now go through each fixture
- for (command, issue, repo) in tasks_for_issues(issues_data):
- result = output_task(command, issue, repo, verbose)
- if result is not None:
- results.append(result)
-
- return results
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/bots/issues-review b/bots/issues-review
deleted file mode 100755
index fa15e7409..000000000
--- a/bots/issues-review
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import time
-
-from task import github
-
-def issues_review(api, opts):
- now = time.time()
- treshold = opts.age * 86400
- count = 100
- page = 1
- while count == 100:
- issues = api.get("issues?filter=all&page=%i&per_page=%i" % (page, count))
- page += 1
- count = len(issues)
- for issue in issues:
- age = now - time.mktime(time.strptime(issue["updated_at"], "%Y-%m-%dT%H:%M:%SZ"))
- if age >= treshold:
- print("Labelling #%i last updated at %s" % (issue["number"], issue["updated_at"]))
- api.post("issues/%i/labels" % issue["number"], [opts.label])
-
-
-def main():
- parser = argparse.ArgumentParser(description='Add review label to stale issues')
- parser.add_argument('-a', '--age', metavar='DAYS', default=90,
- help='Label issues whose last update is older than given number of days (default: %(default)s)')
- parser.add_argument('-l', '--label', default=time.strftime('review-%Y-%m'),
- help='Label name (default: %(default)s)')
- parser.add_argument('--repo', help='Work on this GitHub repository (owner/name)')
- opts = parser.parse_args()
-
- api = github.GitHub(repo=opts.repo)
- issues_review(api, opts)
-
-
-if __name__ == '__main__':
- main()
diff --git a/bots/learn-tests b/bots/learn-tests
deleted file mode 100755
index 07542192a..000000000
--- a/bots/learn-tests
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/usr/bin/env python3
-
-# This file is part of Cockpit.
-#
-# Copyright (C) 2017 Slavek Kabrda
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-# The name and version of the training data
-TRAINING_DATA = "tests-train-1.jsonl.gz"
-
-# The number of days in history to learn from. This is different from
-# the amount of data we gather in tests-data, and can be adjusted
-# independently.
-SINCE = 21
-
-import os
-import socket
-import ssl
-import subprocess
-import sys
-import time
-import urllib
-
-sys.dont_write_bytecode = True
-
-import task
-
-from machine import testvm
-
-BOTS = os.path.dirname(os.path.realpath(__file__))
-
-def run(url_or_file, verbose=False, dry=False, **kwargs):
- # Default set of training data, retrieve it and use from data directory
- if not url_or_file:
- url_or_file = TRAINING_DATA
-
- # A URL was provided directly, just use it
- if url_or_file.startswith("http"):
- filename = os.path.basename(url_or_file)
- url = url_or_file
-
- else:
- host = os.environ.get("COCKPIT_LEARN_SERVICE_HOST", "learn-cockpit.apps.ci.centos.org")
- port = os.environ.get("COCKPIT_LEARN_SERVICE_PORT", "443")
- url = "{0}://{1}:{2}/train/{3}".format("https" if port == "443" else "http", host, port, os.path.basename(url_or_file))
- filename = url_or_file
-
- if "/" not in filename and not os.path.exists(filename):
- if not dry:
- subprocess.check_call([ os.path.join(BOTS, "image-download"), "--state", filename ])
- filename = os.path.join(testvm.get_images_data_dir(), filename)
- train(filename, url, verbose)
-
-# Does 'tail -F' on an HTTP URL
-def tail(url, until, verbose=False):
- stop = False
- at = 0
-
- while True:
- time.sleep(10)
-
- try:
- req = urllib.request.Request(url, headers={ "Range": "bytes={0}-".format(at) })
- cafile = os.path.join(BOTS, "images", "files", "ca.pem")
- context = ssl.create_default_context(cafile=cafile)
- with urllib.request.urlopen(req, context=context) as f:
- while True:
- data = f.read(2048)
- if not data:
- break
- at += len(data)
- if verbose:
- sys.stderr.buffer.write(data)
- except urllib.error.HTTPError as ex:
- if ex.code != 404 and ex.code != 416:
- sys.stderr.write("{0}: {1}\n".format(url, ex))
- except (ConnectionResetError, urllib.error.URLError, socket.gaierror) as ex:
- sys.stderr.write("{0}: {1}\n".format(url, ex))
-
- if stop:
- break
-
- # Note that we do one more loop after we stop, to make sure to get all of url
- stop = until()
-
-def train(filename, url, verbose=False):
- if verbose:
- sys.stderr.write(" ^ {0}\n".format(url))
-
- cmd = [ os.path.join(BOTS, "image-upload"), "--state", filename, "--store", url ]
-
- # Passing through a non terminal stdout is necessary to make progress work
- subprocess.check_call(cmd)
-
- # We run until the file disappears, which means training has taken place
- def until():
- try:
- req = urllib.request.Request(url, method="HEAD")
- with urllib.request.urlopen(req, cafile=os.path.join(BOTS, "images", "files", "ca.pem")) as f:
- f.read()
- except urllib.error.HTTPError as ex:
- if ex.code == 404:
- return True
- sys.stderr.write("{0}: {1}\n".format(url, ex))
- except (ConnectionResetError, urllib.error.URLError, socket.gaierror) as ex:
- sys.stderr.write("{0}: {1}\n".format(url, ex))
- return False
-
- # Now tail the logs until above happens
- log = urllib.parse.urljoin(url, "../log")
- tail(log, until, verbose)
-
-if __name__ == '__main__':
- task.main(function=run, title="Learn from testing data", verbose=True)
diff --git a/bots/learn-trigger b/bots/learn-trigger
deleted file mode 100755
index fbe29d2fc..000000000
--- a/bots/learn-trigger
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python3
-
-# This file is part of Cockpit.
-#
-# Copyright (C) 2017 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-# Number of days between learning attempts
-DAYS = 7
-
-import argparse
-import sys
-import time
-
-sys.dont_write_bytecode = True
-
-import task
-
-# The name and version of the training data
-TRAINING_DATA = "tests-train-1.jsonl.gz"
-
-def main():
- parser = argparse.ArgumentParser(description='Ensure necessary issue learning from tests')
- parser.add_argument('-v', '--verbose', action="store_true", default=False,
- help="Print verbose information")
- parser.parse_args()
-
- title = "Update machine learning from test logs"
- body = "Perform machine learning tasks such as retrieving new test logs and updating"
-
- since = time.time() - (DAYS * 86400)
- items = [
- "tests-data " + TRAINING_DATA,
- # We want this to run in both namespaces
- "learn-tests https://learn-cockpit.apps.ci.centos.org/train/" + TRAINING_DATA,
- "flakes-refresh",
- ]
- issue = task.issue(title, body, items[1], items=items, state="all", since=since)
-
- if issue:
- sys.stderr.write("#{0}: learn-tests\n".format(issue["number"]))
-
- return 0
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/bots/machine/cloud-init.iso b/bots/machine/cloud-init.iso
deleted file mode 100644
index 3ffbb0828..000000000
Binary files a/bots/machine/cloud-init.iso and /dev/null differ
diff --git a/bots/machine/host_key b/bots/machine/host_key
deleted file mode 100644
index 14f588b45..000000000
--- a/bots/machine/host_key
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAr+bCynyw7hAG03Bwt3joCTPjrexdO+ynsA+HtZRs38N9NCaO
-MZ7j7KCRFUgkezo7GEAp7lRparZWrzAixcyATZNOokwYP55flvsWtwhTSE2wI4gY
-n+0nmNFy+l3qs29VWFzVX8CkCqXBiGw53uo8qLuMEWVdXmstNxR00pHdvlyjOhjl
-BpZBFKD8gMGDx6qClGIosgcSbNtJf6Xl1ceo7BoLNknOoJdyiT9EwdhO53A9aVhx
-kbYjbIRRVWq8P2Cq/kbPioYlUtwgAH2A4aQTVlzsEyssdnriYwIbERddG8eqZ7mn
-UhKU/FH6Of2BSFQA9Rh6bVC0s1Y1KCZupLaBwQIDAQABAoIBAQCbjHLA4NcNDjsb
-CxmCBXcbfDlged5QuYvoEzOtDN3iWlsDnPytQJbJj4v8x9kK54mOfl8WFKtL5IZv
-UR/OznK/Jv6oYqYmzAQ33T5PCRusmpaiNR2hfvQ/HSiR4i9EEbXk9+LwU8g8aivk
-WeArEfQmOgM49uxELH7FcF+GPdtbE9TsHNdkVf1CzCMcGdIeNjeCqEDQrgRSdAq8
-YpCrlvQj76Gv8g6IOUiMZYS/fqbuvMR/XryXSQkEUX/4I5QojZOD1XrzxGA94jJ9
-dOv3Yr1y2+fhPAy5dDIFoqWSuDMib2yGV47jFo+Mqu6ovLVPAt74UWucHKImXgo1
-qvl0wFkJAoGBANwmWqJZ8dxJTU5gcK2KOq3u2JUYSYek3HMkEsPjEezGtht1Okg5
-TjxFEiw+yc4yeUtj0lIOyNU976FA0+5ItiW18/Byw6zWUi2BmrLGsCM0/CL/xwKM
-hVo8DrMXcGrZY6ZSqNiLtAYLmgAUKkJEP+pw8r1Qr0pO5yfVHNeK0v/3AoGBAMyL
-xWIhETGKkmyCuqFSFPELxbmMwjqWagNrzFK23/cqgbFv0aCz6wXhcwQ5JiszFq7B
-Hvz8Wezl9Ur2FdFz3wGz46q+Cdqnw7uQTTGd5WbDWHN/tCS67bKn998BqENpPiWK
-OIgNFXnNcucFtte9o7QDmjSaDd4u0xwveRYwHg4HAoGBAJWPbOV864X3OpCzjfkn
-vmOprvPjUxjW1HlYmXMA0Y2lFdSjmFu2qsLhPc5XPaxat/KStzDOIHxWHnTTYOcx
-+KS37yh8HxlNZPjLYrhvqPvSJDT2xVGi+3lo8aeTlejRFRTKdTDgAAZXXWEOUgNA
-8Jcp8o7QwLVf00RJUNXR1zTTAoGAChy+3WMVHoXjR0oPP/p23pPeapXy5EKbax/h
-MhWobOfFEaidjHxYminTLdpFcM1NycXyaj9vkq6rudEAsyIvXD4wezh59D1nB9bS
-eil8NeBidxNRLJ+xMKvtLTE/yFVjpSd4NAGxlhv6GkHGEFRny3aCISecl+douHQA
-YIBwe/ECgYALzEEkESm8d5Zq2fuFtUhRqFGcOtr/IYR8OgtUIZe2NRImsR+r5ycN
-w4mw7RAnxKqOoXeAtWBwi5MykItiaof2MD3MIe4kxlZQt0NPpyE4dkzsUkYf89kE
-ndu5mUalV7s7KBttm9gn8e+btzERna2VPRfDQh8nHw/zLXtE7lFSUg==
------END RSA PRIVATE KEY-----
diff --git a/bots/machine/host_key.pub b/bots/machine/host_key.pub
deleted file mode 100644
index a05bca32c..000000000
--- a/bots/machine/host_key.pub
+++ /dev/null
@@ -1 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCv5sLKfLDuEAbTcHC3eOgJM+Ot7F077KewD4e1lGzfw300Jo4xnuPsoJEVSCR7OjsYQCnuVGlqtlavMCLFzIBNk06iTBg/nl+W+xa3CFNITbAjiBif7SeY0XL6Xeqzb1VYXNVfwKQKpcGIbDne6jyou4wRZV1eay03FHTSkd2+XKM6GOUGlkEUoPyAwYPHqoKUYiiyBxJs20l/peXVx6jsGgs2Sc6gl3KJP0TB2E7ncD1pWHGRtiNshFFVarw/YKr+Rs+KhiVS3CAAfYDhpBNWXOwTKyx2euJjAhsRF10bx6pnuadSEpT8Ufo5/YFIVAD1GHptULSzVjUoJm6ktoHB
diff --git a/bots/machine/identity b/bots/machine/identity
deleted file mode 100644
index d02b8c671..000000000
--- a/bots/machine/identity
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpQIBAAKCAQEA1DrTSXQRF8isQQfPfK3U+eFC4zBrjur+Iy15kbHUYUeSHf5S
-jXPYbHYqD1lHj4GJajC9okle9rykKFYZMmJKXLI6987wZ8vfucXo9/kwS6BDAJto
-ZpZSj5sWCQ1PI0Ce8CbkazlTp5NIkjRfhXGP8mkNKMEhdNjaYceO49ilnNCIxhpb
-eH5dH5hybmQQNmnzf+CGCCLBFmc4g3sFbWhI1ldyJzES5ZX3ahjJZYRUfnndoUM/
-TzdkHGqZhL1EeFAsv5iV65HuYbchch4vBAn8jDMmHh8G1ixUCL3uAlosfarZLLyo
-3HrZ8U/llq7rXa93PXHyI/3NL/2YP3OMxE8baQIDAQABAoIBAQCxuOUwkKqzsQ9W
-kdTWArfj3RhnKigYEX9qM+2m7TT9lbKtvUiiPc2R3k4QdmIvsXlCXLigyzJkCsqp
-IJiPEbJV98bbuAan1Rlv92TFK36fBgC15G5D4kQXD/ce828/BSFT2C3WALamEPdn
-v8Xx+Ixjokcrxrdeoy4VTcjB0q21J4C2wKP1wEPeMJnuTcySiWQBdAECCbeZ4Vsj
-cmRdcvL6z8fedRPtDW7oec+IPkYoyXPktVt8WsQPYkwEVN4hZVBneJPCcuhikYkp
-T3WGmPV0MxhUvCZ6hSG8D2mscZXRq3itXVlKJsUWfIHaAIgGomWrPuqC23rOYCdT
-5oSZmTvFAoGBAPs1FbbxDDd1fx1hisfXHFasV/sycT6ggP/eUXpBYCqVdxPQvqcA
-ktplm5j04dnaQJdHZ8TPlwtL+xlWhmhFhlCFPtVpU1HzIBkp6DkSmmu0gvA/i07Z
-pzo5Z+HRZFzruTQx6NjDtvWwiXVLwmZn2oiLeM9xSqPu55OpITifEWNjAoGBANhH
-XwV6IvnbUWojs7uiSGsXuJOdB1YCJ+UF6xu8CqdbimaVakemVO02+cgbE6jzpUpo
-krbDKOle4fIbUYHPeyB0NMidpDxTAPCGmiJz7BCS1fCxkzRgC+TICjmk5zpaD2md
-HCrtzIeHNVpTE26BAjOIbo4QqOHBXk/WPen1iC3DAoGBALsD3DSj46puCMJA2ebI
-2EoWaDGUbgZny2GxiwrvHL7XIx1XbHg7zxhUSLBorrNW7nsxJ6m3ugUo/bjxV4LN
-L59Gc27ByMvbqmvRbRcAKIJCkrB1Pirnkr2f+xx8nLEotGqNNYIawlzKnqr6SbGf
-Y2wAGWKmPyEoPLMLWLYkhfdtAoGANsFa/Tf+wuMTqZuAVXCwhOxsfnKy+MNy9jiZ
-XVwuFlDGqVIKpjkmJyhT9KVmRM/qePwgqMSgBvVOnszrxcGRmpXRBzlh6yPYiQyK
-2U4f5dJG97j9W7U1TaaXcCCfqdZDMKnmB7hMn8NLbqK5uLBQrltMIgt1tjIOfofv
-BNx0raECgYEApAvjwDJ75otKz/mvL3rUf/SNpieODBOLHFQqJmF+4hrSOniHC5jf
-f5GS5IuYtBQ1gudBYlSs9fX6T39d2avPsZjfvvSbULXi3OlzWD8sbTtvQPuCaZGI
-Df9PUWMYZ3HRwwdsYovSOkT53fG6guy+vElUEDkrpZYczROZ6GUcx70=
------END RSA PRIVATE KEY-----
diff --git a/bots/machine/identity.pub b/bots/machine/identity.pub
deleted file mode 100644
index d60db7e4c..000000000
--- a/bots/machine/identity.pub
+++ /dev/null
@@ -1 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDUOtNJdBEXyKxBB898rdT54ULjMGuO6v4jLXmRsdRhR5Id/lKNc9hsdioPWUePgYlqML2iSV72vKQoVhkyYkpcsjr3zvBny9+5xej3+TBLoEMAm2hmllKPmxYJDU8jQJ7wJuRrOVOnk0iSNF+FcY/yaQ0owSF02Nphx47j2KWc0IjGGlt4fl0fmHJuZBA2afN/4IYIIsEWZziDewVtaEjWV3InMRLllfdqGMllhFR+ed2hQz9PN2QcapmEvUR4UCy/mJXrke5htyFyHi8ECfyMMyYeHwbWLFQIve4CWix9qtksvKjcetnxT+WWrutdr3c9cfIj/c0v/Zg/c4zETxtp cockpit-test
diff --git a/bots/machine/machine_core/__init__.py b/bots/machine/machine_core/__init__.py
deleted file mode 100644
index a2ee36f93..000000000
--- a/bots/machine/machine_core/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-# Place holder for python module
diff --git a/bots/machine/machine_core/cli.py b/bots/machine/machine_core/cli.py
deleted file mode 100644
index 21184cfb1..000000000
--- a/bots/machine/machine_core/cli.py
+++ /dev/null
@@ -1,49 +0,0 @@
-# This file is part of Cockpit.
-#
-# Copyright (C) 2013 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-import signal
-import argparse
-from . import machine_virtual
-
-def cmd_cli():
- parser = argparse.ArgumentParser(description="Run a VM image until SIGTERM or SIGINT")
- parser.add_argument("--memory", type=int, default=1024,
- help="Memory in MiB to allocate to the VM (default: %(default)s)")
- parser.add_argument("image", help="Image name")
- args = parser.parse_args()
-
- network = machine_virtual.VirtNetwork(0, image=args.image)
- machine = machine_virtual.VirtMachine(image=args.image, networking=network.host(), memory_mb=args.memory)
- machine.start()
- machine.wait_boot()
-
- # run a command to force starting the SSH master
- machine.execute('uptime')
-
- # print ssh command
- print("ssh -o ControlPath=%s -p %s %s@%s" %
- (machine.ssh_master, machine.ssh_port, machine.ssh_user, machine.ssh_address))
- # print Cockpit web address
- print("http://%s:%s" % (machine.web_address, machine.web_port))
- # print marker that the VM is ready; tests can poll for this to wait for the VM
- print("RUNNING")
-
- signal.signal(signal.SIGTERM, lambda sig, frame: machine.stop())
- try:
- signal.pause()
- except KeyboardInterrupt:
- machine.stop()
diff --git a/bots/machine/machine_core/constants.py b/bots/machine/machine_core/constants.py
deleted file mode 100644
index 88cf9a0b1..000000000
--- a/bots/machine/machine_core/constants.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# This file is part of Cockpit.
-#
-# Copyright (C) 2013 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-import os
-
-# Images which are Atomic based
-ATOMIC_IMAGES = ["rhel-atomic", "fedora-atomic", "continuous-atomic"]
-
-MACHINE_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-BOTS_DIR = os.path.dirname(MACHINE_DIR)
-BASE_DIR = os.path.dirname(BOTS_DIR)
-TEST_DIR = os.path.join(BASE_DIR, "test")
-GIT_DIR = os.path.join(BASE_DIR, ".git")
-
-IMAGES_DIR = os.path.join(BOTS_DIR, "images")
-SCRIPTS_DIR = os.path.join(IMAGES_DIR, "scripts")
-
-DEFAULT_IDENTITY_FILE = os.path.join(MACHINE_DIR, "identity")
-
-TEST_OS_DEFAULT = "fedora-30"
-DEFAULT_IMAGE = os.environ.get("TEST_OS", TEST_OS_DEFAULT)
diff --git a/bots/machine/machine_core/directories.py b/bots/machine/machine_core/directories.py
deleted file mode 100644
index 741eb978c..000000000
--- a/bots/machine/machine_core/directories.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# This file is part of Cockpit.
-#
-# Copyright (C) 2019 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-import os
-import subprocess
-
-from .constants import BOTS_DIR, BASE_DIR, GIT_DIR
-
-def get_git_config(variable):
- if not os.path.exists(GIT_DIR):
- return None
-
- try:
- myenv = os.environ.copy()
- myenv["GIT_DIR"] = GIT_DIR
- return subprocess.check_output(["git", "config", variable], universal_newlines=True, env=myenv).strip()
-
- except (OSError, subprocess.CalledProcessError): # 'git' not in PATH, or cmd fails
- return None
-
-_images_data_dir = None
-def get_images_data_dir():
- global _images_data_dir
-
- if _images_data_dir is None:
- _images_data_dir = get_git_config('cockpit.bots.images-data-dir')
-
- if _images_data_dir is None:
- _images_data_dir = os.path.join(os.environ.get("TEST_DATA", BOTS_DIR), "images")
-
- return _images_data_dir
-
-_temp_dir = None
-def get_temp_dir():
- global _temp_dir
-
- if _temp_dir is None:
- _temp_dir = os.path.join(os.environ.get("TEST_DATA", BASE_DIR), "tmp")
- os.makedirs(_temp_dir, exist_ok=True)
-
- return _temp_dir
diff --git a/bots/machine/machine_core/exceptions.py b/bots/machine/machine_core/exceptions.py
deleted file mode 100644
index e7daa8431..000000000
--- a/bots/machine/machine_core/exceptions.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# This file is part of Cockpit.
-#
-# Copyright (C) 2013 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-class Failure(Exception):
- def __init__(self, msg):
- self.msg = msg
-
- def __str__(self):
- return self.msg
-
-
-class RepeatableFailure(Failure):
- pass
diff --git a/bots/machine/machine_core/machine.py b/bots/machine/machine_core/machine.py
deleted file mode 100644
index 4e6f2f1ef..000000000
--- a/bots/machine/machine_core/machine.py
+++ /dev/null
@@ -1,280 +0,0 @@
-# This file is part of Cockpit.
-#
-# Copyright (C) 2013 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-import os
-
-from . import ssh_connection
-from . import timeout
-from .constants import DEFAULT_IDENTITY_FILE, ATOMIC_IMAGES, TEST_DIR
-
-LOGIN_MESSAGE = """
-TTY LOGIN
- User: {ssh_user}/admin Password: foobar
- To quit use Ctrl+], Ctrl+5 (depending on locale)
-
-SSH ACCESS
- $ ssh -p {ssh_port} {ssh_user}@{ssh_address}
- Password: foobar
-
-COCKPIT
- http://{web_address}:{web_port}
-"""
-
-RESOLV_SCRIPT = """
-set -e
-# HACK: Racing with operating systems reading/updating resolv.conf and
-# the fact that resolv.conf can be a symbolic link. Avoid failures like:
-# chattr: Operation not supported while reading flags on /etc/resolv.conf
-mkdir -p /etc/NetworkManager/conf.d
-printf '[main]\ndns=none\n' > /etc/NetworkManager/conf.d/dns.conf
-systemctl reload-or-restart NetworkManager
-printf 'domain {domain}\nsearch {domain}\nnameserver {nameserver}\n' >/etc/resolv2.conf
-chcon -v unconfined_u:object_r:net_conf_t:s0 /etc/resolv2.conf 2> /dev/null || true
-mv /etc/resolv2.conf /etc/resolv.conf
-"""
-
-
-class Machine(ssh_connection.SSHConnection):
- def __init__(self, address="127.0.0.1", image="unknown", verbose=False, label=None, browser=None,
- user="root", identity_file=None, arch="x86_64", ssh_port=22, web_port=9090):
-
- identity_file_old = identity_file
- identity_file = identity_file or DEFAULT_IDENTITY_FILE
-
- if identity_file_old is None:
- os.chmod(identity_file, 0o600)
- if ":" in address:
- (ssh_address, unused, ssh_port) = address.rpartition(":")
- else:
- ssh_address = address
- ssh_port = ssh_port
- if not browser:
- browser = address
-
- super(Machine, self).__init__(user, ssh_address, ssh_port, identity_file, verbose=verbose)
-
- self.arch = arch
- self.image = image
- self.atomic_image = self.image in ATOMIC_IMAGES
- if ":" in browser:
- (self.web_address, unused, self.web_port) = browser.rpartition(":")
- else:
- self.web_address = browser
- self.web_port = web_port
- if label:
- self.label = label
- elif self.image is not "unknown":
- self.label = "{}-{}-{}".format(self.image, self.ssh_address, self.ssh_port)
- else:
- self.label = "{}@{}:{}".format(self.ssh_user, self.ssh_address, self.ssh_port)
-
- # The Linux kernel boot_id
- self.boot_id = None
-
- def diagnose(self):
- keys = {
- "ssh_user": self.ssh_user,
- "ssh_address": self.ssh_address,
- "ssh_port": self.ssh_port,
- "web_address": self.web_address,
- "web_port": self.web_port,
- }
- return LOGIN_MESSAGE.format(**keys)
-
- def start(self):
- """Overridden by machine classes to start the machine"""
- self.message("Assuming machine is already running")
-
- def stop(self):
- """Overridden by machine classes to stop the machine"""
- self.message("Not shutting down already running machine")
-
- def wait_poweroff(self):
- """Overridden by machine classes to wait for a machine to stop"""
- assert False, "Cannot wait for a machine we didn't start"
-
- def kill(self):
- """Overridden by machine classes to unconditionally kill the running machine"""
- assert False, "Cannot kill a machine we didn't start"
-
- def shutdown(self):
- """Overridden by machine classes to gracefully shutdown the running machine"""
- assert False, "Cannot shutdown a machine we didn't start"
-
- def upload(self, sources, dest, relative_dir=TEST_DIR):
- """Upload a file into the test machine
-
- Arguments:
- sources: the array of paths of the file to upload
- dest: the file path in the machine to upload to
- """
- super(Machine, self).upload(sources, dest, relative_dir)
-
- def download(self, source, dest, relative_dir=TEST_DIR):
- """Download a file from the test machine.
- """
- super(Machine, self).download(source, dest, relative_dir)
-
- def download_dir(self, source, dest, relative_dir=TEST_DIR):
- """Download a directory from the test machine, recursively.
- """
- super(Machine, self).download_dir(source, dest, relative_dir)
-
- def journal_cursor(self):
- """Return current journal cursor
-
- This can be passed to journal_messages() or audit_messages().
- """
- return self.execute("journalctl --show-cursor -n0 -o cat | sed 's/^.*cursor: *//'")
-
- def journal_messages(self, syslog_ids, log_level, cursor=None):
- """Return interesting journal messages"""
-
- # give the OS some time to write pending log messages, to make
- # unexpected message detection more reliable; RHEL/CentOS 7 does not
- # yet know about --sync, so ignore failures
- self.execute("journalctl --sync 2>/dev/null || true; sleep 3; journalctl --sync 2>/dev/null || true")
-
- # Journald does not always set trusted fields like
- # _SYSTEMD_UNIT or _EXE correctly for the last few messages of
- # a dying process, so we filter by the untrusted but reliable
- # SYSLOG_IDENTIFIER instead
-
- matches = " ".join(map(lambda id: "SYSLOG_IDENTIFIER=" + id, syslog_ids))
-
- # Some versions of journalctl terminate unsuccessfully when
- # the output is empty. We work around this by ignoring the
- # exit status and including error messages from journalctl
- # itself in the returned messages.
-
- if cursor:
- cursor_arg = "--cursor '%s'" % cursor
- else:
- cursor_arg = ""
-
- cmd = "journalctl 2>&1 %s -o cat -p %d %s || true" % (cursor_arg, log_level, matches)
- messages = self.execute(cmd).splitlines()
- if len(messages) == 1 and ("Cannot assign requested address" in messages[0]
- or "-- No entries --" in messages[0]):
- # No messages
- return [ ]
- else:
- return messages
-
- def audit_messages(self, type_pref, cursor=None):
- if cursor:
- cursor_arg = "--cursor '%s'" % cursor
- else:
- cursor_arg = ""
-
- cmd = "journalctl %s -o cat SYSLOG_IDENTIFIER=kernel 2>&1 | grep 'type=%s.*audit' || true" % (cursor_arg, type_pref, )
- messages = self.execute(cmd).splitlines()
- if len(messages) == 1 and "Cannot assign requested address" in messages[0]:
- messages = [ ]
- return messages
-
- def get_admin_group(self):
- if "debian" in self.image or "ubuntu" in self.image:
- return "sudo"
- else:
- return "wheel"
-
- def start_cockpit(self, atomic_wait_for_host=None, tls=False):
- """Start Cockpit.
-
- Cockpit is not running when the test virtual machine starts up, to
- allow you to make modifications before it starts.
- """
-
- if self.atomic_image:
- # HACK: https://bugzilla.redhat.com/show_bug.cgi?id=1228776
- # we want to run:
- # self.execute("atomic run cockpit/ws --no-tls")
- # but atomic doesn't forward the parameter, so we use the resulting command
- # also we need to wait for cockpit to be up and running
- cmd = """#!/bin/sh
- systemctl start docker &&
- """
- if tls:
- cmd += "/usr/bin/docker run -d --privileged --pid=host -v /:/host cockpit/ws /container/atomic-run --local-ssh\n"
- else:
- cmd += "/usr/bin/docker run -d --privileged --pid=host -v /:/host cockpit/ws /container/atomic-run --local-ssh --no-tls\n"
- with timeout.Timeout(seconds=90, error_message="Timeout while waiting for cockpit/ws to start"):
- self.execute(script=cmd)
- self.wait_for_cockpit_running(atomic_wait_for_host or "localhost")
- elif tls:
- self.execute(script="""#!/bin/sh
- rm -f /etc/systemd/system/cockpit.service.d/notls.conf &&
- systemctl daemon-reload &&
- systemctl stop cockpit.service &&
- systemctl start cockpit.socket
- """)
- else:
- self.execute(script="""#!/bin/sh
- mkdir -p /etc/systemd/system/cockpit.service.d/ &&
- rm -f /etc/systemd/system/cockpit.service.d/notls.conf &&
- printf \"[Service]\nExecStartPre=-/bin/sh -c 'echo 0 > /proc/sys/kernel/yama/ptrace_scope'\nExecStart=\n%s --no-tls\n\" `systemctl cat cockpit.service | grep ExecStart=` > /etc/systemd/system/cockpit.service.d/notls.conf &&
- systemctl daemon-reload &&
- systemctl stop cockpit.service &&
- systemctl start cockpit.socket
- """)
-
- def restart_cockpit(self):
- """Restart Cockpit.
- """
- if self.atomic_image:
- with timeout.Timeout(seconds=90, error_message="timeoutlib.Timeout while waiting for cockpit/ws to restart"):
- self.execute("docker restart `docker ps | grep cockpit/ws | awk '{print $1;}'`")
- self.wait_for_cockpit_running()
- else:
- self.execute("systemctl restart cockpit")
-
- def stop_cockpit(self):
- """Stop Cockpit.
- """
- if self.atomic_image:
- with timeout.Timeout(seconds=60, error_message="Timeout while waiting for cockpit/ws to stop"):
- self.execute("docker kill `docker ps | grep cockpit/ws | awk '{print $1;}'`")
- else:
- self.execute("systemctl stop cockpit.socket cockpit.service")
-
- def set_address(self, address, mac='52:54:01'):
- """Set IP address for the network interface with given mac prefix"""
- cmd = "nmcli con add type ethernet autoconnect yes con-name static-{mac} ifname \"$(grep -l '{mac}' /sys/class/net/*/address | cut -d / -f 5)\" ip4 {address} && ( nmcli conn up static-{mac} || true )"
- self.execute(cmd.format(mac=mac, address=address))
-
- def set_dns(self, nameserver=None, domain=None):
- self.execute(RESOLV_SCRIPT.format(nameserver=nameserver or "127.0.0.1", domain=domain or "cockpit.lan"))
-
- def dhcp_server(self, mac='52:54:01', range=['10.111.112.2', '10.111.127.254']):
- """Sets up a DHCP server on the interface"""
- cmd = "dnsmasq --domain=cockpit.lan --interface=\"$(grep -l '{mac}' /sys/class/net/*/address | cut -d / -f 5)\" --bind-dynamic --dhcp-range=" + ','.join(range) + " && firewall-cmd --add-service=dhcp"
- self.execute(cmd.format(mac=mac))
-
- def dns_server(self, mac='52:54:01'):
- """Sets up a DNS server on the interface"""
- cmd = "dnsmasq --domain=cockpit.lan --interface=\"$(grep -l '{mac}' /sys/class/net/*/address | cut -d / -f 5)\" --bind-dynamic"
- self.execute(cmd.format(mac=mac))
-
- def wait_for_cockpit_running(self, address="localhost", port=9090, seconds=30, tls=False):
- WAIT_COCKPIT_RUNNING = """#!/bin/sh
- until curl --insecure --silent --connect-timeout 2 --max-time 3 %s://%s:%s >/dev/null; do
- sleep 0.5;
- done;
- """ % (tls and "https" or "http", address, port)
- with timeout.Timeout(seconds=seconds, error_message="Timeout while waiting for cockpit to start"):
- self.execute(script=WAIT_COCKPIT_RUNNING)
diff --git a/bots/machine/machine_core/machine_virtual.py b/bots/machine/machine_core/machine_virtual.py
deleted file mode 100644
index 804525f3d..000000000
--- a/bots/machine/machine_core/machine_virtual.py
+++ /dev/null
@@ -1,686 +0,0 @@
-# This file is part of Cockpit.
-#
-# Copyright (C) 2013 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-import contextlib
-import errno
-import fcntl
-import libvirt
-import libvirt_qemu
-import os
-import string
-import socket
-import subprocess
-import tempfile
-import sys
-import time
-
-from .exceptions import Failure, RepeatableFailure
-from .machine import Machine
-from .constants import TEST_DIR, BOTS_DIR
-from .directories import get_temp_dir
-
-MEMORY_MB = 1024
-
-
-# The Atomic variants can't build their own packages, so we build in
-# their non-Atomic siblings. For example, fedora-atomic is built
-# in fedora-29
-def get_build_image(image):
- (test_os, unused) = os.path.splitext(os.path.basename(image))
- if test_os == "fedora-atomic":
- image = "fedora-29"
- elif test_os == "rhel-atomic":
- image = "rhel-7-7"
- elif test_os == "continuous-atomic":
- image = "centos-7"
- return image
-
-
-# some tests have suffixes that run the same image in different modes; map a
-# test context image to an actual physical image name
-def get_test_image(image):
- return image.replace("-distropkg", "")
-
-
-# based on http://stackoverflow.com/a/17753573
-# we use this to quieten down calls
-@contextlib.contextmanager
-def stdchannel_redirected(stdchannel, dest_filename):
- """
- A context manager to temporarily redirect stdout or stderr
- e.g.:
- with stdchannel_redirected(sys.stderr, os.devnull):
- noisy_function()
- """
- try:
- stdchannel.flush()
- oldstdchannel = os.dup(stdchannel.fileno())
- dest_file = open(dest_filename, 'w')
- os.dup2(dest_file.fileno(), stdchannel.fileno())
- yield
- finally:
- if oldstdchannel is not None:
- os.dup2(oldstdchannel, stdchannel.fileno())
- if dest_file is not None:
- dest_file.close()
-
-
-TEST_CONSOLE_XML="""
-
-
-
-"""
-
-TEST_GRAPHICS_XML="""
-
-
-
-
-"""
-
-TEST_DOMAIN_XML="""
-
- {label}
- {cpu}
-
- hvm
-
- {loader}
-
- {memory_in_mib}
- {memory_in_mib}
-
-
-
-
-
-
-
-
- ROOT
-
-
- {console}
-
-
-
-
-
-
- /dev/urandom
-
- {bridgedev}
-
-
- {ethernet}
- {redir}
-
-
-"""
-
-TEST_DISK_XML="""
-
-
-
- %(serial)s
-
-
-
-"""
-
-TEST_KVM_XML="""
-
- {cpus}
-"""
-
-# The main network interface which we use to communicate between VMs
-TEST_MCAST_XML="""
-
-
-
-
-"""
-
-TEST_BRIDGE_XML="""
-
-
-
-
-
-"""
-
-# Used to access SSH from the main host into the virtual machines
-TEST_REDIR_XML="""
-
-
-
-
-"""
-
-class VirtNetwork:
- def __init__(self, network=None, bridge=None, image="generic"):
- self.locked = [ ]
- self.bridge = bridge
- self.image = image
-
- if network is None:
- offset = 0
- force = False
- else:
- offset = network * 100
- force = True
-
- # This is a shared port used as the identifier for the socket mcast network
- self.network = self._lock(5500 + offset, step=100, force=force)
-
- # An offset for other ports allocated later
- self.offset = (self.network - 5500)
-
- # The last machine we allocated
- self.last = 0
-
- # Unique hostnet identifiers
- self.hostnet = 8
-
- def _lock(self, start, step=1, force=False):
- resources = os.path.join(tempfile.gettempdir(), ".cockpit-test-resources")
- try:
- os.mkdir(resources, 0o755)
- except FileExistsError:
- pass
- for port in range(start, start + (100 * step), step):
- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- lockpath = os.path.join(resources, "network-{0}".format(port))
- try:
- lockf = os.open(lockpath, os.O_WRONLY | os.O_CREAT)
- fcntl.flock(lockf, fcntl.LOCK_NB | fcntl.LOCK_EX)
- sock.bind(("127.0.0.1", port))
- self.locked.append(lockf)
- except IOError:
- if force:
- return port
- os.close(lockf)
- continue
- else:
- return port
- finally:
- sock.close()
- raise Failure("Couldn't find unique network port number")
-
- # Create resources for an interface, returns address and XML
- def interface(self, number=None):
- if number is None:
- number = self.last + 1
- if number > self.last:
- self.last = number
- mac = self._lock(10000 + self.offset + number) - (10000 + self.offset)
- hostnet = self.hostnet
- self.hostnet += 1
- result = {
- "number": self.offset + number,
- "mac": '52:54:01:{:02x}:{:02x}:{:02x}'.format((mac >> 16) & 0xff, (mac >> 8) & 0xff, mac & 0xff),
- "name": "m{0}.cockpit.lan".format(mac),
- "mcast": self.network,
- "hostnet": "hostnet{0}".format(hostnet)
- }
- return result
-
- # Create resources for a host, returns address and XML
- def host(self, number=None, restrict=False, isolate=False, forward={ }):
- result = self.interface(number)
- result["mcast"] = self.network
- result["restrict"] = restrict and "on" or "off"
- result["forward"] = { "22": 2200, "9090": 9090 }
- result["forward"].update(forward)
- result["netdriver"] = ("windows" in self.image) and "rtl8139" or "virtio-net-pci"
- forwards = []
- for remote, local in result["forward"].items():
- local = self._lock(int(local) + result["number"])
- result["forward"][remote] = "127.0.0.2:{}".format(local)
- forwards.append("hostfwd=tcp:{}-:{}".format(result["forward"][remote], remote))
- if remote == "22":
- result["control"] = result["forward"][remote]
- elif remote == "9090":
- result["browser"] = result["forward"][remote]
-
- if isolate:
- result["bridge"] = ""
- result["bridgedev"] = ""
- result["ethernet"] = ""
- elif self.bridge:
- result["bridge"] = self.bridge
- result["bridgedev"] = TEST_BRIDGE_XML.format(**result)
- result["ethernet"] = ""
- else:
- result["bridge"] = ""
- result["bridgedev"] = ""
- result["ethernet"] = TEST_MCAST_XML.format(**result)
- result["forwards"] = ",".join(forwards)
- result["redir"] = TEST_REDIR_XML.format(**result)
- return result
-
- def kill(self):
- locked = self.locked
- self.locked = [ ]
- for x in locked:
- os.close(x)
-
-class VirtMachine(Machine):
- network = None
- memory_mb = None
- cpus = None
-
- def __init__(self, image, networking=None, maintain=False, memory_mb=None, cpus=None, graphics=False, **args):
- self.maintain = maintain
-
- # Currently all images are run on x86_64. When that changes we will have
- # an override file for those images that are not
- self.arch = "x86_64"
-
- self.memory_mb = memory_mb or VirtMachine.memory_mb or MEMORY_MB
- self.cpus = cpus or VirtMachine.cpus or 1
- self.graphics = graphics or "windows" in image
-
- # Set up some temporary networking info if necessary
- if networking is None:
- networking = VirtNetwork(image=image).host()
-
- # Allocate network information about this machine
- self.networking = networking
- args["address"] = networking["control"]
- args["browser"] = networking["browser"]
- self.forward = networking["forward"]
-
- # The path to the image file to load, and parse an image name
- if "/" in image:
- self.image_file = image = os.path.abspath(image)
- else:
- self.image_file = os.path.join(TEST_DIR, "images", image)
- if not os.path.lexists(self.image_file):
- self.image_file = os.path.join(BOTS_DIR, "images", image)
- (image, extension) = os.path.splitext(os.path.basename(image))
-
- Machine.__init__(self, image=image, **args)
-
- base_dir = os.path.dirname(BOTS_DIR)
- self.run_dir = os.path.join(get_temp_dir(), "run")
-
- self.virt_connection = self._libvirt_connection(hypervisor = "qemu:///session")
-
- self._disks = [ ]
- self._domain = None
-
- # init variables needed for running a vm
- self._cleanup()
-
- def _libvirt_connection(self, hypervisor, read_only = False):
- tries_left = 5
- connection = None
- if read_only:
- open_function = libvirt.openReadOnly
- else:
- open_function = libvirt.open
- while not connection and (tries_left > 0):
- try:
- connection = open_function(hypervisor)
- except:
- # wait a bit
- time.sleep(1)
- pass
- tries_left -= 1
- if not connection:
- # try again, but if an error occurs, don't catch it
- connection = open_function(hypervisor)
- return connection
-
- def _start_qemu(self):
- self._cleanup()
-
- os.makedirs(self.run_dir, 0o750, exist_ok=True)
-
- def execute(*args):
- self.message(*args)
- return subprocess.check_call(args)
-
- image_to_use = self.image_file
- if not self.maintain:
- (unused, self._transient_image) = tempfile.mkstemp(suffix='.qcow2', prefix="", dir=self.run_dir)
- execute("qemu-img", "create", "-q", "-f", "qcow2",
- "-o", "backing_file=%s" % self.image_file, self._transient_image)
- image_to_use = self._transient_image
-
- keys = {
- "label": self.label,
- "image": self.image,
- "type": "qemu",
- "arch": self.arch,
- "cpu": "",
- "cpus": self.cpus,
- "memory_in_mib": self.memory_mb,
- "drive": image_to_use,
- "iso": os.path.join(BOTS_DIR, "machine", "cloud-init.iso"),
- }
-
- if os.path.exists("/dev/kvm"):
- keys["type"] = "kvm"
- keys["cpu"] = TEST_KVM_XML.format(**keys)
- else:
- sys.stderr.write("WARNING: Starting virtual machine with emulation due to missing KVM\n")
- sys.stderr.write("WARNING: Machine will run about 10-20 times slower\n")
-
- keys.update(self.networking)
- keys["name"] = "{image}-{control}".format(**keys)
-
- # No need or use for redir network on windows
- if "windows" in self.image:
- keys["disk"] = "ide"
- keys["redir"] = ""
- else:
- keys["disk"] = "virtio"
- if self.graphics:
- keys["console"] = TEST_GRAPHICS_XML.format(**keys)
- else:
- keys["console"] = TEST_CONSOLE_XML.format(**keys)
- if "windows-10" in self.image:
- keys["loader"] = "/usr/share/edk2/ovmf/OVMF_CODE.fd"
- else:
- keys["loader"] = ""
- test_domain_desc = TEST_DOMAIN_XML.format(**keys)
-
- # add the virtual machine
- try:
- # print >> sys.stderr, test_domain_desc
- self._domain = self.virt_connection.createXML(test_domain_desc, libvirt.VIR_DOMAIN_START_AUTODESTROY)
- except libvirt.libvirtError as le:
- if 'already exists with uuid' in str(le):
- raise RepeatableFailure("libvirt domain already exists: " + str(le))
- else:
- raise
-
- # start virsh console
- def qemu_console(self, extra_message=""):
- self.message("Started machine {0}".format(self.label))
- if self.maintain:
- message = "\nWARNING: Uncontrolled shutdown can lead to a corrupted image\n"
- else:
- message = "\nWARNING: All changes are discarded, the image file won't be changed\n"
- message += self.diagnose() + extra_message + "\nlogin: "
- message = message.replace("\n", "\r\n")
-
- try:
- proc = subprocess.Popen("virsh -c qemu:///session console %s" % str(self._domain.ID()), shell=True)
-
- # Fill in information into /etc/issue about login access
- pid = 0
- while pid == 0:
- if message:
- try:
- with stdchannel_redirected(sys.stderr, os.devnull):
- Machine.wait_boot(self)
- sys.stderr.write(message)
- except (Failure, subprocess.CalledProcessError):
- pass
- message = None
- (pid, ret) = os.waitpid(proc.pid, message and os.WNOHANG or 0)
-
- try:
- if self.maintain:
- self.shutdown()
- else:
- self.kill()
- except libvirt.libvirtError as le:
- # the domain may have already been freed (shutdown) while the console was running
- self.message("libvirt error during shutdown: %s" % (le.get_error_message()))
-
- except OSError as ex:
- raise Failure("Failed to launch virsh command: {0}".format(ex.strerror))
- finally:
- self._cleanup()
-
- def graphics_console(self):
- self.message("Started machine {0}".format(self.label))
- if self.maintain:
- message = "\nWARNING: Uncontrolled shutdown can lead to a corrupted image\n"
- else:
- message = "\nWARNING: All changes are discarded, the image file won't be changed\n"
- if "bridge" in self.networking:
- message += "\nIn the machine a web browser can access Cockpit on parent host:\n\n"
- message += " https://10.111.112.1:9090\n"
- message = message.replace("\n", "\r\n")
-
- try:
- proc = subprocess.Popen(["virt-viewer", str(self._domain.ID())])
- sys.stderr.write(message)
- proc.wait()
- except OSError as ex:
- raise Failure("Failed to launch virt-viewer command: {0}".format(ex.strerror))
- finally:
- self._cleanup()
-
- def pull(self, image):
- if "/" in image:
- image_file = os.path.abspath(image)
- else:
- image_file = os.path.join(BOTS_DIR, "images", image)
- if not os.path.exists(image_file):
- try:
- subprocess.check_call([ os.path.join(BOTS_DIR, "image-download"), image_file ])
- except OSError as ex:
- if ex.errno != errno.ENOENT:
- raise
- return image_file
-
- def start(self):
- tries = 0
- while True:
- try:
- self._start_qemu()
- if not self._domain.isActive():
- self._domain.start()
- except RepeatableFailure:
- self.kill()
- if tries < 10:
- tries += 1
- time.sleep(tries)
- continue
- else:
- raise
- except:
- self.kill()
- raise
-
- # Normally only one pass
- break
-
- def _diagnose_no_address(self):
- SCRIPT = """
- spawn virsh -c qemu:///session console $argv
- set timeout 300
- expect "Escape character"
- send "\r"
- expect " login: "
- send "root\r"
- expect "Password: "
- send "foobar\r"
- expect " ~]# "
- send "ip addr\r\n"
- expect " ~]# "
- exit 0
- """
- expect = subprocess.Popen(["expect", "--", "-", str(self._domain.ID())], stdin=subprocess.PIPE,
- universal_newlines=True)
- expect.communicate(SCRIPT)
-
- def wait_boot(self, timeout_sec=120):
- """Wait for a machine to boot"""
- try:
- Machine.wait_boot(self, timeout_sec)
- except Failure:
- self._diagnose_no_address()
- raise
-
- def stop(self, timeout_sec=120):
- if self.maintain:
- self.shutdown(timeout_sec=timeout_sec)
- else:
- self.kill()
-
- def _cleanup(self, quick=False):
- self.disconnect()
- try:
- for disk in self._disks:
- self.rem_disk(disk, quick)
-
- self._domain = None
- if hasattr(self, '_transient_image') and self._transient_image and os.path.exists(self._transient_image):
- os.unlink(self._transient_image)
- except:
- (type, value, traceback) = sys.exc_info()
- sys.stderr.write("WARNING: Cleanup failed:%s\n" % value)
-
- def kill(self):
- # stop system immediately, with potential data loss
- # to shutdown gracefully, use shutdown()
- try:
- self.disconnect()
- except Exception:
- pass
- if self._domain:
- try:
- # not graceful
- with stdchannel_redirected(sys.stderr, os.devnull):
- self._domain.destroyFlags(libvirt.VIR_DOMAIN_DESTROY_DEFAULT)
- except:
- pass
- self._cleanup(quick=True)
-
- def wait_poweroff(self, timeout_sec=120):
- # shutdown must have already been triggered
- if self._domain:
- start_time = time.time()
- while (time.time() - start_time) < timeout_sec:
- try:
- with stdchannel_redirected(sys.stderr, os.devnull):
- if not self._domain.isActive():
- break
- except libvirt.libvirtError as le:
- if 'no domain' in str(le) or 'not found' in str(le):
- break
- raise
- time.sleep(1)
- else:
- raise Failure("Waiting for machine poweroff timed out")
- try:
- with stdchannel_redirected(sys.stderr, os.devnull):
- self._domain.destroyFlags(libvirt.VIR_DOMAIN_DESTROY_DEFAULT)
- except libvirt.libvirtError as le:
- if 'not found' not in str(le):
- raise
- self._cleanup(quick=True)
-
- def shutdown(self, timeout_sec=120):
- # shutdown the system gracefully
- # to stop it immediately, use kill()
- self.disconnect()
- try:
- if self._domain:
- self._domain.shutdown()
- self.wait_poweroff(timeout_sec=timeout_sec)
- finally:
- self._cleanup()
-
- def add_disk(self, size=None, serial=None, path=None, type='raw'):
- index = len(self._disks)
-
- os.makedirs(self.run_dir, 0o750, exist_ok=True)
-
- if path:
- (unused, image) = tempfile.mkstemp(suffix='.qcow2', prefix=os.path.basename(path), dir=self.run_dir)
- subprocess.check_call([ "qemu-img", "create", "-q", "-f", "qcow2",
- "-o", "backing_file=" + os.path.realpath(path), image ])
-
- else:
- assert size is not None
- name = "disk-{0}".format(self._domain.name())
- (unused, image) = tempfile.mkstemp(suffix='qcow2', prefix=name, dir=self.run_dir)
- subprocess.check_call(["qemu-img", "create", "-q", "-f", "raw", image, str(size)])
-
- if not serial:
- serial = "DISK{0}".format(index)
- dev = 'sd' + string.ascii_lowercase[index]
- disk_desc = TEST_DISK_XML % {
- 'file': image,
- 'serial': serial,
- 'unit': index,
- 'dev': dev,
- 'type': type,
- }
-
- if self._domain.attachDeviceFlags(disk_desc, libvirt.VIR_DOMAIN_AFFECT_LIVE) != 0:
- raise Failure("Unable to add disk to vm")
-
- disk = {
- "path": image,
- "serial": serial,
- "filename": image,
- "dev": dev,
- "index": index,
- "type": type,
- }
-
- self._disks.append(disk)
- return disk
-
- def rem_disk(self, disk, quick=False):
- if not quick:
- disk_desc = TEST_DISK_XML % {
- 'file': disk["filename"],
- 'serial': disk["serial"],
- 'unit': disk["index"],
- 'dev': disk["dev"],
- 'type': disk["type"]
- }
-
- if self._domain:
- if self._domain.detachDeviceFlags(disk_desc, libvirt.VIR_DOMAIN_AFFECT_LIVE ) != 0:
- raise Failure("Unable to remove disk from vm")
-
- def _qemu_monitor(self, command):
- self.message("& " + command)
- # you can run commands manually using virsh:
- # virsh -c qemu:///session qemu-monitor-command [domain name/id] --hmp [command]
- output = libvirt_qemu.qemuMonitorCommand(self._domain, command, libvirt_qemu.VIR_DOMAIN_QEMU_MONITOR_COMMAND_HMP)
- self.message(output.strip())
- return output
-
- def add_netiface(self, networking=None):
- if not networking:
- networking = VirtNetwork(image=self.image).interface()
- self._qemu_monitor("netdev_add socket,mcast=230.0.0.1:{mcast},id={id}".format(mcast=networking["mcast"], id=networking["hostnet"]))
- cmd = "device_add virtio-net-pci,mac={0},netdev={1}".format(networking["mac"], networking["hostnet"])
- self._qemu_monitor("device_add virtio-net-pci,mac={0},netdev={1}".format(networking["mac"], networking["hostnet"]))
- return networking["mac"]
-
- def needs_writable_usr(self):
- # On atomic systems, we need a hack to change files in /usr/lib/systemd
- if self.atomic_image:
- self.execute(command="mount -o remount,rw /usr")
diff --git a/bots/machine/machine_core/ssh_connection.py b/bots/machine/machine_core/ssh_connection.py
deleted file mode 100644
index f20f69572..000000000
--- a/bots/machine/machine_core/ssh_connection.py
+++ /dev/null
@@ -1,462 +0,0 @@
-# This file is part of Cockpit.
-#
-# Copyright (C) 2013 Red Hat, Inc.
-#
-# Cockpit is free software; you can redistribute it and/or modify it
-# under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation; either version 2.1 of the License, or
-# (at your option) any later version.
-#
-# Cockpit is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Cockpit; If not, see .
-
-
-import os
-import time
-import socket
-import subprocess
-import tempfile
-import select
-import errno
-import sys
-
-from . import exceptions
-from . import timeout as timeoutlib
-from .directories import get_temp_dir
-
-
-class SSHConnection(object):
- def __init__(self, user, address, ssh_port, identity_file, verbose=False):
- self.verbose = verbose
-
- # Currently all images are x86_64. When that changes we will have
- # an override file for those images that are not
- self.ssh_user = user
- self.identity_file = identity_file
- self.ssh_address = address
- self.ssh_port = ssh_port
- self.ssh_master = None
- self.ssh_process = None
- self.ssh_reachable = False
- self.label = "{}@{}:{}".format(self.ssh_user, self.ssh_address, self.ssh_port)
-
- def disconnect(self):
- self.ssh_reachable = False
- self._kill_ssh_master()
-
- def message(self, *args):
- """Prints args if in verbose mode"""
- if not self.verbose:
- return
- print(" ".join(args))
-
- # wait until we can execute something on the machine. ie: wait for ssh
- def wait_execute(self, timeout_sec=120):
- """Try to connect to self.address on ssh port"""
-
- # If connected to machine, kill master connection
- self._kill_ssh_master()
-
- start_time = time.time()
- while (time.time() - start_time) < timeout_sec:
- addrinfo = socket.getaddrinfo(self.ssh_address, self.ssh_port, 0, socket.SOCK_STREAM)
- (family, socktype, proto, canonname, sockaddr) = addrinfo[0]
- sock = socket.socket(family, socktype, proto)
- sock.settimeout(5)
- try:
- sock.connect(sockaddr)
- data = sock.recv(10)
- if len(data):
- self.ssh_reachable = True
- return True
- except IOError:
- pass
- finally:
- sock.close()
- time.sleep(0.5)
- return False
-
- def wait_user_login(self):
- """Wait until logging in as non-root works.
-
- Most tests run as the "admin" user, so we make sure that
- user sessions are allowed (and cockit-ws will let "admin"
- in) before declaring a test machine as "booted".
-
- Returns the boot id of the system, or None if ssh timed out.
- """
- tries_left = 60
- while (tries_left > 0):
- try:
- with timeoutlib.Timeout(seconds=30):
- return self.execute("! test -f /run/nologin && cat /proc/sys/kernel/random/boot_id", direct=True)
- except subprocess.CalledProcessError:
- pass
- except RuntimeError:
- # timeout; assume that ssh just went down during reboot, go back to wait_boot()
- return None
- tries_left = tries_left - 1
- time.sleep(1)
- raise exceptions.Failure("Timed out waiting for /run/nologin to disappear")
-
- def wait_boot(self, timeout_sec=120):
- """Wait for a machine to boot"""
- start_time = time.time()
- boot_id = None
- while (time.time() - start_time) < timeout_sec:
- if self.wait_execute(timeout_sec=15):
- boot_id = self.wait_user_login()
- if boot_id:
- break
- if not boot_id:
- raise exceptions.Failure("Unable to reach machine {0} via ssh: {1}:{2}".format(self.label, self.ssh_address, self.ssh_port))
- self.boot_id = boot_id
-
- def wait_reboot(self, timeout_sec=180):
- self.disconnect()
- assert self.boot_id, "Before using wait_reboot() use wait_boot() successfully"
- boot_id = self.boot_id
- start_time = time.time()
- while (time.time() - start_time) < timeout_sec:
- try:
- self.wait_boot(timeout_sec=timeout_sec)
- if self.boot_id != boot_id:
- break
- except exceptions.Failure:
- pass
- else:
- raise exceptions.Failure("Timeout waiting for system to reboot properly")
-
-
- def _start_ssh_master(self):
- self._kill_ssh_master()
-
- control = os.path.join(get_temp_dir(), "ssh-%h-%p-%r-" + str(os.getpid()))
-
- cmd = [
- "ssh",
- "-p", str(self.ssh_port),
- "-i", self.identity_file,
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "BatchMode=yes",
- "-M", # ControlMaster, no stdin
- "-o", "ControlPath=" + control,
- "-o", "LogLevel=ERROR",
- "-l", self.ssh_user,
- self.ssh_address,
- "/bin/bash -c 'echo READY; read a'"
- ]
-
- # Connection might be refused, so try this 10 times
- tries_left = 10
- while tries_left > 0:
- tries_left = tries_left - 1
- proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
- stdout_fd = proc.stdout.fileno()
- output = ""
- while stdout_fd > -1 and "READY" not in output:
- ret = select.select([stdout_fd], [], [], 10)
- for fd in ret[0]:
- if fd == stdout_fd:
- data = os.read(fd, 1024)
- if not data:
- stdout_fd = -1
- proc.stdout.close()
- output += data.decode('utf-8', 'replace')
-
- if stdout_fd > -1:
- break
-
- # try again if the connection was refused, unless we've used up our tries
- proc.wait()
- if proc.returncode == 255 and tries_left > 0:
- self.message("ssh: connection refused, trying again")
- time.sleep(1)
- continue
- else:
- raise exceptions.Failure("SSH master process exited with code: {0}".format(proc.returncode))
-
- self.ssh_master = control
- self.ssh_process = proc
-
- if not self._check_ssh_master():
- raise exceptions.Failure("Couldn't launch an SSH master process")
-
- def _kill_ssh_master(self):
- if self.ssh_master:
- try:
- os.unlink(self.ssh_master)
- except OSError as e:
- if e.errno != errno.ENOENT:
- raise
- self.ssh_master = None
- if self.ssh_process:
- self.message("killing ssh master process", str(self.ssh_process.pid))
- self.ssh_process.stdin.close()
- self.ssh_process.terminate()
- self.ssh_process.stdout.close()
- with timeoutlib.Timeout(seconds=90, error_message="Timeout while waiting for ssh master to shut down"):
- self.ssh_process.wait()
- self.ssh_process = None
-
- def _check_ssh_master(self):
- if not self.ssh_master:
- return False
- cmd = [
- "ssh",
- "-q",
- "-p", str(self.ssh_port),
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "BatchMode=yes",
- "-S", self.ssh_master,
- "-O", "check",
- "-l", self.ssh_user,
- self.ssh_address
- ]
- with open(os.devnull, 'w') as devnull:
- code = subprocess.call(cmd, stdin=devnull, stdout=devnull, stderr=devnull)
- if code == 0:
- self.ssh_reachable = True
- return True
- return False
-
- def _ensure_ssh_master(self):
- if not self._check_ssh_master():
- self._start_ssh_master()
-
- def execute(self, command=None, script=None, input=None, environment={},
- stdout=None, quiet=False, direct=False, timeout=120,
- ssh_env=["env", "-u", "LANGUAGE", "LC_ALL=C"]):
- """Execute a shell command in the test machine and return its output.
-
- Either specify @command or @script
-
- Arguments:
- command: The string to execute by /bin/sh.
- script: A multi-line script to execute in /bin/sh
- input: Input to send to the command
- environment: Additional environment variables
- timeout: Applies if not already wrapped in a #Timeout context
- Returns:
- The command/script output as a string.
- """
- assert command or script
- assert self.ssh_address
-
- if not direct:
- self._ensure_ssh_master()
-
- env_script = ""
- env_command = []
- if environment and isinstance(environment, dict):
- for name, value in environment.items():
- env_script += "%s='%s'\n" % (name, value)
- env_script += "export %s\n" % name
- env_command.append("{}={}".format(name, value))
- elif environment == {}:
- pass
- else:
- raise Exception("enviroment support dict or list items given: ".format(environment))
- default_ssh_params = [
- "ssh",
- "-p", str(self.ssh_port),
- "-o", "StrictHostKeyChecking=no",
- "-o", "UserKnownHostsFile=/dev/null",
- "-o", "LogLevel=ERROR",
- "-o", "BatchMode=yes",
- "-l", self.ssh_user,
- self.ssh_address
- ]
- additional_ssh_params = []
- cmd = []
-
- if direct:
- additional_ssh_params += ["-i", self.identity_file]
- else:
- additional_ssh_params += ["-o", "ControlPath=" + self.ssh_master]
-
- if command:
- if getattr(command, "strip", None): # Is this a string?
- cmd += [command]
- if not quiet:
- self.message("+", command)
- else:
- cmd += command
- if not quiet:
- self.message("+", *command)
- else:
- assert not input, "input not supported to script"
- cmd += ["sh", "-s"]
- if self.verbose:
- cmd += ["-x"]
- input = env_script
- input += script
- command = "
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Logs
-
- Result directory
- Raw log
-
-
- Done: .
-
-
-
-
-
-