Merge branch 'python-vagrant-copy-caches' into 'master'
complete staging buildserver setup on jenkins.debian.net using nested KVM instances See merge request !176
This commit is contained in:
Hans-Christoph Steiner
commit
361ce5ca41
|
|
@ -28,6 +28,16 @@
|
|||
#
|
||||
# apt_package_cache = True
|
||||
|
||||
# The buildserver can use some local caches to speed up builds,
|
||||
# especially when the internet connection is slow and/or expensive.
|
||||
# If enabled, the buildserver setup will look for standard caches in
|
||||
# your HOME dir and copy them to the buildserver VM. Be aware: this
|
||||
# will reduce the isolation of the buildserver from your host machine,
|
||||
# so the buildserver will provide an environment only as trustworthy
|
||||
# as the host machine's environment.
|
||||
#
|
||||
# copy_caches_from_host = True
|
||||
|
||||
# To specify which Debian mirror the build server VM should use, by
|
||||
# default it uses http.debian.net, which auto-detects which is the
|
||||
# best mirror to use.
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import re
|
|||
import tarfile
|
||||
import traceback
|
||||
import time
|
||||
import json
|
||||
import requests
|
||||
import tempfile
|
||||
from configparser import ConfigParser
|
||||
|
|
@ -37,6 +36,7 @@ from . import common
|
|||
from . import net
|
||||
from . import metadata
|
||||
from . import scanner
|
||||
from . import vmtools
|
||||
from .common import FDroidPopen, SdkToolsPopen
|
||||
from .exception import FDroidException, BuildException, VCSException
|
||||
|
||||
|
|
@ -46,205 +46,6 @@ except ImportError:
|
|||
pass
|
||||
|
||||
|
||||
def get_builder_vm_id():
|
||||
vd = os.path.join('builder', '.vagrant')
|
||||
if os.path.isdir(vd):
|
||||
# Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
|
||||
with open(os.path.join(vd, 'machines', 'default',
|
||||
'virtualbox', 'id')) as vf:
|
||||
id = vf.read()
|
||||
return id
|
||||
else:
|
||||
# Vagrant 1.0 - it's a json file...
|
||||
with open(os.path.join('builder', '.vagrant')) as vf:
|
||||
v = json.load(vf)
|
||||
return v['active']['default']
|
||||
|
||||
|
||||
def got_valid_builder_vm():
|
||||
"""Returns True if we have a valid-looking builder vm
|
||||
"""
|
||||
if not os.path.exists(os.path.join('builder', 'Vagrantfile')):
|
||||
return False
|
||||
vd = os.path.join('builder', '.vagrant')
|
||||
if not os.path.exists(vd):
|
||||
return False
|
||||
if not os.path.isdir(vd):
|
||||
# Vagrant 1.0 - if the directory is there, it's valid...
|
||||
return True
|
||||
# Vagrant 1.2 - the directory can exist, but the id can be missing...
|
||||
if not os.path.exists(os.path.join(vd, 'machines', 'default',
|
||||
'virtualbox', 'id')):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def vagrant(params, cwd=None, printout=False):
|
||||
"""Run a vagrant command.
|
||||
|
||||
:param: list of parameters to pass to vagrant
|
||||
:cwd: directory to run in, or None for current directory
|
||||
:returns: (ret, out) where ret is the return code, and out
|
||||
is the stdout (and stderr) from vagrant
|
||||
"""
|
||||
p = FDroidPopen(['vagrant'] + params, cwd=cwd)
|
||||
return (p.returncode, p.output)
|
||||
|
||||
|
||||
def get_vagrant_sshinfo():
|
||||
"""Get ssh connection info for a vagrant VM
|
||||
|
||||
:returns: A dictionary containing 'hostname', 'port', 'user'
|
||||
and 'idfile'
|
||||
"""
|
||||
if subprocess.call('vagrant ssh-config >sshconfig',
|
||||
cwd='builder', shell=True) != 0:
|
||||
raise BuildException("Error getting ssh config")
|
||||
vagranthost = 'default' # Host in ssh config file
|
||||
sshconfig = paramiko.SSHConfig()
|
||||
sshf = open(os.path.join('builder', 'sshconfig'), 'r')
|
||||
sshconfig.parse(sshf)
|
||||
sshf.close()
|
||||
sshconfig = sshconfig.lookup(vagranthost)
|
||||
idfile = sshconfig['identityfile']
|
||||
if isinstance(idfile, list):
|
||||
idfile = idfile[0]
|
||||
elif idfile.startswith('"') and idfile.endswith('"'):
|
||||
idfile = idfile[1:-1]
|
||||
return {'hostname': sshconfig['hostname'],
|
||||
'port': int(sshconfig['port']),
|
||||
'user': sshconfig['user'],
|
||||
'idfile': idfile}
|
||||
|
||||
|
||||
def get_clean_vm(reset=False):
|
||||
"""Get a clean VM ready to do a buildserver build.
|
||||
|
||||
This might involve creating and starting a new virtual machine from
|
||||
scratch, or it might be as simple (unless overridden by the reset
|
||||
parameter) as re-using a snapshot created previously.
|
||||
|
||||
A BuildException will be raised if anything goes wrong.
|
||||
|
||||
:reset: True to force creating from scratch.
|
||||
:returns: A dictionary containing 'hostname', 'port', 'user'
|
||||
and 'idfile'
|
||||
"""
|
||||
# Reset existing builder machine to a clean state if possible.
|
||||
vm_ok = False
|
||||
if not reset:
|
||||
logging.info("Checking for valid existing build server")
|
||||
|
||||
if got_valid_builder_vm():
|
||||
logging.info("...VM is present")
|
||||
p = FDroidPopen(['VBoxManage', 'snapshot',
|
||||
get_builder_vm_id(), 'list',
|
||||
'--details'], cwd='builder')
|
||||
if 'fdroidclean' in p.output:
|
||||
logging.info("...snapshot exists - resetting build server to "
|
||||
"clean state")
|
||||
retcode, output = vagrant(['status'], cwd='builder')
|
||||
|
||||
if 'running' in output:
|
||||
logging.info("...suspending")
|
||||
vagrant(['suspend'], cwd='builder')
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
|
||||
'restore', 'fdroidclean'],
|
||||
cwd='builder')
|
||||
|
||||
if p.returncode == 0:
|
||||
logging.info("...reset to snapshot - server is valid")
|
||||
retcode, output = vagrant(['up'], cwd='builder')
|
||||
if retcode != 0:
|
||||
raise BuildException("Failed to start build server")
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
sshinfo = get_vagrant_sshinfo()
|
||||
vm_ok = True
|
||||
else:
|
||||
logging.info("...failed to reset to snapshot")
|
||||
else:
|
||||
logging.info("...snapshot doesn't exist - "
|
||||
"VBoxManage snapshot list:\n" + p.output)
|
||||
|
||||
# If we can't use the existing machine for any reason, make a
|
||||
# new one from scratch.
|
||||
if not vm_ok:
|
||||
if os.path.exists('builder'):
|
||||
logging.info("Removing broken/incomplete/unwanted build server")
|
||||
vagrant(['destroy', '-f'], cwd='builder')
|
||||
shutil.rmtree('builder')
|
||||
os.mkdir('builder')
|
||||
|
||||
p = subprocess.Popen(['vagrant', '--version'],
|
||||
universal_newlines=True,
|
||||
stdout=subprocess.PIPE)
|
||||
vver = p.communicate()[0].strip().split(' ')[1]
|
||||
if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4:
|
||||
raise BuildException("Unsupported vagrant version {0}".format(vver))
|
||||
|
||||
with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf:
|
||||
vf.write('Vagrant.configure("2") do |config|\n')
|
||||
vf.write('config.vm.box = "buildserver"\n')
|
||||
vf.write('config.vm.synced_folder ".", "/vagrant", disabled: true\n')
|
||||
vf.write('end\n')
|
||||
|
||||
logging.info("Starting new build server")
|
||||
retcode, _ = vagrant(['up'], cwd='builder')
|
||||
if retcode != 0:
|
||||
raise BuildException("Failed to start build server")
|
||||
|
||||
# Open SSH connection to make sure it's working and ready...
|
||||
logging.info("Connecting to virtual machine...")
|
||||
sshinfo = get_vagrant_sshinfo()
|
||||
sshs = paramiko.SSHClient()
|
||||
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
|
||||
port=sshinfo['port'], timeout=300,
|
||||
look_for_keys=False,
|
||||
key_filename=sshinfo['idfile'])
|
||||
sshs.close()
|
||||
|
||||
logging.info("Saving clean state of new build server")
|
||||
retcode, _ = vagrant(['suspend'], cwd='builder')
|
||||
if retcode != 0:
|
||||
raise BuildException("Failed to suspend build server")
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
|
||||
'take', 'fdroidclean'],
|
||||
cwd='builder')
|
||||
if p.returncode != 0:
|
||||
raise BuildException("Failed to take snapshot")
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
logging.info("Restarting new build server")
|
||||
retcode, _ = vagrant(['up'], cwd='builder')
|
||||
if retcode != 0:
|
||||
raise BuildException("Failed to start build server")
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
# Make sure it worked...
|
||||
p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
|
||||
'list', '--details'],
|
||||
cwd='builder')
|
||||
if 'fdroidclean' not in p.output:
|
||||
raise BuildException("Failed to take snapshot.")
|
||||
|
||||
return sshinfo
|
||||
|
||||
|
||||
def release_vm():
|
||||
"""Release the VM previously started with get_clean_vm().
|
||||
|
||||
This should always be called.
|
||||
"""
|
||||
logging.info("Suspending build server")
|
||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
||||
|
||||
|
||||
# Note that 'force' here also implies test mode.
|
||||
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
||||
"""Do a build on the builder vm.
|
||||
|
|
@ -268,7 +69,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
|||
else:
|
||||
logging.getLogger("paramiko").setLevel(logging.WARN)
|
||||
|
||||
sshinfo = get_clean_vm()
|
||||
sshinfo = vmtools.get_clean_builder('builder')
|
||||
|
||||
try:
|
||||
if not buildserverid:
|
||||
|
|
@ -455,9 +256,9 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
|||
ftp.close()
|
||||
|
||||
finally:
|
||||
|
||||
# Suspend the build server.
|
||||
release_vm()
|
||||
vm = vmtools.get_build_vm('builder')
|
||||
vm.suspend()
|
||||
|
||||
|
||||
def force_gradle_build_tools(build_dir, build_tools):
|
||||
|
|
@ -989,7 +790,7 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir,
|
|||
this is the 'unsigned' directory.
|
||||
:param repo_dir: The repo directory - used for checking if the build is
|
||||
necessary.
|
||||
:paaram also_check_dir: An additional location for checking if the build
|
||||
:param also_check_dir: An additional location for checking if the build
|
||||
is necessary (usually the archive repo)
|
||||
:param test: True if building in test mode, in which case the build will
|
||||
always happen, even if the output already exists. In test mode, the
|
||||
|
|
|
|||
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
'''
|
||||
Python-Tail - Unix tail follow implementation in Python.
|
||||
|
||||
python-tail can be used to monitor changes to a file.
|
||||
|
||||
Example:
|
||||
import tail
|
||||
|
||||
# Create a tail instance
|
||||
t = tail.Tail('file-to-be-followed')
|
||||
|
||||
# Register a callback function to be called when a new line is found in the followed file.
|
||||
# If no callback function is registerd, new lines would be printed to standard out.
|
||||
t.register_callback(callback_function)
|
||||
|
||||
# Follow the file with 5 seconds as sleep time between iterations.
|
||||
# If sleep time is not provided 1 second is used as the default time.
|
||||
t.follow(s=5) '''
|
||||
|
||||
# Author - Kasun Herath <kasunh01 at gmail.com>
|
||||
# Source - https://github.com/kasun/python-tail
|
||||
|
||||
# modified by Hans-Christoph Steiner <hans@eds.org> to add the
|
||||
# background thread and support reading multiple lines per read cycle
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
|
||||
|
||||
class Tail(object):
|
||||
''' Represents a tail command. '''
|
||||
def __init__(self, tailed_file):
|
||||
''' Initiate a Tail instance.
|
||||
Check for file validity, assigns callback function to standard out.
|
||||
|
||||
Arguments:
|
||||
tailed_file - File to be followed. '''
|
||||
|
||||
self.check_file_validity(tailed_file)
|
||||
self.tailed_file = tailed_file
|
||||
self.callback = sys.stdout.write
|
||||
self.t_stop = threading.Event()
|
||||
|
||||
def start(self, s=1):
|
||||
'''Start tailing a file in a background thread.
|
||||
|
||||
Arguments:
|
||||
s - Number of seconds to wait between each iteration; Defaults to 3.
|
||||
'''
|
||||
|
||||
t = threading.Thread(target=self.follow, args=(s,))
|
||||
t.start()
|
||||
|
||||
def stop(self):
|
||||
'''Stop a background tail.
|
||||
'''
|
||||
self.t_stop.set()
|
||||
|
||||
def follow(self, s=1):
|
||||
''' Do a tail follow. If a callback function is registered it is called with every new line.
|
||||
Else printed to standard out.
|
||||
|
||||
Arguments:
|
||||
s - Number of seconds to wait between each iteration; Defaults to 1. '''
|
||||
|
||||
with open(self.tailed_file, encoding='utf8') as file_:
|
||||
# Go to the end of file
|
||||
file_.seek(0, 2)
|
||||
while not self.t_stop.is_set():
|
||||
curr_position = file_.tell()
|
||||
lines = file_.readlines()
|
||||
if len(lines) == 0:
|
||||
file_.seek(curr_position)
|
||||
else:
|
||||
for line in lines:
|
||||
self.callback(line)
|
||||
time.sleep(s)
|
||||
|
||||
def register_callback(self, func):
|
||||
''' Overrides default callback function to provided function. '''
|
||||
self.callback = func
|
||||
|
||||
def check_file_validity(self, file_):
|
||||
''' Check whether the a given file exists, readable and is a file '''
|
||||
if not os.access(file_, os.F_OK):
|
||||
raise TailError("File '%s' does not exist" % (file_))
|
||||
if not os.access(file_, os.R_OK):
|
||||
raise TailError("File '%s' not readable" % (file_))
|
||||
if os.path.isdir(file_):
|
||||
raise TailError("File '%s' is a directory" % (file_))
|
||||
|
||||
|
||||
class TailError(Exception):
|
||||
|
||||
def __init__(self, msg):
|
||||
self.message = msg
|
||||
|
||||
def __str__(self):
|
||||
return self.message
|
||||
|
|
@ -0,0 +1,534 @@
|
|||
#!/usr/bin/env python3
|
||||
#
|
||||
# vmtools.py - part of the FDroid server tools
|
||||
# Copyright (C) 2017 Michael Poehn <michael.poehn@fsfe.org>
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from os import remove as rmfile
|
||||
from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser
|
||||
import os
|
||||
import math
|
||||
import json
|
||||
import tarfile
|
||||
import time
|
||||
import shutil
|
||||
import subprocess
|
||||
import textwrap
|
||||
from .common import FDroidException
|
||||
from logging import getLogger
|
||||
|
||||
logger = getLogger('fdroidserver-vmtools')
|
||||
|
||||
|
||||
def get_clean_builder(serverdir, reset=False):
|
||||
if not os.path.isdir(serverdir):
|
||||
if os.path.islink(serverdir):
|
||||
os.unlink(serverdir)
|
||||
logger.info("buildserver path does not exists, creating %s", serverdir)
|
||||
os.makedirs(serverdir)
|
||||
vagrantfile = os.path.join(serverdir, 'Vagrantfile')
|
||||
if not os.path.isfile(vagrantfile):
|
||||
with open(os.path.join('builder', 'Vagrantfile'), 'w') as f:
|
||||
f.write(textwrap.dedent("""\
|
||||
# generated file, do not change.
|
||||
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vm.box = "buildserver"
|
||||
config.vm.synced_folder ".", "/vagrant", disabled: true
|
||||
end
|
||||
"""))
|
||||
vm = get_build_vm(serverdir)
|
||||
if reset:
|
||||
logger.info('resetting buildserver by request')
|
||||
elif not vm.vagrant_uuid_okay():
|
||||
logger.info('resetting buildserver, bceause vagrant vm is not okay.')
|
||||
reset = True
|
||||
elif not vm.snapshot_exists('fdroidclean'):
|
||||
logger.info("resetting buildserver, because snapshot 'fdroidclean' is not present.")
|
||||
reset = True
|
||||
|
||||
if reset:
|
||||
vm.destroy()
|
||||
vm.up()
|
||||
vm.suspend()
|
||||
|
||||
if reset:
|
||||
logger.info('buildserver recreated: taking a clean snapshot')
|
||||
vm.snapshot_create('fdroidclean')
|
||||
else:
|
||||
logger.info('builserver ok: reverting to clean snapshot')
|
||||
vm.snapshot_revert('fdroidclean')
|
||||
vm.up()
|
||||
|
||||
try:
|
||||
sshinfo = vm.sshinfo()
|
||||
except FDroidBuildVmException:
|
||||
# workaround because libvirt sometimes likes to forget
|
||||
# about ssh connection info even thou the vm is running
|
||||
vm.halt()
|
||||
vm.up()
|
||||
sshinfo = vm.sshinfo()
|
||||
|
||||
return sshinfo
|
||||
|
||||
|
||||
def _check_call(cmd, shell=False, cwd=None):
|
||||
logger.debug(' '.join(cmd))
|
||||
return subprocess.check_call(cmd, shell=shell, cwd=cwd)
|
||||
|
||||
|
||||
def _check_output(cmd, shell=False, cwd=None):
|
||||
logger.debug(' '.join(cmd))
|
||||
return subprocess.check_output(cmd, shell=shell, cwd=cwd)
|
||||
|
||||
|
||||
def get_build_vm(srvdir, provider=None):
|
||||
"""Factory function for getting FDroidBuildVm instances.
|
||||
|
||||
This function tries to figure out what hypervisor should be used
|
||||
and creates an object for controlling a build VM.
|
||||
|
||||
:param srvdir: path to a directory which contains a Vagrantfile
|
||||
:param provider: optionally this parameter allows specifiying an
|
||||
spesific vagrant provider.
|
||||
:returns: FDroidBuildVm instance.
|
||||
"""
|
||||
abssrvdir = abspath(srvdir)
|
||||
|
||||
# use supplied provider
|
||||
if provider:
|
||||
if provider == 'libvirt':
|
||||
logger.debug('build vm provider \'libvirt\' selected')
|
||||
return LibvirtBuildVm(abssrvdir)
|
||||
elif provider == 'virtualbox':
|
||||
logger.debug('build vm provider \'virtualbox\' selected')
|
||||
return VirtualboxBuildVm(abssrvdir)
|
||||
else:
|
||||
logger.warn('build vm provider not supported: \'%s\'', provider)
|
||||
|
||||
# try guessing provider from installed software
|
||||
try:
|
||||
kvm_installed = 0 == _check_call(['which', 'kvm'])
|
||||
except subprocess.CalledProcessError:
|
||||
kvm_installed = False
|
||||
try:
|
||||
kvm_installed |= 0 == _check_call(['which', 'qemu'])
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
try:
|
||||
vbox_installed = 0 == _check_call(['which', 'VBoxHeadless'])
|
||||
except subprocess.CalledProcessError:
|
||||
vbox_installed = False
|
||||
if kvm_installed and vbox_installed:
|
||||
logger.debug('both kvm and vbox are installed.')
|
||||
elif kvm_installed:
|
||||
logger.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'')
|
||||
return LibvirtBuildVm(abssrvdir)
|
||||
elif vbox_installed:
|
||||
logger.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'')
|
||||
return VirtualboxBuildVm(abssrvdir)
|
||||
else:
|
||||
logger.debug('could not confirm that either virtualbox or kvm/libvirt are installed')
|
||||
|
||||
# try guessing provider from .../srvdir/.vagrant internals
|
||||
has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant',
|
||||
'machines', 'default', 'libvirt'))
|
||||
has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant',
|
||||
'machines', 'default', 'libvirt'))
|
||||
if has_libvirt_machine and has_vbox_machine:
|
||||
logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'')
|
||||
return VirtualboxBuildVm(abssrvdir)
|
||||
elif has_libvirt_machine:
|
||||
logger.debug('build vm provider lookup found \'libvirt\'')
|
||||
return LibvirtBuildVm(abssrvdir)
|
||||
elif has_vbox_machine:
|
||||
logger.debug('build vm provider lookup found \'virtualbox\'')
|
||||
return VirtualboxBuildVm(abssrvdir)
|
||||
|
||||
logger.info('build vm provider lookup could not determine provider, defaulting to \'virtualbox\'')
|
||||
return VirtualboxBuildVm(abssrvdir)
|
||||
|
||||
|
||||
class FDroidBuildVmException(FDroidException):
|
||||
pass
|
||||
|
||||
|
||||
class FDroidBuildVm():
|
||||
"""Abstract base class for working with FDroids build-servers.
|
||||
|
||||
Use the factory method `fdroidserver.vmtools.get_build_vm()` for
|
||||
getting correct instances of this class.
|
||||
|
||||
This is intended to be a hypervisor independant, fault tolerant
|
||||
wrapper around the vagrant functions we use.
|
||||
"""
|
||||
|
||||
def __init__(self, srvdir):
|
||||
"""Create new server class.
|
||||
"""
|
||||
self.srvdir = srvdir
|
||||
self.srvname = basename(srvdir) + '_default'
|
||||
self.vgrntfile = joinpath(srvdir, 'Vagrantfile')
|
||||
self.srvuuid = self._vagrant_fetch_uuid()
|
||||
if not isdir(srvdir):
|
||||
raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir))
|
||||
if not isfile(self.vgrntfile):
|
||||
raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile))
|
||||
import vagrant
|
||||
self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
|
||||
|
||||
def up(self, provision=True):
|
||||
try:
|
||||
self.vgrnt.up(provision=provision)
|
||||
logger.info('...waiting a sec...')
|
||||
time.sleep(10)
|
||||
self.srvuuid = self._vagrant_fetch_uuid()
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e
|
||||
|
||||
def suspend(self):
|
||||
logger.info('suspending buildserver')
|
||||
try:
|
||||
self.vgrnt.suspend()
|
||||
logger.info('...waiting a sec...')
|
||||
time.sleep(10)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e
|
||||
|
||||
def halt(self):
|
||||
self.vgrnt.halt(force=True)
|
||||
|
||||
def destroy(self):
|
||||
"""Remove every trace of this VM from the system.
|
||||
|
||||
This includes deleting:
|
||||
* hypervisor specific definitions
|
||||
* vagrant state informations (eg. `.vagrant` folder)
|
||||
* images related to this vm
|
||||
"""
|
||||
logger.info("destroying vm '%s'", self.srvname)
|
||||
try:
|
||||
self.vgrnt.destroy()
|
||||
logger.debug('vagrant destroy completed')
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.exception('vagrant destroy failed: %s', e)
|
||||
vgrntdir = joinpath(self.srvdir, '.vagrant')
|
||||
try:
|
||||
shutil.rmtree(vgrntdir)
|
||||
logger.debug('deleted vagrant dir: %s', vgrntdir)
|
||||
except Exception as e:
|
||||
logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e)
|
||||
try:
|
||||
_check_call(['vagrant', 'global-status', '--prune'])
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.debug('pruning global vagrant status failed: %s', e)
|
||||
|
||||
def package(self, output=None):
|
||||
self.vgrnt.package(output=output)
|
||||
|
||||
def vagrant_uuid_okay(self):
|
||||
'''Having an uuid means that vagrant up has run successfully.'''
|
||||
if self.srvuuid is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _vagrant_file_name(self, name):
|
||||
return name.replace('/', '-VAGRANTSLASH-')
|
||||
|
||||
def _vagrant_fetch_uuid(self):
|
||||
if isfile(joinpath(self.srvdir, '.vagrant')):
|
||||
# Vagrant 1.0 - it's a json file...
|
||||
with open(joinpath(self.srvdir, '.vagrant')) as f:
|
||||
id = json.load(f)['active']['default']
|
||||
logger.debug('vm uuid: %s', id)
|
||||
return id
|
||||
elif isfile(joinpath(self.srvdir, '.vagrant', 'machines',
|
||||
'default', self.provider, 'id')):
|
||||
# Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
|
||||
with open(joinpath(self.srvdir, '.vagrant', 'machines',
|
||||
'default', self.provider, 'id')) as f:
|
||||
id = f.read()
|
||||
logger.debug('vm uuid: %s', id)
|
||||
return id
|
||||
else:
|
||||
logger.debug('vm uuid is None')
|
||||
return None
|
||||
|
||||
def box_add(self, boxname, boxfile, force=True):
|
||||
"""Add vagrant box to vagrant.
|
||||
|
||||
:param boxname: name assigned to local deployment of box
|
||||
:param boxfile: path to box file
|
||||
:param force: overwrite existing box image (default: True)
|
||||
"""
|
||||
boxfile = abspath(boxfile)
|
||||
if not isfile(boxfile):
|
||||
raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile)
|
||||
self.vgrnt.box_add(boxname, abspath(boxfile), force=force)
|
||||
|
||||
def box_remove(self, boxname):
|
||||
try:
|
||||
_check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname])
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.debug('tried removing box %s, but is did not exist: %s', boxname, e)
|
||||
boxpath = joinpath(expanduser('~'), '.vagrant',
|
||||
self._vagrant_file_name(boxname))
|
||||
if isdir(boxpath):
|
||||
logger.info("attempting to remove box '%s' by deleting: %s",
|
||||
boxname, boxpath)
|
||||
shutil.rmtree(boxpath)
|
||||
|
||||
def sshinfo(self):
|
||||
"""Get ssh connection info for a vagrant VM
|
||||
|
||||
:returns: A dictionary containing 'hostname', 'port', 'user'
|
||||
and 'idfile'
|
||||
"""
|
||||
import paramiko
|
||||
try:
|
||||
_check_call(['vagrant ssh-config > sshconfig'],
|
||||
cwd=self.srvdir, shell=True)
|
||||
vagranthost = 'default' # Host in ssh config file
|
||||
sshconfig = paramiko.SSHConfig()
|
||||
with open(joinpath(self.srvdir, 'sshconfig'), 'r') as f:
|
||||
sshconfig.parse(f)
|
||||
sshconfig = sshconfig.lookup(vagranthost)
|
||||
idfile = sshconfig['identityfile']
|
||||
if isinstance(idfile, list):
|
||||
idfile = idfile[0]
|
||||
elif idfile.startswith('"') and idfile.endswith('"'):
|
||||
idfile = idfile[1:-1]
|
||||
return {'hostname': sshconfig['hostname'],
|
||||
'port': int(sshconfig['port']),
|
||||
'user': sshconfig['user'],
|
||||
'idfile': idfile}
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise FDroidBuildVmException("Error getting ssh config") from e
|
||||
|
||||
def snapshot_create(self, snapshot_name):
|
||||
raise NotImplementedError('not implemented, please use a sub-type instance')
|
||||
|
||||
def snapshot_list(self):
|
||||
raise NotImplementedError('not implemented, please use a sub-type instance')
|
||||
|
||||
def snapshot_exists(self, snapshot_name):
|
||||
raise NotImplementedError('not implemented, please use a sub-type instance')
|
||||
|
||||
def snapshot_revert(self, snapshot_name):
|
||||
raise NotImplementedError('not implemented, please use a sub-type instance')
|
||||
|
||||
|
||||
class LibvirtBuildVm(FDroidBuildVm):
|
||||
def __init__(self, srvdir):
|
||||
self.provider = 'libvirt'
|
||||
super().__init__(srvdir)
|
||||
import libvirt
|
||||
|
||||
try:
|
||||
self.conn = libvirt.open('qemu:///system')
|
||||
except libvirt.libvirtError as e:
|
||||
raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e))
|
||||
|
||||
def destroy(self):
|
||||
|
||||
super().destroy()
|
||||
|
||||
# resorting to virsh instead of libvirt python bindings, because
|
||||
# this is way more easy and therefore fault tolerant.
|
||||
# (eg. lookupByName only works on running VMs)
|
||||
try:
|
||||
_check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname))
|
||||
logger.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.info("could not force libvirt domain '%s' off: %s", self.srvname, e)
|
||||
try:
|
||||
# libvirt python bindings do not support all flags required
|
||||
# for undefining domains correctly.
|
||||
_check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata'))
|
||||
logger.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e)
|
||||
|
||||
def package(self, output=None, keep_box_file=False):
|
||||
if not output:
|
||||
output = "buildserver.box"
|
||||
logger.debug('no output name set for packaging \'%s\',' +
|
||||
'defaulting to %s', self.srvname, output)
|
||||
storagePool = self.conn.storagePoolLookupByName('default')
|
||||
if storagePool:
|
||||
|
||||
if isfile('metadata.json'):
|
||||
rmfile('metadata.json')
|
||||
if isfile('Vagrantfile'):
|
||||
rmfile('Vagrantfile')
|
||||
if isfile('box.img'):
|
||||
rmfile('box.img')
|
||||
|
||||
logger.debug('preparing box.img for box %s', output)
|
||||
vol = storagePool.storageVolLookupByName(self.srvname + '.img')
|
||||
imagepath = vol.path()
|
||||
# TODO use a libvirt storage pool to ensure the img file is readable
|
||||
_check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
|
||||
shutil.copy2(imagepath, 'box.img')
|
||||
_check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
|
||||
img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])
|
||||
img_info = json.loads(img_info_raw.decode('utf-8'))
|
||||
metadata = {"provider": "libvirt",
|
||||
"format": img_info['format'],
|
||||
"virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)),
|
||||
}
|
||||
|
||||
logger.debug('preparing metadata.json for box %s', output)
|
||||
with open('metadata.json', 'w') as fp:
|
||||
fp.write(json.dumps(metadata))
|
||||
logger.debug('preparing Vagrantfile for box %s', output)
|
||||
vagrantfile = textwrap.dedent("""\
|
||||
Vagrant.configure("2") do |config|
|
||||
config.ssh.username = "vagrant"
|
||||
config.ssh.password = "vagrant"
|
||||
|
||||
config.vm.provider :libvirt do |libvirt|
|
||||
|
||||
libvirt.driver = "kvm"
|
||||
libvirt.host = ""
|
||||
libvirt.connect_via_ssh = false
|
||||
libvirt.storage_pool_name = "default"
|
||||
|
||||
end
|
||||
end""")
|
||||
with open('Vagrantfile', 'w') as fp:
|
||||
fp.write(vagrantfile)
|
||||
with tarfile.open(output, 'w:gz') as tar:
|
||||
logger.debug('adding metadata.json to box %s ...', output)
|
||||
tar.add('metadata.json')
|
||||
logger.debug('adding Vagrantfile to box %s ...', output)
|
||||
tar.add('Vagrantfile')
|
||||
logger.debug('adding box.img to box %s ...', output)
|
||||
tar.add('box.img')
|
||||
|
||||
if not keep_box_file:
|
||||
logger.debug('box packaging complete, removing temporary files.')
|
||||
rmfile('metadata.json')
|
||||
rmfile('Vagrantfile')
|
||||
rmfile('box.img')
|
||||
|
||||
else:
|
||||
logger.warn('could not connect to storage-pool \'default\',' +
|
||||
'skipping packaging buildserver box')
|
||||
|
||||
def box_add(self, boxname, boxfile, force=True):
|
||||
boximg = '%s_vagrant_box_image_0.img' % (boxname)
|
||||
if force:
|
||||
try:
|
||||
_check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', boximg])
|
||||
logger.debug("removed old box image '%s' from libvirt storeage pool", boximg)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.debug("tired removing old box image '%s', file was not present in first place", boximg, exc_info=e)
|
||||
super().box_add(boxname, boxfile, force)
|
||||
|
||||
def box_remove(self, boxname):
|
||||
super().box_remove(boxname)
|
||||
try:
|
||||
_check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)])
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.debug("tired removing '%s', file was not present in first place", boxname, exc_info=e)
|
||||
|
||||
def snapshot_create(self, snapshot_name):
|
||||
logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
|
||||
try:
|
||||
_check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name])
|
||||
logger.info('...waiting a sec...')
|
||||
time.sleep(10)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise FDroidBuildVmException("could not cerate snapshot '%s' "
|
||||
"of libvirt vm '%s'"
|
||||
% (snapshot_name, self.srvname)) from e
|
||||
|
||||
def snapshot_list(self):
|
||||
import libvirt
|
||||
try:
|
||||
dom = self.conn.lookupByName(self.srvname)
|
||||
return dom.listAllSnapshots()
|
||||
except libvirt.libvirtError as e:
|
||||
raise FDroidBuildVmException('could not list snapshots for domain \'%s\'' % self.srvname) from e
|
||||
|
||||
def snapshot_exists(self, snapshot_name):
|
||||
import libvirt
|
||||
try:
|
||||
dom = self.conn.lookupByName(self.srvname)
|
||||
return dom.snapshotLookupByName(snapshot_name) is not None
|
||||
except libvirt.libvirtError:
|
||||
return False
|
||||
|
||||
def snapshot_revert(self, snapshot_name):
|
||||
logger.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name)
|
||||
import libvirt
|
||||
try:
|
||||
dom = self.conn.lookupByName(self.srvname)
|
||||
snap = dom.snapshotLookupByName(snapshot_name)
|
||||
dom.revertToSnapshot(snap)
|
||||
logger.info('...waiting a sec...')
|
||||
time.sleep(10)
|
||||
except libvirt.libvirtError as e:
|
||||
raise FDroidBuildVmException('could not revert domain \'%s\' to snapshot \'%s\''
|
||||
% (self.srvname, snapshot_name)) from e
|
||||
|
||||
|
||||
class VirtualboxBuildVm(FDroidBuildVm):
|
||||
|
||||
def __init__(self, srvdir):
|
||||
self.provider = 'virtualbox'
|
||||
super().__init__(srvdir)
|
||||
|
||||
def snapshot_create(self, snapshot_name):
|
||||
logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname)
|
||||
try:
|
||||
_check_call(['VBoxManage', 'snapshot', self.srvuuid, 'take', 'fdroidclean'], cwd=self.srvdir)
|
||||
logger.info('...waiting a sec...')
|
||||
time.sleep(10)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise FDroidBuildVmException('could not cerate snapshot '
|
||||
'of virtualbox vm %s'
|
||||
% self.srvname) from e
|
||||
|
||||
def snapshot_list(self):
|
||||
try:
|
||||
o = _check_output(['VBoxManage', 'snapshot',
|
||||
self.srvuuid, 'list',
|
||||
'--details'], cwd=self.srvdir)
|
||||
return o
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise FDroidBuildVmException("could not list snapshots "
|
||||
"of virtualbox vm '%s'"
|
||||
% (self.srvname)) from e
|
||||
|
||||
def snapshot_exists(self, snapshot_name):
|
||||
try:
|
||||
return str(snapshot_name) in str(self.snapshot_list())
|
||||
except FDroidBuildVmException:
|
||||
return False
|
||||
|
||||
def snapshot_revert(self, snapshot_name):
|
||||
logger.info("reverting vm '%s' to snapshot '%s'",
|
||||
self.srvname, snapshot_name)
|
||||
try:
|
||||
_check_call(['VBoxManage', 'snapshot', self.srvuuid,
|
||||
'restore', 'fdroidclean'], cwd=self.srvdir)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise FDroidBuildVmException("could not load snapshot "
|
||||
"'fdroidclean' for vm '%s'"
|
||||
% (self.srvname)) from e
|
||||
|
|
@ -9,9 +9,77 @@ if [ `dirname $0` != "." ]; then
|
|||
exit
|
||||
fi
|
||||
|
||||
# jenkins.debian.net slaves do not export WORKSPACE
|
||||
if [ -z $WORKSPACE ]; then
|
||||
export WORKSPACE=`pwd`
|
||||
fi
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
# this is a local repo on the Guardian Project Jenkins server
|
||||
cd tests
|
||||
./complete-ci-tests /var/www/fdroid
|
||||
#./complete-ci-tests /var/www/fdroid
|
||||
|
||||
|
||||
# report info about virtualization
|
||||
(dmesg | grep -i -e hypervisor -e qemu -e kvm) || true
|
||||
(lspci | grep -i -e virtio -e virtualbox -e qemu -e kvm) || true
|
||||
lsmod
|
||||
if systemd-detect-virt -q ; then
|
||||
echo "Virtualization is used:" `systemd-detect-virt`
|
||||
else
|
||||
echo "No virtualization is used."
|
||||
fi
|
||||
sudo /bin/chmod -R a+rX /var/lib/libvirt/images
|
||||
ls -ld /var/lib/libvirt/images
|
||||
ls -l /var/lib/libvirt/images || echo no access
|
||||
ls -lR ~/.vagrant.d/ || echo no access
|
||||
virsh --connect qemu:///system list --all || echo cannot virsh list
|
||||
cat /etc/issue
|
||||
|
||||
/sbin/ifconfig || true
|
||||
hostname || true
|
||||
|
||||
# point to the Vagrant/VirtualBox configs created by reproducible_setup_fdroid_build_environment.sh
|
||||
# these variables are actually set in fdroidserver/jenkins-build-makebuildserver
|
||||
export SETUP_WORKSPACE=$(dirname $WORKSPACE)/fdroid/fdroidserver
|
||||
export XDG_CONFIG_HOME=$SETUP_WORKSPACE
|
||||
export VBOX_USER_HOME=$SETUP_WORKSPACE/VirtualBox
|
||||
export VAGRANT_HOME=$SETUP_WORKSPACE/vagrant.d
|
||||
|
||||
# let's see what is actually there:
|
||||
find $SETUP_WORKSPACE | grep -v fdroiddata/metadata/ | cut -b43-9999
|
||||
|
||||
# the way we handle jenkins slaves doesn't copy the workspace to the slaves
|
||||
# so we need to "manually" clone the git repo here…
|
||||
cd $WORKSPACE
|
||||
|
||||
# set up Android SDK to use the Debian packages in stretch
|
||||
export ANDROID_HOME=/usr/lib/android-sdk
|
||||
|
||||
# ignore username/password prompt for non-existant repos
|
||||
git config --global url."https://fakeusername:fakepassword@github.com".insteadOf https://github.com
|
||||
git config --global url."https://fakeusername:fakepassword@gitlab.com".insteadOf https://gitlab.com
|
||||
git config --global url."https://fakeusername:fakepassword@bitbucket.org".insteadOf https://bitbucket.org
|
||||
|
||||
# now build the whole archive
|
||||
cd $WORKSPACE
|
||||
|
||||
# this can be handled in the jenkins job, or here:
|
||||
if [ -e fdroiddata ]; then
|
||||
cd fdroiddata
|
||||
git remote update -p
|
||||
git checkout master
|
||||
git reset --hard origin/master
|
||||
else
|
||||
git clone https://gitlab.com/fdroid/fdroiddata.git fdroiddata
|
||||
cd fdroiddata
|
||||
fi
|
||||
|
||||
echo "build_server_always = True" > config.py
|
||||
$WORKSPACE/fdroid build --verbose --latest --no-tarball --all
|
||||
|
||||
vagrant global-status
|
||||
cd builder
|
||||
vagrant status
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ cleanup_all() {
|
|||
set +e
|
||||
echo "$(date -u) - cleanup in progress..."
|
||||
ps auxww | grep -e VBox -e qemu
|
||||
virsh --connect qemu:///system list --all
|
||||
ls -hl /var/lib/libvirt/images
|
||||
cd $WORKSPACE/buildserver
|
||||
vagrant halt
|
||||
sleep 5
|
||||
|
|
@ -49,7 +51,14 @@ cd $WORKSPACE
|
|||
echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py
|
||||
echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py
|
||||
echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py
|
||||
./makebuildserver --verbose --clean
|
||||
echo "copy_caches_from_host = True" >> $WORKSPACE/makebuildserver.config.py
|
||||
./makebuildserver -vv --clean
|
||||
|
||||
if [ -z "`vagrant box list | egrep '^buildserver\s+\((libvirt|virtualbox), [0-9]+\)$'`" ]; then
|
||||
vagrant box list
|
||||
echo "ERROR: buildserver box does not exist!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# this can be handled in the jenkins job, or here:
|
||||
if [ -e fdroiddata ]; then
|
||||
|
|
@ -70,7 +79,7 @@ if [ -z $ANDROID_HOME ]; then
|
|||
. ~/.android/bashrc
|
||||
else
|
||||
echo "ANDROID_HOME must be set!"
|
||||
exit
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
|
|||
480
makebuildserver
480
makebuildserver
|
|
@ -2,61 +2,59 @@
|
|||
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import requests
|
||||
import stat
|
||||
import sys
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import vagrant
|
||||
import hashlib
|
||||
import yaml
|
||||
import json
|
||||
import logging
|
||||
from clint.textui import progress
|
||||
from optparse import OptionParser
|
||||
import fdroidserver.tail
|
||||
import fdroidserver.vmtools
|
||||
|
||||
|
||||
parser = OptionParser()
|
||||
parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1,
|
||||
help="Spew out even more information than normal")
|
||||
parser.add_option('-q', action='store_const', const=0, dest='verbosity')
|
||||
parser.add_option("-c", "--clean", action="store_true", default=False,
|
||||
help="Build from scratch, rather than attempting to update the existing server")
|
||||
parser.add_option('--skip-cache-update', action="store_true", default=False,
|
||||
help="""Skip downloading and checking cache."""
|
||||
"""This assumes that the cache is already downloaded completely.""")
|
||||
parser.add_option('--keep-box-file', action="store_true", default=False,
|
||||
help="""Box file will not be deleted after adding it to box storage"""
|
||||
""" (KVM-only).""")
|
||||
options, args = parser.parse_args()
|
||||
|
||||
logger = logging.getLogger('fdroidserver-makebuildserver')
|
||||
if options.verbosity >= 2:
|
||||
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
elif options.verbosity == 1:
|
||||
logging.basicConfig(format='%(message)s', level=logging.INFO)
|
||||
logger.setLevel(logging.INFO)
|
||||
elif options.verbosity <= 0:
|
||||
logging.basicConfig(format='%(message)s', level=logging.WARNING)
|
||||
logger.setLevel(logging.WARNING)
|
||||
|
||||
|
||||
if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'):
|
||||
print('This must be run as ./makebuildserver in fdroidserver.git!')
|
||||
logger.critical('This must be run as ./makebuildserver in fdroidserver.git!')
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def vagrant(params, cwd=None, printout=False):
|
||||
"""Run vagrant.
|
||||
|
||||
:param: list of parameters to pass to vagrant
|
||||
:cwd: directory to run in, or None for current directory
|
||||
:printout: True to print output in realtime, False to just
|
||||
return it
|
||||
:returns: (ret, out) where ret is the return code, and out
|
||||
is the stdout (and stderr) from vagrant
|
||||
"""
|
||||
p = subprocess.Popen(['vagrant'] + params, cwd=cwd,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
universal_newlines=True)
|
||||
out = ''
|
||||
if printout:
|
||||
while True:
|
||||
line = p.stdout.readline()
|
||||
if len(line) == 0:
|
||||
break
|
||||
print(line.rstrip())
|
||||
out += line
|
||||
p.wait()
|
||||
else:
|
||||
out = p.communicate()[0]
|
||||
return (p.returncode, out)
|
||||
|
||||
|
||||
boxfile = 'buildserver.box'
|
||||
serverdir = 'buildserver'
|
||||
|
||||
parser = OptionParser()
|
||||
parser.add_option("-v", "--verbose", action="store_true", default=False,
|
||||
help="Spew out even more information than normal")
|
||||
parser.add_option("-c", "--clean", action="store_true", default=False,
|
||||
help="Build from scratch, rather than attempting to update the existing server")
|
||||
options, args = parser.parse_args()
|
||||
tail = None
|
||||
|
||||
# set up default config
|
||||
cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
|
||||
logger.debug('cachedir set to: %s', cachedir)
|
||||
|
||||
config = {
|
||||
'basebox': 'jessie64',
|
||||
'baseboxurl': [
|
||||
|
|
@ -65,6 +63,7 @@ config = {
|
|||
],
|
||||
'debian_mirror': 'http://http.debian.net/debian/',
|
||||
'apt_package_cache': False,
|
||||
'copy_caches_from_host': False,
|
||||
'boot_timeout': 600,
|
||||
'cachedir': cachedir,
|
||||
'cpus': 1,
|
||||
|
|
@ -79,10 +78,11 @@ if os.path.isfile('/usr/bin/systemd-detect-virt'):
|
|||
except subprocess.CalledProcessError as e:
|
||||
virt = 'none'
|
||||
if virt == 'qemu' or virt == 'kvm' or virt == 'bochs':
|
||||
print('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
|
||||
logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt')
|
||||
config['vm_provider'] = 'libvirt'
|
||||
elif virt != 'none':
|
||||
print('Running in an unsupported VM guest (' + virt + ')!')
|
||||
logger.info('Running in an unsupported VM guest (%s)!', virt)
|
||||
logger.debug('detected virt: %s', virt)
|
||||
|
||||
# load config file, if present
|
||||
if os.path.exists('makebuildserver.config.py'):
|
||||
|
|
@ -92,33 +92,36 @@ elif os.path.exists('makebs.config.py'):
|
|||
exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
|
||||
if '__builtins__' in config:
|
||||
del(config['__builtins__']) # added by compile/exec
|
||||
|
||||
if os.path.exists(boxfile):
|
||||
os.remove(boxfile)
|
||||
|
||||
if options.clean:
|
||||
vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose)
|
||||
if config['vm_provider'] == 'libvirt':
|
||||
subprocess.call(['virsh', 'undefine', 'buildserver_default'])
|
||||
subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img'])
|
||||
logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True))
|
||||
|
||||
# Update cached files.
|
||||
cachedir = config['cachedir']
|
||||
if not os.path.exists(cachedir):
|
||||
os.makedirs(cachedir, 0o755)
|
||||
logger.debug('created cachedir %s because it did not exists.', cachedir)
|
||||
|
||||
if config['vm_provider'] == 'libvirt':
|
||||
tmp = cachedir
|
||||
while tmp != '/':
|
||||
mode = os.stat(tmp).st_mode
|
||||
if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode):
|
||||
print('ERROR:', tmp, 'will not be accessible to the VM! To fix, run:')
|
||||
print(' chmod a+X', tmp)
|
||||
logger.critical('ERROR: %s will not be accessible to the VM! To fix, run:', tmp)
|
||||
logger.critical(' chmod a+X %s', tmp)
|
||||
sys.exit(1)
|
||||
tmp = os.path.dirname(tmp)
|
||||
logger.debug('cache dir %s is accessible for libvirt vm.', cachedir)
|
||||
|
||||
if config['apt_package_cache']:
|
||||
config['aptcachedir'] = cachedir + '/apt/archives'
|
||||
logger.debug('aptcachedir is set to %s', config['aptcachedir'])
|
||||
aptcachelock = os.path.join(config['aptcachedir'], 'lock')
|
||||
if os.path.isfile(aptcachelock):
|
||||
logger.info('apt cache dir is locked, removing lock')
|
||||
os.remove(aptcachelock)
|
||||
aptcachepartial = os.path.join(config['aptcachedir'], 'partial')
|
||||
if os.path.isdir(aptcachepartial):
|
||||
logger.info('removing partial downloads from apt cache dir')
|
||||
shutil.rmtree(aptcachepartial)
|
||||
|
||||
cachefiles = [
|
||||
('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip',
|
||||
|
|
@ -325,164 +328,233 @@ def sha256_for_file(path):
|
|||
return s.hexdigest()
|
||||
|
||||
|
||||
for srcurl, shasum in cachefiles:
|
||||
filename = os.path.basename(srcurl)
|
||||
local_filename = os.path.join(cachedir, filename)
|
||||
|
||||
if os.path.exists(local_filename):
|
||||
local_length = os.path.getsize(local_filename)
|
||||
def run_via_vagrant_ssh(v, cmdlist):
|
||||
if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)):
|
||||
cmd = cmdlist
|
||||
else:
|
||||
local_length = -1
|
||||
cmd = ' '.join(cmdlist)
|
||||
v._run_vagrant_command(['ssh', '-c', cmd])
|
||||
|
||||
resume_header = {}
|
||||
download = True
|
||||
|
||||
try:
|
||||
r = requests.head(srcurl, allow_redirects=True, timeout=60)
|
||||
if r.status_code == 200:
|
||||
content_length = int(r.headers.get('content-length'))
|
||||
def update_cache(cachedir, cachefiles):
|
||||
for srcurl, shasum in cachefiles:
|
||||
filename = os.path.basename(srcurl)
|
||||
local_filename = os.path.join(cachedir, filename)
|
||||
|
||||
if os.path.exists(local_filename):
|
||||
local_length = os.path.getsize(local_filename)
|
||||
else:
|
||||
content_length = local_length # skip the download
|
||||
except requests.exceptions.RequestException as e:
|
||||
content_length = local_length # skip the download
|
||||
print(e)
|
||||
local_length = -1
|
||||
|
||||
if local_length == content_length:
|
||||
download = False
|
||||
elif local_length > content_length:
|
||||
print('deleting corrupt file from cache: ' + local_filename)
|
||||
os.remove(local_filename)
|
||||
print("Downloading " + filename + " to cache")
|
||||
elif local_length > -1 and local_length < content_length:
|
||||
print("Resuming download of " + local_filename)
|
||||
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
|
||||
else:
|
||||
print("Downloading " + filename + " to cache")
|
||||
resume_header = {}
|
||||
download = True
|
||||
|
||||
if download:
|
||||
r = requests.get(srcurl, headers=resume_header,
|
||||
stream=True, verify=False, allow_redirects=True)
|
||||
content_length = int(r.headers.get('content-length'))
|
||||
with open(local_filename, 'ab') as f:
|
||||
for chunk in progress.bar(r.iter_content(chunk_size=65536),
|
||||
expected_size=(content_length / 65536) + 1):
|
||||
if chunk: # filter out keep-alive new chunks
|
||||
f.write(chunk)
|
||||
|
||||
v = sha256_for_file(local_filename)
|
||||
if v == shasum:
|
||||
print("\t...shasum verified for " + local_filename)
|
||||
else:
|
||||
print("Invalid shasum of '" + v + "' detected for " + local_filename)
|
||||
os.remove(local_filename)
|
||||
sys.exit(1)
|
||||
|
||||
local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
|
||||
print("Setting executable bit for " + local_qt_filename)
|
||||
os.chmod(local_qt_filename, 0o755)
|
||||
|
||||
# use VirtualBox software virtualization if hardware is not available,
|
||||
# like if this is being run in kvm or some other VM platform, like
|
||||
# http://jenkins.debian.net, the values are 'on' or 'off'
|
||||
if sys.platform.startswith('darwin'):
|
||||
# all < 10 year old Macs work, and OSX servers as VM host are very
|
||||
# rare, but this could also be auto-detected if someone codes it
|
||||
config['hwvirtex'] = 'on'
|
||||
elif os.path.exists('/proc/cpuinfo'):
|
||||
with open('/proc/cpuinfo') as f:
|
||||
contents = f.read()
|
||||
if 'vmx' in contents or 'svm' in contents:
|
||||
config['hwvirtex'] = 'on'
|
||||
|
||||
# Check against the existing Vagrantfile.yaml, and if they differ, we
|
||||
# need to create a new box:
|
||||
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
||||
writevf = True
|
||||
if os.path.exists(vf):
|
||||
print('Halting', serverdir)
|
||||
vagrant(['halt'], serverdir)
|
||||
with open(vf, 'r', encoding='utf-8') as f:
|
||||
oldconfig = yaml.load(f)
|
||||
if config != oldconfig:
|
||||
print("Server configuration has changed, rebuild from scratch is required")
|
||||
vagrant(['destroy', '-f'], serverdir)
|
||||
else:
|
||||
print("Re-provisioning existing server")
|
||||
writevf = False
|
||||
else:
|
||||
print("No existing server - building from scratch")
|
||||
if writevf:
|
||||
with open(vf, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(config, f)
|
||||
|
||||
if config['vm_provider'] == 'libvirt':
|
||||
returncode, out = vagrant(['box', 'list'], serverdir, printout=options.verbose)
|
||||
found_basebox = False
|
||||
needs_mutate = False
|
||||
for line in out.splitlines():
|
||||
if line.startswith(config['basebox']):
|
||||
found_basebox = True
|
||||
if line.split('(')[1].split(',')[0] != 'libvirt':
|
||||
needs_mutate = True
|
||||
continue
|
||||
if not found_basebox:
|
||||
if isinstance(config['baseboxurl'], str):
|
||||
baseboxurl = config['baseboxurl']
|
||||
else:
|
||||
baseboxurl = config['baseboxurl'][0]
|
||||
print('Adding', config['basebox'], 'from', baseboxurl)
|
||||
vagrant(['box', 'add', '--name', config['basebox'], baseboxurl],
|
||||
serverdir, printout=options.verbose)
|
||||
needs_mutate = True
|
||||
if needs_mutate:
|
||||
print('Converting', config['basebox'], 'to libvirt format')
|
||||
vagrant(['mutate', config['basebox'], 'libvirt'],
|
||||
serverdir, printout=options.verbose)
|
||||
print('Removing virtualbox format copy of', config['basebox'])
|
||||
vagrant(['box', 'remove', '--provider', 'virtualbox', config['basebox']],
|
||||
serverdir, printout=options.verbose)
|
||||
|
||||
print("Configuring build server VM")
|
||||
returncode, out = vagrant(['up', '--provision'], serverdir, printout=True)
|
||||
with open(os.path.join(serverdir, 'up.log'), 'w') as log:
|
||||
log.write(out)
|
||||
if returncode != 0:
|
||||
print("Failed to configure server")
|
||||
sys.exit(1)
|
||||
|
||||
print("Writing buildserver ID")
|
||||
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
buildserverid = p.communicate()[0].strip()
|
||||
print("...ID is " + buildserverid)
|
||||
subprocess.call(
|
||||
['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"'
|
||||
.format(buildserverid)],
|
||||
cwd=serverdir)
|
||||
|
||||
print("Stopping build server VM")
|
||||
vagrant(['halt'], serverdir)
|
||||
|
||||
print("Waiting for build server VM to be finished")
|
||||
ready = False
|
||||
while not ready:
|
||||
time.sleep(2)
|
||||
returncode, out = vagrant(['status'], serverdir)
|
||||
if returncode != 0:
|
||||
print("Error while checking status")
|
||||
sys.exit(1)
|
||||
for line in out.splitlines():
|
||||
if line.startswith("default"):
|
||||
if line.find("poweroff") != -1 or line.find("shutoff") != 1:
|
||||
ready = True
|
||||
try:
|
||||
r = requests.head(srcurl, allow_redirects=True, timeout=60)
|
||||
if r.status_code == 200:
|
||||
content_length = int(r.headers.get('content-length'))
|
||||
else:
|
||||
print("Status: " + line)
|
||||
content_length = local_length # skip the download
|
||||
except requests.exceptions.RequestException as e:
|
||||
content_length = local_length # skip the download
|
||||
logger.warn('%s', e)
|
||||
|
||||
print("Packaging")
|
||||
vagrant(['package', '--output', os.path.join('..', boxfile)], serverdir,
|
||||
printout=options.verbose)
|
||||
print("Adding box")
|
||||
vagrant(['box', 'add', 'buildserver', boxfile, '-f'],
|
||||
printout=options.verbose)
|
||||
if local_length == content_length:
|
||||
download = False
|
||||
elif local_length > content_length:
|
||||
logger.info('deleting corrupt file from cache: %s', local_filename)
|
||||
os.remove(local_filename)
|
||||
logger.info("Downloading %s to cache", filename)
|
||||
elif local_length > -1 and local_length < content_length:
|
||||
logger.info("Resuming download of %s", local_filename)
|
||||
resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)}
|
||||
else:
|
||||
logger.info("Downloading %s to cache", filename)
|
||||
|
||||
os.remove(boxfile)
|
||||
if download:
|
||||
r = requests.get(srcurl, headers=resume_header,
|
||||
stream=True, verify=False, allow_redirects=True)
|
||||
content_length = int(r.headers.get('content-length'))
|
||||
with open(local_filename, 'ab') as f:
|
||||
for chunk in progress.bar(r.iter_content(chunk_size=65536),
|
||||
expected_size=(content_length / 65536) + 1):
|
||||
if chunk: # filter out keep-alive new chunks
|
||||
f.write(chunk)
|
||||
|
||||
v = sha256_for_file(local_filename)
|
||||
if v == shasum:
|
||||
logger.info("\t...shasum verified for %s", local_filename)
|
||||
else:
|
||||
logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename)
|
||||
os.remove(local_filename)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def debug_log_vagrant_vm(vm_dir, config):
|
||||
if options.verbosity >= 3:
|
||||
_vagrant_dir = os.path.join(vm_dir, '.vagrant')
|
||||
logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir))
|
||||
logger.debug('> vagrant status')
|
||||
subprocess.call(['vagrant', 'status'], cwd=vm_dir)
|
||||
logger.debug('> vagrant box list')
|
||||
subprocess.call(['vagrant', 'box', 'list'])
|
||||
if config['vm_provider'] == 'libvirt':
|
||||
logger.debug('> virsh -c qmeu:///system list --all')
|
||||
subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all'])
|
||||
domain = 'buildserver_default'
|
||||
logger.debug('> virsh -c qemu:///system snapshot-list %s', domain)
|
||||
subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain])
|
||||
|
||||
|
||||
def main():
|
||||
global cachedir, cachefiles, config, tail
|
||||
|
||||
if options.skip_cache_update:
|
||||
logger.info('skipping cache update and verification...')
|
||||
else:
|
||||
update_cache(cachedir, cachefiles)
|
||||
|
||||
local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run')
|
||||
logger.info("Setting executable bit for %s", local_qt_filename)
|
||||
os.chmod(local_qt_filename, 0o755)
|
||||
|
||||
# use VirtualBox software virtualization if hardware is not available,
|
||||
# like if this is being run in kvm or some other VM platform, like
|
||||
# http://jenkins.debian.net, the values are 'on' or 'off'
|
||||
if sys.platform.startswith('darwin'):
|
||||
# all < 10 year old Macs work, and OSX servers as VM host are very
|
||||
# rare, but this could also be auto-detected if someone codes it
|
||||
config['hwvirtex'] = 'on'
|
||||
logger.info('platform is darwnin -> hwvirtex = \'on\'')
|
||||
elif os.path.exists('/proc/cpuinfo'):
|
||||
with open('/proc/cpuinfo') as f:
|
||||
contents = f.read()
|
||||
if 'vmx' in contents or 'svm' in contents:
|
||||
config['hwvirtex'] = 'on'
|
||||
logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'')
|
||||
|
||||
serverdir = os.path.join(os.getcwd(), 'buildserver')
|
||||
logfilename = os.path.join(serverdir, 'up.log')
|
||||
if not os.path.exists(logfilename):
|
||||
open(logfilename, 'a').close() # create blank file
|
||||
log_cm = vagrant.make_file_cm(logfilename)
|
||||
v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm)
|
||||
|
||||
if options.verbosity >= 2:
|
||||
tail = fdroidserver.tail.Tail(logfilename)
|
||||
tail.start()
|
||||
|
||||
vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider'])
|
||||
if options.clean:
|
||||
vm.destroy()
|
||||
|
||||
# Check against the existing Vagrantfile.yaml, and if they differ, we
|
||||
# need to create a new box:
|
||||
vf = os.path.join(serverdir, 'Vagrantfile.yaml')
|
||||
writevf = True
|
||||
if os.path.exists(vf):
|
||||
logger.info('Halting %s', serverdir)
|
||||
v.halt()
|
||||
with open(vf, 'r', encoding='utf-8') as f:
|
||||
oldconfig = yaml.load(f)
|
||||
if config != oldconfig:
|
||||
logger.info("Server configuration has changed, rebuild from scratch is required")
|
||||
vm.destroy()
|
||||
else:
|
||||
logger.info("Re-provisioning existing server")
|
||||
writevf = False
|
||||
else:
|
||||
logger.info("No existing server - building from scratch")
|
||||
if writevf:
|
||||
with open(vf, 'w', encoding='utf-8') as f:
|
||||
yaml.dump(config, f)
|
||||
|
||||
if config['vm_provider'] == 'libvirt':
|
||||
found_basebox = False
|
||||
needs_mutate = False
|
||||
for box in v.box_list():
|
||||
if box.name == config['basebox']:
|
||||
found_basebox = True
|
||||
if box.provider != 'libvirt':
|
||||
needs_mutate = True
|
||||
continue
|
||||
if not found_basebox:
|
||||
if isinstance(config['baseboxurl'], str):
|
||||
baseboxurl = config['baseboxurl']
|
||||
else:
|
||||
baseboxurl = config['baseboxurl'][0]
|
||||
logger.info('Adding %s from %s', config['basebox'], baseboxurl)
|
||||
v.box_add(config['basebox'], baseboxurl)
|
||||
needs_mutate = True
|
||||
if needs_mutate:
|
||||
logger.info('Converting %s to libvirt format', config['basebox'])
|
||||
v._call_vagrant_command(['mutate', config['basebox'], 'libvirt'])
|
||||
logger.info('Removing virtualbox format copy of %s', config['basebox'])
|
||||
v.box_remove(config['basebox'], 'virtualbox')
|
||||
|
||||
logger.info("Configuring build server VM")
|
||||
debug_log_vagrant_vm(serverdir, config)
|
||||
try:
|
||||
v.up(provision=True)
|
||||
except fdroidserver.vmtools.FDroidBuildVmException as e:
|
||||
debug_log_vagrant_vm(serverdir, config)
|
||||
logger.exception('could not bring buildserver vm up. %s', e)
|
||||
sys.exit(1)
|
||||
|
||||
if config['copy_caches_from_host']:
|
||||
ssh_config = v.ssh_config()
|
||||
user = re.search(r'User ([^ \n]+)', ssh_config).group(1)
|
||||
hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1)
|
||||
port = re.search(r'Port ([0-9]+)', ssh_config).group(1)
|
||||
key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1)
|
||||
|
||||
for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'):
|
||||
fullpath = os.path.join(os.getenv('HOME'), d)
|
||||
if os.path.isdir(fullpath):
|
||||
# TODO newer versions of vagrant provide `vagrant rsync`
|
||||
run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d])
|
||||
subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e',
|
||||
'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port),
|
||||
fullpath + '/',
|
||||
user + '@' + hostname + ':~/' + d + '/'])
|
||||
|
||||
# this file changes every time but should not be cached
|
||||
run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock'])
|
||||
run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/'])
|
||||
|
||||
p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
buildserverid = p.communicate()[0].strip()
|
||||
logger.info("Writing buildserver ID ...ID is %s", buildserverid)
|
||||
run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid)
|
||||
|
||||
logger.info("Stopping build server VM")
|
||||
v.halt()
|
||||
|
||||
logger.info("Packaging")
|
||||
boxfile = os.path.join(os.getcwd(), 'buildserver.box')
|
||||
if os.path.exists(boxfile):
|
||||
os.remove(boxfile)
|
||||
|
||||
vm.package(output=boxfile)
|
||||
|
||||
logger.info("Adding box")
|
||||
vm.box_add('buildserver', boxfile, force=True)
|
||||
|
||||
if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'):
|
||||
logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile)
|
||||
sys.exit(1)
|
||||
|
||||
if not options.keep_box_file:
|
||||
logger.debug('box added to vagrant, ' +
|
||||
'removing generated box file \'%s\'',
|
||||
boxfile)
|
||||
os.remove(boxfile)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
if tail is not None:
|
||||
tail.stop()
|
||||
|
|
|
|||
1
setup.py
1
setup.py
|
|
@ -36,6 +36,7 @@ setup(name='fdroidserver',
|
|||
'apache-libcloud >= 0.14.1',
|
||||
'pyasn1',
|
||||
'pyasn1-modules',
|
||||
'python-vagrant',
|
||||
'PyYAML',
|
||||
'requests < 2.11',
|
||||
'docker-py == 1.9.0',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,140 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import textwrap
|
||||
import tempfile
|
||||
import inspect
|
||||
from argparse import ArgumentParser
|
||||
|
||||
localmodule = os.path.realpath(
|
||||
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..', '..'))
|
||||
print('localmodule: ' + localmodule)
|
||||
if localmodule not in sys.path:
|
||||
sys.path.insert(0, localmodule)
|
||||
|
||||
from fdroidserver.vmtools import get_build_vm
|
||||
|
||||
|
||||
def main(args):
|
||||
|
||||
if args.provider != None:
|
||||
if args.provider not in ('libvirt', 'virtualbox'):
|
||||
logging.critical('provider: %s not supported.', args.provider)
|
||||
sys.exit(1)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
|
||||
# define a simple vagrant vm 'x'
|
||||
x_dir = os.path.join(tmpdir, 'x')
|
||||
os.makedirs(x_dir)
|
||||
with open(os.path.join(x_dir, 'Vagrantfile'), 'w') as f:
|
||||
f.write(textwrap.dedent("""\
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vm.box = "debian/jessie64"
|
||||
config.vm.synced_folder ".", "/vagrant", disabled: true
|
||||
config.ssh.insert_key = false
|
||||
end
|
||||
"""))
|
||||
# define another simple vagrant vm 'y' which uses 'x' as a base box
|
||||
y_dir = os.path.join(tmpdir, 'y')
|
||||
os.makedirs(y_dir)
|
||||
with open(os.path.join(y_dir, 'Vagrantfile'), 'w') as f:
|
||||
f.write(textwrap.dedent("""\
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vm.box = "x"
|
||||
config.vm.synced_folder ".", "/vagrant", disabled: true
|
||||
end
|
||||
"""))
|
||||
|
||||
# vagrant file for packaging 'x' box
|
||||
vgrntf=textwrap.dedent("""\
|
||||
Vagrant.configure("2") do |config|
|
||||
|
||||
config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: "4", nfs_udp: false
|
||||
|
||||
config.vm.provider :libvirt do |libvirt|
|
||||
libvirt.driver = "kvm"
|
||||
libvirt.connect_via_ssh = false
|
||||
libvirt.username = "root"
|
||||
libvirt.storage_pool_name = "default"
|
||||
end
|
||||
end
|
||||
""")
|
||||
|
||||
# create a box: x
|
||||
if not args.skip_create_x:
|
||||
x = get_build_vm(x_dir, provider=args.provider)
|
||||
x.destroy()
|
||||
x.up(provision=True)
|
||||
x.halt()
|
||||
x.package(output='x.box', vagrantfile=vgrntf, keep_box_file=False)
|
||||
x.box_remove('x')
|
||||
x.box_add('x', 'x.box')
|
||||
|
||||
# use previously created box to spin up a new vm
|
||||
if not args.skip_create_y:
|
||||
y = get_build_vm(y_dir, provider=args.provider)
|
||||
y.destroy()
|
||||
y.up()
|
||||
|
||||
# create and restore a snapshot
|
||||
if not args.skip_snapshot_y:
|
||||
y = get_build_vm(y_dir, provider=args.provider)
|
||||
|
||||
if y.snapshot_exists('clean'):
|
||||
y.destroy()
|
||||
y.up()
|
||||
|
||||
y.suspend()
|
||||
y.snapshot_create('clean')
|
||||
y.up()
|
||||
|
||||
logging.info('snapshot \'clean\' exsists: %r', y.snapshot_exists('clean'))
|
||||
|
||||
# test if snapshot exists
|
||||
se = y.snapshot_exists('clean')
|
||||
logging.info('snapshot \'clean\' available: %r', se)
|
||||
|
||||
# revert snapshot
|
||||
y.suspend()
|
||||
logging.info('asdf %s', y.snapshot_revert('clean'))
|
||||
y.resume()
|
||||
|
||||
# cleanup
|
||||
if not args.skip_clean:
|
||||
x = get_build_vm(x_dir, provider=args.provider)
|
||||
y = get_build_vm(y_dir, provider=args.provider)
|
||||
y.destroy()
|
||||
x.destroy()
|
||||
x.box_remove('x')
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(format='%(message)s', level=logging.DEBUG)
|
||||
|
||||
parser = ArgumentParser(description="""\
|
||||
This is intended for manually testing vmtools.py
|
||||
|
||||
NOTE: Should this test-run fail it might leave traces of vagrant VMs or boxes
|
||||
on your system. Those vagrant VMs are named 'x' and 'y'.
|
||||
""")
|
||||
parser.add_argument('--provider', help="Force this script use supplied "
|
||||
"provider instead using our auto provider lookup. "
|
||||
"Supported values: 'libvirt', 'virtualbox'")
|
||||
parser.add_argument('--skip-create-x', action="store_true", default=False,
|
||||
help="Skips: Creating 'x' vm, packaging it into a "
|
||||
"a box and adding it to vagrant.")
|
||||
parser.add_argument('--skip-create-y', action="store_true", default=False,
|
||||
help="Skips: Creating 'y' vm. Depends on having "
|
||||
"box 'x' added to vagrant.")
|
||||
parser.add_argument('--skip-snapshot-y', action="store_true", default=False,
|
||||
help="Skips: Taking a snapshot and restoring a "
|
||||
"a snapshot of 'y' vm. Requires 'y' mv to be "
|
||||
"present.")
|
||||
parser.add_argument('--skip-clean', action="store_true", default=False,
|
||||
help="Skips: Cleaning up mv images and vagrant "
|
||||
"metadata on the system.")
|
||||
args = parser.parse_args()
|
||||
|
||||
main(args)
|
||||
Loading…
Reference in New Issue