opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

revised build server creation

This commit is contained in:
Michael Pöhn 2017-04-06 11:42:35 +02:00 committed by Hans-Christoph Steiner
parent 8abd3f1cbc
commit 3187d2cbcf
2 changed files with 133 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
fdroidserver/build.py
View File

@ -29,6 +29,7 @@ import time
import json
import requests
import tempfile
import textwrap
from configparser import ConfigParser
from argparse import ArgumentParser
import logging
@ -223,6 +224,45 @@ def vm_test_ssh_into_builder():
sshs.close()
def vm_new_get_clean_builder(serverdir, reset=False):
if not os.path.isdir(serverdir):
logging.info("buildserver path does not exists, creating %s", serverdir)
os.makedirs(serverdir)
vagrantfile = os.path.join(serverdir, 'Vagrantfile')
if not os.path.isfile(vagrantfile):
with open(os.path.join('builder', 'Vagrantfile'), 'w') as f:
f.write(textwrap.dedent("""\
# generated file, do not change.
Vagrant.configure("2") do |config|
config.vm.box = "buildserver"
config.vm.synced_folder ".", "/vagrant", disabled: true
end
"""))
vm = vmtools.get_build_vm(serverdir)
if reset:
logging.info('resetting buildserver by request')
elif not vm.check_okay():
logging.info('resetting buildserver because it appears to be absent or broken')
reset = True
elif not vm.snapshot_exists('fdroidclean'):
logging.info("resetting buildserver, because snapshot 'fdroidclean' is not present")
reset = True
if reset:
vm.destroy()
vm.up()
vm.suspend()
if reset:
vm.snapshot_create('fdroidclean')
else:
vm.snapshot_revert('droidclean')
vm.resume()
return get_vagrant_sshinfo()
def vm_get_clean_builder(reset=False):
"""Get a clean VM ready to do a buildserver build.
@ -374,7 +414,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
else:
logging.getLogger("paramiko").setLevel(logging.WARN)
sshinfo = vm_get_clean_builder()
# sshinfo = vm_get_clean_builder()
sshinfo = vm_new_get_clean_builder('builder')
try:
if not buildserverid:

102
fdroidserver/vmtools.py
View File

@ -17,13 +17,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from os import remove as rmfile
from os.path import isdir, isfile, join as joinpath, basename, abspath
from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser
import math
import json
import tarfile
import time
import shutil
import vagrant
import subprocess
from .common import FDroidException
from logging import getLogger
@ -132,10 +131,11 @@ class FDroidBuildVm():
raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir))
if not isfile(self.vgrntfile):
raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile))
import vagrant
self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm)
def isUpAndRunning(self):
raise NotImplementedError('TODO implement this')
def check_okay(self):
return True
def up(self, provision=True):
try:
@ -143,6 +143,15 @@ class FDroidBuildVm():
except subprocess.CalledProcessError as e:
logger.info('could not bring vm up: %s', e)
def snapshot_create(self, name):
raise NotImplementedError('not implemented, please use a sub-type instance')
def suspend(self):
self.vgrnt.suspend()
def resume(self):
self.vgrnt.resume()
def halt(self):
self.vgrnt.halt(force=True)
@ -177,6 +186,9 @@ class FDroidBuildVm():
shutil.rmtree(previous_tmp_dir)
self.vgrnt.package(output=output, vagrantfile=vagrantfile)
def _vagrant_file_name(self, name):
return name.replace('/', '-VAGRANTSLASH-')
def box_add(self, boxname, boxfile, force=True):
"""Add vagrant box to vagrant.
@ -194,8 +206,12 @@ class FDroidBuildVm():
_check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname])
except subprocess.CalledProcessError as e:
logger.debug('tried removing box %s, but is did not exist: %s', boxname, e)
# TODO: remove box files manually
# nesessary when Vagrantfile in ~/.vagrant.d/... is broken.
boxpath = joinpath(expanduser('~'), '.vagrant',
self._vagrant_file_name(boxname))
if isdir(boxpath):
logger.info("attempting to remove box '%s' by deleting: %s",
boxname, boxpath)
shutil.rmtree(boxpath)
class LibvirtBuildVm(FDroidBuildVm):
@ -208,6 +224,22 @@ class LibvirtBuildVm(FDroidBuildVm):
except libvirt.libvirtError as e:
raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e))
def check_okay(self):
import libvirt
imagepath = joinpath('var', 'lib', 'libvirt', 'images',
'%s.img' % self._vagrant_file_name(self.srvname))
image_present = False
if isfile(imagepath):
image_present = True
try:
self.conn.lookupByName(self.srvname)
domain_defined = True
except libvirt.libvirtError:
pass
if image_present and domain_defined:
return True
return False
def destroy(self):
super().destroy()
@ -235,9 +267,7 @@ class LibvirtBuildVm(FDroidBuildVm):
output = "buildserver.box"
logger.debug('no output name set for packaging \'%s\',' +
'defaulting to %s', self.srvname, output)
import libvirt
virConnect = libvirt.open('qemu:///system')
storagePool = virConnect.storagePoolLookupByName('default')
storagePool = self.conn.storagePoolLookupByName('default')
if storagePool:
if isfile('metadata.json'):
@ -258,7 +288,7 @@ class LibvirtBuildVm(FDroidBuildVm):
img_info = json.loads(img_info_raw.decode('utf-8'))
metadata = {"provider": "libvirt",
"format": img_info['format'],
"virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)) + 1,
"virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)),
}
if not vagrantfile:
@ -296,6 +326,56 @@ class LibvirtBuildVm(FDroidBuildVm):
except subprocess.CalledProcessError as e:
logger.info('tired removing \'%s\', file was not present in first place: %s', boxname, e)
def snapshot_create(self, snapshot_name):
try:
_check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name])
logger.info('...waiting a sec...')
time.sleep(10)
except subprocess.CalledProcessError as e:
raise FDroidBuildVmException("could not cerate snapshot '%s' "
"of libvirt vm '%s'"
% (snapshot_name, self.srvname)) from e
def snapshot_list(self):
import libvirt
try:
dom = self.conn.lookupByName(self.srvname)
return dom.listAllSnapshots()
except libvirt.libvirtError as e:
raise FDroidBuildVmException('could not list snapshots for domain \'%s\'' % self.srvname) from e
def snapshot_exists(self, snapshot_name):
import libvirt
try:
dom = self.conn.lookupByName(self.srvname)
return dom.snapshotLookupByName(snapshot_name) is not None
except libvirt.libvirtError:
return False
def snapshot_revert(self, snapshot_name):
import libvirt
try:
dom = self.conn.lookupByName(self.srvname)
snap = dom.snapshotLookupByName(snapshot_name)
dom.revertToSnapshot(snap)
logger.info('...waiting a sec...')
time.sleep(10)
except libvirt.libvirtError as e:
raise FDroidBuildVmException('could not revert domain \'%s\' to snapshot \'%s\''
% (self.srvname, snapshot_name)) from e
class VirtualboxBuildVm(FDroidBuildVm):
pass
def snapshot_create(self, snapshot_name):
raise NotImplemented('TODO')
try:
_check_call(['VBoxManage', 'snapshot', self.srvname, 'take', 'fdroidclean'], cwd=self.srvdir)
logger.info('...waiting a sec...')
time.sleep(10)
except subprocess.CalledProcessError as e:
raise FDroidBuildVmException('could not cerate snapshot '
'of virtualbox vm %s'
% self.srvname) from e
def snapshot_available(self, snapshot_name):
raise NotImplemented('TODO')