opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

deploying build logs to server after each individual build run

This commit is contained in:
Michael Pöhn 2018-06-12 16:18:21 +02:00
parent 4c53c71fcf
commit 88e64df3ef
3 changed files with 52 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
fdroidserver/build.py
View File

@ -75,6 +75,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
sshinfo = vmtools.get_clean_builder('builder', options.reset_server)
output = None
try:
if not buildserverid:
buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
@ -279,6 +280,13 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
vm = vmtools.get_build_vm('builder')
vm.suspend()
# deploy logfile to repository web server
if output:
common.publish_build_log_with_rsync(app.id, build.versionCode, output)
else:
logging.debug('skip publishing full build logs: '
'no output present')
def force_gradle_build_tools(build_dir, build_tools):
for root, dirs, files in os.walk(build_dir):
@ -1137,7 +1145,7 @@ def main():
raise FDroidException(
'Downloading Binaries from %s failed. %s' % (url, e))
# Now we check weather the build can be verified to
# Now we check whether the build can be verified to
# match the supplied binary or not. Should the
# comparison fail, we mark this build as a failure
# and remove everything from the unsigend folder.

42
fdroidserver/common.py
View File

@ -3072,31 +3072,49 @@ def local_rsync(options, fromdir, todir):
raise FDroidException()
def publish_build_log_with_rsync(appid, vercode, log_path, timestamp=int(time.time())):
"""Upload build log of one individual app build to an fdroid repository."""
def publish_build_log_with_rsync(appid, vercode, log_content,
timestamp=int(time.time())):
"""Upload build log of one individual app build to an fdroid repository.
:param appid: package name for dientifying to which app this log belongs.
:param vercode: version of the app to which this build belongs.
:param log_content: Content of the log which is about to be posted.
Should be either a string or bytes. (bytes will
be decoded as 'utf-8')
:param timestamp: timestamp for avoiding logfile name collisions.
"""
# check if publishing logs is enabled in config
if 'publish_build_logs' not in config:
logging.debug('publishing full build logs not enabled')
if not config.get('publish_build_logs', False):
logging.debug('skip publishing full build logs: not enabled in config')
return
if not os.path.isfile(log_path):
logging.warning('skip uploading "{}" (not a file)'.format(log_path))
if not log_content:
logging.warning('skip publishing full build logs: log content is empty')
return
if not (isinstance(timestamp, int) or isinstance(timestamp, float)):
raise ValueError("supplied timestamp '{}' is not a unix timestamp"
.format(timestamp))
with tempfile.TemporaryDirectory() as tmpdir:
# gzip compress log file
log_gz_path = os.path.join(
tmpdir, '{pkg}_{ver}_{ts}.log.gz'.format(pkg=appid,
ver=vercode,
ts=timestamp))
with open(log_path, 'rb') as i, gzip.open(log_gz_path, 'wb') as o:
shutil.copyfileobj(i, o)
ts=int(timestamp)))
with gzip.open(log_gz_path, 'wb') as f:
if isinstance(log_content, str):
f.write(bytes(log_content, 'utf-8'))
else:
f.write(log_content)
# TODO: sign compressed log file, if a signing key is configured
for webroot in config.get('serverwebroot', []):
dest_path = os.path.join(webroot, "buildlogs")
if not dest_path.endswith('/'):
dest_path += '/' # make sure rsync knows this is a directory
cmd = ['rsync',
'--archive',
'--delete-after',
@ -3111,7 +3129,11 @@ def publish_build_log_with_rsync(appid, vercode, log_path, timestamp=int(time.ti
# TODO: also publish signature file if present
subprocess.call(cmd)
retcode = subprocess.call(cmd)
if retcode:
logging.warning("failded publishing build logs to '{}'".format(webroot))
else:
logging.info("published build logs to '{}'".format(webroot))
def get_per_app_repos():

22
tests/common.TestCase
View File

@ -13,6 +13,7 @@ import tempfile
import unittest
import textwrap
import yaml
import gzip
from zipfile import ZipFile
from unittest import mock
@ -792,11 +793,11 @@ class CommonTest(unittest.TestCase):
def test_publish_build_log_with_rsync_with_id_file(self):
mocklogcontent = textwrap.dedent("""\
mocklogcontent = bytes(textwrap.dedent("""\
build started
building...
build completed
profit!""")
profit!"""), 'utf-8')
fdroidserver.common.options = mock.Mock()
fdroidserver.common.options.verbose = False
@ -824,6 +825,8 @@ class CommonTest(unittest.TestCase):
'example.com:/var/www/fdroid/repo/buildlogs'],
cmd)
self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
with gzip.open(cmd[6], 'r') as f:
self.assertTrue(f.read(), mocklogcontent)
elif assert_subprocess_call_iteration == 1:
self.assertListEqual(['rsync',
'--archive',
@ -835,21 +838,18 @@ class CommonTest(unittest.TestCase):
'example.com:/var/www/fdroid/archive/buildlogs'],
cmd)
self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
with gzip.open(cmd[6], 'r') as f:
self.assertTrue(f.read(), mocklogcontent)
else:
self.fail('unexpected subprocess.call invocation ({})'
.format(assert_subprocess_call_iteration))
assert_subprocess_call_iteration += 1
return 0
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
log_path = os.path.join(tmpdir, 'mock.log')
with open(log_path, 'w') as f:
f.write(mocklogcontent)
with mock.patch('subprocess.call',
side_effect=assert_subprocess_call):
fdroidserver.common.publish_build_log_with_rsync(
'com.example.app', '4711', log_path, 1)
with mock.patch('subprocess.call',
side_effect=assert_subprocess_call):
fdroidserver.common.publish_build_log_with_rsync(
'com.example.app', '4711', mocklogcontent, 1.1)
if __name__ == "__main__":