added helper function for uploading build logs with rsync

examples/config.py

@@ -163,6 +163,11 @@ The repository of older versions of applications from the main demo repository.
 #     'bar.info:/var/www/fdroid',
 #     }
 
+# Uncomment this option if you want to publish build logs to your repository
+# server(s). Logs get published to all servers configured in 'serverwebroot'.
+#
+# publish_build_logs = True
+
 # The full URL to a git remote repository. You can include
 # multiple servers to mirror to by wrapping the whole thing in {} or [], and
 # including the servergitmirrors strings in a comma-separated list.

fdroidserver/common.py

@@ -25,6 +25,7 @@ import os
 import sys
 import re
 import ast
+import gzip
 import shutil
 import glob
 import stat
@@ -100,6 +101,7 @@ default_config = {
     'per_app_repos': False,
     'make_current_version_link': True,
     'current_version_name_source': 'Name',
+    'publish_build_logs': False,
     'update_stats': False,
     'stats_ignore': [],
     'stats_server': None,
@@ -3070,6 +3072,48 @@ def local_rsync(options, fromdir, todir):
         raise FDroidException()
 
 
+def publish_build_log_with_rsync(appid, vercode, log_path, timestamp=int(time.time())):
+    """Upload build log of one individual app build to an fdroid repository."""
+
+    # check if publishing logs is enabled in config
+    if 'publish_build_logs' not in config:
+        logging.debug('publishing full build logs not enabled')
+        return
+
+    if not os.path.isfile(log_path):
+        logging.warning('skip uploading "{}" (not a file)'.format(log_path))
+        return
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        # gzip compress log file
+        log_gz_path = os.path.join(
+            tmpdir, '{pkg}_{ver}_{ts}.log.gz'.format(pkg=appid,
+                                                     ver=vercode,
+                                                     ts=timestamp))
+        with open(log_path, 'rb') as i, gzip.open(log_gz_path, 'wb') as o:
+            shutil.copyfileobj(i, o)
+
+        # TODO: sign compressed log file, if a signing key is configured
+
+        for webroot in config.get('serverwebroot', []):
+            dest_path = os.path.join(webroot, "buildlogs")
+            cmd = ['rsync',
+                   '--archive',
+                   '--delete-after',
+                   '--safe-links']
+            if options.verbose:
+                cmd += ['--verbose']
+            if options.quiet:
+                cmd += ['--quiet']
+            if 'identity_file' in config:
+                cmd += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
+            cmd += [log_gz_path, dest_path]
+
+            # TODO: also publish signature file if present
+
+            subprocess.call(cmd)
+
+
 def get_per_app_repos():
     '''per-app repos are dirs named with the packageName of a single app'''
 

tests/common.TestCase

@@ -14,6 +14,7 @@ import unittest
 import textwrap
 import yaml
 from zipfile import ZipFile
+from unittest import mock
 
 
 localmodule = os.path.realpath(
@@ -789,6 +790,67 @@ class CommonTest(unittest.TestCase):
         with self.assertRaises(SyntaxError):
             fdroidserver.common.calculate_math_string('1-1 # no comment')
 
+    def test_publish_build_log_with_rsync_with_id_file(self):
+
+        mocklogcontent = textwrap.dedent("""\
+            build started
+            building...
+            build completed
+            profit!""")
+
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = False
+        fdroidserver.common.config = {}
+        fdroidserver.common.config['serverwebroot'] = [
+            'example.com:/var/www/fdroid/repo/',
+            'example.com:/var/www/fdroid/archive/']
+        fdroidserver.common.config['publish_build_logs'] = True
+        fdroidserver.common.config['identity_file'] = 'ssh/id_rsa'
+
+        assert_subprocess_call_iteration = 0
+
+        def assert_subprocess_call(cmd):
+            nonlocal assert_subprocess_call_iteration
+            logging.debug(cmd)
+            if assert_subprocess_call_iteration == 0:
+                self.assertListEqual(['rsync',
+                                      '--archive',
+                                      '--delete-after',
+                                      '--safe-links',
+                                      '-e',
+                                      'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ssh/id_rsa',
+                                      cmd[6],
+                                      'example.com:/var/www/fdroid/repo/buildlogs'],
+                                     cmd)
+                self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
+            elif assert_subprocess_call_iteration == 1:
+                self.assertListEqual(['rsync',
+                                      '--archive',
+                                      '--delete-after',
+                                      '--safe-links',
+                                      '-e',
+                                      'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ssh/id_rsa',
+                                      cmd[6],
+                                      'example.com:/var/www/fdroid/archive/buildlogs'],
+                                     cmd)
+                self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
+            else:
+                self.fail('unexpected subprocess.call invocation ({})'
+                          .format(assert_subprocess_call_iteration))
+            assert_subprocess_call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            log_path = os.path.join(tmpdir, 'mock.log')
+            with open(log_path, 'w') as f:
+                f.write(mocklogcontent)
+
+            with mock.patch('subprocess.call',
+                            side_effect=assert_subprocess_call):
+                fdroidserver.common.publish_build_log_with_rsync(
+                    'com.example.app', '4711', log_path, 1)
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))