Merge branch 'serverwebroot-as-dict' into 'master'
config: convert serverwebroot: to list-of-dicts format See merge request fdroid/fdroidserver!1430
This commit is contained in:
commit
d60f36c30e
|
@ -178,6 +178,12 @@
|
|||
# serverwebroot:
|
||||
# - foo.com:/usr/share/nginx/www/fdroid
|
||||
# - bar.info:/var/www/fdroid
|
||||
#
|
||||
# There is a special mode to only deploy the index file:
|
||||
#
|
||||
# serverwebroot:
|
||||
# - url: 'me@b.az:/srv/fdroid'
|
||||
# indexOnly: true
|
||||
|
||||
|
||||
# When running fdroid processes on a remote server, it is possible to
|
||||
|
|
|
@ -55,7 +55,9 @@ scan_apk # NOQA: B101
|
|||
scan_repo_files # NOQA: B101
|
||||
from fdroidserver.deploy import (update_awsbucket,
|
||||
update_servergitmirrors,
|
||||
update_serverwebroots,
|
||||
update_serverwebroot) # NOQA: E402
|
||||
update_awsbucket # NOQA: B101
|
||||
update_servergitmirrors # NOQA: B101
|
||||
update_serverwebroots # NOQA: B101
|
||||
update_serverwebroot # NOQA: B101
|
||||
|
|
|
@ -462,18 +462,22 @@ def read_config(opts=None):
|
|||
|
||||
if 'serverwebroot' in config:
|
||||
if isinstance(config['serverwebroot'], str):
|
||||
roots = [config['serverwebroot']]
|
||||
roots = [{'url': config['serverwebroot']}]
|
||||
elif all(isinstance(item, str) for item in config['serverwebroot']):
|
||||
roots = [{'url': i} for i in config['serverwebroot']]
|
||||
elif all(isinstance(item, dict) for item in config['serverwebroot']):
|
||||
roots = config['serverwebroot']
|
||||
else:
|
||||
raise TypeError(_('only accepts strings, lists, and tuples'))
|
||||
rootlist = []
|
||||
for rootstr in roots:
|
||||
for d in roots:
|
||||
# since this is used with rsync, where trailing slashes have
|
||||
# meaning, ensure there is always a trailing slash
|
||||
rootstr = d['url']
|
||||
if rootstr[-1] != '/':
|
||||
rootstr += '/'
|
||||
rootlist.append(rootstr.replace('//', '/'))
|
||||
d['url'] = rootstr.replace('//', '/')
|
||||
rootlist.append(d)
|
||||
config['serverwebroot'] = rootlist
|
||||
|
||||
if 'servergitmirrors' in config:
|
||||
|
@ -4052,7 +4056,8 @@ def rsync_status_file_to_repo(path, repo_subdir=None):
|
|||
logging.debug(_('skip deploying full build logs: not enabled in config'))
|
||||
return
|
||||
|
||||
for webroot in config.get('serverwebroot', []):
|
||||
for d in config.get('serverwebroot', []):
|
||||
webroot = d['url']
|
||||
cmd = ['rsync',
|
||||
'--archive',
|
||||
'--delete-after',
|
||||
|
|
|
@ -284,34 +284,63 @@ def update_serverwebroot(serverwebroot, repo_section):
|
|||
_('rsync is missing or broken: {error}').format(error=e)
|
||||
) from e
|
||||
rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
|
||||
if not options.no_checksum:
|
||||
if not options or not options.no_checksum:
|
||||
rsyncargs.append('--checksum')
|
||||
if options.verbose:
|
||||
if options and options.verbose:
|
||||
rsyncargs += ['--verbose']
|
||||
if options.quiet:
|
||||
if options and options.quiet:
|
||||
rsyncargs += ['--quiet']
|
||||
if options.identity_file is not None:
|
||||
if options and options.identity_file:
|
||||
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file]
|
||||
elif 'identity_file' in config:
|
||||
elif config and config.get('identity_file'):
|
||||
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
|
||||
logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
|
||||
url = serverwebroot['url']
|
||||
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0:
|
||||
if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
|
||||
raise FDroidException()
|
||||
if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
|
||||
if subprocess.call(rsyncargs + [repo_section, url]) != 0:
|
||||
raise FDroidException()
|
||||
# upload "current version" symlinks if requested
|
||||
if config['make_current_version_link'] and repo_section == 'repo':
|
||||
if config and config.get('make_current_version_link') and repo_section == 'repo':
|
||||
links_to_upload = []
|
||||
for f in glob.glob('*.apk') \
|
||||
+ glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
|
||||
if os.path.islink(f):
|
||||
links_to_upload.append(f)
|
||||
if len(links_to_upload) > 0:
|
||||
if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0:
|
||||
if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True):
|
||||
for d in serverwebroots:
|
||||
# this supports both an ssh host:path and just a path
|
||||
serverwebroot = d['url']
|
||||
s = serverwebroot.rstrip('/').split(':')
|
||||
if len(s) == 1:
|
||||
fdroiddir = s[0]
|
||||
elif len(s) == 2:
|
||||
host, fdroiddir = s
|
||||
else:
|
||||
logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot)
|
||||
sys.exit(1)
|
||||
repobase = os.path.basename(fdroiddir)
|
||||
if standardwebroot and repobase != 'fdroid':
|
||||
logging.error(
|
||||
_(
|
||||
'serverwebroot: path does not end with "fdroid", perhaps you meant one of these:'
|
||||
)
|
||||
+ '\n\t'
|
||||
+ serverwebroot.rstrip('/')
|
||||
+ '/fdroid\n\t'
|
||||
+ serverwebroot.rstrip('/').rstrip(repobase)
|
||||
+ 'fdroid'
|
||||
)
|
||||
sys.exit(1)
|
||||
update_serverwebroot(d, repo_section)
|
||||
|
||||
|
||||
def sync_from_localcopy(repo_section, local_copy_dir):
|
||||
"""Sync the repo from "local copy dir" filesystem to this box.
|
||||
|
||||
|
@ -747,24 +776,6 @@ def main():
|
|||
else:
|
||||
standardwebroot = True
|
||||
|
||||
for serverwebroot in config.get('serverwebroot', []):
|
||||
# this supports both an ssh host:path and just a path
|
||||
s = serverwebroot.rstrip('/').split(':')
|
||||
if len(s) == 1:
|
||||
fdroiddir = s[0]
|
||||
elif len(s) == 2:
|
||||
host, fdroiddir = s
|
||||
else:
|
||||
logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot)
|
||||
sys.exit(1)
|
||||
repobase = os.path.basename(fdroiddir)
|
||||
if standardwebroot and repobase != 'fdroid':
|
||||
logging.error('serverwebroot path does not end with "fdroid", '
|
||||
+ 'perhaps you meant one of these:\n\t'
|
||||
+ serverwebroot.rstrip('/') + '/fdroid\n\t'
|
||||
+ serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
|
||||
sys.exit(1)
|
||||
|
||||
if options.local_copy_dir is not None:
|
||||
local_copy_dir = options.local_copy_dir
|
||||
elif config.get('local_copy_dir'):
|
||||
|
@ -825,8 +836,10 @@ def main():
|
|||
sync_from_localcopy(repo_section, local_copy_dir)
|
||||
else:
|
||||
update_localcopy(repo_section, local_copy_dir)
|
||||
for serverwebroot in config.get('serverwebroot', []):
|
||||
update_serverwebroot(serverwebroot, repo_section)
|
||||
if config.get('serverwebroot'):
|
||||
update_serverwebroots(
|
||||
config['serverwebroot'], repo_section, standardwebroot
|
||||
)
|
||||
if config.get('servergitmirrors', []):
|
||||
# update_servergitmirrors will take care of multiple mirrors so don't need a foreach
|
||||
servergitmirrors = config.get('servergitmirrors', [])
|
||||
|
|
|
@ -1655,8 +1655,8 @@ class CommonTest(unittest.TestCase):
|
|||
fdroidserver.common.options.quiet = False
|
||||
fdroidserver.common.config = {}
|
||||
fdroidserver.common.config['serverwebroot'] = [
|
||||
'example.com:/var/www/fdroid/',
|
||||
'example.com:/var/www/fbot/',
|
||||
{'url': 'example.com:/var/www/fdroid/'},
|
||||
{'url': 'example.com:/var/www/fbot/'},
|
||||
]
|
||||
fdroidserver.common.config['deploy_process_logs'] = True
|
||||
fdroidserver.common.config['identity_file'] = 'ssh/id_rsa'
|
||||
|
@ -1718,7 +1718,7 @@ class CommonTest(unittest.TestCase):
|
|||
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.config = {}
|
||||
fdroidserver.common.config['serverwebroot'] = [fakeserver]
|
||||
fdroidserver.common.config['serverwebroot'] = [{'url': fakeserver}]
|
||||
fdroidserver.common.config['identity_file'] = 'ssh/id_rsa'
|
||||
|
||||
def assert_subprocess_call(cmd):
|
||||
|
@ -2868,6 +2868,30 @@ class CommonTest(unittest.TestCase):
|
|||
with self.assertRaises(TypeError):
|
||||
fdroidserver.common.config_type_check('config/mirrors.yml', str())
|
||||
|
||||
def test_config_serverwebroot_str(self):
|
||||
os.chdir(self.testdir)
|
||||
Path('config.yml').write_text("""serverwebroot: 'foo@example.com:/var/www'""")
|
||||
self.assertEqual(
|
||||
[{'url': 'foo@example.com:/var/www/'}],
|
||||
fdroidserver.common.read_config()['serverwebroot'],
|
||||
)
|
||||
|
||||
def test_config_serverwebroot_list(self):
|
||||
os.chdir(self.testdir)
|
||||
Path('config.yml').write_text("""serverwebroot:\n - foo@example.com:/var/www""")
|
||||
self.assertEqual(
|
||||
[{'url': 'foo@example.com:/var/www/'}],
|
||||
fdroidserver.common.read_config()['serverwebroot'],
|
||||
)
|
||||
|
||||
def test_config_serverwebroot_dict(self):
|
||||
os.chdir(self.testdir)
|
||||
Path('config.yml').write_text("""serverwebroot:\n - url: 'foo@example.com:/var/www'""")
|
||||
self.assertEqual(
|
||||
[{'url': 'foo@example.com:/var/www/'}],
|
||||
fdroidserver.common.read_config()['serverwebroot'],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
|
|
|
@ -32,29 +32,76 @@ class DeployTest(unittest.TestCase):
|
|||
self._td = mkdtemp()
|
||||
self.testdir = self._td.name
|
||||
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.config = {}
|
||||
|
||||
def tearDown(self):
|
||||
self._td.cleanup()
|
||||
|
||||
def test_update_serverwebroots_bad_None(self):
|
||||
with self.assertRaises(TypeError):
|
||||
fdroidserver.deploy.update_serverwebroots(None, 'repo')
|
||||
|
||||
def test_update_serverwebroots_bad_int(self):
|
||||
with self.assertRaises(TypeError):
|
||||
fdroidserver.deploy.update_serverwebroots(9, 'repo')
|
||||
|
||||
def test_update_serverwebroots_bad_float(self):
|
||||
with self.assertRaises(TypeError):
|
||||
fdroidserver.deploy.update_serverwebroots(1.0, 'repo')
|
||||
|
||||
def test_update_serverwebroots(self):
|
||||
"""rsync works with file paths, so this test uses paths for the URLs"""
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir()
|
||||
fake_apk = repo / 'fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
url0 = Path('url0/fdroid')
|
||||
url0.mkdir(parents=True)
|
||||
url1 = Path('url1/fdroid')
|
||||
url1.mkdir(parents=True)
|
||||
|
||||
dest_apk0 = url0 / fake_apk
|
||||
dest_apk1 = url1 / fake_apk
|
||||
self.assertFalse(dest_apk0.is_file())
|
||||
self.assertFalse(dest_apk1.is_file())
|
||||
fdroidserver.deploy.update_serverwebroots(
|
||||
[
|
||||
{'url': str(url0)},
|
||||
{'url': str(url1)},
|
||||
],
|
||||
str(repo),
|
||||
)
|
||||
self.assertTrue(dest_apk0.is_file())
|
||||
self.assertTrue(dest_apk1.is_file())
|
||||
|
||||
def test_update_serverwebroots_url_does_not_end_with_fdroid(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo')
|
||||
|
||||
def test_update_serverwebroots_bad_ssh_url(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
fdroidserver.deploy.update_serverwebroots(
|
||||
[{'url': 'f@b.ar::/path/to/fdroid'}], 'repo'
|
||||
)
|
||||
|
||||
def test_update_serverwebroots_unsupported_ssh_url(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
|
||||
|
||||
def test_update_serverwebroot(self):
|
||||
"""rsync works with file paths, so this test uses paths for the URLs"""
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True)
|
||||
fake_apk = repo / 'fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
serverwebroot = Path('serverwebroot')
|
||||
serverwebroot.mkdir()
|
||||
url = Path('url')
|
||||
url.mkdir()
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk = Path(serverwebroot) / fake_apk
|
||||
dest_apk = url / fake_apk
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo')
|
||||
fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
|
||||
self.assertTrue(dest_apk.is_file())
|
||||
|
||||
@mock.patch.dict(os.environ, clear=True)
|
||||
|
@ -66,13 +113,13 @@ class DeployTest(unittest.TestCase):
|
|||
|
||||
def test_update_serverwebroot_make_cur_version_link(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = True
|
||||
serverwebroot = "example.com:/var/www/fdroid"
|
||||
fdroidserver.deploy.config = {'make_current_version_link': True}
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'repo'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
|
@ -123,7 +170,7 @@ class DeployTest(unittest.TestCase):
|
|||
'--safe-links',
|
||||
'--quiet',
|
||||
'repo',
|
||||
serverwebroot,
|
||||
url,
|
||||
],
|
||||
)
|
||||
elif call_iteration == 2:
|
||||
|
@ -152,18 +199,19 @@ class DeployTest(unittest.TestCase):
|
|||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||
fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
|
||||
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
||||
self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
|
||||
|
||||
def test_update_serverwebroot_with_id_file(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.no_chcksum = False
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = True
|
||||
fdroidserver.deploy.options.quiet = False
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['identity_file'] = './id_rsa'
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
serverwebroot = "example.com:/var/www/fdroid"
|
||||
fdroidserver.deploy.config = {'identity_file': './id_rsa'}
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'archive'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
|
@ -204,7 +252,7 @@ class DeployTest(unittest.TestCase):
|
|||
'--exclude',
|
||||
'archive/index.xml',
|
||||
'archive',
|
||||
serverwebroot,
|
||||
url,
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
|
@ -220,7 +268,7 @@ class DeployTest(unittest.TestCase):
|
|||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
'archive',
|
||||
serverwebroot,
|
||||
url,
|
||||
],
|
||||
)
|
||||
else:
|
||||
|
@ -229,7 +277,7 @@ class DeployTest(unittest.TestCase):
|
|||
return 0
|
||||
|
||||
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||
fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
|
||||
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
||||
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
||||
|
||||
@unittest.skipIf(
|
||||
|
|
Loading…
Reference in New Issue