Merge branch 'switching-s3cmd-for-rclone' into 'master'
Adding rclone as an option to fdroid deploy Closes #1095 See merge request fdroid/fdroidserver!1095
This commit is contained in:
paul mayero
commit
bf3de63a84
|
|
@ -267,10 +267,20 @@
|
|||
# sync_from_local_copy_dir: true
|
||||
|
||||
|
||||
# To upload the repo to an Amazon S3 bucket using `fdroid server
|
||||
# update`. Warning, this deletes and recreates the whole fdroid/
|
||||
# directory each time. This prefers s3cmd, but can also use
|
||||
# apache-libcloud. To customize how s3cmd interacts with the cloud
|
||||
# To upload the repo to an Amazon S3 bucket using `fdroid deploy'
|
||||
# . rclone, s3cmd and apache libcloud are the available options.
|
||||
# If rclone and s3cmd are not installed, apache libcloud is used.
|
||||
# To use apache libcloud, add the following options to this file
|
||||
# (config.yml)
|
||||
#
|
||||
# awsbucket: myawsfdroid
|
||||
# awsaccesskeyid: SEE0CHAITHEIMAUR2USA
|
||||
# awssecretkey: {env: awssecretkey}
|
||||
#
|
||||
# In case s3cmd is installed and rclone is not installed,
|
||||
# s3cmd will be the preferred sync option.
|
||||
# It will delete and recreate the whole fdroid directory each time.
|
||||
# To customize how s3cmd interacts with the cloud
|
||||
# provider, create a 's3cfg' file next to this file (config.yml), and
|
||||
# those settings will be used instead of any 'aws' variable below.
|
||||
# Secrets can be fetched from environment variables to ensure that
|
||||
|
|
@ -279,6 +289,47 @@
|
|||
# awsbucket: myawsfdroid
|
||||
# awsaccesskeyid: SEE0CHAITHEIMAUR2USA
|
||||
# awssecretkey: {env: awssecretkey}
|
||||
#
|
||||
# In case rclone is installed and s3cmd is not installed,
|
||||
# rclone will be the preferred sync option.
|
||||
# It will sync the local folders with remote folders without
|
||||
# deleting anything in one go.
|
||||
# To ensure success, install rclone as per
|
||||
# the instructions at https://rclone.org/install/ and also configure for
|
||||
# object storage services as detailed at https://rclone.org/s3/#configuration
|
||||
# By default rclone uses the configuration file at ~/.config/rclone/rclone.conf
|
||||
# To specify a custom configuration file, please add the full path to the
|
||||
# configuration file as below
|
||||
#
|
||||
# path_to_custom_rclone_config: /home/mycomputer/somedir/example.conf
|
||||
#
|
||||
# This setting will ignore the default rclone config found at
|
||||
# ~/.config/rclone/rclone.conf
|
||||
#
|
||||
# Please note that rclone_config can be assigned a string or list
|
||||
#
|
||||
# awsbucket: myawsfdroid
|
||||
# rclone_config: aws-sample-config
|
||||
#
|
||||
# or
|
||||
#
|
||||
# awsbucket: myawsfdroid
|
||||
# rclone_config: [aws-sample-config, rclone-supported-service-config]
|
||||
#
|
||||
# In case both rclone and s3cmd are installed, the preferred sync
|
||||
# tool can be specified in this file (config.yml)
|
||||
# if s3cmd is preferred, set it as below
|
||||
#
|
||||
# s3cmd: true
|
||||
#
|
||||
# if rclone is preferred, set it as below
|
||||
#
|
||||
# rclone: true
|
||||
#
|
||||
# Please note that only one can be set to true at any time
|
||||
# Also, in the event that both s3cmd and rclone are installed
|
||||
# and both are missing from the config.yml file, the preferred
|
||||
# tool will be s3cmd.
|
||||
|
||||
|
||||
# If you want to force 'fdroid server' to use a non-standard serverwebroot.
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import urllib
|
|||
import yaml
|
||||
from argparse import ArgumentParser
|
||||
import logging
|
||||
from shlex import split
|
||||
import shutil
|
||||
|
||||
from . import _
|
||||
|
|
@ -45,6 +46,7 @@ BINARY_TRANSPARENCY_DIR = 'binary_transparency'
|
|||
|
||||
AUTO_S3CFG = '.fdroid-deploy-s3cfg'
|
||||
USER_S3CFG = 's3cfg'
|
||||
USER_RCLONE_CONF = None
|
||||
REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
|
||||
|
||||
INDEX_FILES = [
|
||||
|
|
@ -100,8 +102,25 @@ def update_awsbucket(repo_section):
|
|||
logging.debug('Syncing "' + repo_section + '" to Amazon S3 bucket "'
|
||||
+ config['awsbucket'] + '"')
|
||||
|
||||
if common.set_command_in_config('s3cmd'):
|
||||
if common.set_command_in_config('s3cmd') and \
|
||||
common.set_command_in_config('rclone'):
|
||||
logging.info('Both rclone and s3cmd are installed.'
|
||||
'Checking config.yml for preference.')
|
||||
if config['s3cmd'] is not True and config['rclone'] is not True:
|
||||
logging.warning('No syncing tool set in config.yml!. Defaulting to using s3cmd')
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
if config['s3cmd'] is True and config['rclone'] is True:
|
||||
logging.warning('Both syncing tools set in config.yml!. Defaulting to using s3cmd')
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
if config['s3cmd'] is True and config['rclone'] is not True:
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
if config['rclone'] is True and config['s3cmd'] is not True:
|
||||
update_remote_storage_with_rclone(repo_section)
|
||||
|
||||
elif common.set_command_in_config('s3cmd'):
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
elif common.set_command_in_config('rclone'):
|
||||
update_remote_storage_with_rclone(repo_section)
|
||||
else:
|
||||
update_awsbucket_libcloud(repo_section)
|
||||
|
||||
|
|
@ -178,6 +197,99 @@ def update_awsbucket_s3cmd(repo_section):
|
|||
raise FDroidException()
|
||||
|
||||
|
||||
def update_remote_storage_with_rclone(repo_section):
|
||||
"""
|
||||
Upload fdroid repo folder to remote storage using rclone sync.
|
||||
|
||||
Rclone sync can send the files to any supported remote storage
|
||||
service once without numerous polling.
|
||||
If remote storage is s3 e.g aws s3, wasabi, filebase then path will be
|
||||
bucket_name/fdroid/repo where bucket_name will be an s3 bucket
|
||||
If remote storage is storage drive/sftp e.g google drive, rsync.net
|
||||
the new path will be bucket_name/fdroid/repo where bucket_name
|
||||
will be a folder
|
||||
|
||||
Better than the s3cmd command as it does the syncing in one command
|
||||
Check https://rclone.org/docs/#config-config-file (optional config file)
|
||||
"""
|
||||
logging.debug(_('Using rclone to sync with: {url}')
|
||||
.format(url=config['awsbucket']))
|
||||
|
||||
if config.get('path_to_custom_rclone_config') is not None:
|
||||
USER_RCLONE_CONF = config['path_to_custom_rclone_config']
|
||||
if os.path.exists(USER_RCLONE_CONF):
|
||||
logging.info("'path_to_custom_rclone_config' found in config.yml")
|
||||
logging.info(_('Using "{path}" for syncing with remote storage.')
|
||||
.format(path=USER_RCLONE_CONF))
|
||||
configfilename = USER_RCLONE_CONF
|
||||
else:
|
||||
logging.info('Custom configuration not found.')
|
||||
logging.info('Using default configuration at {}'
|
||||
.format(subprocess.getoutput('rclone config file')))
|
||||
configfilename = None
|
||||
else:
|
||||
logging.warning("'path_to_custom_rclone_config' not found in config.yml")
|
||||
logging.info('Custom configuration not found.')
|
||||
logging.info('Using default configuration at {}'
|
||||
.format(subprocess.getoutput('rclone config file')))
|
||||
configfilename = None
|
||||
|
||||
upload_dir = 'fdroid/' + repo_section
|
||||
|
||||
if not config.get('rclone_config') or not config.get('awsbucket'):
|
||||
raise FDroidException(
|
||||
_('To use rclone, rclone_config and awsbucket must be set in config.yml!'))
|
||||
|
||||
if isinstance(config['rclone_config'], str):
|
||||
rclone_sync_command = 'rclone sync ' + repo_section + \
|
||||
' ' + config['rclone_config'] + ':' + \
|
||||
config['awsbucket'] + '/' + upload_dir
|
||||
|
||||
rclone_sync_command = split(rclone_sync_command)
|
||||
|
||||
if options.verbose:
|
||||
rclone_sync_command += ['--verbose']
|
||||
elif options.quiet:
|
||||
rclone_sync_command += ['--quiet']
|
||||
|
||||
if configfilename:
|
||||
rclone_sync_command += split('--config=' + configfilename)
|
||||
|
||||
complete_remote_path = config['rclone_config'] + ':' + \
|
||||
config['awsbucket'] + '/' + upload_dir
|
||||
|
||||
logging.debug("rclone sync all files in " + repo_section + ' to '
|
||||
+ complete_remote_path)
|
||||
|
||||
if subprocess.call(rclone_sync_command) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
if isinstance(config['rclone_config'], list):
|
||||
for remote_config in config['rclone_config']:
|
||||
rclone_sync_command = 'rclone sync ' + repo_section + \
|
||||
' ' + remote_config + ':' + \
|
||||
config['awsbucket'] + '/' + upload_dir
|
||||
|
||||
rclone_sync_command = split(rclone_sync_command)
|
||||
|
||||
if options.verbose:
|
||||
rclone_sync_command += ['--verbose']
|
||||
elif options.quiet:
|
||||
rclone_sync_command += ['--quiet']
|
||||
|
||||
if configfilename:
|
||||
rclone_sync_command += split('--config=' + configfilename)
|
||||
|
||||
complete_remote_path = remote_config + ':' + \
|
||||
config['awsbucket'] + '/' + upload_dir
|
||||
|
||||
logging.debug("rclone sync all files in " + repo_section + ' to '
|
||||
+ complete_remote_path)
|
||||
|
||||
if subprocess.call(rclone_sync_command) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
def update_awsbucket_libcloud(repo_section):
|
||||
"""No summary.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import configparser
|
||||
import inspect
|
||||
import logging
|
||||
import optparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
|
@ -22,6 +24,11 @@ from fdroidserver.exception import FDroidException
|
|||
from testcommon import TmpCwd, mkdtemp
|
||||
|
||||
|
||||
class Options:
|
||||
quiet = False
|
||||
verbose = False
|
||||
|
||||
|
||||
class DeployTest(unittest.TestCase):
|
||||
'''fdroidserver/deploy.py'''
|
||||
|
||||
|
|
@ -32,61 +39,50 @@ class DeployTest(unittest.TestCase):
|
|||
self._td = mkdtemp()
|
||||
self.testdir = self._td.name
|
||||
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.config = {}
|
||||
fdroidserver.deploy.USER_RCLONE_CONF = False
|
||||
|
||||
def tearDown(self):
|
||||
self._td.cleanup()
|
||||
|
||||
def test_update_serverwebroots_bad_None(self):
|
||||
with self.assertRaises(TypeError):
|
||||
fdroidserver.deploy.update_serverwebroots(None, 'repo')
|
||||
|
||||
def test_update_serverwebroots_bad_int(self):
|
||||
with self.assertRaises(TypeError):
|
||||
fdroidserver.deploy.update_serverwebroots(9, 'repo')
|
||||
|
||||
def test_update_serverwebroots_bad_float(self):
|
||||
with self.assertRaises(TypeError):
|
||||
fdroidserver.deploy.update_serverwebroots(1.0, 'repo')
|
||||
|
||||
def test_update_serverwebroots(self):
|
||||
"""rsync works with file paths, so this test uses paths for the URLs"""
|
||||
@unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
|
||||
def test_update_remote_storage_with_rclone(self):
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir()
|
||||
fake_apk = repo / 'fake.apk'
|
||||
repo.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fake_apk = repo / 'another_fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
url0 = Path('url0/fdroid')
|
||||
url0.mkdir(parents=True)
|
||||
url1 = Path('url1/fdroid')
|
||||
url1.mkdir(parents=True)
|
||||
|
||||
dest_apk0 = url0 / fake_apk
|
||||
dest_apk1 = url1 / fake_apk
|
||||
self.assertFalse(dest_apk0.is_file())
|
||||
self.assertFalse(dest_apk1.is_file())
|
||||
fdroidserver.deploy.update_serverwebroots(
|
||||
[
|
||||
{'url': str(url0)},
|
||||
{'url': str(url1)},
|
||||
],
|
||||
str(repo),
|
||||
)
|
||||
self.assertTrue(dest_apk0.is_file())
|
||||
self.assertTrue(dest_apk1.is_file())
|
||||
# write out rclone config for test use
|
||||
rclone_config = configparser.ConfigParser()
|
||||
rclone_config.add_section("test-local-config")
|
||||
rclone_config.set("test-local-config", "type", "local")
|
||||
|
||||
def test_update_serverwebroots_url_does_not_end_with_fdroid(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo')
|
||||
rclone_config_path = Path('rclone_config_path')
|
||||
rclone_config_path.mkdir(parents=True, exist_ok=True)
|
||||
rclone_file = rclone_config_path / 'rclone.conf'
|
||||
with open(rclone_file, 'w') as configfile:
|
||||
rclone_config.write(configfile)
|
||||
|
||||
def test_update_serverwebroots_bad_ssh_url(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
fdroidserver.deploy.update_serverwebroots(
|
||||
[{'url': 'f@b.ar::/path/to/fdroid'}], 'repo'
|
||||
)
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
|
||||
fdroidserver.deploy.config['rclone'] = True
|
||||
fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
|
||||
fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
|
||||
fdroidserver.deploy.options = Options
|
||||
|
||||
def test_update_serverwebroots_unsupported_ssh_url(self):
|
||||
with self.assertRaises(SystemExit):
|
||||
fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
|
||||
# write out destination path
|
||||
destination = Path('some_bucket_folder/fdroid')
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
dest_path = Path(destination) / fake_apk
|
||||
self.assertFalse(dest_path.is_file())
|
||||
repo_section = str(repo)
|
||||
# fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
|
||||
fdroidserver.deploy.update_remote_storage_with_rclone(repo_section)
|
||||
self.assertFalse(dest_path.is_file())
|
||||
|
||||
def test_update_serverwebroot(self):
|
||||
"""rsync works with file paths, so this test uses paths for the URLs"""
|
||||
|
|
@ -101,6 +97,8 @@ class DeployTest(unittest.TestCase):
|
|||
|
||||
dest_apk = url / fake_apk
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
|
||||
self.assertTrue(dest_apk.is_file())
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue