Merge branch 'index-only-deploy-mode' into 'master'
Add index only deployment mode Closes #1181 See merge request fdroid/fdroidserver!1420
This commit is contained in:
proletarius101
commit
35f30c01e1
|
|
@ -183,7 +183,7 @@
|
|||
#
|
||||
# serverwebroot:
|
||||
# - url: 'me@b.az:/srv/fdroid'
|
||||
# indexOnly: true
|
||||
# index_only: true
|
||||
|
||||
|
||||
# When running fdroid processes on a remote server, it is possible to
|
||||
|
|
@ -209,7 +209,7 @@
|
|||
# servergitmirrors:
|
||||
# - url: https://github.com/user/repo
|
||||
# - url: https://gitlab.com/user/repo
|
||||
# indexOnly: true
|
||||
# index_only: true
|
||||
|
||||
# Most git hosting services have hard size limits for each git repo.
|
||||
# `fdroid deploy` will delete the git history when the git mirror repo
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@
|
|||
# libraries here as they will become a requirement for all commands.
|
||||
|
||||
import difflib
|
||||
from typing import List
|
||||
import git
|
||||
import glob
|
||||
import io
|
||||
|
|
@ -4037,7 +4038,7 @@ def get_app_display_name(app):
|
|||
return app.get('AutoName') or app['id']
|
||||
|
||||
|
||||
def local_rsync(options, fromdir, todir):
|
||||
def local_rsync(options, from_paths: List[str], todir: str):
|
||||
"""Rsync method for local to local copying of things.
|
||||
|
||||
This is an rsync wrapper with all the settings for safe use within
|
||||
|
|
@ -4054,8 +4055,8 @@ def local_rsync(options, fromdir, todir):
|
|||
rsyncargs += ['--verbose']
|
||||
if options.quiet:
|
||||
rsyncargs += ['--quiet']
|
||||
logging.debug(' '.join(rsyncargs + [fromdir, todir]))
|
||||
if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
|
||||
logging.debug(' '.join(rsyncargs + from_paths + [todir]))
|
||||
if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -25,8 +25,9 @@ import re
|
|||
import subprocess
|
||||
import time
|
||||
import urllib
|
||||
from typing import Optional
|
||||
import yaml
|
||||
from argparse import ArgumentParser
|
||||
from argparse import ArgumentParser, Namespace
|
||||
import logging
|
||||
import shutil
|
||||
|
||||
|
|
@ -36,7 +37,7 @@ from . import index
|
|||
from .exception import FDroidException
|
||||
|
||||
config = None
|
||||
options = None
|
||||
options: Optional[Namespace] = None
|
||||
start_timestamp = time.gmtime()
|
||||
|
||||
GIT_BRANCH = 'master'
|
||||
|
|
@ -61,27 +62,19 @@ INDEX_FILES = [
|
|||
]
|
||||
|
||||
|
||||
def _get_index_excludes(repo_section):
|
||||
def _get_index_file_paths(base_dir):
|
||||
"""Return the list of files to be synced last, since they finalize the deploy.
|
||||
|
||||
The process of pushing all the new packages to the various
|
||||
services can take a while. So the index files should be updated
|
||||
last. That ensures that the package files are available when the
|
||||
client learns about them from the new index files.
|
||||
|
||||
"""
|
||||
indexes = [
|
||||
os.path.join(repo_section, 'entry.jar'),
|
||||
os.path.join(repo_section, 'entry.json'),
|
||||
os.path.join(repo_section, 'entry.json.asc'),
|
||||
os.path.join(repo_section, 'index-v1.jar'),
|
||||
os.path.join(repo_section, 'index-v1.json'),
|
||||
os.path.join(repo_section, 'index-v1.json.asc'),
|
||||
os.path.join(repo_section, 'index-v2.json'),
|
||||
os.path.join(repo_section, 'index-v2.json.asc'),
|
||||
os.path.join(repo_section, 'index.jar'),
|
||||
os.path.join(repo_section, 'index.xml'),
|
||||
]
|
||||
return [os.path.join(base_dir, filename) for filename in INDEX_FILES]
|
||||
|
||||
|
||||
def _get_index_excludes(base_dir):
|
||||
indexes = _get_index_file_paths(base_dir)
|
||||
index_excludes = []
|
||||
for f in indexes:
|
||||
index_excludes.append('--exclude')
|
||||
|
|
@ -89,7 +82,16 @@ def _get_index_excludes(repo_section):
|
|||
return index_excludes
|
||||
|
||||
|
||||
def update_awsbucket(repo_section):
|
||||
def _get_index_includes(base_dir):
|
||||
indexes = _get_index_file_paths(base_dir)
|
||||
index_includes = []
|
||||
for f in indexes:
|
||||
index_includes.append('--include')
|
||||
index_includes.append(f)
|
||||
return index_includes
|
||||
|
||||
|
||||
def update_awsbucket(repo_section, index_only=False):
|
||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||
|
||||
The contents of that subdir of the
|
||||
|
|
@ -101,12 +103,12 @@ def update_awsbucket(repo_section):
|
|||
+ config['awsbucket'] + '"')
|
||||
|
||||
if common.set_command_in_config('s3cmd'):
|
||||
update_awsbucket_s3cmd(repo_section)
|
||||
update_awsbucket_s3cmd(repo_section, index_only)
|
||||
else:
|
||||
update_awsbucket_libcloud(repo_section)
|
||||
update_awsbucket_libcloud(repo_section, index_only)
|
||||
|
||||
|
||||
def update_awsbucket_s3cmd(repo_section):
|
||||
def update_awsbucket_s3cmd(repo_section, index_only=False):
|
||||
"""Upload using the CLI tool s3cmd, which provides rsync-like sync.
|
||||
|
||||
The upload is done in multiple passes to reduce the chance of
|
||||
|
|
@ -149,36 +151,52 @@ def update_awsbucket_s3cmd(repo_section):
|
|||
s3cmd_sync += ['--quiet']
|
||||
|
||||
s3url = s3bucketurl + '/fdroid/'
|
||||
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
|
||||
logging.debug(_('Running first pass with MD5 checking disabled'))
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
returncode = subprocess.call(
|
||||
s3cmd_sync
|
||||
+ excludes
|
||||
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
|
||||
)
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
||||
returncode = subprocess.call(
|
||||
s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
|
||||
)
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
|
||||
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
|
||||
.format(path=repo_section, url=s3url))
|
||||
s3cmd_sync.append('--delete-removed')
|
||||
s3cmd_sync.append('--delete-after')
|
||||
if options.no_checksum:
|
||||
s3cmd_sync.append('--no-check-md5')
|
||||
if index_only:
|
||||
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
|
||||
.format(path=repo_section, url=s3url))
|
||||
sync_indexes_flags = []
|
||||
sync_indexes_flags.extend(_get_index_includes(repo_section))
|
||||
sync_indexes_flags.append('--delete-removed')
|
||||
sync_indexes_flags.append('--delete-after')
|
||||
if options.no_checksum:
|
||||
sync_indexes_flags.append('--no-check-md5')
|
||||
else:
|
||||
sync_indexes_flags.append('--check-md5')
|
||||
returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
else:
|
||||
s3cmd_sync.append('--check-md5')
|
||||
if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
|
||||
raise FDroidException()
|
||||
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
|
||||
logging.debug(_('Running first pass with MD5 checking disabled'))
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
returncode = subprocess.call(
|
||||
s3cmd_sync
|
||||
+ excludes
|
||||
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
|
||||
)
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
||||
returncode = subprocess.call(
|
||||
s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
|
||||
)
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
|
||||
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
|
||||
.format(path=repo_section, url=s3url))
|
||||
s3cmd_sync.append('--delete-removed')
|
||||
s3cmd_sync.append('--delete-after')
|
||||
if options.no_checksum:
|
||||
s3cmd_sync.append('--no-check-md5')
|
||||
else:
|
||||
s3cmd_sync.append('--check-md5')
|
||||
if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
def update_awsbucket_libcloud(repo_section):
|
||||
def update_awsbucket_libcloud(repo_section, index_only=False):
|
||||
"""No summary.
|
||||
|
||||
Upload the contents of the directory `repo_section` (including
|
||||
|
|
@ -221,47 +239,53 @@ def update_awsbucket_libcloud(repo_section):
|
|||
if obj.name.startswith(upload_dir + '/'):
|
||||
objs[obj.name] = obj
|
||||
|
||||
for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
|
||||
for name in files:
|
||||
upload = False
|
||||
file_to_upload = os.path.join(root, name)
|
||||
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
|
||||
if object_name not in objs:
|
||||
if index_only:
|
||||
index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
|
||||
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
||||
files_to_upload = list(set(files_to_upload) & set(index_files))
|
||||
|
||||
else:
|
||||
files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
|
||||
|
||||
for file_to_upload in files_to_upload:
|
||||
upload = False
|
||||
object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
|
||||
if object_name not in objs:
|
||||
upload = True
|
||||
else:
|
||||
obj = objs.pop(object_name)
|
||||
if obj.size != os.path.getsize(file_to_upload):
|
||||
upload = True
|
||||
else:
|
||||
obj = objs.pop(object_name)
|
||||
if obj.size != os.path.getsize(file_to_upload):
|
||||
# if the sizes match, then compare by MD5
|
||||
md5 = hashlib.md5() # nosec AWS uses MD5
|
||||
with open(file_to_upload, 'rb') as f:
|
||||
while True:
|
||||
data = f.read(8192)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
if obj.hash != md5.hexdigest():
|
||||
s3url = 's3://' + awsbucket + '/' + obj.name
|
||||
logging.info(' deleting ' + s3url)
|
||||
if not driver.delete_object(obj):
|
||||
logging.warning('Could not delete ' + s3url)
|
||||
upload = True
|
||||
else:
|
||||
# if the sizes match, then compare by MD5
|
||||
md5 = hashlib.md5() # nosec AWS uses MD5
|
||||
with open(file_to_upload, 'rb') as f:
|
||||
while True:
|
||||
data = f.read(8192)
|
||||
if not data:
|
||||
break
|
||||
md5.update(data)
|
||||
if obj.hash != md5.hexdigest():
|
||||
s3url = 's3://' + awsbucket + '/' + obj.name
|
||||
logging.info(' deleting ' + s3url)
|
||||
if not driver.delete_object(obj):
|
||||
logging.warning('Could not delete ' + s3url)
|
||||
upload = True
|
||||
|
||||
if upload:
|
||||
logging.debug(' uploading "' + file_to_upload + '"...')
|
||||
extra = {'acl': 'public-read'}
|
||||
if file_to_upload.endswith('.sig'):
|
||||
extra['content_type'] = 'application/pgp-signature'
|
||||
elif file_to_upload.endswith('.asc'):
|
||||
extra['content_type'] = 'application/pgp-signature'
|
||||
logging.info(' uploading ' + os.path.relpath(file_to_upload)
|
||||
+ ' to s3://' + awsbucket + '/' + object_name)
|
||||
with open(file_to_upload, 'rb') as iterator:
|
||||
obj = driver.upload_object_via_stream(iterator=iterator,
|
||||
container=container,
|
||||
object_name=object_name,
|
||||
extra=extra)
|
||||
if upload:
|
||||
logging.debug(' uploading "' + file_to_upload + '"...')
|
||||
extra = {'acl': 'public-read'}
|
||||
if file_to_upload.endswith('.sig'):
|
||||
extra['content_type'] = 'application/pgp-signature'
|
||||
elif file_to_upload.endswith('.asc'):
|
||||
extra['content_type'] = 'application/pgp-signature'
|
||||
logging.info(' uploading ' + os.path.relpath(file_to_upload)
|
||||
+ ' to s3://' + awsbucket + '/' + object_name)
|
||||
with open(file_to_upload, 'rb') as iterator:
|
||||
obj = driver.upload_object_via_stream(iterator=iterator,
|
||||
container=container,
|
||||
object_name=object_name,
|
||||
extra=extra)
|
||||
# delete the remnants in the bucket, they do not exist locally
|
||||
while objs:
|
||||
object_name, obj = objs.popitem()
|
||||
|
|
@ -308,22 +332,30 @@ def update_serverwebroot(serverwebroot, repo_section):
|
|||
elif config and config.get('identity_file'):
|
||||
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
|
||||
url = serverwebroot['url']
|
||||
index_only = serverwebroot.get('index_only', False)
|
||||
logging.info('rsyncing ' + repo_section + ' to ' + url)
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
|
||||
raise FDroidException()
|
||||
if subprocess.call(rsyncargs + [repo_section, url]) != 0:
|
||||
raise FDroidException()
|
||||
# upload "current version" symlinks if requested
|
||||
if config and config.get('make_current_version_link') and repo_section == 'repo':
|
||||
links_to_upload = []
|
||||
for f in glob.glob('*.apk') \
|
||||
+ glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
|
||||
if os.path.islink(f):
|
||||
links_to_upload.append(f)
|
||||
if len(links_to_upload) > 0:
|
||||
if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
|
||||
raise FDroidException()
|
||||
if index_only:
|
||||
rsyncargs += _get_index_file_paths(repo_section)
|
||||
rsyncargs += [f'{url}/{repo_section}/']
|
||||
logging.info(rsyncargs)
|
||||
if subprocess.call(rsyncargs) != 0:
|
||||
raise FDroidException()
|
||||
else:
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
|
||||
raise FDroidException()
|
||||
if subprocess.call(rsyncargs + [repo_section, url]) != 0:
|
||||
raise FDroidException()
|
||||
# upload "current version" symlinks if requested
|
||||
if config['make_current_version_link'] and repo_section == 'repo':
|
||||
links_to_upload = []
|
||||
for f in glob.glob('*.apk') \
|
||||
+ glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
|
||||
if os.path.islink(f):
|
||||
links_to_upload.append(f)
|
||||
if len(links_to_upload) > 0:
|
||||
if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True):
|
||||
|
|
@ -364,10 +396,11 @@ def sync_from_localcopy(repo_section, local_copy_dir):
|
|||
|
||||
"""
|
||||
logging.info('Syncing from local_copy_dir to this repo.')
|
||||
|
||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||
# make sure both paths have exactly one trailing slash
|
||||
common.local_rsync(options,
|
||||
os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
|
||||
[os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'],
|
||||
repo_section.rstrip('/') + '/')
|
||||
|
||||
offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
|
||||
|
|
@ -385,7 +418,7 @@ def update_localcopy(repo_section, local_copy_dir):
|
|||
|
||||
"""
|
||||
# local_copy_dir is guaranteed to have a trailing slash in main() below
|
||||
common.local_rsync(options, repo_section, local_copy_dir)
|
||||
common.local_rsync(options, [repo_section], local_copy_dir)
|
||||
|
||||
offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
|
||||
if os.path.isdir(os.path.join(offline_copy, '.git')):
|
||||
|
|
@ -445,9 +478,10 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
|
||||
shutil.rmtree(archive_path, ignore_errors=True)
|
||||
|
||||
# rsync is very particular about trailing slashes
|
||||
# trailing slashes have a meaning in rsync which is not needed here, so
|
||||
# make sure both paths have exactly one trailing slash
|
||||
common.local_rsync(options,
|
||||
repo_section.rstrip('/') + '/',
|
||||
[repo_section.rstrip('/') + '/'],
|
||||
git_repodir.rstrip('/') + '/')
|
||||
|
||||
# use custom SSH command if identity_file specified
|
||||
|
|
@ -457,28 +491,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
elif 'identity_file' in config:
|
||||
ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
|
||||
|
||||
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
|
||||
|
||||
enabled_remotes = []
|
||||
for d in servergitmirrors:
|
||||
remote_url = d['url']
|
||||
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||
enabled_remotes.append(name)
|
||||
r = git.remote.Remote(repo, name)
|
||||
if r in repo.remotes:
|
||||
r = repo.remote(name)
|
||||
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
|
||||
r.set_url(remote_url)
|
||||
else:
|
||||
repo.create_remote(name, remote_url)
|
||||
logging.info('Mirroring to: ' + remote_url)
|
||||
|
||||
# sadly index.add don't allow the --all parameter
|
||||
logging.debug('Adding all files to git mirror')
|
||||
repo.git.add(all=True)
|
||||
logging.debug('Committing all files into git mirror')
|
||||
repo.index.commit("fdroidserver git-mirror")
|
||||
|
||||
if options.verbose:
|
||||
progressbar = progress.Bar()
|
||||
|
||||
|
|
@ -490,23 +502,78 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
else:
|
||||
progress = None
|
||||
|
||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||
gitlab_ci_job_name = 'pages'
|
||||
else:
|
||||
gitlab_ci_job_name = 'GitLab Artifacts'
|
||||
logging.warning(
|
||||
_(
|
||||
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
|
||||
)
|
||||
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
||||
)
|
||||
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
|
||||
|
||||
# push for every remote. This will overwrite the git history
|
||||
for remote in repo.remotes:
|
||||
if remote.name not in enabled_remotes:
|
||||
repo.delete_remote(remote)
|
||||
continue
|
||||
# An initial commit of the git tree is required be for other operations
|
||||
initial_branch_ref = repo.head.ref
|
||||
repo.index.commit('Initial commit')
|
||||
|
||||
enabled_remotes = []
|
||||
for d in servergitmirrors:
|
||||
# Test
|
||||
print(f"d: {d}")
|
||||
print("files:")
|
||||
print(glob.glob('.' + '/**/*', recursive=True))
|
||||
|
||||
index_only = d.get('index_only', False)
|
||||
|
||||
# Use a separate branch for the index only mode as it needs a different set of files to commit
|
||||
if index_only:
|
||||
branch_name = 'index_only'
|
||||
else:
|
||||
branch_name = 'full'
|
||||
if not branch_name in repo.heads:
|
||||
repo.create_head(branch_name, initial_branch_ref)
|
||||
repo.head.reference = repo.heads[branch_name]
|
||||
|
||||
# test
|
||||
print(repo.git.status())
|
||||
|
||||
remote_url = d['url']
|
||||
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||
enabled_remotes.append(name)
|
||||
r = git.remote.Remote(repo, name)
|
||||
if r in repo.remotes:
|
||||
r = repo.remote(name)
|
||||
if 'set_url' in dir(r): # force remote URL if using GitPython 2.x
|
||||
r.set_url(remote_url)
|
||||
else:
|
||||
repo.create_remote(name, remote_url)
|
||||
logging.info('Mirroring to: ' + remote_url)
|
||||
|
||||
if index_only:
|
||||
# test
|
||||
print(glob.glob('.' + '/**/*', recursive=True))
|
||||
|
||||
logging.debug('Adding index files to git mirror')
|
||||
repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
|
||||
else:
|
||||
# sadly index.add don't allow the --all parameter
|
||||
logging.debug('Adding all files to git mirror')
|
||||
repo.git.add(all=True)
|
||||
|
||||
logging.debug('Committing files into git mirror')
|
||||
repo.index.commit("fdroidserver git-mirror")
|
||||
|
||||
# Test
|
||||
print(f"In index-only: {index_only} mode")
|
||||
print(repo.git.status())
|
||||
print(repo.head.log())
|
||||
|
||||
# only deploy to GitLab Artifacts if too big for GitLab Pages
|
||||
if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
|
||||
gitlab_ci_job_name = 'pages'
|
||||
else:
|
||||
gitlab_ci_job_name = 'GitLab Artifacts'
|
||||
logging.warning(
|
||||
_(
|
||||
'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
|
||||
)
|
||||
% (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
|
||||
)
|
||||
|
||||
# push. This will overwrite the git history
|
||||
remote = repo.remote(name)
|
||||
if remote.name == 'gitlab':
|
||||
logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
|
||||
with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
|
||||
|
|
@ -526,7 +593,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
default_flow_style=False,
|
||||
)
|
||||
|
||||
repo.git.add(all=True)
|
||||
repo.index.add(['.gitlab-ci.yml'])
|
||||
repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
|
||||
|
||||
logging.debug(_('Pushing to {url}').format(url=remote.url))
|
||||
|
|
@ -548,6 +615,10 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
else:
|
||||
logging.debug(remote.url + ': ' + pushinfo.summary)
|
||||
|
||||
# Switch to the initial branch and unstage all files
|
||||
repo.head.reference = initial_branch_ref
|
||||
repo.head.reset(index=True, working_tree=False)
|
||||
repo.delete_head(repo.branches[branch_name], force=True)
|
||||
if progress:
|
||||
progressbar.done()
|
||||
|
||||
|
|
@ -782,6 +853,12 @@ def main():
|
|||
help=_("Don't use rsync checksums"))
|
||||
parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
|
||||
help=_("If a git mirror gets to big, allow the archive to be deleted"))
|
||||
parser.add_argument(
|
||||
"--index-only",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Only deploy the index files entry.* and index-v*.json",
|
||||
)
|
||||
options = parser.parse_args()
|
||||
config = common.read_config(options)
|
||||
|
||||
|
|
|
|||
|
|
@ -324,11 +324,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||
|
||||
os.chdir(repo_basedir)
|
||||
if os.path.isdir(git_mirror_repodir):
|
||||
common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
|
||||
common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
|
||||
if os.path.isdir(git_mirror_metadatadir):
|
||||
common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
|
||||
common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
|
||||
if os.path.isdir(git_mirror_statsdir):
|
||||
common.local_rsync(options, git_mirror_statsdir + '/', 'stats/')
|
||||
common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
|
||||
|
||||
ssh_private_key_file = _ssh_key_from_debug_keystore()
|
||||
# this is needed for GitPython to find the SSH key
|
||||
|
|
@ -430,9 +430,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
|
|||
cwd=repo_basedir,
|
||||
)
|
||||
common.local_rsync(
|
||||
options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
|
||||
options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
|
||||
)
|
||||
common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
|
||||
common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
|
||||
mirror_git_repo.git.add(all=True)
|
||||
mirror_git_repo.index.commit("update app metadata")
|
||||
|
||||
|
|
|
|||
|
|
@ -60,6 +60,10 @@ class DeployTest(unittest.TestCase):
|
|||
url1 = Path('url1/fdroid')
|
||||
url1.mkdir(parents=True)
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk0 = url0 / fake_apk
|
||||
dest_apk1 = url1 / fake_apk
|
||||
self.assertFalse(dest_apk0.is_file())
|
||||
|
|
@ -96,13 +100,53 @@ class DeployTest(unittest.TestCase):
|
|||
fake_apk = repo / 'fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
url = Path('url')
|
||||
url.mkdir()
|
||||
|
||||
dest_apk = url / fake_apk
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk = Path(url) / fake_apk
|
||||
dest_index = Path(url) / fake_index
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertFalse(dest_index.is_file())
|
||||
|
||||
fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
|
||||
self.assertTrue(dest_apk.is_file())
|
||||
self.assertTrue(dest_index.is_file())
|
||||
|
||||
def test_update_serverwebroot_in_index_only_mode(self):
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir()
|
||||
fake_apk = repo / 'fake.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
for i in fdroidserver.deploy.INDEX_FILES:
|
||||
fake_index = repo / i
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
url = Path('url')
|
||||
url.mkdir()
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
|
||||
dest_apk = Path(url) / fake_apk
|
||||
dest_index = Path(url) / fake_index
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertFalse(dest_index.is_file())
|
||||
|
||||
fdroidserver.deploy.update_serverwebroot(
|
||||
{'url': str(url), 'index_only': True}, 'repo'
|
||||
)
|
||||
self.assertFalse(dest_apk.is_file())
|
||||
self.assertTrue(dest_index.is_file())
|
||||
|
||||
@mock.patch.dict(os.environ, clear=True)
|
||||
def test_update_serverwebroot_no_rsync_error(self):
|
||||
|
|
@ -118,7 +162,8 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.config = {'make_current_version_link': True}
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = True
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'repo'
|
||||
|
||||
|
|
@ -202,6 +247,89 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
||||
self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
|
||||
|
||||
def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['make_current_version_link'] = True
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'repo'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
call_iteration = 0
|
||||
|
||||
def update_server_webroot_call(cmd):
|
||||
nonlocal call_iteration
|
||||
if call_iteration == 0:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
'rsync',
|
||||
'--archive',
|
||||
'--delete-after',
|
||||
'--safe-links',
|
||||
'--quiet',
|
||||
'repo/entry.jar',
|
||||
'repo/entry.json',
|
||||
'repo/entry.json.asc',
|
||||
'repo/index-v1.jar',
|
||||
'repo/index-v1.json',
|
||||
'repo/index-v1.json.asc',
|
||||
'repo/index-v2.json',
|
||||
'repo/index-v2.json.asc',
|
||||
'repo/index.jar',
|
||||
'repo/index.xml',
|
||||
'example.com:/var/www/fdroid/repo/',
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
'rsync',
|
||||
'--archive',
|
||||
'--delete-after',
|
||||
'--safe-links',
|
||||
'--quiet',
|
||||
'repo',
|
||||
url,
|
||||
],
|
||||
)
|
||||
elif call_iteration == 2:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
'rsync',
|
||||
'--archive',
|
||||
'--delete-after',
|
||||
'--safe-links',
|
||||
'--quiet',
|
||||
'Sym.apk',
|
||||
'Sym.apk.asc',
|
||||
'Sym.apk.sig',
|
||||
'example.com:/var/www/fdroid',
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
os.mkdir('repo')
|
||||
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||
fdroidserver.deploy.update_serverwebroot(
|
||||
{'url': url, 'index_only': True}, repo_section
|
||||
)
|
||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||
|
||||
def test_update_serverwebroot_with_id_file(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
|
|
@ -210,7 +338,8 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.deploy.options.verbose = True
|
||||
fdroidserver.deploy.options.quiet = False
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config = {'identity_file': './id_rsa'}
|
||||
fdroidserver.deploy.config['identity_file'] = './id_rsa'
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'archive'
|
||||
|
||||
|
|
@ -280,6 +409,73 @@ class DeployTest(unittest.TestCase):
|
|||
fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
|
||||
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
||||
|
||||
def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.no_chcksum = False
|
||||
fdroidserver.deploy.options.verbose = True
|
||||
fdroidserver.deploy.options.quiet = False
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.config['identity_file'] = './id_rsa'
|
||||
fdroidserver.deploy.config['make_current_version_link'] = False
|
||||
url = "example.com:/var/www/fdroid"
|
||||
repo_section = 'archive'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
call_iteration = 0
|
||||
|
||||
def update_server_webroot_call(cmd):
|
||||
nonlocal call_iteration
|
||||
if call_iteration == 0:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
'rsync',
|
||||
'--archive',
|
||||
'--delete-after',
|
||||
'--safe-links',
|
||||
'--verbose',
|
||||
'-e',
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
'archive/entry.jar',
|
||||
'archive/entry.json',
|
||||
'archive/entry.json.asc',
|
||||
'archive/index-v1.jar',
|
||||
'archive/index-v1.json',
|
||||
'archive/index-v1.json.asc',
|
||||
'archive/index-v2.json',
|
||||
'archive/index-v2.json.asc',
|
||||
'archive/index.jar',
|
||||
'archive/index.xml',
|
||||
"example.com:/var/www/fdroid/archive/",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
'rsync',
|
||||
'--archive',
|
||||
'--delete-after',
|
||||
'--safe-links',
|
||||
'--verbose',
|
||||
'-e',
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
"example.com:/var/www/fdroid/archive/",
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
|
||||
fdroidserver.deploy.update_serverwebroot(
|
||||
{'url': url, 'index_only': True}, repo_section
|
||||
)
|
||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||
|
||||
@unittest.skipIf(
|
||||
not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
|
||||
)
|
||||
|
|
@ -300,14 +496,352 @@ class DeployTest(unittest.TestCase):
|
|||
name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
|
||||
)
|
||||
|
||||
def test_update_servergitmirrors(self):
|
||||
def test_update_awsbucket_s3cmd(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
call_iteration = 0
|
||||
|
||||
def update_awsbucket_s3cmd_call(cmd):
|
||||
nonlocal call_iteration
|
||||
if call_iteration == 0:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'info',
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'sync',
|
||||
'--acl-public',
|
||||
'--quiet',
|
||||
'--exclude',
|
||||
'repo/entry.jar',
|
||||
'--exclude',
|
||||
'repo/entry.json',
|
||||
'--exclude',
|
||||
'repo/entry.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v1.jar',
|
||||
'--exclude',
|
||||
'repo/index-v1.json',
|
||||
'--exclude',
|
||||
'repo/index-v1.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v2.json',
|
||||
'--exclude',
|
||||
'repo/index-v2.json.asc',
|
||||
'--exclude',
|
||||
'repo/index.jar',
|
||||
'--exclude',
|
||||
'repo/index.xml',
|
||||
'--no-check-md5',
|
||||
'--skip-existing',
|
||||
repo_section,
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 2:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'sync',
|
||||
'--acl-public',
|
||||
'--quiet',
|
||||
'--exclude',
|
||||
'repo/entry.jar',
|
||||
'--exclude',
|
||||
'repo/entry.json',
|
||||
'--exclude',
|
||||
'repo/entry.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v1.jar',
|
||||
'--exclude',
|
||||
'repo/index-v1.json',
|
||||
'--exclude',
|
||||
'repo/index-v1.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v2.json',
|
||||
'--exclude',
|
||||
'repo/index-v2.json.asc',
|
||||
'--exclude',
|
||||
'repo/index.jar',
|
||||
'--exclude',
|
||||
'repo/index.xml',
|
||||
'--no-check-md5',
|
||||
repo_section,
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 3:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'sync',
|
||||
'--acl-public',
|
||||
'--quiet',
|
||||
'--delete-removed',
|
||||
'--delete-after',
|
||||
'--no-check-md5',
|
||||
repo_section,
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
os.mkdir('repo')
|
||||
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
|
||||
fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
|
||||
self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call')
|
||||
|
||||
def test_update_awsbucket_s3cmd_in_index_only_mode(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
call_iteration = 0
|
||||
|
||||
def update_awsbucket_s3cmd_call(cmd):
|
||||
nonlocal call_iteration
|
||||
if call_iteration == 0:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'info',
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}",
|
||||
],
|
||||
)
|
||||
elif call_iteration == 1:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
's3cmd',
|
||||
f"--config={fdroidserver.deploy.AUTO_S3CFG}",
|
||||
'sync',
|
||||
'--acl-public',
|
||||
'--quiet',
|
||||
'--include',
|
||||
'repo/entry.jar',
|
||||
'--include',
|
||||
'repo/entry.json',
|
||||
'--include',
|
||||
'repo/entry.json.asc',
|
||||
'--include',
|
||||
'repo/index-v1.jar',
|
||||
'--include',
|
||||
'repo/index-v1.json',
|
||||
'--include',
|
||||
'repo/index-v1.json.asc',
|
||||
'--include',
|
||||
'repo/index-v2.json',
|
||||
'--include',
|
||||
'repo/index-v2.json.asc',
|
||||
'--include',
|
||||
'repo/index.jar',
|
||||
'--include',
|
||||
'repo/index.xml',
|
||||
'--delete-removed',
|
||||
'--delete-after',
|
||||
'--no-check-md5',
|
||||
repo_section,
|
||||
f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
os.mkdir('repo')
|
||||
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
|
||||
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
|
||||
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
|
||||
with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
|
||||
fdroidserver.deploy.update_awsbucket_s3cmd(
|
||||
repo_section, index_only=True
|
||||
)
|
||||
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
|
||||
|
||||
def test_update_awsbucket_libcloud(self):
|
||||
from libcloud.storage.base import Container
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True)
|
||||
fake_apk = repo / 'Sym.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
|
||||
with mock.patch(
|
||||
'libcloud.storage.drivers.s3.S3StorageDriver'
|
||||
) as mock_driver_class:
|
||||
mock_driver = mock_driver_class.return_value
|
||||
mock_container = mock.MagicMock(spec=Container)
|
||||
mock_container.list_objects.return_value = [
|
||||
mock.MagicMock(name='Sym.apk'),
|
||||
mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
|
||||
]
|
||||
|
||||
mock_driver.get_container.return_value = mock_container
|
||||
mock_driver.upload_object_via_stream.return_value = None
|
||||
|
||||
fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
|
||||
|
||||
mock_driver.get_container.assert_called_once_with(
|
||||
container_name=fdroidserver.deploy.config["awsbucket"]
|
||||
)
|
||||
mock_container.list_objects.assert_called_once_with()
|
||||
files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar']
|
||||
calls = [
|
||||
mock.call(
|
||||
iterator=mock.ANY,
|
||||
container=mock_container,
|
||||
object_name=file,
|
||||
extra={'acl': 'public-read'},
|
||||
)
|
||||
for file in files_to_upload
|
||||
]
|
||||
mock_driver.upload_object_via_stream.assert_has_calls(calls, any_order=True)
|
||||
assert mock_driver.upload_object_via_stream.call_count == 2
|
||||
|
||||
def test_update_awsbucket_libcloud_in_index_only_mode(self):
|
||||
from libcloud.storage.base import Container
|
||||
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options = mock.Mock()
|
||||
fdroidserver.deploy.options.no_checksum = True
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["awsbucket"] = "bucket"
|
||||
fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
|
||||
fdroidserver.deploy.config["awssecretkey"] = "secretkey"
|
||||
fdroidserver.deploy.config["s3cmd"] = "s3cmd"
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True)
|
||||
fake_apk = repo / 'Sym.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
|
||||
with mock.patch(
|
||||
'libcloud.storage.drivers.s3.S3StorageDriver'
|
||||
) as mock_driver_class:
|
||||
mock_driver = mock_driver_class.return_value
|
||||
mock_container = mock.MagicMock(spec=Container)
|
||||
mock_container.list_objects.return_value = [
|
||||
mock.MagicMock(name='Sym.apk'),
|
||||
mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
|
||||
]
|
||||
|
||||
mock_driver.get_container.return_value = mock_container
|
||||
mock_driver.upload_object_via_stream.return_value = None
|
||||
|
||||
fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
|
||||
|
||||
mock_driver.get_container.assert_called_once_with(
|
||||
container_name=fdroidserver.deploy.config["awsbucket"]
|
||||
)
|
||||
mock_container.list_objects.assert_called_once_with()
|
||||
files_to_upload = ['fdroid/repo/entry.jar']
|
||||
calls = [
|
||||
mock.call(
|
||||
iterator=mock.ANY,
|
||||
container=mock_container,
|
||||
object_name=file,
|
||||
extra={'acl': 'public-read'},
|
||||
)
|
||||
for file in files_to_upload
|
||||
]
|
||||
mock_driver.upload_object_via_stream.assert_has_calls(
|
||||
calls,
|
||||
any_order=False,
|
||||
)
|
||||
assert mock_driver.upload_object_via_stream.call_count == 1
|
||||
|
||||
def test_update_servergitmirrors(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
fdroidserver.deploy.options.index_only = False
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
|
|
@ -317,8 +851,7 @@ class DeployTest(unittest.TestCase):
|
|||
repo_section = 'repo'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
update_servergitmirrors_call_iteration = 0
|
||||
remote_push_call_iteration = 0
|
||||
call_iteration = 0
|
||||
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
|
|
@ -331,8 +864,8 @@ class DeployTest(unittest.TestCase):
|
|||
fp.write('not an index, but has the right filename')
|
||||
|
||||
def update_servergitmirrors_call(cmd):
|
||||
nonlocal update_servergitmirrors_call_iteration
|
||||
if update_servergitmirrors_call_iteration == 0:
|
||||
nonlocal call_iteration
|
||||
if call_iteration == 0:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
|
|
@ -351,34 +884,67 @@ class DeployTest(unittest.TestCase):
|
|||
)
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
update_servergitmirrors_call_iteration += 1
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
def remote_push_call(ref, force=False, set_upstream=False, **_args):
|
||||
nonlocal remote_push_call_iteration
|
||||
if remote_push_call_iteration == 0:
|
||||
self.assertEqual([ref, force, set_upstream], ['master', True, True])
|
||||
with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
|
||||
fdroidserver.deploy.update_servergitmirrors([], repo_section)
|
||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||
|
||||
def test_update_servergitmirrors_in_index_only_mode(self):
|
||||
# setup parameters for this test run
|
||||
fdroidserver.deploy.options.identity_file = None
|
||||
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
|
||||
fdroidserver.deploy.options.verbose = False
|
||||
fdroidserver.deploy.options.quiet = True
|
||||
|
||||
config = {}
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
fdroidserver.deploy.config = config
|
||||
fdroidserver.deploy.config["servergitmirrors"] = []
|
||||
|
||||
repo_section = 'repo'
|
||||
|
||||
# setup function for asserting subprocess.call invocations
|
||||
call_iteration = 0
|
||||
|
||||
os.chdir(self.testdir)
|
||||
repo = Path('repo')
|
||||
repo.mkdir(parents=True)
|
||||
fake_apk = repo / 'Sym.apk'
|
||||
with fake_apk.open('w') as fp:
|
||||
fp.write('not an APK, but has the right filename')
|
||||
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
|
||||
with fake_index.open('w') as fp:
|
||||
fp.write('not an index, but has the right filename')
|
||||
|
||||
def update_servergitmirrors_call(cmd):
|
||||
nonlocal call_iteration
|
||||
if call_iteration == 0:
|
||||
self.assertListEqual(
|
||||
cmd,
|
||||
[
|
||||
'rsync',
|
||||
'--recursive',
|
||||
'--safe-links',
|
||||
'--times',
|
||||
'--perms',
|
||||
'--one-file-system',
|
||||
'--delete',
|
||||
'--chmod=Da+rx,Fa-x,a+r,u+w',
|
||||
'--quiet',
|
||||
'repo/',
|
||||
"git-mirror/fdroid/repo/",
|
||||
],
|
||||
)
|
||||
else:
|
||||
self.fail('unexpected git.Remote.push invocation')
|
||||
remote_push_call_iteration += 1
|
||||
return []
|
||||
self.fail('unexpected subprocess.call invocation')
|
||||
call_iteration += 1
|
||||
return 0
|
||||
|
||||
with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
|
||||
with mock.patch(
|
||||
'git.Remote.push', side_effect=remote_push_call
|
||||
) as mock_remote_push:
|
||||
mock_remote_push.return_value = []
|
||||
fdroidserver.deploy.update_servergitmirrors(
|
||||
[{'url': 'https://github.com/user/repo'}], repo_section
|
||||
)
|
||||
self.assertEqual(
|
||||
update_servergitmirrors_call_iteration,
|
||||
1,
|
||||
'expected 1 invocations of subprocess.call',
|
||||
)
|
||||
self.assertEqual(
|
||||
remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
|
||||
)
|
||||
fdroidserver.deploy.update_servergitmirrors([], repo_section)
|
||||
self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Reference in New Issue