opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'servergitmirrors-as-dict' into 'master' 

feat: add servergitmirrors as a dict support

See merge request fdroid/fdroidserver!1438
This commit is contained in:
Hans-Christoph Steiner 2024-02-14 16:50:40 +00:00
parent 12692b76b7 947217549a
commit 094764398a
10 changed files with 118 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitlab-ci.yml
View File

@ -542,7 +542,7 @@ servergitmirrors:
- cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
- cd /tmp/fdroid
- touch fdroid-icon.png
- printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
- printf "servergitmirrors:\n-\ url:\ $SERVER_GIT_MIRROR\n" >> config.yml
- $PYTHONPATH/fdroid update --verbose --create-metadata
- $PYTHONPATH/fdroid deploy --verbose
- export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`

8
examples/config.yml
View File

@ -198,14 +198,14 @@
# deploy_process_logs: true
# The full URL to a git remote repository. You can include
# multiple servers to mirror to by wrapping the whole thing in {} or [], and
# including the servergitmirrors strings in a comma-separated list.
# multiple servers to mirror to by adding strings to a YAML list or map.
# Servers listed here will also be automatically inserted in the mirrors list.
#
# servergitmirrors: https://github.com/user/repo
# servergitmirrors:
# - https://github.com/user/repo
# - https://gitlab.com/user/repo
# - url: https://github.com/user/repo
# - url: https://gitlab.com/user/repo
# indexOnly: true
# Most git hosting services have hard size limits for each git repo.
# `fdroid deploy` will delete the git history when the git mirror repo

6
fdroidserver/common.py
View File

@ -67,7 +67,7 @@ from pyasn1.error import PyAsn1Error
import fdroidserver.metadata
import fdroidserver.lint
from fdroidserver import _
from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException, \
BuildException, VerificationException, MetaDataException
from .asynchronousfilereader import AsynchronousFileReader
from .looseversion import LooseVersion
@ -482,8 +482,10 @@ def read_config(opts=None):
if 'servergitmirrors' in config:
if isinstance(config['servergitmirrors'], str):
roots = [config['servergitmirrors']]
roots = [{"url": config['servergitmirrors']}]
elif all(isinstance(item, str) for item in config['servergitmirrors']):
roots = [{'url': i} for i in config['servergitmirrors']]
elif all(isinstance(item, dict) for item in config['servergitmirrors']):
roots = config['servergitmirrors']
else:
raise TypeError(_('only accepts strings, lists, and tuples'))

21
fdroidserver/deploy.py
View File

@ -47,6 +47,19 @@ AUTO_S3CFG = '.fdroid-deploy-s3cfg'
USER_S3CFG = 's3cfg'
REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
INDEX_FILES = [
"entry.jar",
"entry.json",
"entry.json.asc",
"index-v1.jar",
"index-v1.json",
"index-v1.json.asc",
"index-v2.json",
"index-v2.json.asc",
"index.jar",
"index.xml",
]
def _get_index_excludes(repo_section):
"""Return the list of files to be synced last, since they finalize the deploy.
@ -447,7 +460,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
enabled_remotes = []
for remote_url in servergitmirrors:
for d in servergitmirrors:
remote_url = d['url']
name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
enabled_remotes.append(name)
r = git.remote.Remote(repo, name)
@ -840,10 +854,9 @@ def main():
update_serverwebroots(
config['serverwebroot'], repo_section, standardwebroot
)
if config.get('servergitmirrors', []):
if config.get('servergitmirrors'):
# update_servergitmirrors will take care of multiple mirrors so don't need a foreach
servergitmirrors = config.get('servergitmirrors', [])
update_servergitmirrors(servergitmirrors, repo_section)
update_servergitmirrors(config['servergitmirrors'], repo_section)
if config.get('awsbucket'):
update_awsbucket(repo_section)
if config.get('androidobservatory'):

3
fdroidserver/index.py
View File

@ -1478,7 +1478,7 @@ def add_mirrors_to_repodict(repo_section, repodict):
repodict['mirrors'].insert(0, {'isPrimary': True, 'url': repodict['address']})
def get_mirror_service_urls(url):
def get_mirror_service_urls(mirror):
"""Get direct URLs from git service for use by fdroidclient.
Via 'servergitmirrors', fdroidserver can create and push a mirror
@ -1496,6 +1496,7 @@ def get_mirror_service_urls(url):
information about the repo available to end user.
"""
url = mirror['url']
if url.startswith('git@'):
url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url)

2
fdroidserver/nightly.py
View File

@ -358,7 +358,7 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
'archive_url': repo_base + '/archive',
'archive_description': 'Old nightly builds that have been archived.',
'archive_older': options.archive_older,
'servergitmirrors': servergitmirror,
'servergitmirrors': [{"url": servergitmirror}],
'keystore': KEYSTORE_FILE,
'repo_keyalias': KEY_ALIAS,
'keystorepass': PASSWORD,

80
tests/deploy.TestCase
View File

@ -300,6 +300,86 @@ class DeployTest(unittest.TestCase):
name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
)
def test_update_servergitmirrors(self):
# setup parameters for this test run
fdroidserver.deploy.options = mock.Mock()
fdroidserver.deploy.options.identity_file = None
fdroidserver.deploy.options.no_keep_git_mirror_archive = False
fdroidserver.deploy.options.verbose = False
fdroidserver.deploy.options.quiet = True
fdroidserver.deploy.options.index_only = False
config = {}
fdroidserver.common.fill_config_defaults(config)
fdroidserver.deploy.config = config
fdroidserver.deploy.config["servergitmirrors"] = []
repo_section = 'repo'
# setup function for asserting subprocess.call invocations
update_servergitmirrors_call_iteration = 0
remote_push_call_iteration = 0
os.chdir(self.testdir)
repo = Path('repo')
repo.mkdir(parents=True)
fake_apk = repo / 'Sym.apk'
with fake_apk.open('w') as fp:
fp.write('not an APK, but has the right filename')
fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
with fake_index.open('w') as fp:
fp.write('not an index, but has the right filename')
def update_servergitmirrors_call(cmd):
nonlocal update_servergitmirrors_call_iteration
if update_servergitmirrors_call_iteration == 0:
self.assertListEqual(
cmd,
[
'rsync',
'--recursive',
'--safe-links',
'--times',
'--perms',
'--one-file-system',
'--delete',
'--chmod=Da+rx,Fa-x,a+r,u+w',
'--quiet',
'repo/',
"git-mirror/fdroid/repo/",
],
)
else:
self.fail('unexpected subprocess.call invocation')
update_servergitmirrors_call_iteration += 1
return 0
def remote_push_call(ref, force=False, set_upstream=False, **_args):
nonlocal remote_push_call_iteration
if remote_push_call_iteration == 0:
self.assertEqual([ref, force, set_upstream], ['master', True, True])
else:
self.fail('unexpected git.Remote.push invocation')
remote_push_call_iteration += 1
return []
with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
with mock.patch(
'git.Remote.push', side_effect=remote_push_call
) as mock_remote_push:
mock_remote_push.return_value = []
fdroidserver.deploy.update_servergitmirrors(
[{'url': 'https://github.com/user/repo'}], repo_section
)
self.assertEqual(
update_servergitmirrors_call_iteration,
1,
'expected 1 invocations of subprocess.call',
)
self.assertEqual(
remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
)
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))

10
tests/index.TestCase
View File

@ -575,7 +575,7 @@ class IndexTest(unittest.TestCase):
]:
self.assertEqual(
['https://raw.githubusercontent.com/foo/bar/master/fdroid'],
index.get_mirror_service_urls(url),
index.get_mirror_service_urls({"url": url}),
)
@patch.dict(os.environ, clear=True)
@ -603,13 +603,13 @@ class IndexTest(unittest.TestCase):
]
self.assertEqual(
expected,
index.get_mirror_service_urls(url),
index.get_mirror_service_urls({"url": url}),
)
with patch.dict(os.environ, clear=True):
os.environ['CI_JOB_ID'] = ci_job_id
self.assertEqual(
expected + [artifacts_url],
index.get_mirror_service_urls(url),
index.get_mirror_service_urls({"url": url}),
)
with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 10):
expected = [
@ -617,13 +617,13 @@ class IndexTest(unittest.TestCase):
]
self.assertEqual(
expected,
index.get_mirror_service_urls(url),
index.get_mirror_service_urls({"url": url}),
)
with patch.dict(os.environ, clear=True):
os.environ['CI_JOB_ID'] = ci_job_id
self.assertEqual(
expected + [artifacts_url],
index.get_mirror_service_urls(url),
index.get_mirror_service_urls({"url": url}),
)
def test_make_website(self):

6
tests/nightly.TestCase
View File

@ -256,7 +256,7 @@ class NightlyTest(unittest.TestCase):
self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']])
self.assertFalse(os.path.exists('config.py'))
git_url = 'git@github.com:f-droid/test-nightly'
mirror_url = index.get_mirror_service_urls(git_url)[0]
mirror_url = index.get_mirror_service_urls({"url": git_url})[0]
expected = {
'archive_description': 'Old nightly builds that have been archived.',
'archive_name': 'f-droid/test-nightly archive',
@ -271,7 +271,7 @@ class NightlyTest(unittest.TestCase):
'repo_keyalias': 'androiddebugkey',
'repo_name': 'f-droid/test-nightly',
'repo_url': mirror_url + '/repo',
'servergitmirrors': git_url,
'servergitmirrors': [{"url": git_url}],
'update_stats': True,
}
with open('config.yml') as fp:
@ -344,7 +344,7 @@ class NightlyTest(unittest.TestCase):
'repo_keyalias': 'androiddebugkey',
'repo_name': 'fdroid/test-nightly',
'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
'servergitmirrors': 'git@gitlab.com:fdroid/test-nightly',
'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}],
'update_stats': True,
}
with open('config.yml') as fp:

2
tests/run-tests
View File

@ -1154,7 +1154,7 @@ GIT_MIRROR=$REPOROOT/git-mirror
cd $REPOROOT
fdroid_init_with_prebuilt_keystore
printf '\narchive_older: 3\n' >> config.yml
echo "servergitmirrors: $SERVER_GIT_MIRROR" >> config.yml
printf "servergitmirrors: $SERVER_GIT_MIRROR\n" >> config.yml
cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/
$fdroid update --create-metadata