index: fix GitLab Raw URLs with gitlab.com and recent versions

GitLab seems to be moving to always having "-" as the first path segment in
all the project URLs.  So the URL without a "-" is now a redirect.
This commit is contained in:
Hans-Christoph Steiner 2020-12-08 19:44:39 +01:00
parent 531c36f310
commit ba854cbc0f
4 changed files with 85 additions and 4 deletions

View File

@ -291,3 +291,40 @@ fdroid build:
# each `fdroid build --on-server` run expects sudo, then uninstalls it
- apt-get install sudo
- fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid
# test a full update and deploy cycle to gitlab.com
servergitmirrors:
image: debian
<<: *apt-template
only:
- master@fdroid/fdroidserver
script:
- apt-get install
default-jdk-headless
git
openssh-client
openssl
python3-pip
python3-venv
rsync
wget
- python3 -m venv env
- . env/bin/activate
- export PYTHONPATH=`pwd`
- $pip install -e .
- mkdir /root/.ssh/
- ./tests/key-tricks.py
- ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
- test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
- cp tests/config.py tests/keystore.jks /tmp/fdroid/
- cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
- cd /tmp/fdroid
- touch fdroid-icon.png
- printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
- $PYTHONPATH/fdroid update --verbose --create-metadata
- $PYTHONPATH/fdroid deploy --verbose
- export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
- echo $DLURL
- wget $DLURL/index-v1.jar
- diff repo/index-v1.jar index-v1.jar

View File

@ -648,7 +648,7 @@ def get_mirror_service_urls(url):
'''
if url.startswith('git@'):
url = re.sub(r'^git@(.*):(.*)', r'https://\1/\2', url)
url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url)
segments = url.split("/")
@ -676,10 +676,9 @@ def get_mirror_service_urls(url):
# Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
urls.append('/'.join(gitlab_pages))
# Gitlab Raw "https://gitlab.com/user/repo/raw/branch/folder"
gitlab_raw = segments + ['raw', branch, folder]
# GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder"
gitlab_raw = segments + ['-', 'raw', branch, folder]
urls.append('/'.join(gitlab_raw))
return urls
return urls

View File

@ -247,6 +247,26 @@ class IndexTest(unittest.TestCase):
fdroidserver.common.default_config['repo_icon'])))
self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
def test_get_mirror_service_urls(self):
for url in [
'git@github.com:foo/bar',
'git@github.com:foo/bar.git',
'https://github.com/foo/bar',
'https://github.com/foo/bar.git',
]:
self.assertEqual(['https://raw.githubusercontent.com/foo/bar/master/fdroid'],
fdroidserver.index.get_mirror_service_urls(url))
for url in [
'git@gitlab.com:group/project',
'git@gitlab.com:group/project.git',
'https://gitlab.com/group/project',
'https://gitlab.com/group/project.git',
]:
self.assertEqual(['https://group.gitlab.io/project/fdroid',
'https://gitlab.com/group/project/-/raw/master/fdroid'],
fdroidserver.index.get_mirror_service_urls(url))
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))

25
tests/key-tricks.py Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env python3
import os
import fdroidserver
import shutil
import sys
from fdroidserver import common, nightly
if os.getenv('CI') is None:
print('ERROR: This can overwrite SSH keys, so it should only be run in CI')
sys.exit(1)
os.chdir(os.path.dirname(__file__))
config = fdroidserver.common.read_config(common.options)
nightly.PASSWORD = config['keystorepass']
nightly.KEY_ALIAS = config['repo_keyalias']
privkey = nightly._ssh_key_from_debug_keystore('keystore.jks')
print('privkey', privkey)
ssh_private_key_file = os.path.join(os.getenv('HOME'), '.ssh', 'id_rsa')
if os.path.exists(ssh_private_key_file):
print('ERROR:', ssh_private_key_file, 'exists!')
sys.exit(1)
shutil.move(privkey, ssh_private_key_file)
shutil.move(privkey + '.pub', ssh_private_key_file + '.pub')