gitlab-ci: add job to check if Repo URL is a redirect

We need a strong link between the source repo URL and our buildserver. HTTP
redirects have a number of issues, and are unnecessary in our use case.
Once this is complete, then the buildserver can set http.followRedirects to
false, and we could also set the buildserver to fetch source over Tor to
avoid targetting it.  Right now, gitlab.com only allows git fetches over
Tor if it is hitting a direct URL.  gitlab.com/s redirect logic will
trigger Cloudflare captures otherwise.

* https://bugs.debian.org/79002
* https://www.debian.org/security/2019/dsa-4371
* https://googleprojectzero.github.io/0days-in-the-wild//0day-RCAs/2021/CVE-2021-38000.html
* fdroidclient#1041

.gitlab-ci.yml

@@ -112,6 +112,23 @@ checkupdates:
         git --no-pager diff --color=always --exit-code
       fi
 
+git redirect:
+  stage: test
+  needs: []
+  image: debian:bullseye-slim
+  rules: *app_verification_rules
+  script:
+    - apt-get -qy update
+    - apt-get -qy dist-upgrade
+    - apt-get -qy install --no-install-recommends ca-certificates git python3 python3-yaml
+    - *get_changed_apps
+    - test -n "$CHANGED" || exit 0
+    - tools/rewrite-git-redirects.py $CHANGED
+    - git --no-pager diff --color=always --exit-code || {
+          printf "\x1b[31mERROR git URLs should be direct, not redirects!\x1b[0m\n";
+          exit 1;
+      }
+
 fdroid lint:
   stage: test
   needs: []

tools/rewrite-git-redirects.py

@@ -1,20 +1,42 @@
 #!/usr/bin/env python3
 #
-#
-# GitLab gives a warning every time if the git URL is redirected.  So this
-# rewrites all GitLab URLs so they are no longer a redirect.
+# GitLab gives a warning every time if the git URL is redirected.  So
+# this rewrites all GitLab URLs so they are no longer a redirect.
+# This also allows the gitlab.com URLs to be used with Tor, and gives
+# other benefits, like a simplier security attack surface.
 
 import glob
 import os
 import re
+import subprocess
 import sys
-
 import yaml
 
+
+def is_git_redirect(url):
+    """Check if git is served a redirect"""
+    host = url.split('/')[2]
+    p = subprocess.run(
+        [
+            'git',
+            '-c',
+            'url.https://git:nopw@{host}.insteadOf=https://{host}'.format(host=host),
+            '-c',
+            'http.followRedirects=false',
+            'ls-remote',
+            '--heads',
+            url,
+            'main',
+        ],
+        capture_output=True,
+    )
+    return p.returncode != 0
+
+
 os.chdir(os.path.dirname(__file__) + '/../')
 
 if len(sys.argv) > 1:
-    files = sys.argv[1:]
+    files = ['metadata/%s.yml' % f for f in sys.argv[1:]]
 else:
     files = sorted(glob.glob('metadata/*.yml'))
 
@@ -22,15 +44,27 @@ pattern = re.compile(r'Repo: .*')
 for f in files:
     with open(f) as fp:
         data = yaml.safe_load(fp)
-    repo_url = None
-    if 'Repo' in data:
-        repo_url = data['Repo'].strip().rstrip('/')
+    repo_url = data.get('Repo', '').strip()
     if (
-        repo_url
-        and not repo_url.endswith('.git')
-        and repo_url.startswith('https://gitlab')
+        'Repo' not in data
+        or data.get('RepoType') != 'git'
+        or data.get('ArchivePolicy') == '0 versions'
     ):
-        new_url = repo_url + '.git'
+        continue
+
+    if not is_git_redirect(repo_url):
+        continue
+
+    new_url = None
+    for url in [
+        repo_url.rstrip('/') + '.git',
+        repo_url.rstrip('/'),
+    ]:
+        if not is_git_redirect(url):
+            new_url = url
+            break
+
+    if new_url:
         print("Repo:", data['Repo'], "\n -->  " + new_url + "'")
         with open(f) as fp:
             raw = fp.read()