index: do not include GitLab Pages mirror if it can't be deployed

GitLab Pages sites are limited to 1GB on gitlab.com, so the CI/CD job will
fail if the repo is bigger than that.  It should not be included as a
mirror in that case.

https://docs.gitlab.com/ee/user/gitlab_com/#gitlab-pages

fdroidserver/common.py

@@ -111,6 +111,10 @@ MAX_VERSION_CODE = 0x7fffffff  # Java's Integer.MAX_VALUE (2147483647)
 
 XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}'
 
+# https://docs.gitlab.com/ee/user/gitlab_com/#gitlab-pages
+GITLAB_COM_PAGES_MAX_SIZE = 1000000000
+
+
 config = None
 options = None
 env = None
@@ -478,6 +482,13 @@ def parse_human_readable_size(size):
         return int(float(m.group("value")) * units[m.group("unit")])
 
 
+def get_dir_size(path_or_str):
+    """Get the total size of all files in the given directory."""
+    if isinstance(path_or_str, str):
+        path_or_str = Path(path_or_str)
+    return sum(f.stat().st_size for f in path_or_str.glob('**/*') if f.is_file())
+
+
 def assert_config_keystore(config):
     """Check weather keystore is configured correctly and raise exception if not."""
     nosigningkey = False

fdroidserver/index.py

@@ -1003,11 +1003,19 @@ def get_mirror_service_urls(url):
     """Get direct URLs from git service for use by fdroidclient.
 
     Via 'servergitmirrors', fdroidserver can create and push a mirror
-    to certain well known git services like gitlab or github.  This
+    to certain well known git services like GitLab or GitHub.  This
     will always use the 'master' branch since that is the default
     branch in git. The files are then accessible via alternate URLs,
     where they are served in their raw format via a CDN rather than
     from git.
+
+    Both of the GitLab URLs will work with F-Droid, but only the
+    GitLab Pages will work in the browser This is because the "raw"
+    URLs are not served with the correct mime types, so any index.html
+    which is put in the repo will not be rendered. Putting an
+    index.html file in the repo root is a common way for to make
+    information about the repo available to end user.
+
     """
     if url.startswith('git@'):
         url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url)
@@ -1030,14 +1038,17 @@ def get_mirror_service_urls(url):
         segments.extend([branch, folder])
         urls.append('/'.join(segments))
     elif hostname == "gitlab.com":
-        # Both these Gitlab URLs will work with F-Droid, but only the first will work in the browser
-        # This is because the `raw` URLs are not served with the correct mime types, so any
-        # index.html which is put in the repo will not be rendered. Putting an index.html file in
-        # the repo root is a common way for to make information about the repo available to end user.
-
-        # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
-        gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
-        urls.append('/'.join(gitlab_pages))
+        if common.get_dir_size() <= common.GITLAB_COM_PAGES_MAX_SIZE:
+            # Gitlab-like Pages segments "https://user.gitlab.io/repo/folder"
+            gitlab_pages = ["https:", "", user + ".gitlab.io", repo, folder]
+            urls.append('/'.join(gitlab_pages))
+        else:
+            logging.warning(
+                _(
+                    'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
+                )
+                % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
+            )
         # GitLab Raw "https://gitlab.com/user/repo/-/raw/branch/folder"
         gitlab_raw = segments + ['-', 'raw', branch, folder]
         urls.append('/'.join(gitlab_raw))

tests/index.TestCase

@@ -358,7 +358,7 @@ class IndexTest(unittest.TestCase):
         )
         self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
 
-    def test_get_mirror_service_urls(self):
+    def test_github_get_mirror_service_urls(self):
         for url in [
             'git@github.com:foo/bar',
             'git@github.com:foo/bar.git',
@@ -370,19 +370,28 @@ class IndexTest(unittest.TestCase):
                 fdroidserver.index.get_mirror_service_urls(url),
             )
 
+    def test_gitlab_get_mirror_service_urls(self):
         for url in [
             'git@gitlab.com:group/project',
             'git@gitlab.com:group/project.git',
             'https://gitlab.com/group/project',
             'https://gitlab.com/group/project.git',
         ]:
-            self.assertEqual(
-                [
-                    'https://group.gitlab.io/project/fdroid',
-                    'https://gitlab.com/group/project/-/raw/master/fdroid',
-                ],
-                fdroidserver.index.get_mirror_service_urls(url),
-            )
+            with patch('fdroidserver.common.get_dir_size', return_value=100000):
+                self.assertEqual(
+                    [
+                        'https://group.gitlab.io/project/fdroid',
+                        'https://gitlab.com/group/project/-/raw/master/fdroid',
+                    ],
+                    fdroidserver.index.get_mirror_service_urls(url),
+                )
+            with patch('fdroidserver.common.get_dir_size', return_value=1234567890):
+                self.assertEqual(
+                    [
+                        'https://gitlab.com/group/project/-/raw/master/fdroid',
+                    ],
+                    fdroidserver.index.get_mirror_service_urls(url),
+                )
 
     def test_make_website(self):
         tmptestsdir = tempfile.mkdtemp(