import: mv reusable functions to common.py to avoid import_proxy.py

import is a strict keyword in Python, so it is not possible to import a
module called 'import', even with things like:

* import fdroidserver.import
* from fdroidserver import import

fdroidserver/common.py

@@ -37,6 +37,8 @@ import logging
 import hashlib
 import socket
 import base64
+import urllib.parse
+import urllib.request
 import zipfile
 import tempfile
 import json
@@ -84,6 +86,9 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[
                                         re.IGNORECASE)
 ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''')
 
+SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
+GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:([^'"]+)['"]''')
+
 MAX_VERSION_CODE = 0x7fffffff  # Java's Integer.MAX_VALUE (2147483647)
 
 XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}'
@@ -1653,6 +1658,152 @@ def is_strict_application_id(name):
         and '.' in name
 
 
+def get_all_gradle_and_manifests(build_dir):
+    paths = []
+    for root, dirs, files in os.walk(build_dir):
+        for f in sorted(files):
+            if f == 'AndroidManifest.xml' \
+               or f.endswith('.gradle') or f.endswith('.gradle.kts'):
+                full = os.path.join(root, f)
+                paths.append(full)
+    return paths
+
+
+def get_gradle_subdir(build_dir, paths):
+    """get the subdir where the gradle build is based"""
+    first_gradle_dir = None
+    for path in paths:
+        if not first_gradle_dir:
+            first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir)
+        if os.path.exists(path) and SETTINGS_GRADLE_REGEX.match(os.path.basename(path)):
+            with open(path) as fp:
+                for m in GRADLE_SUBPROJECT_REGEX.finditer(fp.read()):
+                    for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')):
+                        with open(f) as fp:
+                            while True:
+                                line = fp.readline()
+                                if not line:
+                                    break
+                                if ANDROID_PLUGIN_REGEX.match(line):
+                                    return os.path.relpath(os.path.dirname(f), build_dir)
+    if first_gradle_dir and first_gradle_dir != '.':
+        return first_gradle_dir
+
+    return ''
+
+
+def getrepofrompage(url):
+    """Get the repo type and address from the given web page.
+
+    The page is scanned in a rather naive manner for 'git clone xxxx',
+    'hg clone xxxx', etc, and when one of these is found it's assumed
+    that's the information we want.  Returns repotype, address, or
+    None, reason
+
+    """
+    if not url.startswith('http'):
+        return (None, _('{url} does not start with "http"!'.format(url=url)))
+    req = urllib.request.urlopen(url)  # nosec B310 non-http URLs are filtered out
+    if req.getcode() != 200:
+        return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
+    page = req.read().decode(req.headers.get_content_charset())
+
+    # Works for BitBucket
+    m = re.search('data-fetch-url="(.*)"', page)
+    if m is not None:
+        repo = m.group(1)
+
+        if repo.endswith('.git'):
+            return ('git', repo)
+
+        return ('hg', repo)
+
+    # Works for BitBucket (obsolete)
+    index = page.find('hg clone')
+    if index != -1:
+        repotype = 'hg'
+        repo = page[index + 9:]
+        index = repo.find('<')
+        if index == -1:
+            return (None, _("Error while getting repo address"))
+        repo = repo[:index]
+        repo = repo.split('"')[0]
+        return (repotype, repo)
+
+    # Works for BitBucket (obsolete)
+    index = page.find('git clone')
+    if index != -1:
+        repotype = 'git'
+        repo = page[index + 10:]
+        index = repo.find('<')
+        if index == -1:
+            return (None, _("Error while getting repo address"))
+        repo = repo[:index]
+        repo = repo.split('"')[0]
+        return (repotype, repo)
+
+    return (None, _("No information found.") + page)
+
+
+def get_app_from_url(url):
+    """Guess basic app metadata from the URL.
+
+    The URL must include a network hostname, unless it is an lp:,
+    file:, or git/ssh URL.  This throws ValueError on bad URLs to
+    match urlparse().
+
+    """
+
+    parsed = urllib.parse.urlparse(url)
+    invalid_url = False
+    if not parsed.scheme or not parsed.path:
+        invalid_url = True
+
+    app = fdroidserver.metadata.App()
+    app.Repo = url
+    if url.startswith('git://') or url.startswith('git@'):
+        app.RepoType = 'git'
+    elif parsed.netloc == 'github.com':
+        app.RepoType = 'git'
+        app.SourceCode = url
+        app.IssueTracker = url + '/issues'
+    elif parsed.netloc == 'gitlab.com':
+        # git can be fussy with gitlab URLs unless they end in .git
+        if url.endswith('.git'):
+            url = url[:-4]
+        app.Repo = url + '.git'
+        app.RepoType = 'git'
+        app.SourceCode = url
+        app.IssueTracker = url + '/issues'
+    elif parsed.netloc == 'notabug.org':
+        if url.endswith('.git'):
+            url = url[:-4]
+        app.Repo = url + '.git'
+        app.RepoType = 'git'
+        app.SourceCode = url
+        app.IssueTracker = url + '/issues'
+    elif parsed.netloc == 'bitbucket.org':
+        if url.endswith('/'):
+            url = url[:-1]
+        app.SourceCode = url + '/src'
+        app.IssueTracker = url + '/issues'
+        # Figure out the repo type and adddress...
+        app.RepoType, app.Repo = getrepofrompage(url)
+    elif url.startswith('https://') and url.endswith('.git'):
+        app.RepoType = 'git'
+
+    if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
+        invalid_url = True
+
+    if invalid_url:
+        raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
+
+    if not app.RepoType:
+        raise FDroidException("Unable to determine vcs type. " + app.Repo)
+
+    return app
+
+
 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
               raw=False, prepare=True, preponly=False, refresh=True,
               build=None):

fdroidserver/import.py

@@ -18,14 +18,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import git
-import glob
 import json
 import os
-import re
 import shutil
 import sys
-import urllib.parse
-import urllib.request
 import yaml
 from argparse import ArgumentParser
 import logging
@@ -40,121 +36,12 @@ from . import common
 from . import metadata
 from .exception import FDroidException
 
-SETTINGS_GRADLE = re.compile(r'settings\.gradle(?:\.kts)?')
-GRADLE_SUBPROJECT = re.compile(r'''['"]:([^'"]+)['"]''')
-
-
-# Get the repo type and address from the given web page. The page is scanned
-# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
-# when one of these is found it's assumed that's the information we want.
-# Returns repotype, address, or None, reason
-def getrepofrompage(url):
-    if not url.startswith('http'):
-        return (None, _('{url} does not start with "http"!'.format(url=url)))
-    req = urllib.request.urlopen(url)  # nosec B310 non-http URLs are filtered out
-    if req.getcode() != 200:
-        return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
-    page = req.read().decode(req.headers.get_content_charset())
-
-    # Works for BitBucket
-    m = re.search('data-fetch-url="(.*)"', page)
-    if m is not None:
-        repo = m.group(1)
-
-        if repo.endswith('.git'):
-            return ('git', repo)
-
-        return ('hg', repo)
-
-    # Works for BitBucket (obsolete)
-    index = page.find('hg clone')
-    if index != -1:
-        repotype = 'hg'
-        repo = page[index + 9:]
-        index = repo.find('<')
-        if index == -1:
-            return (None, _("Error while getting repo address"))
-        repo = repo[:index]
-        repo = repo.split('"')[0]
-        return (repotype, repo)
-
-    # Works for BitBucket (obsolete)
-    index = page.find('git clone')
-    if index != -1:
-        repotype = 'git'
-        repo = page[index + 10:]
-        index = repo.find('<')
-        if index == -1:
-            return (None, _("Error while getting repo address"))
-        repo = repo[:index]
-        repo = repo.split('"')[0]
-        return (repotype, repo)
-
-    return (None, _("No information found.") + page)
-
 
 config = None
 options = None
 
 
-def get_app_from_url(url):
+# WARNING!  This cannot be imported as a Python module, so reuseable functions need to go into common.py!
-    """Guess basic app metadata from the URL.
-
-    The URL must include a network hostname, unless it is an lp:,
-    file:, or git/ssh URL.  This throws ValueError on bad URLs to
-    match urlparse().
-
-    """
-
-    parsed = urllib.parse.urlparse(url)
-    invalid_url = False
-    if not parsed.scheme or not parsed.path:
-        invalid_url = True
-
-    app = metadata.App()
-    app.Repo = url
-    if url.startswith('git://') or url.startswith('git@'):
-        app.RepoType = 'git'
-    elif parsed.netloc == 'github.com':
-        app.RepoType = 'git'
-        app.SourceCode = url
-        app.IssueTracker = url + '/issues'
-    elif parsed.netloc == 'gitlab.com':
-        # git can be fussy with gitlab URLs unless they end in .git
-        if url.endswith('.git'):
-            url = url[:-4]
-        app.Repo = url + '.git'
-        app.RepoType = 'git'
-        app.SourceCode = url
-        app.IssueTracker = url + '/issues'
-    elif parsed.netloc == 'notabug.org':
-        if url.endswith('.git'):
-            url = url[:-4]
-        app.Repo = url + '.git'
-        app.RepoType = 'git'
-        app.SourceCode = url
-        app.IssueTracker = url + '/issues'
-    elif parsed.netloc == 'bitbucket.org':
-        if url.endswith('/'):
-            url = url[:-1]
-        app.SourceCode = url + '/src'
-        app.IssueTracker = url + '/issues'
-        # Figure out the repo type and adddress...
-        app.RepoType, app.Repo = getrepofrompage(url)
-    elif url.startswith('https://') and url.endswith('.git'):
-        app.RepoType = 'git'
-
-    if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
-        invalid_url = True
-
-    if invalid_url:
-        raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
-
-    if not app.RepoType:
-        raise FDroidException("Unable to determine vcs type. " + app.Repo)
-
-    return app
-
 
 def clone_to_tmp_dir(app):
     tmp_dir = 'tmp'
@@ -171,40 +58,6 @@ def clone_to_tmp_dir(app):
     return tmp_dir
 
 
-def get_all_gradle_and_manifests(build_dir):
-    paths = []
-    for root, dirs, files in os.walk(build_dir):
-        for f in sorted(files):
-            if f == 'AndroidManifest.xml' \
-               or f.endswith('.gradle') or f.endswith('.gradle.kts'):
-                full = os.path.join(root, f)
-                paths.append(full)
-    return paths
-
-
-def get_gradle_subdir(build_dir, paths):
-    """get the subdir where the gradle build is based"""
-    first_gradle_dir = None
-    for path in paths:
-        if not first_gradle_dir:
-            first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir)
-        if os.path.exists(path) and SETTINGS_GRADLE.match(os.path.basename(path)):
-            with open(path) as fp:
-                for m in GRADLE_SUBPROJECT.finditer(fp.read()):
-                    for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')):
-                        with open(f) as fp:
-                            while True:
-                                line = fp.readline()
-                                if not line:
-                                    break
-                                if common.ANDROID_PLUGIN_REGEX.match(line):
-                                    return os.path.relpath(os.path.dirname(f), build_dir)
-    if first_gradle_dir and first_gradle_dir != '.':
-        return first_gradle_dir
-
-    return ''
-
-
 def main():
 
     global config, options
@@ -256,7 +109,7 @@ def main():
                 break
         write_local_file = True
     elif options.url:
-        app = get_app_from_url(options.url)
+        app = common.get_app_from_url(options.url)
         tmp_importer_dir = clone_to_tmp_dir(app)
         git_repo = git.repo.Repo(tmp_importer_dir)
         build.disable = 'Generated by import.py - check/set version fields and commit id'
@@ -268,8 +121,8 @@ def main():
     build.commit = common.get_head_commit_id(git_repo)
 
     # Extract some information...
-    paths = get_all_gradle_and_manifests(tmp_importer_dir)
+    paths = common.get_all_gradle_and_manifests(tmp_importer_dir)
-    subdir = get_gradle_subdir(tmp_importer_dir, paths)
+    subdir = common.get_gradle_subdir(tmp_importer_dir, paths)
     if paths:
         versionName, versionCode, package = common.parse_androidmanifests(paths, app)
         if not package:

tests/common.TestCase

@@ -983,6 +983,48 @@ class CommonTest(unittest.TestCase):
         self.assertEqual(('1.0-free', '1', 'com.kunzisoft.fdroidtest.applicationidsuffix'),
                          fdroidserver.common.parse_androidmanifests(paths, app))
 
+    def test_get_all_gradle_and_manifests(self):
+        a = fdroidserver.common.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat'))
+        paths = [
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'),
+            os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'),
+        ]
+        self.assertEqual(sorted(paths), sorted(a))
+
+    def test_get_gradle_subdir(self):
+        subdirs = {
+            'cn.wildfirechat.chat': 'chat',
+            'com.anpmech.launcher': 'app',
+            'org.tasks': 'app',
+            'ut.ewh.audiometrytest': 'app',
+        }
+        for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'):
+            build_dir = os.path.join('source-files', f)
+            paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
+            logging.info(paths)
+            subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths)
+            self.assertEqual(subdirs[f], subdir)
+
+    def test_bad_urls(self):
+        for url in ('asdf',
+                    'file://thing.git',
+                    'https:///github.com/my/project',
+                    'git:///so/many/slashes',
+                    'ssh:/notabug.org/missing/a/slash',
+                    'git:notabug.org/missing/some/slashes',
+                    'https//github.com/bar/baz'):
+            with self.assertRaises(ValueError):
+                fdroidserver.common.get_app_from_url(url)
+
     def test_remove_signing_keys(self):
         testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
         print(testdir)

tests/import.TestCase

@@ -49,53 +49,11 @@ class ImportTest(unittest.TestCase):
             print('Skipping ImportTest!')
             return
 
-        app = import_proxy.get_app_from_url(url)
+        app = fdroidserver.common.get_app_from_url(url)
         import_proxy.clone_to_tmp_dir(app)
         self.assertEqual(app.RepoType, 'git')
         self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git')
 
-    def test_get_all_gradle_and_manifests(self):
-        a = import_proxy.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat'))
-        paths = [
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'),
-            os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'),
-        ]
-        self.assertEqual(sorted(paths), sorted(a))
-
-    def test_get_gradle_subdir(self):
-        subdirs = {
-            'cn.wildfirechat.chat': 'chat',
-            'com.anpmech.launcher': 'app',
-            'org.tasks': 'app',
-            'ut.ewh.audiometrytest': 'app',
-        }
-        for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'):
-            build_dir = os.path.join('source-files', f)
-            paths = import_proxy.get_all_gradle_and_manifests(build_dir)
-            logging.info(paths)
-            subdir = import_proxy.get_gradle_subdir(build_dir, paths)
-            self.assertEqual(subdirs[f], subdir)
-
-    def test_bad_urls(self):
-        for url in ('asdf',
-                    'file://thing.git',
-                    'https:///github.com/my/project',
-                    'git:///so/many/slashes',
-                    'ssh:/notabug.org/missing/a/slash',
-                    'git:notabug.org/missing/some/slashes',
-                    'https//github.com/bar/baz'):
-            with self.assertRaises(ValueError):
-                import_proxy.get_app_from_url(url)
-
     def test_get_app_from_url(self):
         testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
         os.chdir(testdir)
@@ -111,7 +69,7 @@ class ImportTest(unittest.TestCase):
             shutil.copytree(os.path.join(self.basedir, 'source-files', appid),
                             tmp_importer)
 
-            app = import_proxy.get_app_from_url(url)
+            app = fdroidserver.common.get_app_from_url(url)
             with mock.patch('fdroidserver.common.getvcs',
                             lambda a, b, c: fdroidserver.common.vcs(url, testdir)):
                 with mock.patch('fdroidserver.common.vcs.gotorevision',
@@ -122,7 +80,7 @@ class ImportTest(unittest.TestCase):
             self.assertEqual(url, app.Repo)
             self.assertEqual(url, app.SourceCode)
             logging.info(build_dir)
-            paths = import_proxy.get_all_gradle_and_manifests(build_dir)
+            paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
             self.assertNotEqual(paths, [])
             versionName, versionCode, package = fdroidserver.common.parse_androidmanifests(paths, app)
             self.assertEqual(vn, versionName)

tests/import_proxy.py

@@ -19,9 +19,6 @@ module = __import__('fdroidserver.import')
 for name, obj in inspect.getmembers(module):
     if name == 'import':
         clone_to_tmp_dir = obj.clone_to_tmp_dir
-        get_all_gradle_and_manifests = obj.get_all_gradle_and_manifests
-        get_app_from_url = obj.get_app_from_url
-        get_gradle_subdir = obj.get_gradle_subdir
         obj.options = Options()
         options = obj.options
         break