From ab2291475bee32c22a00630bcd62cbef97513566 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Wed, 26 Feb 2020 17:07:12 +0100 Subject: [PATCH] import: mv reusable functions to common.py to avoid import_proxy.py import is a strict keyword in Python, so it is not possible to import a module called 'import', even with things like: * import fdroidserver.import * from fdroidserver import import --- fdroidserver/common.py | 151 +++++++++++++++++++++++++++++++++++++++ fdroidserver/import.py | 155 ++--------------------------------------- tests/common.TestCase | 42 +++++++++++ tests/import.TestCase | 48 +------------ tests/import_proxy.py | 3 - 5 files changed, 200 insertions(+), 199 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 7b256476..b1b4f1c1 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -37,6 +37,8 @@ import logging import hashlib import socket import base64 +import urllib.parse +import urllib.request import zipfile import tempfile import json @@ -84,6 +86,9 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[ re.IGNORECASE) ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''') +SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?') +GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:([^'"]+)['"]''') + MAX_VERSION_CODE = 0x7fffffff # Java's Integer.MAX_VALUE (2147483647) XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}' @@ -1653,6 +1658,152 @@ def is_strict_application_id(name): and '.' in name +def get_all_gradle_and_manifests(build_dir): + paths = [] + for root, dirs, files in os.walk(build_dir): + for f in sorted(files): + if f == 'AndroidManifest.xml' \ + or f.endswith('.gradle') or f.endswith('.gradle.kts'): + full = os.path.join(root, f) + paths.append(full) + return paths + + +def get_gradle_subdir(build_dir, paths): + """get the subdir where the gradle build is based""" + first_gradle_dir = None + for path in paths: + if not first_gradle_dir: + first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir) + if os.path.exists(path) and SETTINGS_GRADLE_REGEX.match(os.path.basename(path)): + with open(path) as fp: + for m in GRADLE_SUBPROJECT_REGEX.finditer(fp.read()): + for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')): + with open(f) as fp: + while True: + line = fp.readline() + if not line: + break + if ANDROID_PLUGIN_REGEX.match(line): + return os.path.relpath(os.path.dirname(f), build_dir) + if first_gradle_dir and first_gradle_dir != '.': + return first_gradle_dir + + return '' + + +def getrepofrompage(url): + """Get the repo type and address from the given web page. + + The page is scanned in a rather naive manner for 'git clone xxxx', + 'hg clone xxxx', etc, and when one of these is found it's assumed + that's the information we want. Returns repotype, address, or + None, reason + + """ + if not url.startswith('http'): + return (None, _('{url} does not start with "http"!'.format(url=url))) + req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out + if req.getcode() != 200: + return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) + page = req.read().decode(req.headers.get_content_charset()) + + # Works for BitBucket + m = re.search('data-fetch-url="(.*)"', page) + if m is not None: + repo = m.group(1) + + if repo.endswith('.git'): + return ('git', repo) + + return ('hg', repo) + + # Works for BitBucket (obsolete) + index = page.find('hg clone') + if index != -1: + repotype = 'hg' + repo = page[index + 9:] + index = repo.find('<') + if index == -1: + return (None, _("Error while getting repo address")) + repo = repo[:index] + repo = repo.split('"')[0] + return (repotype, repo) + + # Works for BitBucket (obsolete) + index = page.find('git clone') + if index != -1: + repotype = 'git' + repo = page[index + 10:] + index = repo.find('<') + if index == -1: + return (None, _("Error while getting repo address")) + repo = repo[:index] + repo = repo.split('"')[0] + return (repotype, repo) + + return (None, _("No information found.") + page) + + +def get_app_from_url(url): + """Guess basic app metadata from the URL. + + The URL must include a network hostname, unless it is an lp:, + file:, or git/ssh URL. This throws ValueError on bad URLs to + match urlparse(). + + """ + + parsed = urllib.parse.urlparse(url) + invalid_url = False + if not parsed.scheme or not parsed.path: + invalid_url = True + + app = fdroidserver.metadata.App() + app.Repo = url + if url.startswith('git://') or url.startswith('git@'): + app.RepoType = 'git' + elif parsed.netloc == 'github.com': + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'gitlab.com': + # git can be fussy with gitlab URLs unless they end in .git + if url.endswith('.git'): + url = url[:-4] + app.Repo = url + '.git' + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'notabug.org': + if url.endswith('.git'): + url = url[:-4] + app.Repo = url + '.git' + app.RepoType = 'git' + app.SourceCode = url + app.IssueTracker = url + '/issues' + elif parsed.netloc == 'bitbucket.org': + if url.endswith('/'): + url = url[:-1] + app.SourceCode = url + '/src' + app.IssueTracker = url + '/issues' + # Figure out the repo type and adddress... + app.RepoType, app.Repo = getrepofrompage(url) + elif url.startswith('https://') and url.endswith('.git'): + app.RepoType = 'git' + + if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'): + invalid_url = True + + if invalid_url: + raise ValueError(_('"{url}" is not a valid URL!'.format(url=url))) + + if not app.RepoType: + raise FDroidException("Unable to determine vcs type. " + app.Repo) + + return app + + def getsrclib(spec, srclib_dir, subdir=None, basepath=False, raw=False, prepare=True, preponly=False, refresh=True, build=None): diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 42ba55e3..cca3063c 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -18,14 +18,10 @@ # along with this program. If not, see . import git -import glob import json import os -import re import shutil import sys -import urllib.parse -import urllib.request import yaml from argparse import ArgumentParser import logging @@ -40,121 +36,12 @@ from . import common from . import metadata from .exception import FDroidException -SETTINGS_GRADLE = re.compile(r'settings\.gradle(?:\.kts)?') -GRADLE_SUBPROJECT = re.compile(r'''['"]:([^'"]+)['"]''') - - -# Get the repo type and address from the given web page. The page is scanned -# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and -# when one of these is found it's assumed that's the information we want. -# Returns repotype, address, or None, reason -def getrepofrompage(url): - if not url.startswith('http'): - return (None, _('{url} does not start with "http"!'.format(url=url))) - req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out - if req.getcode() != 200: - return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode())) - page = req.read().decode(req.headers.get_content_charset()) - - # Works for BitBucket - m = re.search('data-fetch-url="(.*)"', page) - if m is not None: - repo = m.group(1) - - if repo.endswith('.git'): - return ('git', repo) - - return ('hg', repo) - - # Works for BitBucket (obsolete) - index = page.find('hg clone') - if index != -1: - repotype = 'hg' - repo = page[index + 9:] - index = repo.find('<') - if index == -1: - return (None, _("Error while getting repo address")) - repo = repo[:index] - repo = repo.split('"')[0] - return (repotype, repo) - - # Works for BitBucket (obsolete) - index = page.find('git clone') - if index != -1: - repotype = 'git' - repo = page[index + 10:] - index = repo.find('<') - if index == -1: - return (None, _("Error while getting repo address")) - repo = repo[:index] - repo = repo.split('"')[0] - return (repotype, repo) - - return (None, _("No information found.") + page) - config = None options = None -def get_app_from_url(url): - """Guess basic app metadata from the URL. - - The URL must include a network hostname, unless it is an lp:, - file:, or git/ssh URL. This throws ValueError on bad URLs to - match urlparse(). - - """ - - parsed = urllib.parse.urlparse(url) - invalid_url = False - if not parsed.scheme or not parsed.path: - invalid_url = True - - app = metadata.App() - app.Repo = url - if url.startswith('git://') or url.startswith('git@'): - app.RepoType = 'git' - elif parsed.netloc == 'github.com': - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'gitlab.com': - # git can be fussy with gitlab URLs unless they end in .git - if url.endswith('.git'): - url = url[:-4] - app.Repo = url + '.git' - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'notabug.org': - if url.endswith('.git'): - url = url[:-4] - app.Repo = url + '.git' - app.RepoType = 'git' - app.SourceCode = url - app.IssueTracker = url + '/issues' - elif parsed.netloc == 'bitbucket.org': - if url.endswith('/'): - url = url[:-1] - app.SourceCode = url + '/src' - app.IssueTracker = url + '/issues' - # Figure out the repo type and adddress... - app.RepoType, app.Repo = getrepofrompage(url) - elif url.startswith('https://') and url.endswith('.git'): - app.RepoType = 'git' - - if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'): - invalid_url = True - - if invalid_url: - raise ValueError(_('"{url}" is not a valid URL!'.format(url=url))) - - if not app.RepoType: - raise FDroidException("Unable to determine vcs type. " + app.Repo) - - return app - +# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py! def clone_to_tmp_dir(app): tmp_dir = 'tmp' @@ -171,40 +58,6 @@ def clone_to_tmp_dir(app): return tmp_dir -def get_all_gradle_and_manifests(build_dir): - paths = [] - for root, dirs, files in os.walk(build_dir): - for f in sorted(files): - if f == 'AndroidManifest.xml' \ - or f.endswith('.gradle') or f.endswith('.gradle.kts'): - full = os.path.join(root, f) - paths.append(full) - return paths - - -def get_gradle_subdir(build_dir, paths): - """get the subdir where the gradle build is based""" - first_gradle_dir = None - for path in paths: - if not first_gradle_dir: - first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir) - if os.path.exists(path) and SETTINGS_GRADLE.match(os.path.basename(path)): - with open(path) as fp: - for m in GRADLE_SUBPROJECT.finditer(fp.read()): - for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')): - with open(f) as fp: - while True: - line = fp.readline() - if not line: - break - if common.ANDROID_PLUGIN_REGEX.match(line): - return os.path.relpath(os.path.dirname(f), build_dir) - if first_gradle_dir and first_gradle_dir != '.': - return first_gradle_dir - - return '' - - def main(): global config, options @@ -256,7 +109,7 @@ def main(): break write_local_file = True elif options.url: - app = get_app_from_url(options.url) + app = common.get_app_from_url(options.url) tmp_importer_dir = clone_to_tmp_dir(app) git_repo = git.repo.Repo(tmp_importer_dir) build.disable = 'Generated by import.py - check/set version fields and commit id' @@ -268,8 +121,8 @@ def main(): build.commit = common.get_head_commit_id(git_repo) # Extract some information... - paths = get_all_gradle_and_manifests(tmp_importer_dir) - subdir = get_gradle_subdir(tmp_importer_dir, paths) + paths = common.get_all_gradle_and_manifests(tmp_importer_dir) + subdir = common.get_gradle_subdir(tmp_importer_dir, paths) if paths: versionName, versionCode, package = common.parse_androidmanifests(paths, app) if not package: diff --git a/tests/common.TestCase b/tests/common.TestCase index 856b71e7..268d5dd4 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -983,6 +983,48 @@ class CommonTest(unittest.TestCase): self.assertEqual(('1.0-free', '1', 'com.kunzisoft.fdroidtest.applicationidsuffix'), fdroidserver.common.parse_androidmanifests(paths, app)) + def test_get_all_gradle_and_manifests(self): + a = fdroidserver.common.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat')) + paths = [ + os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'), + os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'), + ] + self.assertEqual(sorted(paths), sorted(a)) + + def test_get_gradle_subdir(self): + subdirs = { + 'cn.wildfirechat.chat': 'chat', + 'com.anpmech.launcher': 'app', + 'org.tasks': 'app', + 'ut.ewh.audiometrytest': 'app', + } + for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'): + build_dir = os.path.join('source-files', f) + paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) + logging.info(paths) + subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths) + self.assertEqual(subdirs[f], subdir) + + def test_bad_urls(self): + for url in ('asdf', + 'file://thing.git', + 'https:///github.com/my/project', + 'git:///so/many/slashes', + 'ssh:/notabug.org/missing/a/slash', + 'git:notabug.org/missing/some/slashes', + 'https//github.com/bar/baz'): + with self.assertRaises(ValueError): + fdroidserver.common.get_app_from_url(url) + def test_remove_signing_keys(self): testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) print(testdir) diff --git a/tests/import.TestCase b/tests/import.TestCase index 30b660aa..70fe83df 100755 --- a/tests/import.TestCase +++ b/tests/import.TestCase @@ -49,53 +49,11 @@ class ImportTest(unittest.TestCase): print('Skipping ImportTest!') return - app = import_proxy.get_app_from_url(url) + app = fdroidserver.common.get_app_from_url(url) import_proxy.clone_to_tmp_dir(app) self.assertEqual(app.RepoType, 'git') self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git') - def test_get_all_gradle_and_manifests(self): - a = import_proxy.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat')) - paths = [ - os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'), - os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'), - ] - self.assertEqual(sorted(paths), sorted(a)) - - def test_get_gradle_subdir(self): - subdirs = { - 'cn.wildfirechat.chat': 'chat', - 'com.anpmech.launcher': 'app', - 'org.tasks': 'app', - 'ut.ewh.audiometrytest': 'app', - } - for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'): - build_dir = os.path.join('source-files', f) - paths = import_proxy.get_all_gradle_and_manifests(build_dir) - logging.info(paths) - subdir = import_proxy.get_gradle_subdir(build_dir, paths) - self.assertEqual(subdirs[f], subdir) - - def test_bad_urls(self): - for url in ('asdf', - 'file://thing.git', - 'https:///github.com/my/project', - 'git:///so/many/slashes', - 'ssh:/notabug.org/missing/a/slash', - 'git:notabug.org/missing/some/slashes', - 'https//github.com/bar/baz'): - with self.assertRaises(ValueError): - import_proxy.get_app_from_url(url) - def test_get_app_from_url(self): testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) os.chdir(testdir) @@ -111,7 +69,7 @@ class ImportTest(unittest.TestCase): shutil.copytree(os.path.join(self.basedir, 'source-files', appid), tmp_importer) - app = import_proxy.get_app_from_url(url) + app = fdroidserver.common.get_app_from_url(url) with mock.patch('fdroidserver.common.getvcs', lambda a, b, c: fdroidserver.common.vcs(url, testdir)): with mock.patch('fdroidserver.common.vcs.gotorevision', @@ -122,7 +80,7 @@ class ImportTest(unittest.TestCase): self.assertEqual(url, app.Repo) self.assertEqual(url, app.SourceCode) logging.info(build_dir) - paths = import_proxy.get_all_gradle_and_manifests(build_dir) + paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir) self.assertNotEqual(paths, []) versionName, versionCode, package = fdroidserver.common.parse_androidmanifests(paths, app) self.assertEqual(vn, versionName) diff --git a/tests/import_proxy.py b/tests/import_proxy.py index afe9544e..f230fdb1 100644 --- a/tests/import_proxy.py +++ b/tests/import_proxy.py @@ -19,9 +19,6 @@ module = __import__('fdroidserver.import') for name, obj in inspect.getmembers(module): if name == 'import': clone_to_tmp_dir = obj.clone_to_tmp_dir - get_all_gradle_and_manifests = obj.get_all_gradle_and_manifests - get_app_from_url = obj.get_app_from_url - get_gradle_subdir = obj.get_gradle_subdir obj.options = Options() options = obj.options break