Browse Source

import: mv reusable functions to common.py to avoid import_proxy.py

import is a strict keyword in Python, so it is not possible to import a
module called 'import', even with things like:

* import fdroidserver.import
* from fdroidserver import import
merge-requests/725/head
Hans-Christoph Steiner 9 months ago
parent
commit
ab2291475b
5 changed files with 200 additions and 199 deletions
  1. +151
    -0
      fdroidserver/common.py
  2. +4
    -151
      fdroidserver/import.py
  3. +42
    -0
      tests/common.TestCase
  4. +3
    -45
      tests/import.TestCase
  5. +0
    -3
      tests/import_proxy.py

+ 151
- 0
fdroidserver/common.py View File

@ -37,6 +37,8 @@ import logging
import hashlib
import socket
import base64
import urllib.parse
import urllib.request
import zipfile
import tempfile
import json
@ -84,6 +86,9 @@ VALID_APPLICATION_ID_REGEX = re.compile(r'''(?:^[a-z_]+(?:\d*[a-zA-Z_]*)*)(?:\.[
re.IGNORECASE)
ANDROID_PLUGIN_REGEX = re.compile(r'''\s*(:?apply plugin:|id)\(?\s*['"](android|com\.android\.application)['"]\s*\)?''')
SETTINGS_GRADLE_REGEX = re.compile(r'settings\.gradle(?:\.kts)?')
GRADLE_SUBPROJECT_REGEX = re.compile(r'''['"]:([^'"]+)['"]''')
MAX_VERSION_CODE = 0x7fffffff # Java's Integer.MAX_VALUE (2147483647)
XMLNS_ANDROID = '{http://schemas.android.com/apk/res/android}'
@ -1653,6 +1658,152 @@ def is_strict_application_id(name):
and '.' in name
def get_all_gradle_and_manifests(build_dir):
paths = []
for root, dirs, files in os.walk(build_dir):
for f in sorted(files):
if f == 'AndroidManifest.xml' \
or f.endswith('.gradle') or f.endswith('.gradle.kts'):
full = os.path.join(root, f)
paths.append(full)
return paths
def get_gradle_subdir(build_dir, paths):
"""get the subdir where the gradle build is based"""
first_gradle_dir = None
for path in paths:
if not first_gradle_dir:
first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir)
if os.path.exists(path) and SETTINGS_GRADLE_REGEX.match(os.path.basename(path)):
with open(path) as fp:
for m in GRADLE_SUBPROJECT_REGEX.finditer(fp.read()):
for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')):
with open(f) as fp:
while True:
line = fp.readline()
if not line:
break
if ANDROID_PLUGIN_REGEX.match(line):
return os.path.relpath(os.path.dirname(f), build_dir)
if first_gradle_dir and first_gradle_dir != '.':
return first_gradle_dir
return ''
def getrepofrompage(url):
"""Get the repo type and address from the given web page.
The page is scanned in a rather naive manner for 'git clone xxxx',
'hg clone xxxx', etc, and when one of these is found it's assumed
that's the information we want. Returns repotype, address, or
None, reason
"""
if not url.startswith('http'):
return (None, _('{url} does not start with "http"!'.format(url=url)))
req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
if req.getcode() != 200:
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
page = req.read().decode(req.headers.get_content_charset())
# Works for BitBucket
m = re.search('data-fetch-url="(.*)"', page)
if m is not None:
repo = m.group(1)
if repo.endswith('.git'):
return ('git', repo)
return ('hg', repo)
# Works for BitBucket (obsolete)
index = page.find('hg clone')
if index != -1:
repotype = 'hg'
repo = page[index + 9:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
# Works for BitBucket (obsolete)
index = page.find('git clone')
if index != -1:
repotype = 'git'
repo = page[index + 10:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
return (None, _("No information found.") + page)
def get_app_from_url(url):
"""Guess basic app metadata from the URL.
The URL must include a network hostname, unless it is an lp:,
file:, or git/ssh URL. This throws ValueError on bad URLs to
match urlparse().
"""
parsed = urllib.parse.urlparse(url)
invalid_url = False
if not parsed.scheme or not parsed.path:
invalid_url = True
app = fdroidserver.metadata.App()
app.Repo = url
if url.startswith('git://') or url.startswith('git@'):
app.RepoType = 'git'
elif parsed.netloc == 'github.com':
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'gitlab.com':
# git can be fussy with gitlab URLs unless they end in .git
if url.endswith('.git'):
url = url[:-4]
app.Repo = url + '.git'
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'notabug.org':
if url.endswith('.git'):
url = url[:-4]
app.Repo = url + '.git'
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'bitbucket.org':
if url.endswith('/'):
url = url[:-1]
app.SourceCode = url + '/src'
app.IssueTracker = url + '/issues'
# Figure out the repo type and adddress...
app.RepoType, app.Repo = getrepofrompage(url)
elif url.startswith('https://') and url.endswith('.git'):
app.RepoType = 'git'
if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
invalid_url = True
if invalid_url:
raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
if not app.RepoType:
raise FDroidException("Unable to determine vcs type. " + app.Repo)
return app
def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
raw=False, prepare=True, preponly=False, refresh=True,
build=None):

+ 4
- 151
fdroidserver/import.py View File

@ -18,14 +18,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import git
import glob
import json
import os
import re
import shutil
import sys
import urllib.parse
import urllib.request
import yaml
from argparse import ArgumentParser
import logging
@ -40,121 +36,12 @@ from . import common
from . import metadata
from .exception import FDroidException
SETTINGS_GRADLE = re.compile(r'settings\.gradle(?:\.kts)?')
GRADLE_SUBPROJECT = re.compile(r'''['"]:([^'"]+)['"]''')
# Get the repo type and address from the given web page. The page is scanned
# in a rather naive manner for 'git clone xxxx', 'hg clone xxxx', etc, and
# when one of these is found it's assumed that's the information we want.
# Returns repotype, address, or None, reason
def getrepofrompage(url):
if not url.startswith('http'):
return (None, _('{url} does not start with "http"!'.format(url=url)))
req = urllib.request.urlopen(url) # nosec B310 non-http URLs are filtered out
if req.getcode() != 200:
return (None, 'Unable to get ' + url + ' - return code ' + str(req.getcode()))
page = req.read().decode(req.headers.get_content_charset())
# Works for BitBucket
m = re.search('data-fetch-url="(.*)"', page)
if m is not None:
repo = m.group(1)
if repo.endswith('.git'):
return ('git', repo)
return ('hg', repo)
# Works for BitBucket (obsolete)
index = page.find('hg clone')
if index != -1:
repotype = 'hg'
repo = page[index + 9:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
# Works for BitBucket (obsolete)
index = page.find('git clone')
if index != -1:
repotype = 'git'
repo = page[index + 10:]
index = repo.find('<')
if index == -1:
return (None, _("Error while getting repo address"))
repo = repo[:index]
repo = repo.split('"')[0]
return (repotype, repo)
return (None, _("No information found.") + page)
config = None
options = None
def get_app_from_url(url):
"""Guess basic app metadata from the URL.
The URL must include a network hostname, unless it is an lp:,
file:, or git/ssh URL. This throws ValueError on bad URLs to
match urlparse().
"""
parsed = urllib.parse.urlparse(url)
invalid_url = False
if not parsed.scheme or not parsed.path:
invalid_url = True
app = metadata.App()
app.Repo = url
if url.startswith('git://') or url.startswith('git@'):
app.RepoType = 'git'
elif parsed.netloc == 'github.com':
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'gitlab.com':
# git can be fussy with gitlab URLs unless they end in .git
if url.endswith('.git'):
url = url[:-4]
app.Repo = url + '.git'
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'notabug.org':
if url.endswith('.git'):
url = url[:-4]
app.Repo = url + '.git'
app.RepoType = 'git'
app.SourceCode = url
app.IssueTracker = url + '/issues'
elif parsed.netloc == 'bitbucket.org':
if url.endswith('/'):
url = url[:-1]
app.SourceCode = url + '/src'
app.IssueTracker = url + '/issues'
# Figure out the repo type and adddress...
app.RepoType, app.Repo = getrepofrompage(url)
elif url.startswith('https://') and url.endswith('.git'):
app.RepoType = 'git'
if not parsed.netloc and parsed.scheme in ('git', 'http', 'https', 'ssh'):
invalid_url = True
if invalid_url:
raise ValueError(_('"{url}" is not a valid URL!'.format(url=url)))
if not app.RepoType:
raise FDroidException("Unable to determine vcs type. " + app.Repo)
return app
# WARNING! This cannot be imported as a Python module, so reuseable functions need to go into common.py!
def clone_to_tmp_dir(app):
tmp_dir = 'tmp'
@ -171,40 +58,6 @@ def clone_to_tmp_dir(app):
return tmp_dir
def get_all_gradle_and_manifests(build_dir):
paths = []
for root, dirs, files in os.walk(build_dir):
for f in sorted(files):
if f == 'AndroidManifest.xml' \
or f.endswith('.gradle') or f.endswith('.gradle.kts'):
full = os.path.join(root, f)
paths.append(full)
return paths
def get_gradle_subdir(build_dir, paths):
"""get the subdir where the gradle build is based"""
first_gradle_dir = None
for path in paths:
if not first_gradle_dir:
first_gradle_dir = os.path.relpath(os.path.dirname(path), build_dir)
if os.path.exists(path) and SETTINGS_GRADLE.match(os.path.basename(path)):
with open(path) as fp:
for m in GRADLE_SUBPROJECT.finditer(fp.read()):
for f in glob.glob(os.path.join(os.path.dirname(path), m.group(1), 'build.gradle*')):
with open(f) as fp:
while True:
line = fp.readline()
if not line:
break
if common.ANDROID_PLUGIN_REGEX.match(line):
return os.path.relpath(os.path.dirname(f), build_dir)
if first_gradle_dir and first_gradle_dir != '.':
return first_gradle_dir
return ''
def main():
global config, options
@ -256,7 +109,7 @@ def main():
break
write_local_file = True
elif options.url:
app = get_app_from_url(options.url)
app = common.get_app_from_url(options.url)
tmp_importer_dir = clone_to_tmp_dir(app)
git_repo = git.repo.Repo(tmp_importer_dir)
build.disable = 'Generated by import.py - check/set version fields and commit id'
@ -268,8 +121,8 @@ def main():
build.commit = common.get_head_commit_id(git_repo)
# Extract some information...
paths = get_all_gradle_and_manifests(tmp_importer_dir)
subdir = get_gradle_subdir(tmp_importer_dir, paths)
paths = common.get_all_gradle_and_manifests(tmp_importer_dir)
subdir = common.get_gradle_subdir(tmp_importer_dir, paths)
if paths:
versionName, versionCode, package = common.parse_androidmanifests(paths, app)
if not package:

+ 42
- 0
tests/common.TestCase View File

@ -983,6 +983,48 @@ class CommonTest(unittest.TestCase):
self.assertEqual(('1.0-free', '1', 'com.kunzisoft.fdroidtest.applicationidsuffix'),
fdroidserver.common.parse_androidmanifests(paths, app))
def test_get_all_gradle_and_manifests(self):
a = fdroidserver.common.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat'))
paths = [
os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'),
]
self.assertEqual(sorted(paths), sorted(a))
def test_get_gradle_subdir(self):
subdirs = {
'cn.wildfirechat.chat': 'chat',
'com.anpmech.launcher': 'app',
'org.tasks': 'app',
'ut.ewh.audiometrytest': 'app',
}
for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'):
build_dir = os.path.join('source-files', f)
paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
logging.info(paths)
subdir = fdroidserver.common.get_gradle_subdir(build_dir, paths)
self.assertEqual(subdirs[f], subdir)
def test_bad_urls(self):
for url in ('asdf',
'file://thing.git',
'https:///github.com/my/project',
'git:///so/many/slashes',
'ssh:/notabug.org/missing/a/slash',
'git:notabug.org/missing/some/slashes',
'https//github.com/bar/baz'):
with self.assertRaises(ValueError):
fdroidserver.common.get_app_from_url(url)
def test_remove_signing_keys(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
print(testdir)

+ 3
- 45
tests/import.TestCase View File

@ -49,53 +49,11 @@ class ImportTest(unittest.TestCase):
print('Skipping ImportTest!')
return
app = import_proxy.get_app_from_url(url)
app = fdroidserver.common.get_app_from_url(url)
import_proxy.clone_to_tmp_dir(app)
self.assertEqual(app.RepoType, 'git')
self.assertEqual(app.Repo, 'https://gitlab.com/fdroid/ci-test-app.git')
def test_get_all_gradle_and_manifests(self):
a = import_proxy.get_all_gradle_and_manifests(os.path.join('source-files', 'cn.wildfirechat.chat'))
paths = [
os.path.join('source-files', 'cn.wildfirechat.chat', 'avenginekit', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'chat', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'client', 'src', 'main', 'AndroidManifest.xml'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'emojilibrary', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'gradle', 'build_libraries.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'imagepicker', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'mars-core-release', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'push', 'build.gradle'),
os.path.join('source-files', 'cn.wildfirechat.chat', 'settings.gradle'),
]
self.assertEqual(sorted(paths), sorted(a))
def test_get_gradle_subdir(self):
subdirs = {
'cn.wildfirechat.chat': 'chat',
'com.anpmech.launcher': 'app',
'org.tasks': 'app',
'ut.ewh.audiometrytest': 'app',
}
for f in ('cn.wildfirechat.chat', 'com.anpmech.launcher', 'org.tasks', 'ut.ewh.audiometrytest'):
build_dir = os.path.join('source-files', f)
paths = import_proxy.get_all_gradle_and_manifests(build_dir)
logging.info(paths)
subdir = import_proxy.get_gradle_subdir(build_dir, paths)
self.assertEqual(subdirs[f], subdir)
def test_bad_urls(self):
for url in ('asdf',
'file://thing.git',
'https:///github.com/my/project',
'git:///so/many/slashes',
'ssh:/notabug.org/missing/a/slash',
'git:notabug.org/missing/some/slashes',
'https//github.com/bar/baz'):
with self.assertRaises(ValueError):
import_proxy.get_app_from_url(url)
def test_get_app_from_url(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
@ -111,7 +69,7 @@ class ImportTest(unittest.TestCase):
shutil.copytree(os.path.join(self.basedir, 'source-files', appid),
tmp_importer)
app = import_proxy.get_app_from_url(url)
app = fdroidserver.common.get_app_from_url(url)
with mock.patch('fdroidserver.common.getvcs',
lambda a, b, c: fdroidserver.common.vcs(url, testdir)):
with mock.patch('fdroidserver.common.vcs.gotorevision',
@ -122,7 +80,7 @@ class ImportTest(unittest.TestCase):
self.assertEqual(url, app.Repo)
self.assertEqual(url, app.SourceCode)
logging.info(build_dir)
paths = import_proxy.get_all_gradle_and_manifests(build_dir)
paths = fdroidserver.common.get_all_gradle_and_manifests(build_dir)
self.assertNotEqual(paths, [])
versionName, versionCode, package = fdroidserver.common.parse_androidmanifests(paths, app)
self.assertEqual(vn, versionName)

+ 0
- 3
tests/import_proxy.py View File

@ -19,9 +19,6 @@ module = __import__('fdroidserver.import')
for name, obj in inspect.getmembers(module):
if name == 'import':
clone_to_tmp_dir = obj.clone_to_tmp_dir
get_all_gradle_and_manifests = obj.get_all_gradle_and_manifests
get_app_from_url = obj.get_app_from_url
get_gradle_subdir = obj.get_gradle_subdir
obj.options = Options()
options = obj.options
break

Loading…
Cancel
Save