opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'metadata' into 'master' 

metadata.py: use pathlib and support Windows

See merge request fdroid/fdroidserver!939
This commit is contained in:
Hans-Christoph Steiner 2021-06-08 15:02:54 +00:00
parent 5635815898 490f578d1c
commit 8b17fbf703
8 changed files with 766 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
fdroidserver/common.py
View File

@ -669,8 +669,8 @@ def get_metadata_files(vercodes):
found_invalid = False
metadatafiles = []
for appid in vercodes.keys():
f = os.path.join('metadata', '%s.yml' % appid)
if os.path.exists(f):
f = Path('metadata') / ('%s.yml' % appid)
if f.exists():
metadatafiles.append(f)
else:
found_invalid = True
@ -795,9 +795,9 @@ def get_build_dir(app):
'''get the dir that this app will be built in'''
if app.RepoType == 'srclib':
return os.path.join('build', 'srclib', app.Repo)
return Path('build/srclib') / app.Repo
return os.path.join('build', app.id)
return Path('build') / app.id
class Encoder(json.JSONEncoder):
@ -869,6 +869,8 @@ def get_head_commit_id(git_repo):
def setup_vcs(app):
'''checkout code from VCS and return instance of vcs and the build dir'''
build_dir = get_build_dir(app)
# TODO: Remove this
build_dir = str(build_dir)
# Set up vcs interface and make sure we have the latest code...
logging.debug("Getting {0} vcs interface for {1}"
@ -3982,6 +3984,8 @@ YAML_LINT_CONFIG = {'extends': 'default',
def run_yamllint(path, indent=0):
# TODO: Remove this
path = str(path)
try:
import yamllint.config
import yamllint.linter

97
fdroidserver/metadata.py
View File

@ -19,9 +19,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import git
import os
from pathlib import Path, PurePosixPath
import platform
import re
import glob
import logging
import yaml
try:
@ -31,9 +31,9 @@ except ImportError:
import importlib
from collections import OrderedDict
import fdroidserver.common
from fdroidserver import _
from fdroidserver.exception import MetaDataException, FDroidException
from . import common
from . import _
from .exception import MetaDataException, FDroidException
srclibs = None
warnings_action = None
@ -330,7 +330,7 @@ class Build(dict):
ndk = self.ndk
if isinstance(ndk, list):
ndk = self.ndk[0]
return fdroidserver.common.config['ndk_paths'].get(ndk, '')
return common.config['ndk_paths'].get(ndk, '')
flagtypes = {
@ -470,15 +470,22 @@ def parse_yaml_srclib(metadatapath):
'Subdir': None,
'Prepare': None}
if not os.path.exists(metadatapath):
if not metadatapath.exists():
_warn_or_exception(_("Invalid scrlib metadata: '{file}' "
"does not exist"
.format(file=metadatapath)))
return thisinfo
with open(metadatapath, "r", encoding="utf-8") as f:
with metadatapath.open("r", encoding="utf-8") as f:
try:
data = yaml.load(f, Loader=SafeLoader)
if type(data) is not dict:
if platform.system() == 'Windows':
# Handle symlink on Windows
symlink = metadatapath.parent / metadatapath.read_text()
if symlink.is_file():
with symlink.open("r", encoding="utf-8") as s:
data = yaml.load(s, Loader=SafeLoader)
if type(data) is not dict:
raise yaml.error.YAMLError(_('{file} is blank or corrupt!')
.format(file=metadatapath))
@ -486,8 +493,7 @@ def parse_yaml_srclib(metadatapath):
_warn_or_exception(_("Invalid srclib metadata: could not "
"parse '{file}'")
.format(file=metadatapath) + '\n'
+ fdroidserver.common.run_yamllint(metadatapath,
indent=4),
+ common.run_yamllint(metadatapath, indent=4),
cause=e)
return thisinfo
@ -531,13 +537,11 @@ def read_srclibs():
srclibs = {}
srcdir = 'srclibs'
if not os.path.exists(srcdir):
os.makedirs(srcdir)
srcdir = Path('srclibs')
srcdir.mkdir(exist_ok=True)
for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.yml'))):
srclibname = os.path.basename(metadatapath[:-4])
srclibs[srclibname] = parse_yaml_srclib(metadatapath)
for metadatapath in sorted(srcdir.glob('*.yml')):
srclibs[metadatapath.stem] = parse_yaml_srclib(metadatapath)
def read_metadata(appids={}, sort_by_time=False):
@ -559,18 +563,17 @@ def read_metadata(appids={}, sort_by_time=False):
apps = OrderedDict()
for basedir in ('metadata', 'tmp'):
if not os.path.exists(basedir):
os.makedirs(basedir)
Path(basedir).mkdir(exist_ok=True)
if appids:
vercodes = fdroidserver.common.read_pkg_args(appids)
metadatafiles = fdroidserver.common.get_metadata_files(vercodes)
vercodes = common.read_pkg_args(appids)
metadatafiles = common.get_metadata_files(vercodes)
else:
metadatafiles = (glob.glob(os.path.join('metadata', '*.yml'))
+ glob.glob('.fdroid.yml'))
metadatafiles = list(Path('metadata').glob('*.yml')) + list(
Path('.').glob('.fdroid.yml'))
if sort_by_time:
entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
entries = ((path.stat().st_mtime, path) for path in metadatafiles)
metadatafiles = []
for _ignored, path in sorted(entries, reverse=True):
metadatafiles.append(path)
@ -579,8 +582,8 @@ def read_metadata(appids={}, sort_by_time=False):
metadatafiles = sorted(metadatafiles)
for metadatapath in metadatafiles:
appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
if appid != '.fdroid' and not fdroidserver.common.is_valid_package_name(appid):
appid = metadatapath.stem
if appid != '.fdroid' and not common.is_valid_package_name(appid):
_warn_or_exception(_("{appid} from {path} is not a valid Java Package Name!")
.format(appid=appid, path=metadatapath))
if appid in apps:
@ -684,7 +687,7 @@ def post_metadata_parse(app):
# Parse metadata for a single application.
#
# 'metadatapath' - the filename to read. The "Application ID" aka
# 'metadatapath' - the file path to read. The "Application ID" aka
# "Package Name" for the application comes from this
# filename. Pass None to get a blank entry.
#
@ -729,27 +732,27 @@ def parse_metadata(metadatapath):
the source code.
"""
metadatapath = Path(metadatapath)
app = App()
app.metadatapath = metadatapath
metadata_file = os.path.basename(metadatapath)
name, _ignored = fdroidserver.common.get_extension(metadata_file)
app.metadatapath = str(PurePosixPath(metadatapath))
name = metadatapath.stem
if name != '.fdroid':
app.id = name
if metadatapath.endswith('.yml'):
with open(metadatapath, 'r') as mf:
if metadatapath.suffix == '.yml':
with metadatapath.open('r') as mf:
parse_yaml_metadata(mf, app)
else:
_warn_or_exception(_('Unknown metadata format: {path} (use: *.yml)')
.format(path=metadatapath))
if metadata_file != '.fdroid.yml' and app.Repo:
build_dir = fdroidserver.common.get_build_dir(app)
metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
if os.path.isfile(metadata_in_repo):
if metadatapath.name != '.fdroid.yml' and app.Repo:
build_dir = common.get_build_dir(app)
metadata_in_repo = build_dir / '.fdroid.yml'
if metadata_in_repo.is_file():
try:
commit_id = fdroidserver.common.get_head_commit_id(git.repo.Repo(build_dir))
# TODO: Python3.6: Should accept path-like
commit_id = common.get_head_commit_id(git.Repo(str(build_dir)))
logging.debug(_('Including metadata from %s@%s') % (metadata_in_repo, commit_id))
except git.exc.InvalidGitRepositoryError:
logging.debug(_('Including metadata from {path}').format(metadata_in_repo))
@ -764,11 +767,11 @@ def parse_metadata(metadatapath):
if app.get('Builds'):
build = app['Builds'][-1]
if build.subdir:
root_dir = build.subdir
root_dir = Path(build.subdir)
else:
root_dir = '.'
paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
_ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
root_dir = Path('.')
paths = common.manifest_paths(root_dir, build.gradle)
_ignored, _ignored, app.id = common.parse_androidmanifests(paths, app)
return app
@ -790,8 +793,7 @@ def parse_yaml_metadata(mf, app):
except yaml.YAMLError as e:
_warn_or_exception(_("could not parse '{path}'")
.format(path=mf.name) + '\n'
+ fdroidserver.common.run_yamllint(mf.name,
indent=4),
+ common.run_yamllint(mf.name, indent=4),
cause=e)
deprecated_in_yaml = ['Provides']
@ -801,7 +803,7 @@ def parse_yaml_metadata(mf, app):
if field not in yaml_app_fields + deprecated_in_yaml:
msg = (_("Unrecognised app field '{fieldname}' in '{path}'")
.format(fieldname=field, path=mf.name))
if os.path.basename(mf.name) == '.fdroid.yml':
if Path(mf.name).name == '.fdroid.yml':
logging.error(msg)
del yamldata[field]
else:
@ -978,11 +980,10 @@ build_cont = re.compile(r'^[ \t]')
def write_metadata(metadatapath, app):
# TODO: Remove this
metadatapath = str(metadatapath)
if metadatapath.endswith('.yml'):
metadatapath = Path(metadatapath)
if metadatapath.suffix == '.yml':
if importlib.util.find_spec('ruamel.yaml'):
with open(metadatapath, 'w') as mf:
with metadatapath.open('w') as mf:
return write_yaml(mf, app)
else:
raise FDroidException(_('ruamel.yaml not installed, can not write metadata.'))

16
fdroidserver/rewritemeta.py
View File

@ -18,11 +18,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from argparse import ArgumentParser
import os
import logging
import io
import tempfile
import shutil
from pathlib import Path
from . import _
from . import common
@ -36,9 +36,8 @@ def proper_format(app):
s = io.StringIO()
# TODO: currently reading entire file again, should reuse first
# read in metadata.py
with open(app.metadatapath, 'r') as f:
cur_content = f.read()
if app.metadatapath.endswith('.yml'):
cur_content = Path(app.metadatapath).read_text()
if Path(app.metadatapath).suffix == '.yml':
metadata.write_yaml(s, app)
content = s.getvalue()
s.close()
@ -65,8 +64,8 @@ def main():
apps = common.read_app_args(options.appid, allapps, False)
for appid, app in apps.items():
path = app.metadatapath
if path.endswith('.yml'):
path = Path(app.metadatapath)
if path.suffix == '.yml':
logging.info(_("Rewriting '{appid}'").format(appid=appid))
else:
logging.warning(_('Cannot rewrite "{path}"').format(path=path))
@ -91,9 +90,10 @@ def main():
# rewrite to temporary file before overwriting existsing
# file in case there's a bug in write_metadata
with tempfile.TemporaryDirectory() as tmpdir:
tmp_path = os.path.join(tmpdir, os.path.basename(path))
tmp_path = Path(tmpdir) / path.name
metadata.write_metadata(tmp_path, app)
shutil.move(tmp_path, path)
# TODO: Python3.6: Accept path-lik
shutil.move(str(tmp_path), str(path))
logging.debug(_("Finished"))

163
tests/build.TestCase
View File

@ -67,31 +67,134 @@ class BuildTest(unittest.TestCase):
pass # aapt is not required if androguard is present
testcases = [
('repo/obb.main.twoversions_1101613.apk', 'obb.main.twoversions', '1101613', '0.1', None),
('org.bitbucket.tickytacky.mirrormirror_1.apk', 'org.bitbucket.tickytacky.mirrormirror', '1', '1.0', None),
('org.bitbucket.tickytacky.mirrormirror_2.apk', 'org.bitbucket.tickytacky.mirrormirror', '2', '1.0.1', None),
('org.bitbucket.tickytacky.mirrormirror_3.apk', 'org.bitbucket.tickytacky.mirrormirror', '3', '1.0.2', None),
('org.bitbucket.tickytacky.mirrormirror_4.apk', 'org.bitbucket.tickytacky.mirrormirror', '4', '1.0.3', None),
('org.dyndns.fules.ck_20.apk', 'org.dyndns.fules.ck', '20', 'v1.6pre2',
['arm64-v8a', 'armeabi', 'armeabi-v7a', 'mips', 'mips64', 'x86', 'x86_64']),
(
'repo/obb.main.twoversions_1101613.apk',
'obb.main.twoversions',
'1101613',
'0.1',
None,
),
(
'org.bitbucket.tickytacky.mirrormirror_1.apk',
'org.bitbucket.tickytacky.mirrormirror',
'1',
'1.0',
None,
),
(
'org.bitbucket.tickytacky.mirrormirror_2.apk',
'org.bitbucket.tickytacky.mirrormirror',
'2',
'1.0.1',
None,
),
(
'org.bitbucket.tickytacky.mirrormirror_3.apk',
'org.bitbucket.tickytacky.mirrormirror',
'3',
'1.0.2',
None,
),
(
'org.bitbucket.tickytacky.mirrormirror_4.apk',
'org.bitbucket.tickytacky.mirrormirror',
'4',
'1.0.3',
None,
),
(
'org.dyndns.fules.ck_20.apk',
'org.dyndns.fules.ck',
'20',
'v1.6pre2',
[
'arm64-v8a',
'armeabi',
'armeabi-v7a',
'mips',
'mips64',
'x86',
'x86_64',
],
),
('urzip.apk', 'info.guardianproject.urzip', '100', '0.1', None),
('urzip-badcert.apk', 'info.guardianproject.urzip', '100', '0.1', None),
('urzip-badsig.apk', 'info.guardianproject.urzip', '100', '0.1', None),
('urzip-release.apk', 'info.guardianproject.urzip', '100', '0.1', None),
('urzip-release-unsigned.apk', 'info.guardianproject.urzip', '100', '0.1', None),
(
'urzip-release-unsigned.apk',
'info.guardianproject.urzip',
'100',
'0.1',
None,
),
('repo/com.politedroid_3.apk', 'com.politedroid', '3', '1.2', None),
('repo/com.politedroid_4.apk', 'com.politedroid', '4', '1.3', None),
('repo/com.politedroid_5.apk', 'com.politedroid', '5', '1.4', None),
('repo/com.politedroid_6.apk', 'com.politedroid', '6', '1.5', None),
('repo/duplicate.permisssions_9999999.apk', 'duplicate.permisssions', '9999999', '', None),
('repo/info.zwanenburg.caffeinetile_4.apk', 'info.zwanenburg.caffeinetile', '4', '1.3', None),
('repo/obb.main.oldversion_1444412523.apk', 'obb.main.oldversion', '1444412523', '0.1', None),
('repo/obb.mainpatch.current_1619_another-release-key.apk', 'obb.mainpatch.current', '1619', '0.1', None),
('repo/obb.mainpatch.current_1619.apk', 'obb.mainpatch.current', '1619', '0.1', None),
('repo/obb.main.twoversions_1101613.apk', 'obb.main.twoversions', '1101613', '0.1', None),
('repo/obb.main.twoversions_1101615.apk', 'obb.main.twoversions', '1101615', '0.1', None),
('repo/obb.main.twoversions_1101617.apk', 'obb.main.twoversions', '1101617', '0.1', None),
('repo/urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk', 'info.guardianproject.urzip', '100', '0.1', None),
(
'repo/duplicate.permisssions_9999999.apk',
'duplicate.permisssions',
'9999999',
'',
None,
),
(
'repo/info.zwanenburg.caffeinetile_4.apk',
'info.zwanenburg.caffeinetile',
'4',
'1.3',
None,
),
(
'repo/obb.main.oldversion_1444412523.apk',
'obb.main.oldversion',
'1444412523',
'0.1',
None,
),
(
'repo/obb.mainpatch.current_1619_another-release-key.apk',
'obb.mainpatch.current',
'1619',
'0.1',
None,
),
(
'repo/obb.mainpatch.current_1619.apk',
'obb.mainpatch.current',
'1619',
'0.1',
None,
),
(
'repo/obb.main.twoversions_1101613.apk',
'obb.main.twoversions',
'1101613',
'0.1',
None,
),
(
'repo/obb.main.twoversions_1101615.apk',
'obb.main.twoversions',
'1101615',
'0.1',
None,
),
(
'repo/obb.main.twoversions_1101617.apk',
'obb.main.twoversions',
'1101617',
'0.1',
None,
),
(
'repo/urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk',
'info.guardianproject.urzip',
'100',
'0.1',
None,
),
]
for apkfilename, appid, versionCode, versionName, nativecode in testcases:
app = fdroidserver.metadata.App()
@ -259,14 +362,26 @@ class BuildTest(unittest.TestCase):
with mock.patch('fdroidserver.common.replace_build_vars', wraps=make_fake_apk):
with mock.patch('fdroidserver.common.get_native_code', return_value='x86'):
with mock.patch('fdroidserver.common.get_apk_id',
return_value=(app.id, build.versionCode, build.versionName)):
with mock.patch('fdroidserver.common.is_apk_and_debuggable', return_value=False):
with mock.patch(
'fdroidserver.common.get_apk_id',
return_value=(app.id, build.versionCode, build.versionName),
):
with mock.patch(
'fdroidserver.common.is_apk_and_debuggable', return_value=False
):
fdroidserver.build.build_local(
app, build, vcs,
build_dir=testdir, output_dir=testdir,
log_dir=None, srclib_dir=None, extlib_dir=None, tmp_dir=None,
force=False, onserver=False, refresh=False
app,
build,
vcs,
build_dir=testdir,
output_dir=testdir,
log_dir=None,
srclib_dir=None,
extlib_dir=None,
tmp_dir=None,
force=False,
onserver=False,
refresh=False,
)
self.assertTrue(os.path.exists('foo.aar'))

3
tests/checkupdates.TestCase
View File

@ -220,9 +220,6 @@ class CheckupdatesTest(unittest.TestCase):
if __name__ == "__main__":
# TODO: Python3.6: Accept path-like object.
os.chdir(str(Path(__file__).parent))
parser = optparse.OptionParser()
parser.add_option(
"-v",

3
tests/import.TestCase
View File

@ -121,9 +121,6 @@ class ImportTest(unittest.TestCase):
if __name__ == "__main__":
# TODO: Python3.6: Support added to accept objects implementing the os.PathLike interface.
os.chdir(str(Path(__file__).parent))
parser = optparse.OptionParser()
parser.add_option(
"-v",

911
tests/metadata.TestCase
View File

File diff suppressed because it is too large Load Diff

71
tests/rewritemeta.TestCase
View File

@ -1,6 +1,5 @@
#!/usr/bin/env python3
import inspect
import logging
import optparse
import os
@ -9,14 +8,15 @@ import unittest
import tempfile
import textwrap
from unittest import mock
from pathlib import Path
from testcommon import TmpCwd
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
)
print('localmodule: ' + localmodule)
localmodule = Path(__file__).resolve().parent.parent
print('localmodule: ' + str(localmodule))
if localmodule not in sys.path:
sys.path.insert(0, localmodule)
sys.path.insert(0, str(localmodule))
from fdroidserver import common
from fdroidserver import rewritemeta
@ -28,30 +28,29 @@ class RewriteMetaTest(unittest.TestCase):
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
self.basedir = localmodule / 'tests'
self.tmpdir = localmodule / '.testfiles'
self.tmpdir.mkdir(exist_ok=True)
# TODO: Python3.6: Accepts a path-like object.
os.chdir(str(self.basedir))
def test_rewrite_scenario_trivial(self):
sys.argv = ['rewritemeta', 'a', 'b']
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
os.mkdir('metadata')
with open('metadata/a.yml', 'w') as f:
Path('metadata').mkdir()
with Path('metadata/a.yml').open('w') as f:
f.write('AutoName: a')
with open('metadata/b.yml', 'w') as f:
with Path('metadata/b.yml').open('w') as f:
f.write('AutoName: b')
rewritemeta.main()
with open('metadata/a.yml') as f:
self.assertEqual(
f.read(),
textwrap.dedent(
'''\
self.assertEqual(
Path('metadata/a.yml').read_text(),
textwrap.dedent(
'''\
License: Unknown
AutoName: a
@ -59,14 +58,13 @@ class RewriteMetaTest(unittest.TestCase):
AutoUpdateMode: None
UpdateCheckMode: None
'''
),
)
),
)
with open('metadata/b.yml') as f:
self.assertEqual(
f.read(),
textwrap.dedent(
'''\
self.assertEqual(
Path('metadata/b.yml').read_text(),
textwrap.dedent(
'''\
License: Unknown
AutoName: b
@ -74,14 +72,14 @@ class RewriteMetaTest(unittest.TestCase):
AutoUpdateMode: None
UpdateCheckMode: None
'''
),
)
),
)
def test_rewrite_scenario_yml_no_ruamel(self):
sys.argv = ['rewritemeta', 'a']
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
os.mkdir('metadata')
with open('metadata/a.yml', 'w') as f:
Path('metadata').mkdir()
with Path('metadata/a.yml').open('w') as f:
f.write('AutoName: a')
def boom(*args):
@ -91,19 +89,12 @@ class RewriteMetaTest(unittest.TestCase):
with self.assertRaises(FDroidException):
rewritemeta.main()
with open('metadata/a.yml') as f:
self.assertEqual(
f.read(),
textwrap.dedent(
'''\
AutoName: a'''
),
)
self.assertEqual(
Path('metadata/a.yml').read_text(), 'AutoName: a'
)
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
parser = optparse.OptionParser()
parser.add_option(
"-v",