opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

convert App to subclass of dict to support parsing/dumping libs 

Python is heavily based on its core data types, and dict is one of the more
important ones.  Even classes are basically a wrapper around a dict. This
converts metadata.App to be a subclass of dict so it can behave like a dict
when being dumped and loaded.  This makes its drastically easier to use
different data formats for build metadata and for sending data to the
client.  This approach will ultimately mean we no longer have to maintain
custom parsing and dumping code.

This also means then that the YAML/JSON field names will not have spaces in
them, and they will match exactly what it used as the dict keys once the
data is parsed, as well as matching exactly the instance attribute names:

* CurrentVersion: 1.2.6
* app['CurrentVersion'] == '1.2.6'
* app.CurrentVersion == '1.2.6'

Inspired by:
https://goodcode.io/articles/python-dict-object/
This commit is contained in:
Hans-Christoph Steiner 2016-11-23 17:25:59 +01:00
parent 4625651192
commit b7fc7f2228
10 changed files with 165 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
fdroidserver/lint.py
View File

@ -63,10 +63,10 @@ http_checks = https_enforcings + http_url_shorteners + [
]
regex_checks = {
'Web Site': http_checks,
'Source Code': http_checks,
'WebSite': http_checks,
'SourceCode': http_checks,
'Repo': https_enforcings,
'Issue Tracker': http_checks + [
'IssueTracker': http_checks + [
(re.compile(r'.*github\.com/[^/]+/[^/]+/*$'),
"/issues is missing"),
(re.compile(r'.*gitlab\.com/[^/]+/[^/]+/*$'),
@ -121,7 +121,7 @@ regex_checks = {
def check_regexes(app):
for f, checks in regex_checks.items():
for m, r in checks:
v = app.get_field(f)
v = app.get(f)
t = metadata.fieldtype(f)
if t == metadata.TYPE_MULTILINE:
for l in v.splitlines():
@ -183,8 +183,8 @@ def check_old_links(app):
'code.google.com',
]
if any(s in app.Repo for s in usual_sites):
for f in ['Web Site', 'Source Code', 'Issue Tracker', 'Changelog']:
v = app.get_field(f)
for f in ['WebSite', 'SourceCode', 'IssueTracker', 'Changelog']:
v = app.get(f)
if any(s in v for s in old_sites):
yield "App is in '%s' but has a link to '%s'" % (app.Repo, v)
@ -241,7 +241,7 @@ def check_duplicates(app):
links_seen = set()
for f in ['Source Code', 'Web Site', 'Issue Tracker', 'Changelog']:
v = app.get_field(f)
v = app.get(f)
if not v:
continue
v = v.lower()

208
fdroidserver/metadata.py
View File

@ -98,15 +98,21 @@ app_fields = set([
'Current Version',
'Current Version Code',
'No Source Since',
'Build',
'comments', # For formats that don't do inline comments
'builds', # For formats that do builds as a list
])
class App():
class App(dict):
def __init__(self, copydict=None):
if copydict:
super().__init__(copydict)
return
super().__init__()
def __init__(self):
self.Disabled = None
self.AntiFeatures = []
self.Provides = None
@ -148,94 +154,21 @@ class App():
self.comments = {}
self.added = None
self.lastupdated = None
self._modified = set()
@classmethod
def field_to_attr(cls, f):
"""
Translates human-readable field names to attribute names, e.g.
'Auto Name' to 'AutoName'
"""
return f.replace(' ', '')
@classmethod
def attr_to_field(cls, k):
"""
Translates attribute names to human-readable field names, e.g.
'AutoName' to 'Auto Name'
"""
if k in app_fields:
return k
f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
return f
def field_dict(self):
"""
Constructs an old-fashioned dict with the human-readable field
names. Should only be used for tests.
"""
d = {}
for k, v in self.__dict__.items():
if k == 'builds':
d['builds'] = []
for build in v:
b = {k: v for k, v in build.__dict__.items() if not k.startswith('_')}
d['builds'].append(b)
elif not k.startswith('_'):
f = App.attr_to_field(k)
d[f] = v
return d
def get_field(self, f):
"""Gets the value associated to a field name, e.g. 'Auto Name'"""
if f not in app_fields:
warn_or_exception('Unrecognised app field: ' + f)
k = App.field_to_attr(f)
return getattr(self, k)
def set_field(self, f, v):
"""Sets the value associated to a field name, e.g. 'Auto Name'"""
if f not in app_fields:
warn_or_exception('Unrecognised app field: ' + f)
k = App.field_to_attr(f)
self.__dict__[k] = v
self._modified.add(k)
def append_field(self, f, v):
"""Appends to the value associated to a field name, e.g. 'Auto Name'"""
if f not in app_fields:
warn_or_exception('Unrecognised app field: ' + f)
k = App.field_to_attr(f)
if k not in self.__dict__:
self.__dict__[k] = [v]
def __getattr__(self, name):
if name in self:
return self[name]
else:
self.__dict__[k].append(v)
raise AttributeError("No such attribute: " + name)
def update_fields(self, d):
'''Like dict.update(), but using human-readable field names'''
for f, v in d.items():
if f == 'builds':
for b in v:
build = Build()
build.update_flags(b)
self.builds.append(build)
else:
self.set_field(f, v)
def __setattr__(self, name, value):
self[name] = value
def update(self, d):
'''Like dict.update()'''
for k, v in d.__dict__.items():
if k == '_modified':
continue
elif k == 'builds':
for b in v:
build = Build()
del(b.__dict__['_modified'])
build.update_flags(b.__dict__)
self.builds.append(build)
elif v:
self.__dict__[k] = v
self._modified.add(k)
def __delattr__(self, name):
if name in self:
del self[name]
else:
raise AttributeError("No such attribute: " + name)
def get_last_build(self):
if len(self.builds) > 0:
@ -256,16 +189,17 @@ TYPE_BUILD_V2 = 8
fieldtypes = {
'Description': TYPE_MULTILINE,
'Maintainer Notes': TYPE_MULTILINE,
'MaintainerNotes': TYPE_MULTILINE,
'Categories': TYPE_LIST,
'AntiFeatures': TYPE_LIST,
'Build Version': TYPE_BUILD,
'BuildVersion': TYPE_BUILD,
'Build': TYPE_BUILD_V2,
'Use Built': TYPE_OBSOLETE,
'UseBuilt': TYPE_OBSOLETE,
}
def fieldtype(name):
name = name.replace(' ', '')
if name in fieldtypes:
return fieldtypes[name]
return TYPE_STRING
@ -518,9 +452,7 @@ valuetypes = {
def check_metadata(app):
for v in valuetypes:
for k in v.fields:
if k not in app._modified:
continue
v.check(app.__dict__[k], app.id)
v.check(app[k], app.id)
# Formatter for descriptions. Create an instance, and call parseline() with
@ -896,44 +828,21 @@ def sorted_builds(builds):
esc_newlines = re.compile(r'\\( |\n)')
# This function uses __dict__ to be faster
def post_metadata_parse(app):
for k in app._modified:
v = app.__dict__[k]
# TODO keep native types, convert only for .txt metadata
for k, v in app.items():
if type(v) in (float, int):
app.__dict__[k] = str(v)
app[k] = str(v)
builds = []
for build in app.builds:
if not isinstance(build, Build):
build = Build(build)
builds.append(build)
if 'builds' in app:
for build in app['builds']:
if not isinstance(build, Build):
build = Build(build)
builds.append(build)
for k in build._modified:
v = build.__dict__[k]
if type(v) in (float, int):
build.__dict__[k] = str(v)
continue
ftype = flagtype(k)
if ftype == TYPE_SCRIPT:
build.__dict__[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
elif ftype == TYPE_BOOL:
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if isinstance(v, str):
build.__dict__[k] = _decode_bool(v)
elif ftype == TYPE_STRING:
if isinstance(v, bool) and v:
build.__dict__[k] = 'yes'
elif ftype == TYPE_LIST:
if isinstance(v, bool) and v:
build.__dict__[k] = ['yes']
elif isinstance(v, str):
build.__dict__[k] = [v]
if not app.Description:
app.Description = 'No description available'
if not app.get('Description'):
app['Description'] = 'No description available'
app.builds = sorted_builds(builds)
@ -1039,17 +948,18 @@ def parse_json_metadata(mf, app):
# TODO create schema using https://pypi.python.org/pypi/jsonschema
jsoninfo = json.load(mf, parse_int=lambda s: s,
parse_float=lambda s: s)
app.update_fields(jsoninfo)
app.update(jsoninfo)
for f in ['Description', 'Maintainer Notes']:
v = app.get_field(f)
app.set_field(f, '\n'.join(v))
v = app.get(f)
if v:
app[f] = '\n'.join(v)
return app
def parse_yaml_metadata(mf, app):
yamlinfo = yaml.load(mf, Loader=YamlLoader)
app.update_fields(yamlinfo)
app.update(yamlinfo)
return app
@ -1128,6 +1038,8 @@ def parse_txt_metadata(mf, app):
build = None
vc_seen = set()
app.builds = []
c = 0
for line in mf:
c += 1
@ -1162,12 +1074,17 @@ def parse_txt_metadata(mf, app):
except ValueError:
warn_or_exception("Invalid metadata in " + linedesc)
if f not in app_fields:
warn_or_exception('Unrecognised app field: ' + f)
# Translate obsolete fields...
if f == 'Market Version':
f = 'Current Version'
if f == 'Market Version Code':
f = 'Current Version Code'
f = f.replace(' ', '')
ftype = fieldtype(f)
if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
add_comments(f)
@ -1177,9 +1094,9 @@ def parse_txt_metadata(mf, app):
warn_or_exception("Unexpected text on same line as "
+ f + " in " + linedesc)
elif ftype == TYPE_STRING:
app.set_field(f, v)
app[f] = v
elif ftype == TYPE_LIST:
app.set_field(f, split_list_values(v))
app[f] = split_list_values(v)
elif ftype == TYPE_BUILD:
if v.endswith("\\"):
mode = 2
@ -1212,7 +1129,7 @@ def parse_txt_metadata(mf, app):
elif mode == 1: # Multiline field
if line == '.':
mode = 0
app.set_field(f, '\n'.join(multiline_lines))
app[f] = '\n'.join(multiline_lines)
del multiline_lines[:]
else:
multiline_lines.append(line)
@ -1240,6 +1157,23 @@ def parse_txt_metadata(mf, app):
def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
def field_to_attr(f):
"""
Translates human-readable field names to attribute names, e.g.
'Auto Name' to 'AutoName'
"""
return f.replace(' ', '')
def attr_to_field(k):
"""
Translates attribute names to human-readable field names, e.g.
'AutoName' to 'Auto Name'
"""
if k in app_fields:
return k
f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
return f
def w_comments(key):
if key not in app.comments:
return
@ -1247,15 +1181,17 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
w_comment(line)
def w_field_always(f, v=None):
key = field_to_attr(f)
if v is None:
v = app.get_field(f)
w_comments(f)
v = app.get(key)
w_comments(key)
w_field(f, v)
def w_field_nonempty(f, v=None):
key = field_to_attr(f)
if v is None:
v = app.get_field(f)
w_comments(f)
v = app.get(key)
w_comments(key)
if v:
w_field(f, v)

9
fdroidserver/update.py
View File

@ -99,7 +99,7 @@ def update_wiki(apps, sortedids, apks):
generated_redirects = {}
for appid in sortedids:
app = apps[appid]
app = metadata.App(apps[appid])
wikidata = ''
if app.Disabled:
@ -302,7 +302,7 @@ def delete_disabled_builds(apps, apkcache, repodirs):
:param repodirs: the repo directories to process
"""
for appid, app in apps.items():
for build in app.builds:
for build in app['builds']:
if not build.disable:
continue
apkfilename = appid + '_' + str(build.vercode) + '.apk'
@ -1111,7 +1111,7 @@ def make_index(apps, sortedids, apks, repodir, archive):
element.setAttribute('packageName', packageName)
for appid in sortedids:
app = apps[appid]
app = metadata.App(apps[appid])
if app.Disabled is not None:
continue
@ -1265,7 +1265,7 @@ def make_index(apps, sortedids, apks, repodir, archive):
and config['make_current_version_link'] \
and repodir == 'repo': # only create these
namefield = config['current_version_name_source']
sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get_field(namefield))
sanitized_name = re.sub('''[ '"&%?+=/]''', '', app.get(namefield))
apklinkname = sanitized_name + '.apk'
current_version_path = os.path.join(repodir, current_version_file)
if os.path.islink(apklinkname):
@ -1621,6 +1621,7 @@ def main():
logging.debug("Don't know when " + appid + " was last updated")
if bestver == UNSET_VERSION_CODE:
if app.Name is None:
app.Name = app.AutoName or appid
app.icon = None

2
tests/import.TestCase
View File

@ -38,7 +38,7 @@ class ImportTest(unittest.TestCase):
print('Skipping ImportTest!')
return
app = fdroidserver.metadata.get_default_app_info()
app = fdroidserver.metadata.App()
app.UpdateCheckMode = "Tags"
root_dir, src_dir = import_proxy.get_metadata_from_url(app, url)
self.assertEqual(app.RepoType, 'git')

3
tests/metadata.TestCase
View File

@ -38,9 +38,8 @@ class MetadataTest(unittest.TestCase):
apps = fdroidserver.metadata.read_metadata(xref=True)
for appid in ('org.smssecure.smssecure', 'org.adaway', 'org.videolan.vlc'):
app = apps[appid]
savepath = os.path.join('metadata', 'dump', appid + '.yaml')
frommeta = app.field_dict()
frommeta = dict(apps[appid])
self.assertTrue(appid in apps)
with open(savepath, 'r') as f:
frompickle = yaml.load(f)

38
tests/metadata/dump/org.adaway.yaml
View File

@ -1,17 +1,17 @@
AntiFeatures: []
Archive Policy: null
Author Email: null
Author Name: null
Auto Name: AdAway
Auto Update Mode: Version v%v
ArchivePolicy: null
AuthorEmail: null
AuthorName: null
AutoName: AdAway
AutoUpdateMode: Version v%v
Binaries: null
Bitcoin: null
Categories:
- System
- Security
Changelog: ''
Current Version: '3.0'
Current Version Code: '52'
CurrentVersion: '3.0'
CurrentVersionCode: '52'
Description: 'An ad blocker that uses the hosts file. The hosts file
contains a list of mappings between hostnames and IP addresses. When
@ -38,24 +38,24 @@ Description: 'An ad blocker that uses the hosts file. The hosts file
Disabled: null
Donate: http://sufficientlysecure.org/index.php/adaway
FlattrID: '369138'
Issue Tracker: https://github.com/dschuermann/ad-away/issues
IssueTracker: https://github.com/dschuermann/ad-away/issues
License: GPLv3
Litecoin: null
Maintainer Notes: ''
MaintainerNotes: ''
Name: null
No Source Since: ''
NoSourceSince: ''
Provides: org.sufficientlysecure.adaway
Repo: https://github.com/dschuermann/ad-away.git
Repo Type: git
Requires Root: true
Source Code: https://github.com/dschuermann/ad-away
RepoType: git
RequiresRoot: true
SourceCode: https://github.com/dschuermann/ad-away
Summary: Block advertisements
Update Check Data: null
Update Check Ignore: null
Update Check Mode: Tags
Update Check Name: null
Vercode Operation: null
Web Site: http://sufficientlysecure.org/index.php/adaway
UpdateCheckData: null
UpdateCheckIgnore: null
UpdateCheckMode: Tags
UpdateCheckName: null
VercodeOperation: null
WebSite: http://sufficientlysecure.org/index.php/adaway
added: null
builds:
- antcommands: []

38
tests/metadata/dump/org.smssecure.smssecure.yaml
View File

@ -1,16 +1,16 @@
AntiFeatures: []
Archive Policy: null
Author Email: null
Author Name: null
Auto Name: SMSSecure
Auto Update Mode: Version v%v
ArchivePolicy: null
AuthorEmail: null
AuthorName: null
AutoName: SMSSecure
AutoUpdateMode: Version v%v
Binaries: null
Bitcoin: null
Categories:
- Phone & SMS
Changelog: ''
Current Version: 0.6.0
Current Version Code: '102'
CurrentVersion: 0.6.0
CurrentVersionCode: '102'
Description: 'SMSSecure is an SMS/MMS application that allows you to protect your
privacy while communicating with friends.
@ -35,24 +35,24 @@ Description: 'SMSSecure is an SMS/MMS application that allows you to protect you
Disabled: null
Donate: null
FlattrID: null
Issue Tracker: https://github.com/SMSSecure/SMSSecure/issues
IssueTracker: https://github.com/SMSSecure/SMSSecure/issues
License: GPLv3
Litecoin: null
Maintainer Notes: ''
MaintainerNotes: ''
Name: null
No Source Since: ''
NoSourceSince: ''
Provides: null
Repo: https://github.com/SMSSecure/SMSSecure
Repo Type: git
Requires Root: false
Source Code: https://github.com/SMSSecure/SMSSecure
RepoType: git
RequiresRoot: false
SourceCode: https://github.com/SMSSecure/SMSSecure
Summary: Send encrypted text messages (SMS)
Update Check Data: null
Update Check Ignore: null
Update Check Mode: Tags
Update Check Name: null
Vercode Operation: null
Web Site: http://www.smssecure.org
UpdateCheckData: null
UpdateCheckIgnore: null
UpdateCheckMode: Tags
UpdateCheckName: null
VercodeOperation: null
WebSite: http://www.smssecure.org
added: null
builds:
- antcommands: []

38
tests/metadata/dump/org.videolan.vlc.yaml
View File

@ -1,16 +1,16 @@
AntiFeatures: []
Archive Policy: 9 versions
Author Email: null
Author Name: null
Auto Name: VLC
Auto Update Mode: None
ArchivePolicy: 9 versions
AuthorEmail: null
AuthorName: null
AutoName: VLC
AutoUpdateMode: None
Binaries: null
Bitcoin: null
Categories:
- Multimedia
Changelog: ''
Current Version: 1.2.6
Current Version Code: '1030005'
CurrentVersion: 1.2.6
CurrentVersionCode: '1030005'
Description: 'Video and audio player that supports a wide range of formats,
for both local and remote playback.
@ -22,10 +22,10 @@ Description: 'Video and audio player that supports a wide range of formats,
Disabled: null
Donate: http://www.videolan.org/contribute.html#money
FlattrID: null
Issue Tracker: http://www.videolan.org/support/index.html#bugs
IssueTracker: http://www.videolan.org/support/index.html#bugs
License: GPLv3
Litecoin: null
Maintainer Notes: 'Instructions and dependencies here: http://wiki.videolan.org/AndroidCompile
MaintainerNotes: 'Instructions and dependencies here: http://wiki.videolan.org/AndroidCompile
see http://buildbot.videolan.org/builders/ for version code scheme
@ -42,19 +42,19 @@ Maintainer Notes: 'Instructions and dependencies here: http://wiki.videolan.org/
'
Name: null
No Source Since: ''
NoSourceSince: ''
Provides: null
Repo: git://git.videolan.org/vlc-ports/android.git
Repo Type: git
Requires Root: false
Source Code: http://git.videolan.org/?p=vlc-ports/android.git;a=summary
RepoType: git
RequiresRoot: false
SourceCode: http://git.videolan.org/?p=vlc-ports/android.git;a=summary
Summary: Media player
Update Check Data: null
Update Check Ignore: null
Update Check Mode: Tags
Update Check Name: null
Vercode Operation: '%c + 5'
Web Site: http://www.videolan.org/vlc/download-android.html
UpdateCheckData: null
UpdateCheckIgnore: null
UpdateCheckMode: Tags
UpdateCheckName: null
VercodeOperation: '%c + 5'
WebSite: http://www.videolan.org/vlc/download-android.html
added: null
builds:
- antcommands: []

20
tests/metadata/org.adaway.json
View File

@ -1,9 +1,9 @@
{
"Auto Name": "AdAway",
"Auto Update Mode": "Version v%v",
"AutoName": "AdAway",
"AutoUpdateMode": "Version v%v",
"Categories": ["System", "Security"],
"Current Version": "3.0",
"Current Version Code": 52,
"CurrentVersion": "3.0",
"CurrentVersionCode": 52,
"Description": [
"An ad blocker that uses the hosts file. The hosts file",
"contains a list of mappings between hostnames and IP addresses. When",
@ -21,16 +21,16 @@
],
"Donate": "http://sufficientlysecure.org/index.php/adaway",
"FlattrID": "369138",
"Issue Tracker": "https://github.com/dschuermann/ad-away/issues",
"IssueTracker": "https://github.com/dschuermann/ad-away/issues",
"License": "GPLv3",
"Provides": "org.sufficientlysecure.adaway",
"Repo": "https://github.com/dschuermann/ad-away.git",
"Repo Type": "git",
"Requires Root": true,
"Source Code": "https://github.com/dschuermann/ad-away",
"RepoType": "git",
"RequiresRoot": true,
"SourceCode": "https://github.com/dschuermann/ad-away",
"Summary": "Block advertisements",
"Update Check Mode": "Tags",
"Web Site": "http://sufficientlysecure.org/index.php/adaway",
"UpdateCheckMode": "Tags",
"WebSite": "http://sufficientlysecure.org/index.php/adaway",
"builds": [
{

24
tests/metadata/org.videolan.vlc.yml
View File

@ -1,12 +1,12 @@
Categories:
- Multimedia
License: GPLv3
Web Site: http://www.videolan.org/vlc/download-android.html
Source Code: http://git.videolan.org/?p=vlc-ports/android.git;a=summary
Issue Tracker: "http://www.videolan.org/support/index.html#bugs"
WebSite: http://www.videolan.org/vlc/download-android.html
SourceCode: http://git.videolan.org/?p=vlc-ports/android.git;a=summary
IssueTracker: "http://www.videolan.org/support/index.html#bugs"
Donate: "http://www.videolan.org/contribute.html#money"
Auto Name: VLC
AutoName: VLC
Summary: Media player
Description: |
Video and audio player that supports a wide range of formats,
@ -14,7 +14,7 @@ Description: |
[http://git.videolan.org/?p=vlc-ports/android.git;a=blob_plain;f=NEWS NEWS]
Repo Type: git
RepoType: git
Repo: git://git.videolan.org/vlc-ports/android.git
builds:
@ -875,7 +875,7 @@ builds:
buildjni: no
ndk: r10d
Maintainer Notes: |
MaintainerNotes: |
Instructions and dependencies here: http://wiki.videolan.org/AndroidCompile
see http://buildbot.videolan.org/builders/ for version code scheme
The VLC srclib commit can be found out from TESTED_HASH value in compile.sh
@ -902,10 +902,10 @@ Maintainer Notes: |
# +2: x86
# +3: arm
# +4: armv7 (CV)
Archive Policy: 9 versions
Auto Update Mode: None
Update Check Mode: Tags
ArchivePolicy: 9 versions
AutoUpdateMode: None
UpdateCheckMode: Tags
# Only use higher vercode ops, if we do build those arches
Vercode Operation: "%c + 5"
Current Version: 1.2.6
Current Version Code: 1030005
VercodeOperation: "%c + 5"
CurrentVersion: 1.2.6
CurrentVersionCode: 1030005