opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

consolidate all post-parse metadata checks 

This puts all of the needed post parsing checks on the metadata into a
single method that is used by all parsing methods (.txt, JSON, XML, YAML).
This provides the single place to normalize the internal representation of
the metadata.

It would be good to also change the internal representation to use more Python
bool/int types so that less post parsing is needed for JSON, XML, and YAML.

The SMSSecure test .pickle was changed to account for the use of lstrip()
and rstrip() on all 'script' types.

This also changes the example JSON to use ints for versionCodes
This commit is contained in:
Hans-Christoph Steiner 2015-07-23 15:09:57 -07:00
parent 014df8426f
commit 0425788b7d
4 changed files with 1449 additions and 1445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
fdroidserver/metadata.py
View File

@ -585,14 +585,58 @@ def get_default_app_info_list():
def post_metadata_parse(thisinfo):
supported_metadata = app_defaults.keys() + ['comments', 'builds', 'id']
for k, v in thisinfo.iteritems():
if k not in supported_metadata:
raise MetaDataException("Unrecognised metadata: {0}: {1}"
.format(k, v))
if type(v) in (float, int):
thisinfo[k] = str(v)
# convert to the odd internal format
for k in ('Description', 'Maintainer Notes'):
if isinstance(thisinfo[k], basestring):
text = thisinfo[k].rstrip().lstrip()
thisinfo[k] = text.split('\n')
supported_flags = (flag_defaults.keys()
+ ['vercode', 'version', 'versionCode', 'versionName'])
esc_newlines = re.compile('\\\\( |\\n)')
for build in thisinfo['builds']:
for k, v in build.iteritems():
for k, v in build.items():
if k not in supported_flags:
raise MetaDataException("Unrecognised build flag: {0}={1}"
.format(k, v))
if k == 'versionCode':
build['vercode'] = str(v)
del build['versionCode']
elif k == 'versionName':
build['version'] = str(v)
del build['versionName']
elif type(v) in (float, int):
build[k] = str(v)
else:
keyflagtype = flagtype(k)
if keyflagtype == 'list':
# these can be bools, strings or lists, but ultimately are lists
if isinstance(v, basestring):
build[k] = [v]
elif isinstance(v, bool):
if v:
build[k] = ['yes']
else:
build[k] = ['no']
elif keyflagtype == 'script':
build[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
elif keyflagtype == 'bool':
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if isinstance(v, basestring):
if v == 'true':
build[k] = True
else:
build[k] = False
if not thisinfo['Description']:
thisinfo['Description'].append('No description available')
@ -667,27 +711,12 @@ def parse_json_metadata(metafile):
# fdroid metadata is only strings and booleans, no floats or ints. And
# json returns unicode, and fdroidserver still uses plain python strings
# TODO create schema using https://pypi.python.org/pypi/jsonschema
jsoninfo = json.load(open(metafile, 'r'),
object_hook=_decode_dict,
parse_int=lambda s: s,
parse_float=lambda s: s)
supported_metadata = app_defaults.keys() + ['builds', 'comments']
for k, v in jsoninfo.iteritems():
if k not in supported_metadata:
logging.warn(metafile + ' contains unknown metadata key, ignoring: ' + k)
thisinfo.update(jsoninfo)
for build in thisinfo['builds']:
for k, v in build.iteritems():
if k in ('buildjni', 'gradle', 'maven', 'kivy'):
# convert standard types to mixed datatype legacy format
if isinstance(v, bool):
if v:
build[k] = ['yes']
else:
build[k] = ['no']
# TODO create schema using https://pypi.python.org/pypi/jsonschema
post_metadata_parse(thisinfo)
return (appid, thisinfo)
@ -740,30 +769,6 @@ def parse_xml_metadata(metafile):
else:
thisinfo['Requires Root'] = False
# convert to the odd internal format
for k in ('Description', 'Maintainer Notes'):
if isinstance(thisinfo[k], basestring):
text = thisinfo[k].rstrip().lstrip()
thisinfo[k] = text.split('\n')
supported_flags = flag_defaults.keys() + ['versionCode', 'versionName']
for build in thisinfo['builds']:
for k, v in build.iteritems():
if k not in supported_flags:
raise MetaDataException("Unrecognised build flag: {0}={1}"
.format(k, v))
keyflagtype = flagtype(k)
if keyflagtype == 'bool':
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if isinstance(v, basestring):
if v == 'true':
build[k] = True
else:
build[k] = False
elif keyflagtype == 'list':
if isinstance(v, basestring):
build[k] = [v]
post_metadata_parse(thisinfo)
return (appid, thisinfo)

58
tests/metadata/org.adaway.json
View File

@ -3,7 +3,7 @@
"Auto Update Mode": "Version v%v",
"Categories": ["System", "Security"],
"Current Version": "3.0",
"Current Version Code": "52",
"Current Version Code": 52,
"Description": [
"An ad blocker that uses the hosts file. The hosts file",
"contains a list of mappings between hostnames and IP addresses. When",
@ -37,7 +37,7 @@
"buildjni": true,
"commit": "ea5378a94ee0dc1d99d2cec95fae7e6d81afb2b9",
"subdir": "org_adaway/",
"versionCode": "13",
"versionCode": 13,
"versionName": "1.12"
},
{
@ -45,7 +45,7 @@
"commit": "4128e59da2eac5c2904c7c7568d298ca51e79540",
"patch": ["defprop.patch"],
"subdir": "org_adaway/",
"versionCode": "16",
"versionCode": 16,
"versionName": "1.15"
},
{
@ -53,7 +53,7 @@
"commit": "0b9985398b9eef7baf6aadd0dbb12002bc199d2e",
"patch": ["defprop.patch"],
"subdir": "org_adaway/",
"versionCode": "19",
"versionCode": 19,
"versionName": "1.18"
},
{
@ -61,7 +61,7 @@
"commit": "ab27f4dab5f3ea5e228cfb4a6b0e1fbf53695f22",
"patch": ["defprop.patch"],
"subdir": "org_adaway/",
"versionCode": "20",
"versionCode": 20,
"versionName": "1.19"
},
{
@ -69,7 +69,7 @@
"commit": "695e3801e4081026c8f7213a2345fc451d5eb89c",
"patch": ["defprop.patch"],
"subdir": "org_adaway/",
"versionCode": "21",
"versionCode": 21,
"versionName": "1.20"
},
{
@ -77,13 +77,13 @@
"commit": "65138c11cc8b6affd28b68e125fbc1dff0886a4e",
"patch": ["defprop.patch"],
"subdir": "org_adaway/",
"versionCode": "22",
"versionCode": 22,
"versionName": "1.21"
},
{
"commit": "unknown - see disabled",
"disable": "no source in repo",
"versionCode": "24",
"versionCode": 24,
"versionName": "1.23"
},
{
@ -91,7 +91,7 @@
"commit": "f811e53e1e1d2ee047b18715fd7d2072b90ae76b",
"prebuild": "android update project -p ../com_actionbarsherlock",
"subdir": "org_adaway/",
"versionCode": "25",
"versionCode": 25,
"versionName": "1.24"
},
{
@ -99,7 +99,7 @@
"commit": "ff97932761cdee68638dc2550751a64b2cbe18e7",
"prebuild": "android update project -p ../com_actionbarsherlock",
"subdir": "org_adaway/",
"versionCode": "26",
"versionCode": 26,
"versionName": "1.25"
},
{
@ -107,7 +107,7 @@
"commit": "33d4d80998f30bafc88c04c80cbae00b03916f99",
"prebuild": "android update project -p ../com_actionbarsherlock",
"subdir": "org_adaway/",
"versionCode": "27",
"versionCode": 27,
"versionName": "1.26"
},
{
@ -115,7 +115,7 @@
"commit": "743d25a7e287505461f33f4b8e57e4cf988fffea",
"prebuild": "android update project -p ../com_actionbarsherlock",
"subdir": "org_adaway/",
"versionCode": "28",
"versionCode": 28,
"versionName": "1.27"
},
{
@ -123,7 +123,7 @@
"commit": "eaa07f4",
"prebuild": "android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*",
"subdir": "org_adaway/",
"versionCode": "30",
"versionCode": 30,
"versionName": "1.29"
},
{
@ -131,7 +131,7 @@
"commit": "71ced3f",
"prebuild": "android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/* && rm libs/android-support-v4.jar",
"subdir": "org_adaway/",
"versionCode": "33",
"versionCode": 33,
"versionName": "1.32"
},
{
@ -139,7 +139,7 @@
"commit": "9d63c18",
"prebuild": "android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*",
"subdir": "org_adaway/",
"versionCode": "34",
"versionCode": 34,
"versionName": "1.33"
},
{
@ -147,7 +147,7 @@
"commit": "f2568b1",
"prebuild": "android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/* && android update project -p ../org_donations",
"subdir": "org_adaway/",
"versionCode": "35",
"versionCode": 35,
"versionName": "1.34"
},
{
@ -155,7 +155,7 @@
"commit": "7442d5d",
"prebuild": "android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/* && android update project -p ../org_donations",
"subdir": "org_adaway/",
"versionCode": "36",
"versionCode": 36,
"versionName": "1.35"
},
{
@ -163,7 +163,7 @@
"commit": "83fc713",
"prebuild": "android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/* && android update project -p ../org_donations",
"subdir": "org_adaway/",
"versionCode": "37",
"versionCode": 37,
"versionName": "1.36"
},
{
@ -171,7 +171,7 @@
"commit": "70da32b567122b701cdcb1609b780eb85732028f",
"prebuild": "android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/* && android update project -p ../org_donations",
"subdir": "org_adaway/",
"versionCode": "38",
"versionCode": 38,
"versionName": "1.37"
},
{
@ -186,7 +186,7 @@
"android-libs/Donations",
"android-libs/ActionBarSherlock",
"android-libs/HtmlSpanner/htmlspanner"],
"versionCode": "40",
"versionCode": 40,
"versionName": "2.1"
},
{
@ -201,7 +201,7 @@
"android-libs/Donations",
"android-libs/ActionBarSherlock",
"android-libs/HtmlSpanner/htmlspanner"],
"versionCode": "42",
"versionCode": 42,
"versionName": "2.3"
},
{
@ -210,7 +210,7 @@
"gradle": true,
"preassemble": ["renameExecutables"],
"subdir": "AdAway",
"versionCode": "45",
"versionCode": 45,
"versionName": "2.6"
},
{
@ -219,7 +219,7 @@
"gradle": true,
"preassemble": ["renameExecutables"],
"subdir": "AdAway",
"versionCode": "46",
"versionCode": 46,
"versionName": "2.7"
},
{
@ -228,7 +228,7 @@
"gradle": true,
"preassemble": ["renameExecutables"],
"subdir": "AdAway",
"versionCode": "47",
"versionCode": 47,
"versionName": "2.8"
},
{
@ -237,7 +237,7 @@
"gradle": true,
"preassemble": ["renameExecutables"],
"subdir": "AdAway",
"versionCode": "48",
"versionCode": 48,
"versionName": "2.8.1"
},
{
@ -246,7 +246,7 @@
"gradle": true,
"preassemble": ["renameExecutables"],
"subdir": "AdAway",
"versionCode": "49",
"versionCode": 49,
"versionName": "2.9"
},
{
@ -255,7 +255,7 @@
"gradle": true,
"preassemble": ["renameExecutables"],
"subdir": "AdAway",
"versionCode": "50",
"versionCode": 50,
"versionName": "2.9.1"
},
{
@ -264,7 +264,7 @@
"gradle": true,
"preassemble": ["renameExecutables"],
"subdir": "AdAway",
"versionCode": "51",
"versionCode": 51,
"versionName": "2.9.2"
},
{
@ -273,7 +273,7 @@
"gradle": true,
"preassemble": ["renameExecutables"],
"subdir": "AdAway",
"versionCode": "52",
"versionCode": 52,
"versionName": "3.0"
}
],

2747
tests/metadata/org.adaway.pickle
View File

File diff suppressed because it is too large Load Diff

2
tests/metadata/org.smssecure.smssecure.pickle
View File

@ -218,7 +218,7 @@ S'yes'
p111
asS'prebuild'
p112
S"touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar && sed -i -e '20,22d' build.gradle && pushd $$PreferenceFragment$$ && gradle uploadArchives && popd && sed -i -e '/5470f5872514a6226fa1fc6f4e000991f38805691c534cf0bd2778911fc773ad/d' build.gradle && mkdir smil && pushd smil && wget -c http://www.w3.org/TR/smil-boston-dom/java-binding.zip && unzip java-binding.zip && popd && cp -fR smil/java/org src/ && rm -fR smil && sed -i -e '/org.w3c.smil/d' build.gradle && cp -fR $$AospMms$$/src/org src/ "
S"touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar && sed -i -e '20,22d' build.gradle && pushd $$PreferenceFragment$$ && gradle uploadArchives && popd && sed -i -e '/5470f5872514a6226fa1fc6f4e000991f38805691c534cf0bd2778911fc773ad/d' build.gradle && mkdir smil && pushd smil && wget -c http://www.w3.org/TR/smil-boston-dom/java-binding.zip && unzip java-binding.zip && popd && cp -fR smil/java/org src/ && rm -fR smil && sed -i -e '/org.w3c.smil/d' build.gradle && cp -fR $$AospMms$$/src/org src/"
p113
sS'novcheck'
p114