use pyyaml for writing metadata instead of ruamel

This commit is contained in:
Michael Pöhn 2019-03-06 08:42:17 +01:00
parent 81f1bcf752
commit 63afc0acb5
5 changed files with 270 additions and 166 deletions

View File

@ -103,7 +103,6 @@ packages="
python3-setuptools
python3-smmap
python3-yaml
python3-ruamel.yaml
quilt
rsync
scons

View File

@ -248,6 +248,13 @@ def fieldtype(name):
return TYPE_STRING
def yml_fieldtype(name):
if name == 'CurrentVersionCode':
return TYPE_INT
else:
return fieldtype(name)
# In the order in which they are laid out on files
build_flags_order = [
'disable',
@ -1115,117 +1122,123 @@ def write_yaml(mf, app):
:param app: app metadata to written to the yaml file
"""
# import rumael.yaml and check version
try:
import ruamel.yaml
except ImportError as e:
raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
if not ruamel.yaml.__version__:
raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
m = re.match(r'(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
ruamel.yaml.__version__)
if not m:
raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
# suiteable version ruamel.yaml imported successfully
def format_yml_field(field, value, typ, width=80, offset=0):
# TODO: use CSafeDumper if available
_yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
'true', 'True', 'TRUE',
'on', 'On', 'ON')
_yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
'false', 'False', 'FALSE',
'off', 'Off', 'OFF')
_yaml_bools_plus_lists = []
_yaml_bools_plus_lists.extend(_yaml_bools_true)
_yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
_yaml_bools_plus_lists.extend(_yaml_bools_false)
_yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
bool_list_hack = ['gradle']
def _class_as_dict_representer(dumper, data):
'''Creates a YAML representation of a App/Build instance'''
return dumper.represent_dict(data)
data = {}
default_style = None
def _field_to_yaml(typ, value):
if typ is TYPE_STRING:
if value in _yaml_bools_plus_lists:
return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
return str(value)
elif typ is TYPE_INT:
return int(value)
elif typ is TYPE_MULTILINE:
if '\n' in value:
return ruamel.yaml.scalarstring.preserve_literal(str(value))
if typ == TYPE_INT:
data[field] = int(value)
elif typ == TYPE_BOOL:
data[field] = bool(value)
elif typ == TYPE_SCRIPT:
if type(value) != list:
value = value.split(' && ')
if len(value) > 1:
data[field] = value
else:
return str(value)
elif typ is TYPE_SCRIPT:
if type(value) == list:
if len(value) == 1:
return value[0]
else:
return value
else:
script_lines = value.split(' && ')
if len(script_lines) > 1:
return script_lines
else:
return value
data[field] = value[0]
elif typ == TYPE_LIST:
if field in bool_list_hack:
if value in [True, 'yes', ['yes']]:
value = [True]
data[field] = value
elif typ == TYPE_MULTILINE:
default_style = '|'
data[field] = value.strip()
else:
return value
data[field] = value
def _app_to_yaml(app):
cm = ruamel.yaml.comments.CommentedMap()
insert_newline = False
for field in yaml_app_field_order:
if field == '\n':
# next iteration will need to insert a newline
insert_newline = True
else:
if app.get(field) or field == 'Builds':
# .txt calls it 'builds' internally, everywhere else its 'Builds'
if field == 'Builds':
if app.get('builds'):
cm.update({field: _builds_to_yaml(app)})
elif field == 'CurrentVersionCode':
cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
raw_yml = yaml.dump(data, Dumper=yaml.SafeDumper,
default_flow_style=False, width=width,
default_style=default_style, indent=4)
# this is just for debugging purposes remove in production code
# debug_print_yaml_item(typ, data, default_style, offset, raw_yml)
if raw_yml:
# remove all encapsulation newlines
raw_yml = re.sub('\n+$', '', raw_yml)
raw_yml = re.sub('^\n+', '', raw_yml)
if offset > 0:
lines = raw_yml.split('\n')
for i in range(len(lines)):
if lines[i] != '':
lines[i] = ' ' * offset + lines[i]
raw_yml = '\n'.join(lines)
if typ == TYPE_LIST or typ == TYPE_SCRIPT:
# indent sequence items by 2 spaces
new_raw_yml = []
for line in raw_yml.split('\n'):
if re.match(r'^\s*$', line):
new_raw_yml.append('')
elif len(new_raw_yml) > 0:
new_raw_yml.append(' ' + line)
else:
cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
new_raw_yml.append(line)
raw_yml = '\n'.join(new_raw_yml)
elif typ == TYPE_MULTILINE:
# remove quotes from mulit line keys
raw_yml = re.sub(r'^(\s*)"([a-zA-Z0-9]+)":(\s*\|-)$', '\\1\\2:\\3', raw_yml, flags=re.MULTILINE)
if insert_newline:
# we need to prepend a newline in front of this field
insert_newline = False
# inserting empty lines is not supported so we add a
# bogus comment and over-write its value
cm.yaml_set_comment_before_after_key(field, 'bogus')
cm.ca.items[field][1][-1].value = '\n'
return cm
return raw_yml
def _builds_to_yaml(app):
builds = ruamel.yaml.comments.CommentedSeq()
for build in app.builds:
b = ruamel.yaml.comments.CommentedMap()
for field in build_flags:
value = getattr(build, field)
if hasattr(build, field) and value:
if field == 'gradle' and value == ['off']:
value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
if field in ('maven', 'buildozer'):
if value == 'no':
continue
elif value == 'yes':
value = 'yes'
b.update({field: _field_to_yaml(flagtype(field), value)})
builds.append(b)
def format_yml_build_entry(build):
result = []
for field in build_flags:
value = getattr(build, field, None)
if value:
typ = flagtype(field)
line = format_yml_field(field, value, typ, width=0xffffffff, offset=4)
if re.match(r'^\s+$', line):
result.append('')
else:
result.append(line)
result[0] = ' - ' + result[0][4:]
return '\n'.join(result)
# insert extra empty lines between build entries
for i in range(1, len(builds)):
builds.yaml_set_comment_before_after_key(i, 'bogus')
builds.ca.items[i][1][-1].value = '\n'
result = []
for field in yaml_app_field_order:
if field == '\n':
if result[-1] != '':
result.append('')
continue
else:
value = app.get(field)
if field == 'Builds' and app.builds:
result.append('Builds:')
first_build_entry = True
for build in app.builds:
if first_build_entry:
first_build_entry = False
else:
result.append('')
result.append(format_yml_build_entry(build))
if value:
result.append(format_yml_field(field, value, yml_fieldtype(field)))
mf.write('\n'.join(result))
return builds
yaml_app = _app_to_yaml(app)
ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
def debug_print_yaml_item(typ, data, default_style, offset, raw_yml):
"""function debugging indiviudal lines during yaml writing"""
typTable = {0: 'TYPE_UNKNOWN',
1: 'TYPE_OBSOLETE',
2: 'TYPE_STRING',
3: 'TYPE_BOOL',
4: 'TYPE_LIST',
5: 'TYPE_SCRIPT',
6: 'TYPE_MULTILINE',
7: 'TYPE_BUILD',
8: 'TYPE_INT'}
print('###', '(' + typTable[typ] + ', default_style='
+ str(default_style) + ', offset=' + str(offset) + ')\n',
data, '->', raw_yml if raw_yml else '')
build_line_sep = re.compile(r'(?<!\\),')

View File

@ -80,7 +80,6 @@ setup(name='fdroidserver',
'python-vagrant',
'PyYAML',
'qrcode',
'ruamel.yaml >= 0.13',
'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0',
'docker-py >= 1.9, < 2.0',
],

View File

@ -199,14 +199,14 @@ Builds:
commit: 0.53-test
submodules: true
scandelete:
- yes
- 'yes'
- versionName: '0.54'
versionCode: 540
commit: '0.54'
submodules: true
scandelete:
- yes
- 'yes'
- versionName: '0.55'
versionCode: 550
@ -498,7 +498,7 @@ Builds:
subdir: F-Droid
submodules: true
gradle:
- yes
- true
- versionName: 0.95-alpha1
versionCode: 95001
@ -506,7 +506,7 @@ Builds:
subdir: F-Droid
submodules: true
gradle:
- yes
- true
- versionName: 0.95-alpha2
versionCode: 95002
@ -514,7 +514,7 @@ Builds:
subdir: F-Droid
submodules: true
gradle:
- yes
- true
- versionName: '0.95'
versionCode: 95050
@ -522,7 +522,7 @@ Builds:
subdir: F-Droid
submodules: true
gradle:
- yes
- true
- versionName: 0.95.1
versionCode: 95150
@ -530,56 +530,56 @@ Builds:
subdir: F-Droid
submodules: true
gradle:
- yes
- true
- versionName: 0.96-alpha1
versionCode: 96001
commit: v0.96-alpha1
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.96-alpha2
versionCode: 96002
commit: v0.96-alpha2
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.96-alpha3
versionCode: 96003
commit: v0.96-alpha3
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.96-alpha4
versionCode: 96004
commit: v0.96-alpha4
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.96-alpha5
versionCode: 96005
commit: v0.96-alpha5
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.96-alpha6
versionCode: 96006
commit: v0.96-alpha6
subdir: F-Droid
gradle:
- yes
- true
- versionName: '0.96'
versionCode: 96050
commit: v0.96
subdir: F-Droid
gradle:
- yes
- true
scanignore:
- extern/AndroidPinning/res/raw/cacerts
@ -588,7 +588,7 @@ Builds:
commit: v0.96.1
subdir: F-Droid
gradle:
- yes
- true
scanignore:
- extern/AndroidPinning/res/raw/cacerts
@ -597,7 +597,7 @@ Builds:
commit: v0.97-alpha1
subdir: F-Droid
gradle:
- yes
- true
scanignore:
- extern/AndroidPinning/res/raw/cacerts
@ -606,343 +606,343 @@ Builds:
commit: v0.97-alpha2
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.97-alpha3
versionCode: 97003
commit: v0.97-alpha3
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.97-alpha4
versionCode: 97004
commit: v0.97-alpha4
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.97-alpha5
versionCode: 97005
commit: v0.97-alpha5
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.97-alpha6
versionCode: 97006
commit: v0.97-alpha6
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.97-alpha7
versionCode: 97007
commit: v0.97-alpha7
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.97-alpha8
versionCode: 97008
commit: v0.97-alpha8
subdir: F-Droid
gradle:
- yes
- true
- versionName: '0.97'
versionCode: 97050
commit: v0.97
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.98-alpha1
versionCode: 98001
commit: v0.98-alpha1
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.98-alpha2
versionCode: 98002
commit: v0.98-alpha2
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.98-alpha3
versionCode: 98003
commit: v0.98-alpha3
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.98-alpha4
versionCode: 98004
commit: v0.98-alpha4
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.98-alpha5
versionCode: 98005
commit: v0.98-alpha5
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.98-alpha6
versionCode: 98006
commit: v0.98-alpha6
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.98-alpha7
versionCode: 98007
commit: v0.98-alpha7
subdir: F-Droid
gradle:
- yes
- true
- versionName: '0.98'
versionCode: 98050
commit: v0.98
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.98.1
versionCode: 98150
commit: v0.98.1
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.99-alpha1
versionCode: 99001
commit: v0.99-alpha1
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.99-alpha2
versionCode: 99002
commit: v0.99-alpha2
subdir: F-Droid
gradle:
- yes
- true
- versionName: '0.99'
versionCode: 99050
commit: v0.99
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.99.1
versionCode: 99150
commit: v0.99.1
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.99.2
versionCode: 99250
commit: v0.99.2
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.100-alpha1
versionCode: 100001
commit: v0.100-alpha1
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.100-alpha2
versionCode: 100002
commit: v0.100-alpha2
subdir: F-Droid
gradle:
- yes
- true
- versionName: 0.100-alpha3
versionCode: 100003
commit: v0.100-alpha3
subdir: app
gradle:
- yes
- true
- versionName: 0.100-alpha4
versionCode: 100004
commit: v0.100-alpha4
subdir: app
gradle:
- yes
- true
- versionName: 0.100-alpha5
versionCode: 100005
commit: v0.100-alpha5
subdir: app
gradle:
- yes
- true
- versionName: 0.100-alpha6
versionCode: 100006
commit: v0.100-alpha6
subdir: app
gradle:
- yes
- true
- versionName: 0.100-alpha7
versionCode: 100007
commit: v0.100-alpha7
subdir: app
gradle:
- yes
- true
- versionName: 0.100-alpha8
versionCode: 100008
commit: v0.100-alpha8
subdir: app
gradle:
- yes
- true
- versionName: '0.100'
versionCode: 100050
commit: v0.100
subdir: app
gradle:
- yes
- true
- versionName: 0.100.1
versionCode: 100150
commit: v0.100.1
subdir: app
gradle:
- yes
- true
- versionName: 0.101-alpha1
versionCode: 101001
commit: v0.101-alpha1
subdir: app
gradle:
- yes
- true
- versionName: 0.101-alpha2
versionCode: 101002
commit: v0.101-alpha2
subdir: app
gradle:
- yes
- true
- versionName: 0.101-alpha3
versionCode: 101003
commit: v0.101-alpha3
subdir: app
gradle:
- yes
- true
- versionName: 0.101-alpha4
versionCode: 101004
commit: v0.101-alpha4
subdir: app
gradle:
- yes
- true
- versionName: 0.101-alpha5
versionCode: 101005
commit: v0.101-alpha5
subdir: app
gradle:
- yes
- true
- versionName: 0.101-alpha6
versionCode: 101006
commit: v0.101-alpha6
subdir: app
gradle:
- yes
- true
- versionName: '0.101'
versionCode: 101050
commit: v0.101
subdir: app
gradle:
- yes
- true
- versionName: 0.102-alpha1
versionCode: 102001
commit: v0.102-alpha1
subdir: app
gradle:
- yes
- true
- versionName: 0.102-alpha2
versionCode: 102002
commit: v0.102-alpha2
subdir: app
gradle:
- yes
- true
- versionName: 0.102-alpha3
versionCode: 102003
commit: v0.102-alpha3
subdir: app
gradle:
- yes
- true
- versionName: '0.102'
versionCode: 102050
commit: v0.102
subdir: app
gradle:
- yes
- true
- versionName: 0.102.1
versionCode: 102150
commit: v0.102.1
subdir: app
gradle:
- yes
- true
- versionName: 0.102.2
versionCode: 102250
commit: v0.102.2
subdir: app
gradle:
- yes
- true
- versionName: 0.102.3
versionCode: 102350
commit: v0.102.3
subdir: app
gradle:
- yes
- true
- versionName: 0.103-alpha1
versionCode: 103001
commit: v0.103-alpha1
subdir: app
gradle:
- yes
- true
- versionName: 0.103-alpha2
versionCode: 103002
commit: v0.103-alpha2
subdir: app
gradle:
- yes
- true
- versionName: 0.103-alpha3
versionCode: 103003
commit: v0.103-alpha3
subdir: app
gradle:
- yes
- true
ArchivePolicy: 12 versions
AutoUpdateMode: None

View File

@ -307,6 +307,68 @@ class MetadataTest(unittest.TestCase):
'prebuild': "a && b && "
"sed -i 's,a,b,'"}]})
def test_write_yaml_description_with_trailing_whitespace(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app.Description = "this evil description has a trailing whitespace "
app.builds = []
build = fdroidserver.metadata.Build()
build.versionCode = 102030
build.versionName = 'v1.2.3'
build.build = "./gradlew compile"
app.builds.append(build)
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.maxDiff = None
self.assertEqual(mf.read(), textwrap.dedent("""\
Categories:
- None
License: Unknown
Description: |-
this evil description has a trailing whitespace
Builds:
- versionName: v1.2.3
versionCode: 102030
build: ./gradlew compile
AutoUpdateMode: None
UpdateCheckMode: None
"""))
def test_write_yaml_long_description(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app.Description = "long description is long; " * 20
app.builds = []
build = fdroidserver.metadata.Build()
build.versionCode = 102030
build.versionName = 'v1.2.3'
build.build = "./gradlew compile"
app.builds.append(build)
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.maxDiff = None
self.assertEqual(mf.read(), textwrap.dedent("""\
Categories:
- None
License: Unknown
Description: |-
long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long; long description is long;
Builds:
- versionName: v1.2.3
versionCode: 102030
build: ./gradlew compile
AutoUpdateMode: None
UpdateCheckMode: None
"""))
def test_write_yaml_1_line_scripts_as_string(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
@ -461,6 +523,37 @@ class MetadataTest(unittest.TestCase):
UpdateCheckMode: None
"""))
def test_write_yaml_very_long_script(self):
mf = io.StringIO()
app = fdroidserver.metadata.App()
app.Categories = ['None']
app.builds = []
build = fdroidserver.metadata.Build()
build.versionCode = 102030
build.versionName = 'v1.2.3'
build.build = "./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile && long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long;"
app.builds.append(build)
fdroidserver.metadata.write_yaml(mf, app)
mf.seek(0)
self.maxDiff = None
self.assertEqual(mf.read(), textwrap.dedent("""\
Categories:
- None
License: Unknown
Builds:
- versionName: v1.2.3
versionCode: 102030
build:
- ./gradlew someSpecialTask
- sed -i 'd/that wrong config/' gradle.properties
- ./gradlew compile
- long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long; long script is very long;
AutoUpdateMode: None
UpdateCheckMode: None
"""))
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))