rewrite to yaml works for app data now (builds still missing)

fdroidserver/metadata.py

@@ -25,7 +25,7 @@ import html
 import logging
 import textwrap
 import io
-
+import ruamel.yaml
 import yaml
 # use libyaml if it is available
 try:
@@ -35,6 +35,8 @@ except ImportError:
     from yaml import Loader
     YamlLoader = Loader
 
+from collections import OrderedDict
+
 import fdroidserver.common
 from fdroidserver.exception import MetaDataException
 
@@ -953,6 +955,69 @@ def parse_yaml_metadata(mf, app):
     app.update(yamlinfo)
     return app
 
+def write_yaml(mf, app):
+
+    def _class_as_dict_representer(dumper, data):
+        '''Creates a YAML representation of a App/Build instance'''
+        return dumper.represent_dict(data)
+
+    empty_keys = [k for k, v in app.items() if not v]
+    for k in empty_keys:
+        del app[k]
+
+    for k in ['added', 'lastUpdated', 'id', 'metadatapath']:
+        if k in app:
+            del app[k]
+
+    #yaml.add_representer(fdroidserver.metadata.App, _class_as_dict_representer)
+    #ruamel.yaml.add_representer(fdroidserver.metadata.Build, _class_as_dict_representer)
+    #yaml.dump(app.asOrderedDict(), mf, default_flow_style=False, Dumper=yamlordereddictloader.Dumper)
+
+    yaml_app_field_order = [
+        'Categories',
+        'License',
+        'Web Site',
+        'Source Code',
+        'Issue Tracker',
+        'Donate',
+        'Bitcoin',
+        '\n',
+        'Auto Name',
+        'Summary',
+        'Description',
+        '\n',
+        'Repo Type',
+        'Repo',
+        '\n',
+        'Auto Update Mode',
+        'Update Check Mode',
+        'Current Version',
+        'Current Version Code',
+    ]
+
+    preformated = ruamel.yaml.comments.CommentedMap()
+    insert_newline = False
+    for field in yaml_app_field_order:
+        if field is '\n':
+            insert_newline = True
+        else:
+            f = field.replace(' ', '')
+            if hasattr(app, f) and getattr(app, f):
+                if f in ['Description']:
+                    preformated.update({f: ruamel.yaml.scalarstring.preserve_literal(getattr(app, f))})
+                else:
+                    preformated.update({f: getattr(app, f)})
+                if insert_newline:
+                    insert_newline = False
+                    # inserting empty lines is not supported so we add a
+                    # bogus comment and over-write its value
+                    preformated.yaml_set_comment_before_after_key(f, 'bogus')
+                    preformated.ca.items[f][1][0].value = '\n'
+    # TODO implement dump for builds
+    del(preformated['builds'])
+
+    ruamel.yaml.round_trip_dump(preformated, mf, indent=4, block_seq_indent=2)
+
 
 def write_yaml(mf, app):
 

tests/rewrite-metadata/fake.ota.update.yml

@@ -0,0 +1,37 @@
+Categories:
+  - System
+License: Apache-2.0
+WebSite: https://f-droid.org
+SourceCode: https://gitlab.com/fdroid/privileged-extension
+IssueTracker: https://gitlab.com/fdroid/privileged-extension/issues
+Donate: https://f-droid.org/about
+
+AutoName: Fake OTA Update
+Summary: Tests whether OTA ZIP files are being include
+Description: |-
+    F-Droid can make use of system privileges or permissions to
+    install, update and remove applications on its own. The only way to obtain those
+    privileges is to become a system app.
+
+    This is where the Privileged Extension comes in - being a separate app and much
+    smaller, it can be installed as a system app and communicate with the main app
+    via AIDL IPC.
+
+    This has several advantages:
+
+    * Reduced disk usage in the system partition
+    * System updates don't remove F-Droid
+    * The process of installing into system via root is safer
+
+    This is packaged as an OTA (Over-The-Air) update ZIP file.  It must be installed
+    using TWRP or other Android recovery that can flash updates to the system from
+    the /data/data/org.fdroid.fdroid folder on the /data partition. The standalone
+    APK is called F-Droid Privileged Extension.
+
+RepoType: git
+Repo: https://gitlab.com/fdroid/privileged-extension.git
+
+AutoUpdateMode: Version %v
+UpdateCheckMode: Tags
+CurrentVersion: 0.2.1
+CurrentVersionCode: '2000'

tests/rewritemeta.TestCase

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import inspect
+import optparse
+import os
+import sys
+import shutil
+import unittest
+import yaml
+import tempfile
+import textwrap
+
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+
+import fdroidserver.common
+import fdroidserver.metadata
+
+
+class RewritemetaTest(unittest.TestCase):
+    '''    fdroidserver/metadata.py'''
+
+    def test_rewrite_yaml(self):
+
+        # setup/reset test dir if necessary and setup params
+        tmpdir = os.path.join(os.path.dirname(__file__), '..', '.testfiles')
+        if not os.path.exists(tmpdir):
+            os.makedirs(tmpdir)
+        testdir = tempfile.mkdtemp(prefix='test_rewrite_metadata_', dir=tmpdir)
+        fdroidserver.common.config = {'accepted_formats': ['txt', 'yml']}
+
+        # rewrite metadata
+        allapps = fdroidserver.metadata.read_metadata(xref=True)
+        for appid, app in allapps.items():
+            if appid == 'fake.ota.update':
+                fdroidserver.metadata.write_metadata(os.path.join(testdir, appid + '.yml'), app)
+
+        # assert rewrite result
+        with open(os.path.join(testdir, 'fake.ota.update.yml'), 'r', encoding='utf-8') as result:
+            with open('rewrite-metadata/fake.ota.update.yml', 'r', encoding='utf-8') as orig:
+                self.maxDiff = None
+                self.assertEqual(result.read(), orig.read())
+
+if __name__ == "__main__":
+    parser = optparse.OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
+
+    newSuite = unittest.TestSuite()
+    newSuite.addTest(unittest.makeSuite(RewritemetaTest))
+    unittest.main()