allow metadata to be embedded in source repos via .fdroid.yml

This allows a source repo to include a complete metadata file so that it
can be built directly in place using `fdroid build`.  If that app is then
included in fdroiddata, it will first load the source repo type and URL
from fdroiddata, then read .fdroid.yml if it exists, then include the rest
of the metadata as specified in fdroiddata, so that fdroiddata has
precedence over the metadata in the source code.

This lets `fdroid build` apps without having a whole fdroiddata setup, but
instead just directly in place in the source code.  This also lets devs
optionallu maintain the fdroid metadata as part of their app, rather than
in fdroiddata without loosing any control.  This should make it easier to
spread around the maintenance load.

fdroidserver/build.py

@@ -1100,9 +1100,10 @@ def main():
     extlib_dir = os.path.join(build_dir, 'extlib')
 
     # Read all app and srclib metadata
-    allapps = metadata.read_metadata(xref=not options.onserver)
-
+    pkgs = common.read_pkg_args(options.appid, True)
+    allapps = metadata.read_metadata(not options.onserver, pkgs)
     apps = common.read_app_args(options.appid, allapps, True)
+
     for appid, app in list(apps.items()):
         if (app.Disabled and not options.force) or not app.RepoType or not app.builds:
             del apps[appid]
@@ -1141,16 +1142,7 @@ def main():
                 # the source repo. We can reuse it on subsequent builds, if
                 # there are any.
                 if first:
-                    if app.RepoType == 'srclib':
-                        build_dir = os.path.join('build', 'srclib', app.Repo)
-                    else:
-                        build_dir = os.path.join('build', appid)
-
-                    # Set up vcs interface and make sure we have the latest code...
-                    logging.debug("Getting {0} vcs interface for {1}"
-                                  .format(app.RepoType, app.Repo))
-                    vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
-
+                    vcs, build_dir = common.setup_vcs(app)
                     first = False
 
                 logging.debug("Checking " + build.version)

fdroidserver/common.py

@@ -486,6 +486,27 @@ def getcvname(app):
     return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
 
 
+def get_build_dir(app):
+    '''get the dir that this app will be built in'''
+
+    if app.RepoType == 'srclib':
+        return os.path.join('build', 'srclib', app.Repo)
+
+    return os.path.join('build', app.id)
+
+
+def setup_vcs(app):
+    '''checkout code from VCS and return instance of vcs and the build dir'''
+    build_dir = get_build_dir(app)
+
+    # Set up vcs interface and make sure we have the latest code...
+    logging.debug("Getting {0} vcs interface for {1}"
+                  .format(app.RepoType, app.Repo))
+    vcs = getvcs(app.RepoType, app.Repo, build_dir)
+
+    return vcs, build_dir
+
+
 def getvcs(vcstype, remote, local):
     if vcstype == 'git':
         return vcs_git(remote, local)

fdroidserver/metadata.py

@@ -224,6 +224,21 @@ class App():
             else:
                 self.set_field(f, v)
 
+    def update(self, d):
+        '''Like dict.update()'''
+        for k, v in d.__dict__.items():
+            if k == '_modified':
+                continue
+            elif k == 'builds':
+                for b in v:
+                    build = Build()
+                    del(b.__dict__['_modified'])
+                    build.update_flags(b.__dict__)
+                    self.builds.append(build)
+            elif v:
+                self.__dict__[k] = v
+                self._modified.add(k)
+
 
 TYPE_UNKNOWN = 0
 TYPE_OBSOLETE = 1
@@ -778,10 +793,12 @@ def read_srclibs():
         srclibs[srclibname] = parse_srclib(metadatapath)
 
 
-def read_metadata(xref=True):
+def read_metadata(xref=True, check_vcs=[]):
     """
     Read all metadata. Returns a list of 'app' objects (which are dictionaries as
     returned by the parse_txt_metadata function.
+
+    check_vcs is the list of packageNames to check for .fdroid.yml in source
     """
 
     # Always read the srclibs before the apps, since they can use a srlib as
@@ -809,7 +826,7 @@ def read_metadata(xref=True):
         packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
         if packageName in apps:
             warn_or_exception("Found multiple metadata files for " + packageName)
-        app = parse_metadata(metadatapath)
+        app = parse_metadata(metadatapath, packageName in check_vcs)
         check_metadata(app)
         apps[app.id] = app
 
@@ -962,7 +979,9 @@ def _decode_bool(s):
     warn_or_exception("Invalid bool '%s'" % s)
 
 
-def parse_metadata(metadatapath):
+def parse_metadata(metadatapath, check_vcs=False):
+    '''parse metadata file, optionally checking the git repo for metadata first'''
+
     _, ext = fdroidserver.common.get_extension(metadatapath)
     accepted = fdroidserver.common.config['accepted_formats']
     if ext not in accepted:
@@ -971,7 +990,11 @@ def parse_metadata(metadatapath):
 
     app = App()
     app.metadatapath = metadatapath
-    app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+    name, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+    if name == '.fdroid':
+        check_vcs = False
+    else:
+        app.id = name
 
     with open(metadatapath, 'r', encoding='utf-8') as mf:
         if ext == 'txt':
@@ -985,7 +1008,28 @@ def parse_metadata(metadatapath):
         else:
             warn_or_exception('Unknown metadata format: %s' % metadatapath)
 
+    if check_vcs and app.Repo:
+        build_dir = fdroidserver.common.get_build_dir(app)
+        metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
+        if not os.path.isfile(metadata_in_repo):
+            vcs, build_dir = fdroidserver.common.setup_vcs(app)
+            vcs.gotorevision('HEAD')  # HEAD since we can't know where else to go
+        if os.path.isfile(metadata_in_repo):
+            logging.debug('Including metadata from ' + metadata_in_repo)
+            app.update(parse_metadata(metadata_in_repo))
+
     post_metadata_parse(app)
+
+    if not app.id:
+        if app.builds:
+            build = app.builds[-1]
+            if build.subdir:
+                root_dir = build.subdir
+            else:
+                root_dir = '.'
+            paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
+            _, _, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
+
     return app
 
 

tests/import.TestCase

@@ -25,6 +25,7 @@ class ImportTest(unittest.TestCase):
     '''fdroid import'''
 
     def test_import_gitlab(self):
+        os.chdir(os.path.dirname(__file__))
         # FDroidPopen needs some config to work
         config = dict()
         fdroidserver.common.fill_config_defaults(config)

tests/metadata/info.guardianproject.checkey.txt

@@ -0,0 +1,21 @@
+Categories:Development,GuardianProject
+License:GPLv3
+Web Site:https://dev.guardianproject.info/projects/checkey
+Source Code:https://github.com/guardianproject/checkey
+Issue Tracker:https://dev.guardianproject.info/projects/checkey/issues
+Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+
+Auto Name:Checkey
+Summary:Info on local apps
+Description:
+Checkey is a utility for getting information about the APKs that are installed
+on your device. Starting with a list of all of the apps that you have
+installed on your device, it will show you the APK signature with a single
+touch, and provides links to virustotal.com and androidobservatory.org to
+easily access the profiles of that APK. It will also let you export the
+signing certificate and generate ApkSignaturePin pin files for use with the
+TrustedIntents library.
+.
+
+
+Current Version Code:9999999

tests/metadata/org.fdroid.ci.test.app.txt

@@ -0,0 +1,2 @@
+Repo Type:git
+Repo:https://gitlab.com/eighthave/ci-test-app

tests/metadata/raw.template.txt

@@ -0,0 +1,9 @@
+License:Unknown
+Web Site:
+Source Code:
+Issue Tracker:
+Changelog:
+Summary:Template
+Description:
+Template
+.

tests/run-tests

@@ -100,6 +100,8 @@ echo_header "test python getsig replacement"
 
 cd $WORKSPACE/tests/getsig
 ./make.sh
+
+cd $WORKSPACE/tests
 for testcase in $WORKSPACE/tests/*.TestCase; do
     $testcase
 done
@@ -138,6 +140,19 @@ $fdroid readmeta
 $fdroid update
 
 
+#------------------------------------------------------------------------------#
+echo_header 'run `fdroid build` in fresh git checkout from import.TestCase'
+
+cd $WORKSPACE/tests/tmp/importer
+if [ -d $ANDROID_HOME/platforms/android-23 ]; then
+    echo "build_tools = '`ls -1 $ANDROID_HOME/build-tools/ | sort -n | tail -1`'" > config.py
+    echo "force_build_tools = True" >> config.py
+    $fdroid build --verbose org.fdroid.ci.test.app:300
+else
+    echo 'WARNING: Skipping `fdroid build` test since android-23 is missing!'
+fi
+
+
 #------------------------------------------------------------------------------#
 echo_header "copy tests/repo, generate java/gpg keys, update, and gpgsign"