build: sort `fdroid build --all` by most recently changed first

fdroidserver/build.py

@@ -1111,7 +1111,7 @@ def main():
 
     # Read all app and srclib metadata
     pkgs = common.read_pkg_args(options.appid, True)
-    allapps = metadata.read_metadata(not options.onserver, pkgs)
+    allapps = metadata.read_metadata(not options.onserver, pkgs, sort_by_time=True)
     apps = common.read_app_args(options.appid, allapps, True)
 
     for appid, app in list(apps.items()):

fdroidserver/metadata.py

@@ -26,6 +26,7 @@ import logging
 import textwrap
 import io
 import yaml
+from collections import OrderedDict
 # use libyaml if it is available
 try:
     from yaml import CLoader
@@ -703,35 +704,50 @@ def read_srclibs():
         srclibs[srclibname] = parse_srclib(metadatapath)
 
 
-def read_metadata(xref=True, check_vcs=[]):
-    """
-    Read all metadata. Returns a list of 'app' objects (which are dictionaries as
-    returned by the parse_txt_metadata function.
+def read_metadata(xref=True, check_vcs=[], sort_by_time=False):
+    """Return a list of App instances sorted newest first
+
+    This reads all of the metadata files in a 'data' repository, then
+    builds a list of App instances from those files.  The list is
+    sorted based on creation time, newest first.  Most of the time,
+    the newer files are the most interesting.
+
+    If there are multiple metadata files for a single appid, then the first
+    file that is parsed wins over all the others, and the rest throw an
+    exception. So the original .txt format is parsed first, at least until
+    newer formats stabilize.
 
     check_vcs is the list of packageNames to check for .fdroid.yml in source
+
     """
 
     # Always read the srclibs before the apps, since they can use a srlib as
     # their source repository.
     read_srclibs()
 
-    apps = {}
+    apps = OrderedDict()
 
     for basedir in ('metadata', 'tmp'):
         if not os.path.exists(basedir):
             os.makedirs(basedir)
 
-    # If there are multiple metadata files for a single appid, then the first
-    # file that is parsed wins over all the others, and the rest throw an
-    # exception. So the original .txt format is parsed first, at least until
-    # newer formats stabilize.
+    metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
+                     + glob.glob(os.path.join('metadata', '*.json'))
+                     + glob.glob(os.path.join('metadata', '*.yml'))
+                     + glob.glob('.fdroid.txt')
+                     + glob.glob('.fdroid.json')
+                     + glob.glob('.fdroid.yml'))
 
-    for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt'))
-                               + glob.glob(os.path.join('metadata', '*.json'))
-                               + glob.glob(os.path.join('metadata', '*.yml'))
-                               + glob.glob('.fdroid.txt')
-                               + glob.glob('.fdroid.json')
-                               + glob.glob('.fdroid.yml')):
+    if sort_by_time:
+        entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
+        metadatafiles = []
+        for _ignored, path in sorted(entries, reverse=True):
+            metadatafiles.append(path)
+    else:
+        # most things want the index alpha sorted for stability
+        metadatafiles = sorted(metadatafiles)
+
+    for metadatapath in metadatafiles:
         packageName, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
         if packageName in apps:
             warn_or_exception(_("Found multiple metadata files for {appid}")

tests/metadata.TestCase

@@ -2,9 +2,12 @@
 
 # http://www.drdobbs.com/testing/unit-testing-with-python/240165163
 
+import glob
 import inspect
 import optparse
 import os
+import random
+import shutil
 import sys
 import unittest
 import yaml
@@ -120,6 +123,36 @@ class MetadataTest(unittest.TestCase):
                 self.maxDiff = None
                 self.assertEqual(result.read(), orig.read())
 
+    def test_read_metadata_sort_by_time(self):
+        # setup/reset test dir if necessary and setup params
+        testbasedir = os.path.dirname(__file__)
+        tmpdir = os.path.join(testbasedir, '..', '.testfiles')
+        if not os.path.exists(tmpdir):
+            os.makedirs(tmpdir)
+        testdir = tempfile.mkdtemp(prefix='test_read_metadata_sort_by_time_', dir=tmpdir)
+        metadatadir = os.path.join(testdir, 'metadata')
+        os.makedirs(metadatadir)
+        fdroidserver.common.config = {'accepted_formats': ['txt']}
+
+        randomlist = []
+        randomapps = glob.glob(os.path.join(testbasedir, 'metadata', '*.txt'))
+        random.shuffle(randomapps)
+        i = 1
+        for f in randomapps:
+            shutil.copy(f, metadatadir)
+            new = os.path.join(metadatadir, os.path.basename(f))
+            stat = os.stat(new)
+            os.utime(new, (stat.st_ctime, stat.st_mtime + i))
+            # prepend new item so newest is always first
+            randomlist = [os.path.basename(f)[:-4]] + randomlist
+            i += 1
+        os.chdir(testdir)
+        allapps = fdroidserver.metadata.read_metadata(xref=True, sort_by_time=True)
+        allappids = []
+        for appid, app in allapps.items():
+            allappids.append(appid)
+        self.assertEqual(randomlist, allappids)
+
 
 if __name__ == "__main__":
     parser = optparse.OptionParser()