server: --sync-from-local-copy-dir for updating from offline signing repo

To support a fully offline build/signing machine, there is the "local copy
dir".  The repo is generated on the offline machine and then copied to a
local dir where a thumb drive or SD Card is mounted.  Then on the online
machine, using `fdroid server update --sync-from-local-copy-dir` allows
the whole server update process to happen in a single command:

0. read config.py on online machine's repo
1. rsync from the local_copy_dir to the current dir
2. copy to serverwebroot, awsbucket, etc.

examples/config.py

@@ -124,6 +124,14 @@ keyaliases['com.example.another.plugin'] = '@com.example.another'
 # local_copy_dir = '/media/MyUSBThumbDrive/fdroid'
 
 
+# If you are using local_copy_dir on an offline build/signing server, once the
+# thumb drive has been plugged into the online machine, it will need to be
+# synced to the copy on the online machine.  To make that happen
+# automatically, set sync_from_local_copy_dir to True:
+#
+# sync_from_local_copy_dir = True
+
+
 # To upload the repo to an Amazon S3 bucket using `fdroid server update`.
 # Warning, this deletes and recreates the whole fdroid/ directory each
 # time. This is based on apache-libcloud, which supports basically all cloud

fdroidserver/common.py

@@ -47,6 +47,7 @@ def get_default_config():
         'mvn3': "mvn",
         'gradle': 'gradle',
         'archive_older': 0,
+        'sync_from_local_copy_dir': False,
         'update_stats': False,
         'stats_to_carbon': False,
         'repo_maxage': 0,

fdroidserver/server.py

@@ -140,8 +140,8 @@ def update_serverwebroot(repo_section):
         sys.exit(1)
 
 
-def update_localcopy(repo_section, local_copy_dir):
-    rsyncargs = ['rsync', '--update', '--recursive', '--delete']
+def _local_sync(fromdir, todir):
+    rsyncargs = ['rsync', '--archive', '--one-file-system', '--delete']
     # use stricter rsync checking on all files since people using offline mode
     # are already prioritizing security above ease and speed
     rsyncargs += ['--checksum']
@@ -149,11 +149,24 @@ def update_localcopy(repo_section, local_copy_dir):
         rsyncargs += ['--verbose']
     if options.quiet:
         rsyncargs += ['--quiet']
-    # local_copy_dir is guaranteed to have a trailing slash in main() below
-    if subprocess.call(rsyncargs + [repo_section, local_copy_dir]) != 0:
+    logging.debug(' '.join(rsyncargs + [fromdir, todir]))
+    if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
         sys.exit(1)
 
 
+def sync_from_localcopy(repo_section, local_copy_dir):
+    logging.info('Syncing from local_copy_dir to this repo.')
+    # trailing slashes have a meaning in rsync which is not needed here, so
+    # remove them all
+    _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/'),
+                repo_section.rstrip('/'))
+
+
+def update_localcopy(repo_section, local_copy_dir):
+    # local_copy_dir is guaranteed to have a trailing slash in main() below
+    _local_sync(repo_section, local_copy_dir)
+
+
 def main():
     global config, options
 
@@ -163,6 +176,8 @@ def main():
                       help="Specify an identity file to provide to SSH for rsyncing")
     parser.add_option("--local-copy-dir", default=None,
                       help="Specify a local folder to sync the repo to")
+    parser.add_option("--sync-from-local-copy-dir", action="store_true", default=False,
+                      help="Before uploading to servers, sync from local copy dir")
     parser.add_option("-v", "--verbose", action="store_true", default=False,
                       help="Spew out even more information than normal")
     parser.add_option("-q", "--quiet", action="store_true", default=False,
@@ -251,12 +266,15 @@ def main():
                     sys.exit(1)
     elif args[0] == 'update':
         for repo_section in repo_sections:
+            if local_copy_dir is not None:
+                if config['sync_from_local_copy_dir'] and os.path.exists(repo_section):
+                    sync_from_localcopy(repo_section, local_copy_dir)
+                else:
+                    update_localcopy(repo_section, local_copy_dir)
             if config.get('serverwebroot'):
                 update_serverwebroot(repo_section)
             if config.get('awsbucket'):
                 update_awsbucket(repo_section)
-            if local_copy_dir is not None:
-                update_localcopy(repo_section, local_copy_dir)
 
     sys.exit(0)
 

tests/run-tests

@@ -106,7 +106,7 @@ set -e
 
 
 #------------------------------------------------------------------------------#
-echo_header "setup a new repo from scratch using ANDROID_HOME"
+echo_header "setup a new repo from scratch using ANDROID_HOME and do a local sync"
 
 REPOROOT=`create_test_dir`
 cd $REPOROOT
@@ -115,6 +115,13 @@ copy_apks_into_repo $REPOROOT
 $fdroid update --create-metadata
 grep -F '<application id=' repo/index.xml
 
+LOCALCOPYDIR=`create_test_dir`/fdroid
+$fdroid server update --local-copy-dir=$LOCALCOPYDIR
+NEWREPOROOT=`create_test_dir`
+cd $NEWREPOROOT
+$fdroid init
+$fdroid server update --local-copy-dir=$LOCALCOPYDIR --sync-from-local-copy-dir
+
 
 #------------------------------------------------------------------------------#
 # check that --android-home fails when dir does not exist or is not a dir