opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Rework build into a class 

This simplifies usage, goes from

    build['flag']
to
    build.flag

Also makes static analyzers able to detect invalid attributes as the set
is now limited in the class definition.

As a bonus, setting of the default field values is now done in the
constructor, not separately and manually.

While at it, unify "build", "thisbuild", "info", "thisinfo", etc into
just "build".
This commit is contained in:
Daniel Martí 2015-11-28 17:55:27 +01:00
parent 2c12485aeb
commit bf8518ee8f
15 changed files with 6997 additions and 8636 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
fdroidserver/build.py
View File

@ -245,7 +245,7 @@ def release_vm():
# Note that 'force' here also implies test mode.
def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
def build_server(app, build, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
try:
@ -333,9 +333,9 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
if thisbuild['extlibs']:
if build.extlibs:
ftp.chdir(homedir + '/build/extlib')
for lib in thisbuild['extlibs']:
for lib in build.extlibs:
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
@ -350,8 +350,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if thisbuild['srclibs']:
for lib in thisbuild['srclibs']:
if build.srclibs:
for lib in build.srclibs:
srclibpaths.append(
common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False))
@ -389,7 +389,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
cmdline += " %s:%s" % (app.id, thisbuild['vercode'])
cmdline += " %s:%s" % (app.id, build.vercode)
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
while not chan.exit_status_ready():
@ -406,7 +406,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
if returncode != 0:
raise BuildException(
"Build.py failed on server for {0}:{1}".format(
app.id, thisbuild['version']), output)
app.id, build.version), output)
# Retrieve the built files...
logging.info("Retrieving build output...")
@ -414,8 +414,8 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
ftp.chdir(homedir + '/tmp')
else:
ftp.chdir(homedir + '/unsigned')
apkfile = common.getapkname(app, thisbuild)
tarball = common.getsrcname(app, thisbuild)
apkfile = common.getapkname(app, build)
tarball = common.getsrcname(app, build)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
if not options.notarball:
@ -423,7 +423,7 @@ def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
except:
raise BuildException(
"Build failed for %s:%s - missing output files".format(
app.id, thisbuild['version']), output)
app.id, build.version), output)
ftp.close()
finally:
@ -457,32 +457,33 @@ def capitalize_intact(string):
return string[0].upper() + string[1:]
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
"""Do a build locally."""
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if not thisbuild['ndk_path']:
logging.critical("Android NDK version '%s' could not be found!" % thisbuild['ndk'])
ndk_path = build.ndk_path()
if build.buildjni and build.buildjni != ['no']:
if not ndk_path:
logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r10e')
logging.critical("Configured versions:")
for k, v in config['ndk_paths'].iteritems():
if k.endswith("_orig"):
continue
logging.critical(" %s: %s" % (k, v))
sys.exit(3)
elif not os.path.isdir(thisbuild['ndk_path']):
logging.critical("Android NDK '%s' is not a directory!" % thisbuild['ndk_path'])
elif not os.path.isdir(ndk_path):
logging.critical("Android NDK '%s' is not a directory!" % ndk_path)
sys.exit(3)
# Set up environment vars that depend on each build
for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
common.env[n] = thisbuild['ndk_path']
common.env[n] = ndk_path
common.reset_env_path()
# Set up the current NDK to the PATH
common.add_to_env_path(thisbuild['ndk_path'])
common.add_to_env_path(ndk_path)
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
root_dir, srclibpaths = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, onserver, refresh)
@ -490,26 +491,27 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
# different from the default ones
p = None
gradletasks = []
if thisbuild['type'] == 'maven':
method = build.method()
if method == 'maven':
logging.info("Cleaning Maven project...")
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1])
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif thisbuild['type'] == 'gradle':
elif method == 'gradle':
logging.info("Cleaning Gradle project...")
if thisbuild['preassemble']:
gradletasks += thisbuild['preassemble']
if build.preassemble:
gradletasks += build.preassemble
flavours = thisbuild['gradle']
flavours = build.gradle
if flavours == ['yes']:
flavours = []
@ -522,8 +524,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
adapt_gradle(libpath)
cmd = [config['gradle']]
if thisbuild['gradleprops']:
cmd += ['-P'+kv for kv in thisbuild['gradleprops']]
if build.gradleprops:
cmd += ['-P'+kv for kv in build.gradleprops]
for task in gradletasks:
parts = task.split(':')
@ -534,16 +536,16 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'kivy':
elif method == 'kivy':
pass
elif thisbuild['type'] == 'ant':
elif method == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
(app.id, thisbuild['version']), p.output)
(app.id, build.version), p.output)
for root, dirs, files in os.walk(build_dir):
@ -575,12 +577,12 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
del_dirs(['obj'])
if options.skipscan:
if thisbuild['scandelete']:
if build.scandelete:
raise BuildException("Refusing to skip source scan since scandelete is present")
else:
# Scan before building...
logging.info("Scanning source for common problems...")
count = scanner.scan_source(build_dir, root_dir, thisbuild)
count = scanner.scan_source(build_dir, root_dir, build)
if count > 0:
if force:
logging.warn('Scanner found %d problems' % count)
@ -590,7 +592,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
tarname = common.getsrcname(app, thisbuild)
tarname = common.getsrcname(app, build)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
@ -599,9 +601,9 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
tarball.close()
# Run a build command if one is required...
if thisbuild['build']:
if build.build:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = common.replace_config_vars(thisbuild['build'], thisbuild)
cmd = common.replace_config_vars(build.build, build)
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
@ -612,16 +614,16 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
(app.id, thisbuild['version']), p.output)
(app.id, build.version), p.output)
# Build native stuff if required...
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if build.buildjni and build.buildjni != ['no']:
logging.info("Building the native code")
jni_components = thisbuild['buildjni']
jni_components = build.buildjni
if jni_components == ['yes']:
jni_components = ['']
cmd = [os.path.join(thisbuild['ndk_path'], "ndk-build"), "-j1"]
cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
@ -640,15 +642,15 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
raise BuildException("NDK build failed for %s:%s" % (app.id, thisbuild['version']), p.output)
raise BuildException("NDK build failed for %s:%s" % (app.id, build.version), p.output)
p = None
# Build the release...
if thisbuild['type'] == 'maven':
if method == 'maven':
logging.info("Building Maven project...")
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1])
if '@' in build.maven:
maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1])
else:
maven_dir = root_dir
@ -656,12 +658,12 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true',
'package']
if thisbuild['target']:
target = thisbuild["target"].split('-')[1]
if build.target:
target = build.target.split('-')[1]
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(root_dir, 'pom.xml'))
if '@' in thisbuild['maven']:
if '@' in build.maven:
common.regsub_file(r'<platform>[0-9]*</platform>',
r'<platform>%s</platform>' % target,
os.path.join(maven_dir, 'pom.xml'))
@ -670,7 +672,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
bindir = os.path.join(root_dir, 'target')
elif thisbuild['type'] == 'kivy':
elif method == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
@ -690,8 +692,8 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + thisbuild['ndk_path']
cmd += ' ANDROIDNDKVER=' + thisbuild['ndk']
cmd += ' ANDROIDNDK=' + ndk_path
cmd += ' ANDROIDNDKVER=' + build.ndk
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
@ -731,27 +733,27 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild['type'] == 'gradle':
elif method == 'gradle':
logging.info("Building Gradle project...")
# Avoid having to use lintOptions.abortOnError false
if thisbuild['gradlepluginver'] >= LooseVersion('0.7'):
if build.gradlepluginver >= LooseVersion('0.7'):
with open(os.path.join(root_dir, 'build.gradle'), "a") as f:
f.write("\nandroid { lintOptions { checkReleaseBuilds false } }\n")
cmd = [config['gradle']]
if thisbuild['gradleprops']:
cmd += ['-P'+kv for kv in thisbuild['gradleprops']]
if build.gradleprops:
cmd += ['-P'+kv for kv in build.gradleprops]
cmd += gradletasks
p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'ant':
elif method == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
if thisbuild['antcommands']:
cmd += thisbuild['antcommands']
if build.antcommands:
cmd += build.antcommands
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
@ -759,10 +761,10 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app.id, thisbuild['version']), p.output)
logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app.id)
raise BuildException("Build failed for %s:%s" % (app.id, build.version), p.output)
logging.info("Successfully built version " + build.version + ' of ' + app.id)
if thisbuild['type'] == 'maven':
if method == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if any(
a in line for a in ('.apk', '.ap_', '.jar'))])
@ -782,14 +784,14 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif thisbuild['type'] == 'kivy':
elif method == 'kivy':
src = os.path.join('python-for-android', 'dist', 'default', 'bin',
'{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'),
bconfig.get('app', 'version')))
elif thisbuild['type'] == 'gradle':
elif method == 'gradle':
if thisbuild['gradlepluginver'] >= LooseVersion('0.11'):
if build.gradlepluginver >= LooseVersion('0.11'):
apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk')
else:
apks_dir = os.path.join(root_dir, 'build', 'apk')
@ -801,14 +803,14 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if len(apks) < 1:
raise BuildException('Failed to find gradle output in %s' % apks_dir)
src = apks[0]
elif thisbuild['type'] == 'ant':
elif method == 'ant':
stdout_apk = '\n'.join([
line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
elif thisbuild['type'] == 'raw':
src = os.path.join(root_dir, thisbuild['output'])
elif method == 'raw':
src = os.path.join(root_dir, build.output)
src = os.path.normpath(src)
# Make sure it's not debuggable...
@ -850,12 +852,12 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if build.buildjni and build.buildjni != ['no']:
if nativecode is None:
raise BuildException("Native code should have been built but none was packaged")
if thisbuild['novcheck']:
vercode = thisbuild['vercode']
version = thisbuild['version']
if build.novcheck:
vercode = build.vercode
version = build.version
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
@ -871,13 +873,13 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
if (version != build.version or
vercode != build.vercode):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
% (version, str(vercode), thisbuild['version'],
str(thisbuild['vercode']))
% (version, str(vercode), build.version,
str(build.vercode))
)
# Add information for 'fdroid verify' to be able to reproduce the build
@ -895,7 +897,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
dest = os.path.join(output_dir, common.getapkname(app, build))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
@ -904,7 +906,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d
os.path.join(output_dir, tarname))
def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
def trybuild(app, build, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh):
"""
Build a particular version of an application, if it needs building.
@ -923,7 +925,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir,
:returns: True if the build was done, False if it wasn't necessary.
"""
dest_apk = common.getapkname(app, thisbuild)
dest_apk = common.getapkname(app, build)
dest = os.path.join(output_dir, dest_apk)
dest_repo = os.path.join(repo_dir, dest_apk)
@ -937,20 +939,20 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir,
if os.path.exists(dest_also):
return False
if thisbuild['disable'] and not options.force:
if build.disable and not options.force:
return False
logging.info("Building version %s (%s) of %s" % (
thisbuild['version'], thisbuild['vercode'], app.id))
build.version, build.vercode, app.id))
if server:
# When using server mode, still keep a local cache of the repo, by
# grabbing the source now.
vcs.gotorevision(thisbuild['commit'])
vcs.gotorevision(build.commit)
build_server(app, thisbuild, vcs, build_dir, output_dir, force)
build_server(app, build, vcs, build_dir, output_dir, force)
else:
build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
return True
@ -1060,7 +1062,7 @@ def main():
if options.latest:
for app in apps.itervalues():
for build in reversed(app.builds):
if build['disable'] and not options.force:
if build.disable and not options.force:
continue
app.builds = [build]
break
@ -1078,7 +1080,7 @@ def main():
first = True
for thisbuild in app.builds:
for build in app.builds:
wikilog = None
try:
@ -1098,8 +1100,8 @@ def main():
first = False
logging.debug("Checking " + thisbuild['version'])
if trybuild(app, thisbuild, build_dir, output_dir,
logging.debug("Checking " + build.version)
if trybuild(app, build, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir,
tmp_dir, repo_dir, vcs, options.test,
options.server, options.force,
@ -1112,10 +1114,10 @@ def main():
# alongside our built one in the 'unsigend'
# directory.
url = app.Binaries
url = url.replace('%v', thisbuild['version'])
url = url.replace('%c', str(thisbuild['vercode']))
url = url.replace('%v', build.version)
url = url.replace('%c', str(build.vercode))
logging.info("...retrieving " + url)
of = "{0}_{1}.apk.binary".format(app.id, thisbuild['vercode'])
of = "{0}_{1}.apk.binary".format(app.id, build.vercode)
of = os.path.join(output_dir, of)
net.download_file(url, local_filename=of)
@ -1148,7 +1150,7 @@ def main():
if options.wiki and wikilog:
try:
# Write a page with the last build log for this version code
lastbuildpage = appid + '/lastbuild_' + thisbuild['vercode']
lastbuildpage = appid + '/lastbuild_' + build.vercode
newpage = site.Pages[lastbuildpage]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + wikilog
newpage.save(txt, summary='Build log')

59
fdroidserver/checkupdates.py
View File

@ -108,10 +108,12 @@ def check_tags(app, pattern):
vcs.gotorevision(None)
flavours = []
last_build = metadata.Build()
if len(app.builds) > 0:
if app.builds[-1]['gradle']:
flavours = app.builds[-1]['gradle']
last_build = app.builds[-1]
if last_build.submodules:
vcs.initsubmodules()
hpak = None
htag = None
@ -143,7 +145,7 @@ def check_tags(app, pattern):
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, flavours)
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
@ -199,10 +201,12 @@ def check_repomanifest(app, branch=None):
elif repotype == 'bzr':
vcs.gotorevision(None)
flavours = []
last_build = metadata.Build()
if len(app.builds) > 0:
if app.builds[-1]['gradle']:
flavours = app.builds[-1]['gradle']
last_build = app.builds[-1]
if last_build.submodules:
vcs.initsubmodules()
hpak = None
hver = None
@ -212,7 +216,7 @@ def check_repomanifest(app, branch=None):
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, flavours)
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
@ -313,14 +317,12 @@ def possible_subdirs(app):
else:
build_dir = os.path.join('build', app.id)
flavours = []
last_build = metadata.Build()
if len(app.builds) > 0:
build = app.builds[-1]
if build['gradle']:
flavours = build['gradle']
last_build = app.builds[-1]
for d in dirs_with_manifest(build_dir):
m_paths = common.manifest_paths(d, flavours)
m_paths = common.manifest_paths(d, last_build.gradle)
package = common.parse_androidmanifests(m_paths, app)[2]
if package is not None:
subdir = os.path.relpath(d, build_dir)
@ -344,10 +346,9 @@ def fetch_autoname(app, tag):
except VCSException:
return None
flavours = []
last_build = metadata.Build()
if len(app.builds) > 0:
if app.builds[-1]['gradle']:
flavours = app.builds[-1]['gradle']
last_build = app.builds[-1]
logging.debug("...fetch auto name from " + build_dir)
new_name = None
@ -356,7 +357,7 @@ def fetch_autoname(app, tag):
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
new_name = common.fetch_real_name(root_dir, flavours)
new_name = common.fetch_real_name(root_dir, last_build.gradle)
if new_name is not None:
break
commitmsg = None
@ -458,25 +459,25 @@ def checkupdates_app(app, first=True):
gotcur = False
latest = None
for build in app.builds:
if int(build['vercode']) >= int(app.CurrentVersionCode):
if int(build.vercode) >= int(app.CurrentVersionCode):
gotcur = True
if not latest or int(build['vercode']) > int(latest['vercode']):
if not latest or int(build.vercode) > int(latest.vercode):
latest = build
if int(latest['vercode']) > int(app.CurrentVersionCode):
if int(latest.vercode) > int(app.CurrentVersionCode):
logging.info("Refusing to auto update, since the latest build is newer")
if not gotcur:
newbuild = latest.copy()
if 'origlines' in newbuild:
del newbuild['origlines']
newbuild['disable'] = False
newbuild['vercode'] = app.CurrentVersionCode
newbuild['version'] = app.CurrentVersion + suffix
logging.info("...auto-generating build for " + newbuild['version'])
commit = pattern.replace('%v', newbuild['version'])
commit = commit.replace('%c', newbuild['vercode'])
newbuild['commit'] = commit
if newbuild.origlines:
del newbuild.origlines[:]
newbuild.disable = False
newbuild.vercode = app.CurrentVersionCode
newbuild.version = app.CurrentVersion + suffix
logging.info("...auto-generating build for " + newbuild.version)
commit = pattern.replace('%v', newbuild.version)
commit = commit.replace('%c', newbuild.vercode)
newbuild.commit = commit
app.builds.append(newbuild)
name = common.getappname(app)
ver = common.getcvname(app)

116
fdroidserver/common.py
View File

@ -221,15 +221,6 @@ def read_config(opts, config_file='config.py'):
return config
def get_ndk_path(version):
if version is None:
version = 'r10e' # falls back to latest
paths = config['ndk_paths']
if version not in paths:
return ''
return paths[version] or ''
def find_sdk_tools_cmd(cmd):
'''find a working path to a tool from the Android SDK'''
@ -363,10 +354,10 @@ def read_app_args(args, allapps, allow_vercodes=False):
vc = vercodes[appid]
if not vc:
continue
app.builds = [b for b in app.builds if b['vercode'] in vc]
app.builds = [b for b in app.builds if b.vercode in vc]
if len(app.builds) != len(vercodes[appid]):
error = True
allvcs = [b['vercode'] for b in app.builds]
allvcs = [b.vercode for b in app.builds]
for v in vercodes[appid]:
if v not in allvcs:
logging.critical("No such vercode %s for app %s" % (v, appid))
@ -419,11 +410,11 @@ def apknameinfo(filename):
def getapkname(app, build):
return "%s_%s.apk" % (app.id, build['vercode'])
return "%s_%s.apk" % (app.id, build.vercode)
def getsrcname(app, build):
return "%s_%s_src.tar.gz" % (app.id, build['vercode'])
return "%s_%s_src.tar.gz" % (app.id, build.vercode)
def getappname(app):
@ -1250,17 +1241,17 @@ gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
# Optionally, the actual app source can be in a subdirectory
if build['subdir']:
root_dir = os.path.join(build_dir, build['subdir'])
if build.subdir:
root_dir = os.path.join(build_dir, build.subdir)
else:
root_dir = build_dir
# Get a working copy of the right revision
logging.info("Getting source for revision " + build['commit'])
vcs.gotorevision(build['commit'], refresh)
logging.info("Getting source for revision " + build.commit)
vcs.gotorevision(build.commit, refresh)
# Initialise submodules if required
if build['submodules']:
if build.submodules:
logging.info("Initialising submodules")
vcs.initsubmodules()
@ -1270,19 +1261,19 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
raise BuildException('Missing subdir ' + root_dir)
# Run an init command if one is required
if build['init']:
cmd = replace_config_vars(build['init'], build)
if build.init:
cmd = replace_config_vars(build.init, build)
logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running init command for %s:%s" %
(app.id, build['version']), p.output)
(app.id, build.version), p.output)
# Apply patches if any
if build['patch']:
if build.patch:
logging.info("Applying patches")
for patch in build['patch']:
for patch in build.patch:
patch = patch.strip()
logging.info("Applying " + patch)
patch_path = os.path.join('metadata', app.id, patch)
@ -1292,9 +1283,9 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
# Get required source libraries
srclibpaths = []
if build['srclibs']:
if build.srclibs:
logging.info("Collecting source libraries")
for lib in build['srclibs']:
for lib in build.srclibs:
srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
for name, number, libpath in srclibpaths:
@ -1307,8 +1298,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
# Update the local.properties file
localprops = [os.path.join(build_dir, 'local.properties')]
if build['subdir']:
parts = build['subdir'].split(os.sep)
if build.subdir:
parts = build.subdir.split(os.sep)
cur = build_dir
for d in parts:
cur = os.path.join(cur, d)
@ -1324,26 +1315,27 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
logging.info("Creating local.properties file at %s" % path)
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary
if build['oldsdkloc']:
if build.oldsdkloc:
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S | re.M).group(1)
props += "sdk-location=%s\n" % sdkloc
else:
props += "sdk.dir=%s\n" % config['sdk_path']
props += "sdk-location=%s\n" % config['sdk_path']
if build['ndk_path']:
ndk_path = build.ndk_path()
if ndk_path:
# Add ndk location
props += "ndk.dir=%s\n" % build['ndk_path']
props += "ndk-location=%s\n" % build['ndk_path']
props += "ndk.dir=%s\n" % ndk_path
props += "ndk-location=%s\n" % ndk_path
# Add java.encoding if necessary
if build['encoding']:
props += "java.encoding=%s\n" % build['encoding']
if build.encoding:
props += "java.encoding=%s\n" % build.encoding
with open(path, 'w') as f:
f.write(props)
flavours = []
if build['type'] == 'gradle':
flavours = build['gradle']
if build.method() == 'gradle':
flavours = build.gradle
gradlepluginver = None
@ -1372,13 +1364,13 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
break
if gradlepluginver:
build['gradlepluginver'] = LooseVersion(gradlepluginver)
build.gradlepluginver = LooseVersion(gradlepluginver)
else:
logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
build['gradlepluginver'] = LooseVersion('0.11')
build.gradlepluginver = LooseVersion('0.11')
if build['target']:
n = build["target"].split('-')[1]
if build.target:
n = build.target.split('-')[1]
regsub_file(r'compileSdkVersion[ =]+[0-9]+',
r'compileSdkVersion %s' % n,
os.path.join(root_dir, 'build.gradle'))
@ -1387,38 +1379,38 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
remove_debuggable_flags(root_dir)
# Insert version code and number into the manifest if necessary
if build['forceversion']:
if build.forceversion:
logging.info("Changing the version name")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
regsub_file(r'android:versionName="[^"]*"',
r'android:versionName="%s"' % build['version'],
r'android:versionName="%s"' % build.version,
path)
elif has_extension(path, 'gradle'):
regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
r"""\1versionName '%s'""" % build['version'],
r"""\1versionName '%s'""" % build.version,
path)
if build['forcevercode']:
if build.forcevercode:
logging.info("Changing the version code")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
regsub_file(r'android:versionCode="[^"]*"',
r'android:versionCode="%s"' % build['vercode'],
r'android:versionCode="%s"' % build.vercode,
path)
elif has_extension(path, 'gradle'):
regsub_file(r'versionCode[ =]+[0-9]+',
r'versionCode %s' % build['vercode'],
r'versionCode %s' % build.vercode,
path)
# Delete unwanted files
if build['rm']:
if build.rm:
logging.info("Removing specified files")
for part in getpaths(build_dir, build['rm']):
for part in getpaths(build_dir, build.rm):
dest = os.path.join(build_dir, part)
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
@ -1432,12 +1424,12 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
remove_signing_keys(build_dir)
# Add required external libraries
if build['extlibs']:
if build.extlibs:
logging.info("Collecting prebuilt libraries")
libsdir = os.path.join(root_dir, 'libs')
if not os.path.exists(libsdir):
os.mkdir(libsdir)
for lib in build['extlibs']:
for lib in build.extlibs:
lib = lib.strip()
logging.info("...installing extlib {0}".format(lib))
libf = os.path.basename(lib)
@ -1447,10 +1439,10 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
shutil.copyfile(libsrc, os.path.join(libsdir, libf))
# Run a pre-build command if one is required
if build['prebuild']:
if build.prebuild:
logging.info("Running 'prebuild' commands in %s" % root_dir)
cmd = replace_config_vars(build['prebuild'], build)
cmd = replace_config_vars(build.prebuild, build)
# Substitute source library paths into prebuild commands
for name, number, libpath in srclibpaths:
@ -1460,20 +1452,20 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running prebuild command for %s:%s" %
(app.id, build['version']), p.output)
(app.id, build.version), p.output)
# Generate (or update) the ant build file, build.xml...
if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
if build.update and build.update != ['no'] and build.method() == 'ant':
parms = ['android', 'update', 'lib-project']
lparms = ['android', 'update', 'project']
if build['target']:
parms += ['-t', build['target']]
lparms += ['-t', build['target']]
if build['update'] == ['auto']:
update_dirs = ant_subprojects(root_dir) + ['.']
if build.target:
parms += ['-t', build.target]
lparms += ['-t', build.target]
if build.update:
update_dirs = build.update
else:
update_dirs = build['update']
update_dirs = ant_subprojects(root_dir) + ['.']
for d in update_dirs:
subdir = os.path.join(root_dir, d)
@ -1770,9 +1762,9 @@ def replace_config_vars(cmd, build):
cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
if build is not None:
cmd = cmd.replace('$$COMMIT$$', build['commit'])
cmd = cmd.replace('$$VERSION$$', build['version'])
cmd = cmd.replace('$$VERCODE$$', build['vercode'])
cmd = cmd.replace('$$COMMIT$$', build.commit)
cmd = cmd.replace('$$VERSION$$', build.version)
cmd = cmd.replace('$$VERCODE$$', build.vercode)
return cmd

19
fdroidserver/import.py
View File

@ -223,20 +223,15 @@ def main():
sys.exit(1)
# Create a build line...
build = {}
build['version'] = version or '?'
build['vercode'] = vercode or '?'
build['commit'] = '?'
build['disable'] = 'Generated by import.py - check/set version fields and commit id'
build = metadata.Build()
build.version = version or '?'
build.vercode = vercode or '?'
build.commit = '?'
build.disable = 'Generated by import.py - check/set version fields and commit id'
if options.subdir:
build['subdir'] = options.subdir
build.subdir = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = ['yes']
for flag, value in metadata.flag_defaults.iteritems():
if flag in build:
continue
build[flag] = value
build.buildjni = ['yes']
app.builds.append(build)

26
fdroidserver/lint.py
View File

@ -122,11 +122,11 @@ def get_lastbuild(builds):
lowest_vercode = -1
lastbuild = None
for build in builds:
if not build['disable']:
vercode = int(build['vercode'])
if not build.disable:
vercode = int(build.vercode)
if lowest_vercode == -1 or vercode < lowest_vercode:
lowest_vercode = vercode
if not lastbuild or int(build['vercode']) > int(lastbuild['vercode']):
if not lastbuild or int(build.vercode) > int(lastbuild.vercode):
lastbuild = build
return lastbuild
@ -134,14 +134,14 @@ def get_lastbuild(builds):
def check_ucm_tags(app):
lastbuild = get_lastbuild(app.builds)
if (lastbuild is not None
and lastbuild['commit']
and lastbuild.commit
and app.UpdateCheckMode == 'RepoManifest'
and not lastbuild['commit'].startswith('unknown')
and lastbuild['vercode'] == app.CurrentVersionCode
and not lastbuild['forcevercode']
and any(s in lastbuild['commit'] for s in '.,_-/')):
and not lastbuild.commit.startswith('unknown')
and lastbuild.vercode == app.CurrentVersionCode
and not lastbuild.forcevercode
and any(s in lastbuild.commit for s in '.,_-/')):
yield "Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (
lastbuild['commit'], app.UpdateCheckMode)
lastbuild.commit, app.UpdateCheckMode)
def check_char_limits(app):
@ -285,12 +285,12 @@ def check_bulleted_lists(app):
def check_builds(app):
for build in app.builds:
if build['disable']:
if build.disable:
continue
for s in ['master', 'origin', 'HEAD', 'default', 'trunk']:
if build['commit'] and build['commit'].startswith(s):
yield "Branch '%s' used as commit in build '%s'" % (s, build['version'])
for srclib in build['srclibs']:
if build.commit and build.commit.startswith(s):
yield "Branch '%s' used as commit in build '%s'" % (s, build.version)
for srclib in build.srclibs:
ref = srclib.split('@')[1].split('/')[0]
if ref.startswith(s):
yield "Branch '%s' used as commit in srclib '%s'" % (s, srclib)

585
fdroidserver/metadata.py
View File

@ -38,8 +38,6 @@ except ImportError:
# use the C implementation when available
import xml.etree.cElementTree as ElementTree
from collections import OrderedDict
import common
srclibs = None
@ -155,7 +153,16 @@ class App():
# Constructs an old-fashioned dict with the human-readable field
# names. Should only be used for tests.
def field_dict(self):
return {App.attr_to_field(k): v for k, v in self.__dict__.iteritems()}
d = {}
for k, v in self.__dict__.iteritems():
if k == 'builds':
d['builds'] = []
for build in v:
d['builds'].append(build.__dict__)
else:
k = App.attr_to_field(k)
d[k] = v
return d
# Gets the value associated to a field name, e.g. 'Auto Name'
def get_field(self, f):
@ -184,44 +191,156 @@ class App():
# Like dict.update(), but using human-readable field names
def update_fields(self, d):
for f, v in d.iteritems():
self.set_field(f, v)
if f == 'builds':
for b in v:
build = Build()
build.update_flags(b)
self.builds.append(build)
else:
self.set_field(f, v)
def metafieldtype(name):
if name in ['Description', 'Maintainer Notes']:
return 'multiline'
if name in ['Categories', 'AntiFeatures']:
return 'list'
if name == 'Build Version':
return 'build'
if name == 'Build':
return 'buildv2'
if name == 'Use Built':
return 'obsolete'
if name not in app_fields:
return 'unknown'
return 'string'
# In the order in which they are laid out on files
# Sorted by their action and their place in the build timeline
# These variables can have varying datatypes. For example, anything with
# flagtype(v) == 'list' is inited as False, then set as a list of strings.
flag_defaults = OrderedDict([
('disable', False),
('commit', None),
('subdir', None),
('submodules', False),
('init', ''),
('patch', []),
('gradle', False),
('maven', False),
('kivy', False),
('output', None),
('srclibs', []),
('oldsdkloc', False),
('encoding', None),
('forceversion', False),
('forcevercode', False),
('rm', []),
('extlibs', []),
('prebuild', ''),
('update', ['auto']),
('target', None),
('scanignore', []),
('scandelete', []),
('build', ''),
('buildjni', []),
('ndk', 'r10e'), # defaults to latest
('preassemble', []),
('gradleprops', []),
('antcommands', None),
('novcheck', False),
])
build_flags_order = [
'disable',
'commit',
'subdir',
'submodules',
'init',
'patch',
'gradle',
'maven',
'kivy',
'output',
'srclibs',
'oldsdkloc',
'encoding',
'forceversion',
'forcevercode',
'rm',
'extlibs',
'prebuild',
'update',
'target',
'scanignore',
'scandelete',
'build',
'buildjni',
'ndk',
'preassemble',
'gradleprops',
'antcommands',
'novcheck',
]
build_flags = set(build_flags_order + ['version', 'vercode'])
class Build():
def __init__(self):
self.disable = False
self.commit = None
self.subdir = None
self.submodules = False
self.init = ''
self.patch = []
self.gradle = False
self.maven = False
self.kivy = False
self.output = None
self.srclibs = []
self.oldsdkloc = False
self.encoding = None
self.forceversion = False
self.forcevercode = False
self.rm = []
self.extlibs = []
self.prebuild = ''
self.update = None
self.target = None
self.scanignore = []
self.scandelete = []
self.build = ''
self.buildjni = []
self.ndk = None
self.preassemble = []
self.gradleprops = []
self.antcommands = None
self.novcheck = False
def get_flag(self, f):
if f not in build_flags:
raise MetaDataException('Unrecognised build flag: ' + f)
return getattr(self, f)
def set_flag(self, f, v):
if f == 'versionName':
f = 'version'
if f == 'versionCode':
f = 'vercode'
if f not in build_flags:
raise MetaDataException('Unrecognised build flag: ' + f)
setattr(self, f, v)
def append_flag(self, f, v):
if f not in build_flags:
raise MetaDataException('Unrecognised build flag: ' + f)
if f not in self.__dict__:
self.__dict__[f] = [v]
else:
self.__dict__[f].append(v)
def method(self):
for f in ['maven', 'gradle', 'kivy']:
if self.get_flag(f):
return f
if build.output:
return 'raw'
return 'ant'
def ndk_path(self):
version = self.ndk
if not version:
version = 'r10e' # falls back to latest
paths = common.config['ndk_paths']
if version not in paths:
return ''
return paths[version]
def update_flags(self, d):
for f, v in d.iteritems():
self.set_flag(f, v)
def flagtype(name):
if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
'gradleprops']:
return 'list'
if name in ['init', 'prebuild', 'build']:
return 'script'
if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
'novcheck']:
return 'bool'
return 'string'
# Designates a metadata field type and checks that it matches
@ -230,18 +349,18 @@ flag_defaults = OrderedDict([
# 'matching' - List of possible values or regex expression
# 'sep' - Separator to use if value may be a list
# 'fields' - Metadata fields (Field:Value) of this type
# 'attrs' - Build attributes (attr=value) of this type
# 'flags' - Build flags (flag=value) of this type
#
class FieldValidator():
def __init__(self, name, matching, sep, fields, attrs):
def __init__(self, name, matching, sep, fields, flags):
self.name = name
self.matching = matching
if type(matching) is str:
self.compiled = re.compile(matching)
self.sep = sep
self.fields = fields
self.attrs = attrs
self.flags = flags
def _assert_regex(self, values, appid):
for v in values:
@ -257,13 +376,13 @@ class FieldValidator():
% (v, self.name, appid) +
"Possible values: %s" % (", ".join(self.matching)))
def check(self, value, appid):
if type(value) is not str or not value:
def check(self, v, appid):
if type(v) is not str or not v:
return
if self.sep is not None:
values = value.split(self.sep)
values = v.split(self.sep)
else:
values = [value]
values = [v]
if type(self.matching) is list:
self._assert_list(values, appid)
else:
@ -337,11 +456,11 @@ valuetypes = {
# Check an app's metadata information for integrity errors
def check_metadata(app):
for v in valuetypes:
for field in v.fields:
v.check(app.get_field(field), app.id)
for f in v.fields:
v.check(app.get_field(f), app.id)
for build in app.builds:
for attr in v.attrs:
v.check(build[attr], app.id)
for f in v.flags:
v.check(build.get_flag(f), app.id)
# Formatter for descriptions. Create an instance, and call parseline() with
@ -554,14 +673,14 @@ def parse_srclib(metadatapath):
continue
try:
field, value = line.split(':', 1)
f, v = line.split(':', 1)
except ValueError:
raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
if field == "Subdir":
thisinfo[field] = value.split(',')
if f == "Subdir":
thisinfo[f] = v.split(',')
else:
thisinfo[field] = value
thisinfo[f] = v
return thisinfo
@ -640,54 +759,6 @@ def read_metadata(xref=True):
return apps
# Get the type expected for a given metadata field.
def metafieldtype(name):
if name in ['Description', 'Maintainer Notes']:
return 'multiline'
if name in ['Categories', 'AntiFeatures']:
return 'list'
if name == 'Build Version':
return 'build'
if name == 'Build':
return 'buildv2'
if name == 'Use Built':
return 'obsolete'
if name not in app_fields:
return 'unknown'
return 'string'
def flagtype(name):
if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
'gradleprops']:
return 'list'
if name in ['init', 'prebuild', 'build']:
return 'script'
if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
'novcheck']:
return 'bool'
return 'string'
def fill_build_defaults(build):
def get_build_type():
for t in ['maven', 'gradle', 'kivy']:
if build[t]:
return t
if build['output']:
return 'raw'
return 'ant'
for flag, value in flag_defaults.iteritems():
if flag in build:
continue
build[flag] = value
build['type'] = get_build_type()
build['ndk_path'] = common.get_ndk_path(build['ndk'])
def split_list_values(s):
# Port legacy ';' separators
l = [v.strip() for v in s.replace(';', ',').split(',')]
@ -709,7 +780,7 @@ def get_default_app_info(metadatapath=None):
def sorted_builds(builds):
return sorted(builds, key=lambda build: int(build['vercode']))
return sorted(builds, key=lambda build: int(build.vercode))
def post_metadata_parse(app):
@ -726,60 +797,30 @@ def post_metadata_parse(app):
text = v.rstrip().lstrip()
app.set_field(f, text.split('\n'))
supported_flags = (flag_defaults.keys()
+ ['vercode', 'version', 'versionCode', 'versionName',
'type', 'ndk_path'])
esc_newlines = re.compile('\\\\( |\\n)')
for build in app.builds:
for k, v in build.items():
if k not in supported_flags:
raise MetaDataException("Unrecognised build flag: {0}={1}"
.format(k, v))
for k in build_flags:
v = build.get_flag(k)
if k == 'versionCode':
build['vercode'] = str(v)
del build['versionCode']
elif k == 'versionName':
build['version'] = str(v)
del build['versionName']
elif type(v) in (float, int):
build[k] = str(v)
if type(v) in (float, int):
build.set_flag(k, v)
else:
keyflagtype = flagtype(k)
if keyflagtype == 'list':
# these can be bools, strings or lists, but ultimately are lists
if isinstance(v, basestring):
build[k] = [v]
elif isinstance(v, bool):
build[k] = ['yes' if v else 'no']
elif isinstance(v, list):
build[k] = []
for e in v:
if isinstance(e, bool):
build[k].append('yes' if v else 'no')
else:
build[k].append(e)
elif keyflagtype == 'script':
build[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
if keyflagtype == 'script':
build.set_flag(k, re.sub(esc_newlines, '', v).lstrip().rstrip())
elif keyflagtype == 'bool':
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if isinstance(v, basestring):
if v == 'true':
build[k] = True
else:
build[k] = False
if isinstance(v, basestring) and v == 'true':
build.set_flag(k, 'true')
elif keyflagtype == 'string':
if isinstance(v, bool):
build[k] = 'yes' if v else 'no'
if isinstance(v, bool) and v:
build.set_flag(k, 'yes')
if not app.Description:
app.Description = ['No description available']
for build in app.builds:
fill_build_defaults(build)
app.builds = sorted_builds(app.builds)
@ -826,16 +867,16 @@ def _decode_list(data):
def _decode_dict(data):
'''convert items in a dict from unicode to basestring'''
rv = {}
for key, value in data.iteritems():
if isinstance(key, unicode):
key = key.encode('utf-8')
if isinstance(value, unicode):
value = value.encode('utf-8')
elif isinstance(value, list):
value = _decode_list(value)
elif isinstance(value, dict):
value = _decode_dict(value)
rv[key] = value
for k, v in data.iteritems():
if isinstance(k, unicode):
k = k.encode('utf-8')
if isinstance(v, unicode):
v = v.encode('utf-8')
elif isinstance(v, list):
v = _decode_list(v)
elif isinstance(v, dict):
v = _decode_dict(v)
rv[k] = v
return rv
@ -897,16 +938,14 @@ def parse_xml_metadata(metadatapath):
if child.tag == 'string':
app.set_field(name, child.text)
elif child.tag == 'string-array':
items = []
for item in child:
items.append(item.text)
app.set_field(name, items)
app.append_field(name, item.text)
elif child.tag == 'builds':
for build in child:
builddict = dict()
for key in build:
builddict[key.tag] = key.text
app.builds.append(builddict)
for b in child:
build = Build()
for key in b:
build.set_flag(key.tag, key.text)
app.builds.append(build)
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if not isinstance(app.RequiresRoot, bool):
@ -935,7 +974,7 @@ def parse_txt_metadata(metadatapath):
linedesc = None
def add_buildflag(p, thisbuild):
def add_buildflag(p, build):
if not p.strip():
raise MetaDataException("Empty build flag at {1}"
.format(buildlines[0], linedesc))
@ -943,13 +982,10 @@ def parse_txt_metadata(metadatapath):
if len(bv) != 2:
raise MetaDataException("Invalid build flag at {0} in {1}"
.format(buildlines[0], linedesc))
pk, pv = bv
if pk in thisbuild:
raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
.format(pk, thisbuild['version'], linedesc))
pk, pv = bv
pk = pk.lstrip()
if pk not in flag_defaults:
if pk not in build_flags:
raise MetaDataException("Unrecognised build flag at {0} in {1}"
.format(p, linedesc))
t = flagtype(pk)
@ -958,45 +994,45 @@ def parse_txt_metadata(metadatapath):
if pk == 'gradle':
if len(pv) == 1 and pv[0] in ['main', 'yes']:
pv = ['yes']
thisbuild[pk] = pv
build.set_flag(pk, pv)
elif t == 'string' or t == 'script':
thisbuild[pk] = pv
build.set_flag(pk, pv)
elif t == 'bool':
value = pv == 'yes'
if value:
thisbuild[pk] = True
v = pv == 'yes'
if v:
build.set_flag(pk, True)
else:
raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
% (t, p, linedesc))
def parse_buildline(lines):
value = "".join(lines)
v = "".join(lines)
parts = [p.replace("\\,", ",")
for p in re.split(r"(?<!\\),", value)]
for p in re.split(r"(?<!\\),", v)]
if len(parts) < 3:
raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
thisbuild = {}
thisbuild['origlines'] = lines
thisbuild['version'] = parts[0]
thisbuild['vercode'] = parts[1]
raise MetaDataException("Invalid build format: " + v + " in " + metafile.name)
build = Build()
build.origlines = lines
build.version = parts[0]
build.vercode = parts[1]
if parts[2].startswith('!'):
# For backwards compatibility, handle old-style disabling,
# including attempting to extract the commit from the message
thisbuild['disable'] = parts[2][1:]
build.disable = parts[2][1:]
commit = 'unknown - see disabled'
index = parts[2].rfind('at ')
if index != -1:
commit = parts[2][index + 3:]
if commit.endswith(')'):
commit = commit[:-1]
thisbuild['commit'] = commit
build.commit = commit
else:
thisbuild['commit'] = parts[2]
build.commit = parts[2]
for p in parts[3:]:
add_buildflag(p, thisbuild)
add_buildflag(p, build)
return thisbuild
return build
def add_comments(key):
if not curcomments:
@ -1010,7 +1046,7 @@ def parse_txt_metadata(metadatapath):
mode = 0
buildlines = []
curcomments = []
curbuild = None
build = None
vc_seen = {}
c = 0
@ -1020,13 +1056,12 @@ def parse_txt_metadata(metadatapath):
line = line.rstrip('\r\n')
if mode == 3:
if not any(line.startswith(s) for s in (' ', '\t')):
commit = curbuild['commit'] if 'commit' in curbuild else None
if not commit and 'disable' not in curbuild:
if not build.commit and not build.disable:
raise MetaDataException("No commit specified for {0} in {1}"
.format(curbuild['version'], linedesc))
.format(build.version, linedesc))
app.builds.append(curbuild)
add_comments('build:' + curbuild['vercode'])
app.builds.append(build)
add_comments('build:' + build.vercode)
mode = 0
else:
if line.endswith('\\'):
@ -1034,7 +1069,7 @@ def parse_txt_metadata(metadatapath):
else:
buildlines.append(line.lstrip())
bl = ''.join(buildlines)
add_buildflag(bl, curbuild)
add_buildflag(bl, build)
buildlines = []
if mode == 0:
@ -1044,74 +1079,74 @@ def parse_txt_metadata(metadatapath):
curcomments.append(line[1:].strip())
continue
try:
field, value = line.split(':', 1)
f, v = line.split(':', 1)
except ValueError:
raise MetaDataException("Invalid metadata in " + linedesc)
if field != field.strip() or value != value.strip():
if f != f.strip() or v != v.strip():
raise MetaDataException("Extra spacing found in " + linedesc)
# Translate obsolete fields...
if field == 'Market Version':
field = 'Current Version'
if field == 'Market Version Code':
field = 'Current Version Code'
if f == 'Market Version':
f = 'Current Version'
if f == 'Market Version Code':
f = 'Current Version Code'
fieldtype = metafieldtype(field)
fieldtype = metafieldtype(f)
if fieldtype not in ['build', 'buildv2']:
add_comments(field)
add_comments(f)
if fieldtype == 'multiline':
mode = 1
if value:
raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
if v:
raise MetaDataException("Unexpected text on same line as " + f + " in " + linedesc)
elif fieldtype == 'string':
app.set_field(field, value)
app.set_field(f, v)
elif fieldtype == 'list':
app.set_field(field, split_list_values(value))
app.set_field(f, split_list_values(v))
elif fieldtype == 'build':
if value.endswith("\\"):
if v.endswith("\\"):
mode = 2
buildlines = [value[:-1]]
buildlines = [v[:-1]]
else:
curbuild = parse_buildline([value])
app.builds.append(curbuild)
add_comments('build:' + app.builds[-1]['vercode'])
build = parse_buildline([v])
app.builds.append(build)
add_comments('build:' + app.builds[-1].vercode)
elif fieldtype == 'buildv2':
curbuild = {}
vv = value.split(',')
build = Build()
vv = v.split(',')
if len(vv) != 2:
raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
.format(value, linedesc))
curbuild['version'] = vv[0]
curbuild['vercode'] = vv[1]
if curbuild['vercode'] in vc_seen:
.format(v, linedesc))
build.version = vv[0]
build.vercode = vv[1]
if build.vercode in vc_seen:
raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
curbuild['vercode'], linedesc))
vc_seen[curbuild['vercode']] = True
build.vercode, linedesc))
vc_seen[build.vercode] = True
buildlines = []
mode = 3
elif fieldtype == 'obsolete':
pass # Just throw it away!
else:
raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
raise MetaDataException("Unrecognised field type for " + f + " in " + linedesc)
elif mode == 1: # Multiline field
if line == '.':
mode = 0
else:
app.append_field(field, line)
app.append_field(f, line)
elif mode == 2: # Line continuation mode in Build Version
if line.endswith("\\"):
buildlines.append(line[:-1])
else:
buildlines.append(line)
curbuild = parse_buildline(buildlines)
app.builds.append(curbuild)
add_comments('build:' + app.builds[-1]['vercode'])
build = parse_buildline(buildlines)
app.builds.append(build)
add_comments('build:' + app.builds[-1].vercode)
mode = 0
add_comments(None)
# Mode at end of file should always be 0...
if mode == 1:
raise MetaDataException(field + " not terminated in " + metafile.name)
raise MetaDataException(f + " not terminated in " + metafile.name)
elif mode == 2:
raise MetaDataException("Unterminated continuation in " + metafile.name)
elif mode == 3:
@ -1130,18 +1165,18 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
for line in app.comments[key]:
w_comment(line)
def w_field_always(field, value=None):
if value is None:
value = app.get_field(field)
w_comments(field)
w_field(field, value)
def w_field_always(f, v=None):
if v is None:
v = app.get_field(f)
w_comments(f)
w_field(f, v)
def w_field_nonempty(field, value=None):
if value is None:
value = app.get_field(field)
w_comments(field)
if value:
w_field(field, value)
def w_field_nonempty(f, v=None):
if v is None:
v = app.get_field(f)
w_comments(f)
if v:
w_field(f, v)
w_field_nonempty('Disabled')
if app.AntiFeatures:
@ -1175,10 +1210,10 @@ def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
for build in sorted_builds(app.builds):
if build['version'] == "Ignore":
if build.version == "Ignore":
continue
w_comments('build:' + build['vercode'])
w_comments('build:' + build.vercode)
w_build(build)
mf.write('\n')
@ -1211,39 +1246,37 @@ def write_txt_metadata(mf, app):
def w_comment(line):
mf.write("# %s\n" % line)
def w_field(field, value):
t = metafieldtype(field)
def w_field(f, v):
t = metafieldtype(f)
if t == 'list':
value = ','.join(value)
v = ','.join(v)
elif t == 'multiline':
if type(value) == list:
value = '\n' + '\n'.join(value) + '\n.'
if type(v) == list:
v = '\n' + '\n'.join(v) + '\n.'
else:
value = '\n' + value + '\n.'
mf.write("%s:%s\n" % (field, value))
v = '\n' + v + '\n.'
mf.write("%s:%s\n" % (f, v))
def w_build(build):
mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
mf.write("Build:%s,%s\n" % (build.version, build.vercode))
for key in flag_defaults:
value = build[key]
if not value:
continue
if value == flag_defaults[key]:
for f in build_flags_order:
v = build.get_flag(f)
if not v:
continue
t = flagtype(key)
v = ' %s=' % key
t = flagtype(f)
out = ' %s=' % f
if t == 'string':
v += value
out += v
elif t == 'bool':
v += 'yes'
out += 'yes'
elif t == 'script':
v += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
out += '&& \\\n '.join([s.lstrip() for s in v.split('&& ')])
elif t == 'list':
v += ','.join(value) if type(value) == list else value
out += ','.join(v) if type(v) == list else v
mf.write(v)
mf.write(out)
mf.write('\n')
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
@ -1254,26 +1287,26 @@ def write_yaml_metadata(mf, app):
def w_comment(line):
mf.write("# %s\n" % line)
def escape(value):
if not value:
def escape(v):
if not v:
return ''
if any(c in value for c in [': ', '%', '@', '*']):
return "'" + value.replace("'", "''") + "'"
return value
if any(c in v for c in [': ', '%', '@', '*']):
return "'" + v.replace("'", "''") + "'"
return v
def w_field(field, value, prefix='', t=None):
def w_field(f, v, prefix='', t=None):
if t is None:
t = metafieldtype(field)
t = metafieldtype(f)
v = ''
if t == 'list':
v = '\n'
for e in value:
for e in v:
v += prefix + ' - ' + escape(e) + '\n'
elif t == 'multiline':
v = ' |\n'
lines = value
if type(value) == str:
lines = value.splitlines()
lines = v
if type(v) == str:
lines = v.splitlines()
for l in lines:
if l:
v += prefix + ' ' + l + '\n'
@ -1282,16 +1315,16 @@ def write_yaml_metadata(mf, app):
elif t == 'bool':
v = ' yes\n'
elif t == 'script':
cmds = [s + '&& \\' for s in value.split('&& ')]
cmds = [s + '&& \\' for s in v.split('&& ')]
if len(cmds) > 0:
cmds[-1] = cmds[-1][:-len('&& \\')]
w_field(field, cmds, prefix, 'multiline')
w_field(f, cmds, prefix, 'multiline')
return
else:
v = ' ' + escape(value) + '\n'
v = ' ' + escape(v) + '\n'
mf.write(prefix)
mf.write(field)
mf.write(f)
mf.write(":")
mf.write(v)
@ -1304,16 +1337,14 @@ def write_yaml_metadata(mf, app):
mf.write("builds:\n")
first_build = False
w_field('versionName', build['version'], ' - ', 'string')
w_field('versionCode', build['vercode'], ' ', 'strsng')
for key in flag_defaults:
value = build[key]
if not value:
continue
if value == flag_defaults[key]:
w_field('versionName', build.version, ' - ', 'string')
w_field('versionCode', build.vercode, ' ', 'strsng')
for f in build_flags_order:
v = build.get_flag(f)
if not v:
continue
w_field(key, value, ' ', flagtype(key))
w_field(f, v, ' ', flagtype(f))
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)

32
fdroidserver/scanner.py
View File

@ -31,18 +31,18 @@ config = None
options = None
def get_gradle_compile_commands(thisbuild):
def get_gradle_compile_commands(build):
compileCommands = ['compile', 'releaseCompile']
if thisbuild['gradle'] and thisbuild['gradle'] != ['yes']:
compileCommands += [flavor + 'Compile' for flavor in thisbuild['gradle']]
compileCommands += [flavor + 'ReleaseCompile' for flavor in thisbuild['gradle']]
if build.gradle and build.gradle != ['yes']:
compileCommands += [flavor + 'Compile' for flavor in build.gradle]
compileCommands += [flavor + 'ReleaseCompile' for flavor in build.gradle]
return [re.compile(r'\s*' + c, re.IGNORECASE) for c in compileCommands]
# Scan the source code in the given directory (and all subdirectories)
# and return the number of fatal problems encountered
def scan_source(build_dir, root_dir, thisbuild):
def scan_source(build_dir, root_dir, build):
count = 0
@ -85,8 +85,8 @@ def scan_source(build_dir, root_dir, thisbuild):
]
]
scanignore = common.getpaths_map(build_dir, thisbuild['scanignore'])
scandelete = common.getpaths_map(build_dir, thisbuild['scandelete'])
scanignore = common.getpaths_map(build_dir, build.scanignore)
scandelete = common.getpaths_map(build_dir, build.scandelete)
scanignore_worked = set()
scandelete_worked = set()
@ -153,7 +153,7 @@ def scan_source(build_dir, root_dir, thisbuild):
return True
return False
gradle_compile_commands = get_gradle_compile_commands(thisbuild)
gradle_compile_commands = get_gradle_compile_commands(build)
def is_used_by_gradle(line):
return any(command.match(line) for command in gradle_compile_commands)
@ -240,7 +240,7 @@ def scan_source(build_dir, root_dir, thisbuild):
# indicate a problem (if it's not a problem, explicitly use
# buildjni=no to bypass this check)
if (os.path.exists(os.path.join(root_dir, 'jni')) and
not thisbuild['buildjni']):
not build.buildjni):
logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
count += 1
@ -293,24 +293,24 @@ def main():
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
for thisbuild in app.builds:
for build in app.builds:
if thisbuild['disable']:
if build.disable:
logging.info("...skipping version %s - %s" % (
thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:])))
build.version, build.get('disable', build.commit[1:])))
else:
logging.info("...scanning version " + thisbuild['version'])
logging.info("...scanning version " + build.version)
# Prepare the source code...
root_dir, _ = common.prepare_source(vcs, app, thisbuild,
root_dir, _ = common.prepare_source(vcs, app, build,
build_dir, srclib_dir,
extlib_dir, False)
# Do the scan...
count = scan_source(build_dir, root_dir, thisbuild)
count = scan_source(build_dir, root_dir, build)
if count > 0:
logging.warn('Scanner found %d problems in %s (%s)' % (
count, appid, thisbuild['vercode']))
count, appid, build.vercode))
probcount += count
except BuildException as be:

136
fdroidserver/update.py
View File

@ -144,26 +144,26 @@ def update_wiki(apps, sortedids, apks):
gotcurrentver = True
apklist.append(apk)
# Include ones we can't build, as a special case...
for thisbuild in app.builds:
if thisbuild['disable']:
if thisbuild['vercode'] == app.CurrentVersionCode:
for build in app.builds:
if build.disable:
if build.vercode == app.CurrentVersionCode:
cantupdate = True
# TODO: Nasty: vercode is a string in the build, and an int elsewhere
apklist.append({'versioncode': int(thisbuild['vercode']),
'version': thisbuild['version'],
'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(thisbuild['disable']),
apklist.append({'versioncode': int(build.vercode),
'version': build.version,
'buildproblem': "The build for this version was manually disabled. Reason: {0}".format(build.disable),
})
else:
builtit = False
for apk in apklist:
if apk['versioncode'] == int(thisbuild['vercode']):
if apk['versioncode'] == int(build.vercode):
builtit = True
break
if not builtit:
buildfails = True
apklist.append({'versioncode': int(thisbuild['vercode']),
'version': thisbuild['version'],
'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, thisbuild['vercode']),
apklist.append({'versioncode': int(build.vercode),
'version': build.version,
'buildproblem': "The build for this version appears to have failed. Check the [[{0}/lastbuild_{1}|build log]].".format(appid, build.vercode),
})
if app.CurrentVersionCode == '0':
cantupdate = True
@ -291,12 +291,12 @@ def delete_disabled_builds(apps, apkcache, repodirs):
"""
for appid, app in apps.iteritems():
for build in app.builds:
if not build['disable']:
if not build.disable:
continue
apkfilename = appid + '_' + str(build['vercode']) + '.apk'
apkfilename = appid + '_' + str(build.vercode) + '.apk'
iconfilename = "%s.%s.png" % (
appid,
build['vercode'])
build.vercode)
for repodir in repodirs:
files = [
os.path.join(repodir, apkfilename),
@ -453,8 +453,8 @@ def scan_apks(apps, apkcache, repodir, knownapks):
usecache = False
if apkfilename in apkcache:
thisinfo = apkcache[apkfilename]
if thisinfo['sha256'] == shasum:
apk = apkcache[apkfilename]
if apk['sha256'] == shasum:
logging.debug("Reading " + apkfilename + " from cache")
usecache = True
else:
@ -462,17 +462,17 @@ def scan_apks(apps, apkcache, repodir, knownapks):
if not usecache:
logging.debug("Processing " + apkfilename)
thisinfo = {}
thisinfo['apkname'] = apkfilename
thisinfo['sha256'] = shasum
apk = {}
apk['apkname'] = apkfilename
apk['sha256'] = shasum
srcfilename = apkfilename[:-4] + "_src.tar.gz"
if os.path.exists(os.path.join(repodir, srcfilename)):
thisinfo['srcname'] = srcfilename
thisinfo['size'] = os.path.getsize(apkfile)
thisinfo['permissions'] = set()
thisinfo['features'] = set()
thisinfo['icons_src'] = {}
thisinfo['icons'] = {}
apk['srcname'] = srcfilename
apk['size'] = os.path.getsize(apkfile)
apk['permissions'] = set()
apk['features'] = set()
apk['icons_src'] = {}
apk['icons'] = {}
p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
if p.returncode != 0:
if options.delete_unknown:
@ -487,51 +487,51 @@ def scan_apks(apps, apkcache, repodir, knownapks):
for line in p.output.splitlines():
if line.startswith("package:"):
try:
thisinfo['id'] = re.match(name_pat, line).group(1)
thisinfo['versioncode'] = int(re.match(vercode_pat, line).group(1))
thisinfo['version'] = re.match(vername_pat, line).group(1)
apk['id'] = re.match(name_pat, line).group(1)
apk['versioncode'] = int(re.match(vercode_pat, line).group(1))
apk['version'] = re.match(vername_pat, line).group(1)
except Exception, e:
logging.error("Package matching failed: " + str(e))
logging.info("Line was: " + line)
sys.exit(1)
elif line.startswith("application:"):
thisinfo['name'] = re.match(label_pat, line).group(1)
apk['name'] = re.match(label_pat, line).group(1)
# Keep path to non-dpi icon in case we need it
match = re.match(icon_pat_nodpi, line)
if match:
thisinfo['icons_src']['-1'] = match.group(1)
apk['icons_src']['-1'] = match.group(1)
elif line.startswith("launchable-activity:"):
# Only use launchable-activity as fallback to application
if not thisinfo['name']:
thisinfo['name'] = re.match(label_pat, line).group(1)
if '-1' not in thisinfo['icons_src']:
if not apk['name']:
apk['name'] = re.match(label_pat, line).group(1)
if '-1' not in apk['icons_src']:
match = re.match(icon_pat_nodpi, line)
if match:
thisinfo['icons_src']['-1'] = match.group(1)
apk['icons_src']['-1'] = match.group(1)
elif line.startswith("application-icon-"):
match = re.match(icon_pat, line)
if match:
density = match.group(1)
path = match.group(2)
thisinfo['icons_src'][density] = path
apk['icons_src'][density] = path
elif line.startswith("sdkVersion:"):
m = re.match(sdkversion_pat, line)
if m is None:
logging.error(line.replace('sdkVersion:', '')
+ ' is not a valid minSdkVersion!')
else:
thisinfo['sdkversion'] = m.group(1)
apk['sdkversion'] = m.group(1)
elif line.startswith("maxSdkVersion:"):
thisinfo['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
apk['maxsdkversion'] = re.match(sdkversion_pat, line).group(1)
elif line.startswith("native-code:"):
thisinfo['nativecode'] = []
apk['nativecode'] = []
for arch in line[13:].split(' '):
thisinfo['nativecode'].append(arch[1:-1])
apk['nativecode'].append(arch[1:-1])
elif line.startswith("uses-permission:"):
perm = re.match(string_pat, line).group(1)
if perm.startswith("android.permission."):
perm = perm[19:]
thisinfo['permissions'].add(perm)
apk['permissions'].add(perm)
elif line.startswith("uses-feature:"):
perm = re.match(string_pat, line).group(1)
# Filter out this, it's only added with the latest SDK tools and
@ -540,11 +540,11 @@ def scan_apks(apps, apkcache, repodir, knownapks):
and perm != "android.hardware.screen.landscape":
if perm.startswith("android.feature."):
perm = perm[16:]
thisinfo['features'].add(perm)
apk['features'].add(perm)
if 'sdkversion' not in thisinfo:
if 'sdkversion' not in apk:
logging.warn("No SDK version information found in {0}".format(apkfile))
thisinfo['sdkversion'] = 0
apk['sdkversion'] = 0
# Check for debuggable apks...
if common.isApkDebuggable(apkfile, config):
@ -552,20 +552,20 @@ def scan_apks(apps, apkcache, repodir, knownapks):
# Get the signature (or md5 of, to be precise)...
logging.debug('Getting signature of {0}'.format(apkfile))
thisinfo['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
if not thisinfo['sig']:
apk['sig'] = getsig(os.path.join(os.getcwd(), apkfile))
if not apk['sig']:
logging.critical("Failed to get apk signature")
sys.exit(1)
apk = zipfile.ZipFile(apkfile, 'r')
apkzip = zipfile.ZipFile(apkfile, 'r')
# if an APK has files newer than the system time, suggest updating
# the system clock. This is useful for offline systems, used for
# signing, which do not have another source of clock sync info. It
# has to be more than 24 hours newer because ZIP/APK files do not
# store timezone info
info = apk.getinfo('AndroidManifest.xml')
dt_obj = datetime(*info.date_time)
manifest = apkzip.getinfo('AndroidManifest.xml')
dt_obj = datetime(*manifest.date_time)
checkdt = dt_obj - timedelta(1)
if datetime.today() < checkdt:
logging.warn('System clock is older than manifest in: '
@ -573,44 +573,44 @@ def scan_apks(apps, apkcache, repodir, knownapks):
+ 'sudo date -s "' + str(dt_obj) + '"')
iconfilename = "%s.%s.png" % (
thisinfo['id'],
thisinfo['versioncode'])
apk['id'],
apk['versioncode'])
# Extract the icon file...
empty_densities = []
for density in screen_densities:
if density not in thisinfo['icons_src']:
if density not in apk['icons_src']:
empty_densities.append(density)
continue
iconsrc = thisinfo['icons_src'][density]
iconsrc = apk['icons_src'][density]
icon_dir = get_icon_dir(repodir, density)
icondest = os.path.join(icon_dir, iconfilename)
try:
with open(icondest, 'wb') as f:
f.write(apk.read(iconsrc))
thisinfo['icons'][density] = iconfilename
f.write(apkzip.read(iconsrc))
apk['icons'][density] = iconfilename
except:
logging.warn("Error retrieving icon file")
del thisinfo['icons'][density]
del thisinfo['icons_src'][density]
del apk['icons'][density]
del apk['icons_src'][density]
empty_densities.append(density)
if '-1' in thisinfo['icons_src']:
iconsrc = thisinfo['icons_src']['-1']
if '-1' in apk['icons_src']:
iconsrc = apk['icons_src']['-1']
iconpath = os.path.join(
get_icon_dir(repodir, '0'), iconfilename)
with open(iconpath, 'wb') as f:
f.write(apk.read(iconsrc))
f.write(apkzip.read(iconsrc))
try:
im = Image.open(iconpath)
dpi = px_to_dpi(im.size[0])
for density in screen_densities:
if density in thisinfo['icons']:
if density in apk['icons']:
break
if density == screen_densities[-1] or dpi >= int(density):
thisinfo['icons'][density] = iconfilename
apk['icons'][density] = iconfilename
shutil.move(iconpath,
os.path.join(get_icon_dir(repodir, density), iconfilename))
empty_densities.remove(density)
@ -618,10 +618,10 @@ def scan_apks(apps, apkcache, repodir, knownapks):
except Exception, e:
logging.warn("Failed reading {0} - {1}".format(iconpath, e))
if thisinfo['icons']:
thisinfo['icon'] = iconfilename
if apk['icons']:
apk['icon'] = iconfilename
apk.close()
apkzip.close()
# First try resizing down to not lose quality
last_density = None
@ -675,19 +675,19 @@ def scan_apks(apps, apkcache, repodir, knownapks):
# Copy from icons-mdpi to icons since mdpi is the baseline density
baseline = os.path.join(get_icon_dir(repodir, '160'), iconfilename)
if os.path.isfile(baseline):
thisinfo['icons']['0'] = iconfilename
apk['icons']['0'] = iconfilename
shutil.copyfile(baseline,
os.path.join(get_icon_dir(repodir, '0'), iconfilename))
# Record in known apks, getting the added date at the same time..
added = knownapks.recordapk(thisinfo['apkname'], thisinfo['id'])
added = knownapks.recordapk(apk['apkname'], apk['id'])
if added:
thisinfo['added'] = added
apk['added'] = added
apkcache[apkfilename] = thisinfo
apkcache[apkfilename] = apk
cachechanged = True
apks.append(thisinfo)
apks.append(apk)
return apks, cachechanged

23
tests/common.TestCase
View File

@ -114,20 +114,19 @@ class CommonTest(unittest.TestCase):
config = dict()
config['sdk_path'] = os.getenv('ANDROID_HOME')
config['ndk_paths'] = {'r10d': os.getenv('ANDROID_NDK_HOME')}
config['build_tools'] = 'FAKE_BUILD_TOOLS_VERSION'
fdroidserver.common.config = config
app = fdroidserver.metadata.App()
app.id = 'org.fdroid.froid'
build = dict(fdroidserver.metadata.flag_defaults)
build['commit'] = 'master'
build['forceversion'] = True
build['forcevercode'] = True
build['gradle'] = ['yes']
build['ndk_path'] = os.getenv('ANDROID_NDK_HOME')
build['target'] = 'android-' + str(testint)
build['type'] = 'gradle'
build['version'] = teststr
build['vercode'] = testint
build = fdroidserver.metadata.Build()
build.commit = 'master'
build.forceversion = True
build.forcevercode = True
build.gradle = ['yes']
build.target = 'android-' + str(testint)
build.version = teststr
build.vercode = testint
class FakeVcs():
# no need to change to the correct commit here
@ -147,8 +146,8 @@ class CommonTest(unittest.TestCase):
with open(os.path.join(testdir, 'AndroidManifest.xml')) as f:
filedata = f.read()
self.assertIsNone(re.search('android:debuggable', filedata))
self.assertIsNotNone(re.search('android:versionName="%s"' % build['version'], filedata))
self.assertIsNotNone(re.search('android:versionCode="%s"' % build['vercode'], filedata))
self.assertIsNotNone(re.search('android:versionName="%s"' % build.version, filedata))
self.assertIsNotNone(re.search('android:versionCode="%s"' % build.vercode, filedata))
if __name__ == "__main__":

3
tests/metadata.TestCase
View File

@ -46,8 +46,9 @@ class MetadataTest(unittest.TestCase):
frompickle = pickle.load(f)
frommeta = app.field_dict()
self.assertEquals(frommeta, frompickle)
# Uncomment to overwrite
# with open(savepath, 'wb') as f:
# pickle.dump(app, f)
# pickle.dump(frommeta, f)
if __name__ == "__main__":

668
tests/metadata/net.osmand.plus.pickle
View File

@ -10,87 +10,87 @@ S'Tracking'
p5
aS'NonFreeNet'
p6
asS'Web Site'
asS'Litecoin'
p7
S'http://osmand.net'
NsS'comments'
p8
sS'Auto Update Mode'
p9
S'None'
p10
(dp9
sS'Provides'
p11
p10
NsS'Issue Tracker'
p12
p11
S'https://github.com/osmandapp/Osmand/issues'
p13
p12
sS'Donate'
p14
p13
S'https://code.google.com/p/osmand/#Please_support_the_project'
p14
sS'Archive Policy'
p15
sS'id'
NsS'Description'
p16
S'net.osmand.plus'
p17
sS'Description'
p18
(lp19
(lp17
S"Osmand~'s features can be extended by enabling the plugins via the settings,"
p20
p18
aS'which include online maps from many sources, tracking, OpenStreetMap (OSM) editing and'
p21
p19
aS'accessibility enhancements.'
p22
p20
aS''
p23
p21
aS'Map data of both vector and raster types can be stored on the phone memory'
p24
p22
aS'card for offline usage, and navigation by default uses offline methods. Map'
p25
p23
aS'data packages for many territories can be downloaded from within the app and'
p26
p24
aS'there is a desktop program available on the website as well for creating your'
p27
p25
aS'own.'
p28
ag23
p26
ag21
aS'Anti-Features: Tracking - It will send your device and application specs to an'
p29
p27
aS'Analytics server upon downloading the list of maps you can download.'
p30
ag23
p28
ag21
aS'[https://osmandapp.github.io/changes.html Changelog]'
p31
p29
asS'Requires Root'
p32
p30
I00
sS'comments'
sS'lastupdated'
p31
NsS'id'
p32
S'net.osmand.plus'
p33
(dp34
sS'Repo Type'
p35
S'git'
p36
sS'Repo'
p37
p34
S'https://github.com/mvdan/OsmAnd-submodules'
p38
p35
sS'No Source Since'
p39
g23
p36
g21
sS'Repo Type'
p37
S'git'
p38
sS'Auto Name'
p40
g23
p39
g21
sS'Categories'
p41
(lp42
S'Navigation'
p40
(lp41
S'None'
p42
aS'Navigation'
p43
asS'Source Code'
p44
S'https://github.com/osmandapp/Osmand'
p45
sS'Litecoin'
sS'added'
p46
NsS'Update Check Ignore'
p47
@ -104,7 +104,7 @@ S'GPLv3'
p51
sS'Changelog'
p52
g23
g21
sS'Update Check Mode'
p53
S'None'
@ -113,394 +113,344 @@ sS'Summary'
p55
S'Offline/online maps and navigation'
p56
sS'Maintainer Notes'
sS'Current Version'
p57
(lp58
S'No UCMs apply because git never contains actual releases, only pre-releses.'
S'1.9.5'
p58
sS'Maintainer Notes'
p59
ag23
aS'The build instructions have been moved to a script in the root of the repo,'
p60
aS"'build'. This way it can be updated along with the submodules."
(lp60
S'No UCMs apply because git never contains actual releases, only pre-releses.'
p61
asS'Current Version Code'
ag21
aS'The build instructions have been moved to a script in the root of the repo,'
p62
S'197'
aS"'build'. This way it can be updated along with the submodules."
p63
sS'Binaries'
asS'Current Version Code'
p64
NsS'Archive Policy'
S'197'
p65
NsS'builds'
sS'Binaries'
p66
(lp67
(dp68
NsS'builds'
p67
(lp68
(dp69
S'submodules'
p69
I01
sS'vercode'
p70
S'182'
S'true'
p71
sS'forceversion'
sS'kivy'
p72
I00
sS'oldsdkloc'
sS'forceversion'
p73
I00
sS'gradleprops'
sS'oldsdkloc'
p74
(lp75
sS'scanignore'
I00
sS'antcommands'
p75
NsS'scanignore'
p76
(lp77
sS'patch'
sS'gradle'
p78
(lp79
sS'srclibs'
p80
(lp81
sS'output'
p82
S'bin/OsmAnd-release-unsigned.apk'
p83
sS'encoding'
p84
NsS'extlibs'
p85
(lp86
sS'init'
p87
g23
sS'version'
p88
S'1.8.2'
p89
sS'build'
p90
S'./old-ndk-build.sh && ant -Dsdk.dir="$ANDROID_SDK" -Dndk.dir="$ANDROID_NDK" -DBLACKBERRY_BUILD=false -DBUILD_SUFFIX= -DAPK_NUMBER_VERSION=182 "-DFEATURES=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" -DCLEAN_CPP=false -DPACKAGE_TO_BUILT=net.osmand.plus -DAPK_VERSION=1.8.2 -Dnet.osmand.plus= -Dbuild.version=1.8.2 -Dbuild.version.code=182 -Dnativeoff=false "-DversionFeatures=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" clean release'
p91
sS'rm'
p92
(lp93
sS'kivy'
p94
I00
sS'srclibs'
p79
(lp80
sS'encoding'
p81
NsS'extlibs'
p82
(lp83
sS'init'
p84
g21
sS'version'
p85
S'1.8.2'
p86
sS'subdir'
p95
p87
S'android/OsmAnd'
p96
p88
sS'rm'
p89
(lp90
sS'build'
p91
S'./old-ndk-build.sh && ant -Dsdk.dir="$ANDROID_SDK" -Dndk.dir="$ANDROID_NDK" -DBLACKBERRY_BUILD=false -DBUILD_SUFFIX= -DAPK_NUMBER_VERSION=182 "-DFEATURES=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" -DCLEAN_CPP=false -DPACKAGE_TO_BUILT=net.osmand.plus -DAPK_VERSION=1.8.2 -Dnet.osmand.plus= -Dbuild.version=1.8.2 -Dbuild.version.code=182 -Dnativeoff=false "-DversionFeatures=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" clean release'
p92
sS'vercode'
p93
S'182'
p94
sS'forcevercode'
p97
p95
I00
sS'preassemble'
p98
(lp99
p96
(lp97
sS'update'
p100
(lp101
S'auto'
p102
asS'maven'
p103
p98
NsS'maven'
p99
I00
sS'disable'
p104
I00
sS'ndk_path'
p105
g23
sS'scandelete'
p106
(lp107
sS'buildjni'
p108
(lp109
S'no'
p110
asS'ndk'
p111
S'r10e'
p112
sS'target'
p113
NsS'type'
p114
S'raw'
p115
sS'antcommands'
p116
NsS'gradle'
p117
I00
sS'prebuild'
p118
S'sed -i \'s/"OsmAnd+"/"OsmAnd~"/g\' build.xml'
p119
sS'novcheck'
p120
p100
I00
sS'commit'
p121
p101
S'76ada6c8a08afe69acb755503373ac36328ef665'
p122
sa(dp123
S'submodules'
p124
I01
sg70
S'183'
p125
p102
sS'scandelete'
p103
(lp104
sS'buildjni'
p105
S'no'
p106
sS'ndk'
p107
NsS'target'
p108
NsS'gradleprops'
p109
(lp110
sS'patch'
p111
(lp112
sS'prebuild'
p113
S'sed -i \'s/"OsmAnd+"/"OsmAnd~"/g\' build.xml'
p114
sS'novcheck'
p115
I00
sS'output'
p116
S'bin/OsmAnd-release-unsigned.apk'
p117
sa(dp118
g70
g71
sg72
I00
sg73
I00
sg74
(lp126
sg76
g77
I00
sg75
Nsg76
(lp119
sg78
g79
sg80
g81
sS'output'
p127
S'bin/OsmAnd-release-unsigned.apk'
p128
I00
sg79
(lp120
sg81
Nsg82
(lp121
sg84
Nsg85
g86
g21
sg85
S'1.8.3'
p122
sg87
g23
sg88
S'1.8.3'
p129
sS'subdir'
p130
S'android/OsmAnd'
p131
sg92
g93
sg94
I00
sS'build'
p132
p123
sg89
(lp124
sg91
S'../../build'
p133
sg97
p125
sg93
S'183'
p126
sg95
I00
sg96
(lp127
sg98
g99
Nsg99
I00
sg100
g101
sg103
I00
sg104
I00
sg105
g23
sg106
g107
sS'buildjni'
p134
(lp135
S'no'
p136
asg111
g112
sg113
Nsg114
g115
sg116
Nsg117
I00
sS'prebuild'
p137
g23
sg120
I00
sS'commit'
p138
sg101
S'1.8.3'
p128
sg103
(lp129
sg105
S'no'
p130
sg107
Nsg108
Nsg109
(lp131
sg111
(lp132
sg113
g21
sg115
I00
sg116
S'bin/OsmAnd-release-unsigned.apk'
p133
sa(dp134
g70
g71
sg72
I00
sg73
I00
sg74
I00
sg75
Nsg76
(lp135
sg78
I00
sg79
(lp136
sg81
Nsg82
(lp137
sg84
g21
sg85
S'1.9.4'
p138
sg87
S'android/OsmAnd'
p139
sa(dp140
S'submodules'
sg89
(lp140
sg91
S'../../build'
p141
I01
sg70
sg93
S'196'
p142
sg72
sg95
I00
sg73
I00
sg74
sg96
(lp143
sg76
g77
sg78
g79
sg80
g81
sS'output'
p144
S'bin/OsmAnd-release-unsigned.apk'
p145
sg84
Nsg85
g86
sg87
g23
sg88
S'1.9.4'
p146
sS'subdir'
p147
S'android/OsmAnd'
p148
sg92
g93
sg94
I00
sS'build'
p149
S'../../build'
p150
sg97
I00
sg98
g99
Nsg99
I00
sg100
g101
sg103
I00
sg104
I00
sg105
g23
sg106
g107
sS'buildjni'
p151
(lp152
S'no'
p153
asS'ndk'
p154
S'r10d'
p155
sg113
Nsg114
g115
sg116
Nsg117
I00
sg137
g23
sg120
I00
sS'commit'
p156
sg101
S'1.9.4'
p157
sa(dp158
S'submodules'
p159
I01
sg70
S'197'
p160
p144
sg103
(lp145
sg105
S'no'
p146
sg107
S'r10d'
p147
sg108
Nsg109
(lp148
sg111
(lp149
sg113
g21
sg115
I00
sg116
S'bin/OsmAnd-release-unsigned.apk'
p150
sa(dp151
g70
g71
sg72
I00
sg73
I00
sg74
(lp161
sg76
g77
I00
sg75
Nsg76
(lp152
sg78
g79
sg80
g81
sS'output'
p162
S'bin/OsmAnd-release-unsigned.apk'
p163
I00
sg79
(lp153
sg81
Nsg82
(lp154
sg84
Nsg85
g86
g21
sg85
S'1.9.5'
p155
sg87
g23
sg88
S'1.9.5'
p164
sS'subdir'
p165
S'android/OsmAnd'
p166
sg92
g93
sg94
I00
sS'build'
p167
p156
sg89
(lp157
sg91
S'../../build'
p168
sg97
p158
sg93
S'197'
p159
sg95
I00
sg96
(lp160
sg98
g99
Nsg99
I00
sg100
g101
I00
sg101
S'1.9.5'
p161
sg103
I00
sg104
I00
(lp162
sg105
g23
sg106
g107
sS'buildjni'
p169
(lp170
S'no'
p171
asS'ndk'
p172
p163
sg107
S'r10d'
p173
p164
sg108
Nsg109
(lp165
sg111
(lp166
sg113
Nsg114
g115
g21
sg115
I00
sg116
Nsg117
I00
sg137
g23
sg120
I00
sS'commit'
p174
S'1.9.5'
p175
S'bin/OsmAnd-release-unsigned.apk'
p167
sasS'FlattrID'
p176
p168
NsS'metadatapath'
p177
p169
S'metadata/net.osmand.plus.xml'
p178
p170
sS'Disabled'
p179
NsS'added'
p180
NsS'lastupdated'
p181
NsS'Update Check Name'
p182
p171
NsS'Web Site'
p172
S'http://osmand.net'
p173
sS'Update Check Name'
p174
NsS'Vercode Operation'
p183
NsS'Current Version'
p184
S'1.9.5'
p185
s.
p175
NsS'Auto Update Mode'
p176
S'None'
p177
s.

3517
tests/metadata/org.adaway.pickle
View File

File diff suppressed because it is too large Load Diff

1128
tests/metadata/org.smssecure.smssecure.pickle
View File

File diff suppressed because it is too large Load Diff

9124
tests/metadata/org.videolan.vlc.pickle
View File

File diff suppressed because it is too large Load Diff

15
tests/metadata/update-pickle.py
View File

@ -1,15 +0,0 @@
#!/usr/bin/env python2
#
# This script is for updating the .pickle test files when there are changes to
# the default metadata, e.g. adding a new key/tag.
import glob
import pickle
for picklefile in glob.glob('*.pickle'):
p = pickle.load(open(picklefile))
for build in p['builds']:
build['gradleprops'] = []
pickle.dump(p, open(picklefile, 'w'))