From f84818c15e290248713b918072f20f8fe3af838d Mon Sep 17 00:00:00 2001 From: Marcus Date: Mon, 15 Jun 2020 18:03:19 +0000 Subject: [PATCH] scanner: add a simple scan for blacklisted classes after build step add com.android.billing to blacklist, see https://gitlab.com/fdroid/fdroiddata/-/issues/2070#note_360611289 --- completion/bash-completion | 2 +- examples/config.py | 3 +++ fdroidserver/build.py | 5 +++++ fdroidserver/common.py | 2 ++ fdroidserver/scanner.py | 31 ++++++++++++++++++++++++++++++- tests/build.TestCase | 2 ++ tests/scanner.TestCase | 2 ++ 7 files changed, 45 insertions(+), 2 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index 7e2a15da..5f21621e 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -83,7 +83,7 @@ __complete_options() { __complete_build() { opts="-v -q -l -s -t -f -a -w" - lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --no-tarball --force --all --wiki --no-refresh" + lopts="--verbose --quiet --latest --stop --test --server --reset-server --skip-scan --scan-binary --no-tarball --force --all --wiki --no-refresh" case "${prev}" in :) __vercode diff --git a/examples/config.py b/examples/config.py index e5735591..f01a2548 100644 --- a/examples/config.py +++ b/examples/config.py @@ -50,6 +50,9 @@ # Defaults to using an internal gradle wrapper (gradlew-fdroid). # gradle = "gradle" +# Always scan the APKs produced by `fdroid build` for known non-free classes +# scan_binary = True + # Set the maximum age (in days) of an index that a client should accept from # this repo. Setting it to 0 or not setting it at all disables this # functionality. If you do set this to a non-zero value, you need to ensure diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 922fe169..2799dc1b 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -806,6 +806,9 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext " Expected: '%s' / '%s'") % (version, str(vercode), build.versionName, str(build.versionCode))) + if (options.scan_binary or config.get('scan_binary')) and not options.skipscan: + if scanner.scan_binary(src): + raise BuildException("Found blacklisted packages in final apk!") # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... @@ -899,6 +902,8 @@ def parse_commandline(): help=argparse.SUPPRESS) parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False, help=_("Skip scanning the source code for binaries and other problems")) + parser.add_argument("--scan-binary", action="store_true", default=False, + help=_("Scan the resulting APK(s) for known non-free classes.")) parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False, help=_("Don't create a source tarball, useful when testing a build")) parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True, diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 4ce6d86a..48ebe899 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -114,6 +114,7 @@ default_config = { 'build_tools': MINIMUM_AAPT_VERSION, 'force_build_tools': False, 'java_paths': None, + 'scan_binary': False, 'ant': "ant", 'mvn3': "mvn", 'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'), @@ -432,6 +433,7 @@ def find_sdk_tools_cmd(cmd): sdk_tools = os.path.join(config['sdk_path'], 'tools') if os.path.exists(sdk_tools): tooldirs.append(sdk_tools) + tooldirs.append(os.path.join(sdk_tools, 'bin')) sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools') if os.path.exists(sdk_platform_tools): tooldirs.append(sdk_platform_tools) diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py index 9ba94c6c..47526105 100644 --- a/fdroidserver/scanner.py +++ b/fdroidserver/scanner.py @@ -58,6 +58,36 @@ def get_gradle_compile_commands(build): return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands] +def scan_binary(apkfile): + usual_suspects = { + # The `apkanalyzer dex packages` output looks like this: + # M d 1 1 93 + # The first column has P/C/M/F for package, class, methos or field + # The second column has x/k/r/d for removed, kept, referenced and defined. + # We already filter for defined only in the apkanalyzer call. 'r' will be + # for things referenced but not distributed in the apk. + exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [ + r'(com\.google\.firebase[^\s]*)', + r'(com\.google\.android\.gms[^\s]*)', + r'(com\.google\.tagmanager[^\s]*)', + r'(com\.google\.analytics[^\s]*)', + r'(com\.android\.billing[^\s]*)', + ] + } + logging.info("Scanning APK for known non-free classes.") + result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False) + problems = 0 + for suspect, regexp in usual_suspects.items(): + matches = regexp.findall(result.output) + if matches: + for m in set(matches): + logging.debug("Found class '%s'" % m) + problems += 1 + if problems: + logging.critical("Found problems in %s" % apkfile) + return problems + + def scan_source(build_dir, build=metadata.Build()): """Scan the source code in the given directory (and all subdirectories) and return the number of fatal problems encountered @@ -308,7 +338,6 @@ def scan_source(build_dir, build=metadata.Build()): def main(): - global config, options, json_per_build # Parse command line... diff --git a/tests/build.TestCase b/tests/build.TestCase index 0495bb79..183f6661 100755 --- a/tests/build.TestCase +++ b/tests/build.TestCase @@ -131,7 +131,9 @@ class BuildTest(unittest.TestCase): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config + fdroidserver.build.config = config fdroidserver.build.options = mock.Mock() + fdroidserver.build.options.scan_binary = False fdroidserver.build.options.notarball = True fdroidserver.build.options.skipscan = False diff --git a/tests/scanner.TestCase b/tests/scanner.TestCase index 67cef2fd..17684074 100755 --- a/tests/scanner.TestCase +++ b/tests/scanner.TestCase @@ -194,8 +194,10 @@ class ScannerTest(unittest.TestCase): config = dict() fdroidserver.common.fill_config_defaults(config) fdroidserver.common.config = config + fdroidserver.build.config = config fdroidserver.build.options = mock.Mock() fdroidserver.build.options.json = False + fdroidserver.build.options.scan_binary = False fdroidserver.build.options.notarball = True fdroidserver.build.options.skipscan = False fdroidserver.scanner.options = fdroidserver.build.options