From 89614851250c79a05db84070feca6dea033af334 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Fri, 31 Jan 2020 14:16:33 +0100 Subject: [PATCH] remove dscanner subcommand It's unused and unmaintained. It could potentially be revived as a plugin at a later point. --- MANIFEST.in | 7 - README.md | 16 -- completion/bash-completion | 16 -- docker/Dockerfile | 180 -------------- docker/Makefile | 48 ---- docker/README.md | 13 - docker/drozer.py | 35 --- docker/enable_service.py | 16 -- docker/entrypoint.sh | 42 ---- docker/install_agent.py | 63 ----- fdroidserver/__main__.py | 1 - fdroidserver/build.py | 39 --- fdroidserver/dscanner.py | 484 ------------------------------------- locale/POTFILES.in | 1 - setup.py | 1 - tests/main.TestCase | 1 - 16 files changed, 963 deletions(-) delete mode 100644 docker/Dockerfile delete mode 100644 docker/Makefile delete mode 100644 docker/README.md delete mode 100644 docker/drozer.py delete mode 100755 docker/enable_service.py delete mode 100755 docker/entrypoint.sh delete mode 100755 docker/install_agent.py delete mode 100644 fdroidserver/dscanner.py diff --git a/MANIFEST.in b/MANIFEST.in index 8be4dabf..04434d18 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,13 +8,6 @@ include buildserver/setup-env-vars include buildserver/Vagrantfile include CHANGELOG.md include completion/bash-completion -include docker/Dockerfile -include docker/drozer.py -include docker/enable_service.py -include docker/entrypoint.sh -include docker/install_agent.py -include docker/Makefile -include docker/README.md include examples/config.py include examples/fdroid-icon.png include examples/makebuildserver.config.py diff --git a/README.md b/README.md index 00ddb3bc..dac49d72 100644 --- a/README.md +++ b/README.md @@ -85,22 +85,6 @@ RAM. These test scripts are in the root of the project, all starting with _jenkins-_ since they are run on https://jenkins.debian.net. - -### Drozer Scanner - -There is a new feature under development that can scan any APK in a -repo, or any build, using Drozer. Drozer is a dynamic exploit -scanner, it runs an app in the emulator and runs known exploits on it. - -This setup requires specific versions of two Python modules: -_docker-py_ 1.9.0 and _requests_ older than 2.11. Other versions -might cause the docker-py connection to break with the containers. -Newer versions of docker-py might have this fixed already. - -For Debian based distributions: - - apt-get install libffi-dev libssl-dev python-docker - ## Translation Everything can be translated. See diff --git a/completion/bash-completion b/completion/bash-completion index d00a767a..780509db 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -104,22 +104,6 @@ __complete_build() { esac } -__complete_dscanner() { - opts="-v -q -l" - lopts="--verbose --quiet --clean-after --clean-before --clean-only --init-only --latest --repo-path" - case "${cur}" in - -*) - __complete_options - return 0;; - *:) - __vercode - return 0;; - *) - __package - return 0;; - esac -} - __complete_gpgsign() { opts="-v -q" lopts="--verbose --quiet" diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 9b7b4fb6..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,180 +0,0 @@ -# This image is intended to be used with fdroidserver for the purpose -# of dynamic scanning of pre-built APKs during the fdroid build process. - -# Start with ubuntu 12.04 (i386). -FROM ubuntu:14.04 -MAINTAINER fdroid.dscanner - -ENV DROZER_URL https://github.com/mwrlabs/drozer/releases/download/2.3.4/drozer_2.3.4.deb -ENV DROZER_DEB drozer_2.3.4.deb - -ENV AGENT_URL https://github.com/mwrlabs/drozer/releases/download/2.3.4/drozer-agent-2.3.4.apk -ENV AGENT_APK drozer-agent-2.3.4.apk - -# Specially for SSH access and port redirection -ENV ROOTPASSWORD android - -# Expose ADB, ADB control and VNC ports -EXPOSE 22 -EXPOSE 5037 -EXPOSE 5554 -EXPOSE 5555 -EXPOSE 5900 -EXPOSE 5901 - -ENV DEBIAN_FRONTEND noninteractive -RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections -RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections - -# Update packages -RUN apt-get -y update - -# Drozer packages -RUN apt-get install wget python2.7 python-dev python2.7-dev python-openssl python-twisted python-protobuf bash-completion -y - -# First, install add-apt-repository, sshd and bzip2 -RUN apt-get -y install python-software-properties bzip2 ssh net-tools - -# ubuntu 14.04 needs this too -RUN apt-get -y install software-properties-common - -# Add oracle-jdk7 to repositories -RUN add-apt-repository ppa:webupd8team/java - -# Make sure the package repository is up to date -RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list - -# Update apt -RUN apt-get update - -# Add drozer -RUN useradd -ms /bin/bash drozer - -# Install oracle-jdk7 -RUN apt-get -y install oracle-java7-installer - -# Install android sdk -RUN wget http://dl.google.com/android/android-sdk_r23-linux.tgz -RUN tar -xvzf android-sdk_r23-linux.tgz -RUN mv -v android-sdk-linux /usr/local/android-sdk - -# Install apache ant -RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.8.4-bin.tar.gz -RUN tar -xvzf apache-ant-1.8.4-bin.tar.gz -RUN mv -v apache-ant-1.8.4 /usr/local/apache-ant - -# Add android tools and platform tools to PATH -ENV ANDROID_HOME /usr/local/android-sdk -ENV PATH $PATH:$ANDROID_HOME/tools -ENV PATH $PATH:$ANDROID_HOME/platform-tools - -# Add ant to PATH -ENV ANT_HOME /usr/local/apache-ant -ENV PATH $PATH:$ANT_HOME/bin - -# Export JAVA_HOME variable -ENV JAVA_HOME /usr/lib/jvm/java-7-oracle - -# Remove compressed files. -RUN cd /; rm android-sdk_r23-linux.tgz && rm apache-ant-1.8.4-bin.tar.gz - -# Some preparation before update -RUN chown -R root:root /usr/local/android-sdk/ - -# Install latest android tools and system images -RUN echo "y" | android update sdk --filter platform-tool --no-ui --force -RUN echo "y" | android update sdk --filter platform --no-ui --force -RUN echo "y" | android update sdk --filter build-tools-22.0.1 --no-ui -a -RUN echo "y" | android update sdk --filter sys-img-x86-android-19 --no-ui -a -#RUN echo "y" | android update sdk --filter sys-img-x86-android-21 --no-ui -a -#RUN echo "y" | android update sdk --filter sys-img-x86-android-22 --no-ui -a -RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-19 --no-ui -a -#RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-21 --no-ui -a -#RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-22 --no-ui -a - -# Update ADB -RUN echo "y" | android update adb - -# Create fake keymap file -RUN mkdir /usr/local/android-sdk/tools/keymaps -RUN touch /usr/local/android-sdk/tools/keymaps/en-us - -# Run sshd -RUN apt-get install -y openssh-server -RUN mkdir /var/run/sshd -RUN echo "root:$ROOTPASSWORD" | chpasswd -RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -i 's/PermitEmptyPasswords no/PermitEmptyPasswords yes/' /etc/ssh/sshd_config - -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -ENV NOTVISIBLE "in users profile" -RUN echo "export VISIBLE=now" >> /etc/profile - -# Install socat -RUN apt-get install -y socat - -# symlink android bins -RUN ln -sv /usr/local/android-sdk/tools/android /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/tools/emulator /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/tools/ddms /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/tools/scheenshot2 /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/tools/monkeyrunner /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/tools/monitor /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/tools/mksdcard /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/tools/uiautomatorviewer /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/tools/traceview /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/platform-tools/adb /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/platform-tools/fastboot /usr/local/bin/ -RUN ln -sv /usr/local/android-sdk/platform-tools/sqlite3 /usr/local/bin/ - -# Setup DROZER... -# https://labs.mwrinfosecurity.com/tools/drozer/ - -# Run as drozer user -WORKDIR /home/drozer - -# Site lists the shasums, however, I'm not sure the best way to integrate the -# checks here. No real idiomatic way for Dockerfile to do that and most of -# the examples online use chained commands but we want things to *BREAK* when -# the sha doesn't match. So far, I can't seem to reliably make Docker not -# finish the image build process. - -# Download the console -RUN wget -c $DROZER_URL - -# Install the console -RUN dpkg -i $DROZER_DEB - -# Download agent -RUN wget -c $AGENT_URL -# Keep it version agnostic for other scripts such as install_drozer.py -RUN mv -v $AGENT_APK drozer-agent.apk - -# Port forwarding required by drozer -RUN echo 'adb forward tcp:31415 tcp:31415' >> /home/drozer/.bashrc - -# Alias for Drozer -RUN echo "alias drozer='drozer console connect'" >> /home/drozer/.bashrc - -# add extra scripting -COPY install_agent.py /home/drozer/install_agent.py -RUN chmod 755 /home/drozer/install_agent.py -COPY enable_service.py /home/drozer/enable_service.py -RUN chmod 755 /home/drozer/enable_service.py -COPY drozer.py /home/drozer/drozer.py -RUN chmod 755 /home/drozer/drozer.py - -# fix ownerships -RUN chown -R drozer.drozer /home/drozer - -RUN apt-get -y --force-yes install python-pkg-resources=3.3-1ubuntu1 -RUN apt-get -y install python-pip python-setuptools git -RUN pip install "git+https://github.com/dtmilano/AndroidViewClient.git#egg=androidviewclient" -RUN apt-get -y install python-pexpect - -# Add entrypoint -COPY entrypoint.sh /home/drozer/entrypoint.sh -RUN chmod +x /home/drozer/entrypoint.sh -ENTRYPOINT ["/home/drozer/entrypoint.sh"] diff --git a/docker/Makefile b/docker/Makefile deleted file mode 100644 index eacb3268..00000000 --- a/docker/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -SHELL := /bin/bash -ALIAS = "dscanner" -EXISTS := $(shell docker ps -a -q -f name=$(ALIAS)) -RUNNED := $(shell docker ps -q -f name=$(ALIAS)) -ifneq "$(RUNNED)" "" -IP := $(shell docker inspect $(ALIAS) | grep "IPAddress\"" | head -n1 | cut -d '"' -f 4) -endif -STALE_IMAGES := $(shell docker images | grep "" | awk '{print($$3)}') -EMULATOR ?= "android-19" -ARCH ?= "armeabi-v7a" - -COLON := : - -.PHONY = build clean kill info - -all: help - -help: - @echo "usage: make {help|build|clean|kill|info}" - @echo "" - @echo " help this help screen" - @echo " build create docker image" - @echo " clean remove images and containers" - @echo " kill stop running containers" - @echo " info details of running container" - -build: - @docker build -t "dscanner/fdroidserver:latest" . - -clean: kill - @docker ps -a -q | xargs -n 1 -I {} docker rm -f {} -ifneq "$(STALE_IMAGES)" "" - @docker rmi -f $(STALE_IMAGES) -endif - -kill: -ifneq "$(RUNNED)" "" - @docker kill $(ALIAS) -endif - -info: - @docker ps -a -f name=$(ALIAS) -ifneq "$(RUNNED)" "" - $(eval ADBPORT := $(shell docker port $(ALIAS) | grep '5555/tcp' | awk '{split($$3,a,"$(COLON)");print a[2]}')) - @echo -e "Use:\n adb kill-server\n adb connect $(IP):$(ADBPORT)" -else - @echo "Run container" -endif diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index 9f2d657c..00000000 --- a/docker/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# dscanner docker image # - -Use `make help` for up-to-date instructions. - -``` -usage: make {help|build|clean|kill|info} - - help this help screen - build create docker image - clean remove images and containers - kill stop running containers - info details of running container -``` diff --git a/docker/drozer.py b/docker/drozer.py deleted file mode 100644 index d0546934..00000000 --- a/docker/drozer.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python2 - -import pexpect -import sys - -prompt = "dz>" -target = sys.argv[1] - -drozer = pexpect.spawn("drozer console connect") -drozer.logfile = open("/tmp/drozer_report.log", "w") - - -# start -drozer.expect(prompt) - - -def send_command(command, target): - cmd = "run {0} -a {1}".format(command, target) - drozer.sendline(cmd) - drozer.expect(prompt) - -scanners = [ - "scanner.misc.native", # Find native components included in packages - #"scanner.misc.readablefiles", # Find world-readable files in the given folder - #"scanner.misc.secretcodes", # Search for secret codes that can be used from the dialer - #"scanner.misc.sflagbinaries", # Find suid/sgid binaries in the given folder (default is /system). - #"scanner.misc.writablefiles", # Find world-writable files in the given folder - "scanner.provider.finduris", # Search for content providers that can be queried. - "scanner.provider.injection", # Test content providers for SQL injection vulnerabilities. - "scanner.provider.sqltables", # Find tables accessible through SQL injection vulnerabilities. - "scanner.provider.traversal" # Test content providers for basic directory traversal -] - -for scanner in scanners: - send_command(scanner, target) diff --git a/docker/enable_service.py b/docker/enable_service.py deleted file mode 100755 index 803532c9..00000000 --- a/docker/enable_service.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python2 - -from com.dtmilano.android.viewclient import ViewClient - -vc = ViewClient(*ViewClient.connectToDeviceOrExit()) - -button = vc.findViewWithText("OFF") - -if button: - (x, y) = button.getXY() - button.touch() -else: - print("Button not found. Is the app currently running?") - exit() - -print("Done!") diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index 95b5ede1..00000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -if [[ $EMULATOR == "" ]]; then - EMULATOR="android-19" - echo "Using default emulator $EMULATOR" -fi - -if [[ $ARCH == "" ]]; then - ARCH="x86" - echo "Using default arch $ARCH" -fi -echo EMULATOR = "Requested API: ${EMULATOR} (${ARCH}) emulator." -if [[ -n $1 ]]; then - echo "Last line of file specified as non-opt/last argument:" - tail -1 $1 -fi - -# Run sshd -/usr/sbin/sshd -adb start-server - -# Detect ip and forward ADB ports outside to outside interface -ip=$(ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}') -socat tcp-listen:5037,bind=$ip,fork tcp:127.0.0.1:5037 & -socat tcp-listen:5554,bind=$ip,fork tcp:127.0.0.1:5554 & -socat tcp-listen:5555,bind=$ip,fork tcp:127.0.0.1:5555 & - -# Set up and run emulator -if [[ $ARCH == *"x86"* ]] -then - EMU="x86" -else - EMU="arm" -fi - -#FASTDROID_VNC_URL="https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/fastdroid-vnc/fastdroid-vnc" -#wget -c "${FASTDROID_VNC_URL}" - -export PATH="${PATH}:/usr/local/android-sdk/tools/:/usr/local/android-sdk/platform-tools/" - -echo "no" | android create avd -f -n test -t ${EMULATOR} --abi default/${ARCH} -echo "no" | emulator64-${EMU} -avd test -noaudio -no-window -gpu off -verbose -qemu -usbdevice tablet -vnc :0 diff --git a/docker/install_agent.py b/docker/install_agent.py deleted file mode 100755 index 1a0f348a..00000000 --- a/docker/install_agent.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python2 - -import os -from subprocess import call, check_output -from time import sleep - -FNULL = open(os.devnull, 'w') - -print("Ensuring device is online") -call("adb wait-for-device", shell=True) - -print("Installing the drozer agent") -print("If the device just came online it is likely the package manager hasn't booted.") -print("Will try multiple attempts to install.") -print("This may need tweaking depending on hardware.") - - -attempts = 0 -time_to_sleep = 30 - -while attempts < 8: - output = check_output('adb shell "pm list packages"', shell=True) - print("Checking whether the package manager is up...") - if "Could not access the Package Manager" in output: - print("Nope. Sleeping for 30 seconds and then trying again.") - sleep(time_to_sleep) - else: - break - -time_to_sleep = 5 -attempts = 0 - -while attempts < 5: - sleep(time_to_sleep) - try: - install_output = check_output("adb install /home/drozer/drozer-agent.apk", shell=True) - except Exception: - print("Failed. Trying again.") - attempts += 1 - else: - attempts += 1 - if "Error: Could not access the Package Manager" not in install_output: - break - -print("Install attempted. Checking everything worked") - -pm_list_output = check_output('adb shell "pm list packages"', shell=True) - -if "com.mwr.dz" not in pm_list_output: - print(install_output) - exit("APK didn't install properly. Exiting.") - -print("Installed ok.") - -print("Starting the drozer agent main activity: com.mwr.dz/.activities.MainActivity") -call('adb shell "am start com.mwr.dz/.activities.MainActivity"', shell=True, stdout=FNULL) - -print("Starting the service") -# start the service -call("python /home/drozer/enable_service.py", shell=True, stdout=FNULL) - -print("Forward dem ports mon.") -call("adb forward tcp:31415 tcp:31415", shell=True, stdout=FNULL) diff --git a/fdroidserver/__main__.py b/fdroidserver/__main__.py index 953e3bd3..a75236b9 100755 --- a/fdroidserver/__main__.py +++ b/fdroidserver/__main__.py @@ -44,7 +44,6 @@ commands = OrderedDict([ ("rewritemeta", _("Rewrite all the metadata files")), ("lint", _("Warn about possible metadata errors")), ("scanner", _("Scan the source code of a package")), - ("dscanner", _("Dynamically scan APKs post build")), ("stats", _("Update the stats of the repo")), ("server", _("Old, deprecated name for fdroid deploy")), ("signindex", _("Sign indexes created using update --nosign")), diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 0bc08745..a4a1f553 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -882,8 +882,6 @@ def parse_commandline(): help=argparse.SUPPRESS) parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False, help=_("Skip scanning the source code for binaries and other problems")) - parser.add_argument("--dscanner", action="store_true", default=False, - help=_("Setup an emulator, install the APK on it and perform a Drozer scan")) parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False, help=_("Don't create a source tarball, useful when testing a build")) parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True, @@ -1216,43 +1214,6 @@ def main(): for fa in failed_apps: logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa])) - # perform a drozer scan of all successful builds - if options.dscanner and build_succeeded: - from .dscanner import DockerDriver - - docker = DockerDriver() - - try: - for app in build_succeeded: - - logging.info("Need to sign the app before we can install it.") - subprocess.call("fdroid publish {0}".format(app.id)) - - apk_path = None - - for f in os.listdir(repo_dir): - if f.endswith('.apk') and f.startswith(app.id): - apk_path = os.path.join(repo_dir, f) - break - - if not apk_path: - raise Exception("No signed APK found at path: {path}".format(path=apk_path)) - - if not os.path.isdir(repo_dir): - logging.critical("directory does not exists '{path}'".format(path=repo_dir)) - common.force_exit(1) - - logging.info("Performing Drozer scan on {0}.".format(app)) - docker.perform_drozer_scan(apk_path, app.id, repo_dir) - except Exception as e: - logging.error(str(e)) - logging.error("An exception happened. Making sure to clean up") - else: - logging.info("Scan succeeded.") - - logging.info("Cleaning up after ourselves.") - docker.clean() - logging.info(_("Finished")) if len(build_succeeded) > 0: logging.info(ngettext("{} build succeeded", diff --git a/fdroidserver/dscanner.py b/fdroidserver/dscanner.py deleted file mode 100644 index 49362e5f..00000000 --- a/fdroidserver/dscanner.py +++ /dev/null @@ -1,484 +0,0 @@ -#!/usr/bin/env python3 -# -# dscanner.py - part of the FDroid server tools -# Copyright (C) 2016-2017 Shawn Gustaw -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import logging -import os -import json -import sys -from time import sleep -from argparse import ArgumentParser -from subprocess import CalledProcessError, check_output - -from . import _ -from . import common -from . import metadata - -try: - from docker import Client -except ImportError: - logging.error(("Docker client not installed." - "Install it using pip install docker-py")) - -config = None -options = None - - -class DockerConfig: - ALIAS = "dscanner" - CONTAINER = "dscanner/fdroidserver" - EMULATOR = "android-19" - ARCH = "armeabi-v7a" - - -class DockerDriver(object): - """ - Handles all the interactions with the docker container the - Android emulator runs in. - """ - class Commands: - build = ['docker', 'build', '--no-cache=false', '--pull=true', - '--quiet=false', '--rm=true', '-t', - '{0}:latest'.format(DockerConfig.CONTAINER), '.'] - run = [ - 'docker', 'run', - '-e', '"EMULATOR={0}"'.format(DockerConfig.EMULATOR), - '-e', '"ARCH={0}"'.format(DockerConfig.ARCH), - '-d', '-P', '--name', - '{0}'.format(DockerConfig.ALIAS), '--log-driver=json-file', - DockerConfig.CONTAINER] - start = ['docker', 'start', '{0}'.format(DockerConfig.ALIAS)] - inspect = ['docker', 'inspect', '{0}'.format(DockerConfig.ALIAS)] - pm_list = 'adb shell "pm list packages"' - install_drozer = "docker exec {0} python /home/drozer/install_agent.py" - run_drozer = 'python /home/drozer/drozer.py {0}' - copy_to_container = 'docker cp "{0}" {1}:{2}' - copy_from_container = 'docker cp {0}:{1} "{2}"' - - def __init__(self, init_only=False, fresh_start=False, clean_only=False): - self.container_id = None - self.ip_address = None - - self.cli = Client(base_url='unix://var/run/docker.sock') - - if fresh_start or clean_only: - self.clean() - - if clean_only: - logging.info("Cleaned containers and quitting.") - exit(0) - - self.init_docker() - - if init_only: - logging.info("Initialized and quitting.") - exit(0) - - def _copy_to_container(self, src_path, dest_path): - """ - Copies a file (presumed to be an apk) from src_path - to home directory on container. - """ - path = '/home/drozer/{path}.apk'.format(path=dest_path) - command = self.Commands.copy_to_container.format(src_path, - self.container_id, - path) - - try: - check_output(command, shell=True) - except CalledProcessError as e: - logging.error(('Command "{command}" failed with ' - 'error code {code}'.format(command=command, - code=e.returncode))) - raise - - def _copy_from_container(self, src_path, dest_path): - """ - Copies a file from src_path on the container to - dest_path on the host machine. - """ - command = self.Commands.copy_from_container.format(self.container_id, - src_path, - dest_path) - try: - check_output(command, shell=True) - except CalledProcessError as e: - logging.error(('Command "{command}" failed with ' - 'error code {code}'.format(command=command, - code=e.returncode))) - raise - - logging.info("Log stored at {path}".format(path=dest_path)) - - def _adb_install_apk(self, apk_path): - """ - Installs an apk on the device running in the container - using adb. - """ - logging.info("Attempting to install an apk.") - exec_id = self.cli.exec_create( - self.container_id, 'adb install {0}' - .format(apk_path) - )['Id'] - output = self.cli.exec_start(exec_id).decode('utf-8') - - if "INSTALL_PARSE_FAILED_NO_CERTIFICATES" in output: - raise Exception('Install parse failed, no certificates') - elif "INSTALL_FAILED_ALREADY_EXISTS" in output: - logging.info("APK already installed. Skipping.") - elif "Success" not in output: - logging.error("APK didn't install properly") - return False - return True - - def _adb_uninstall_apk(self, app_id): - """ - Uninstalls an application from the device running in the container - via its app_id. - """ - logging.info( - "Uninstalling {app_id} from the emulator." - .format(app_id=app_id) - ) - exec_id = self.cli.exec_create( - self.container_id, - 'adb uninstall {0}'.format(app_id) - )['Id'] - output = self.cli.exec_start(exec_id).decode('utf-8') - - if 'Success' in output: - logging.info("Successfully uninstalled.") - - return True - - def _verify_apk_install(self, app_id): - """ - Checks that the app_id is installed on the device running in the - container. - """ - logging.info( - "Verifying {app} is installed on the device." - .format(app=app_id) - ) - exec_id = self.cli.exec_create( - self.container_id, self.Commands.pm_list - )['Id'] - output = self.cli.exec_start(exec_id).decode('utf-8') - - if ("Could not access the Package Manager" in output - or "device offline" in output): - logging.info("Device or package manager isn't up") - - if app_id.split('_')[0] in output: # TODO: this is a temporary fix - logging.info("{app} is installed.".format(app=app_id)) - return True - - logging.error("APK not found in packages list on emulator.") - - def _delete_file(self, path): - """ - Deletes file off the container to preserve space if scanning many apps - """ - command = "rm {path}".format(path=path) - exec_id = self.cli.exec_create(self.container_id, command)['Id'] - logging.info("Deleting {path} on the container.".format(path=path)) - self.cli.exec_start(exec_id) - - def _install_apk(self, apk_path, app_id): - """ - Installs apk found at apk_path on the emulator. Will then - verify it installed properly by looking up its app_id in - the package manager. - """ - if not all([self.container_id, self.ip_address]): - # TODO: maybe have this fail nicely - raise Exception("Went to install apk and couldn't find container") - - path = "/home/drozer/{app_id}.apk".format(app_id=app_id) - self._copy_to_container(apk_path, app_id) - self._adb_install_apk(path) - self._verify_apk_install(app_id) - self._delete_file(path) - - def _install_drozer(self): - """ - Performs all the initialization of drozer within the emulator. - """ - logging.info("Attempting to install com.mwr.dz on the emulator") - logging.info("This could take a while so be patient...") - logging.info(("We need to wait for the device to boot AND" - " the package manager to come online.")) - command = self.Commands.install_drozer.format(self.container_id) - try: - output = check_output(command, - shell=True).decode('utf-8') - except CalledProcessError as e: - logging.error(('Command "{command}" failed with ' - 'error code {code}'.format(command=command, - code=e.returncode))) - raise - - if 'Installed ok' in output: - return True - - def _run_drozer_scan(self, app): - """ - Runs the drozer agent which connects to the app running - on the emulator. - """ - logging.info("Running the drozer agent") - exec_id = self.cli.exec_create( - self.container_id, - self.Commands.run_drozer.format(app) - )['Id'] - self.cli.exec_start(exec_id) - - def _container_is_running(self): - """ - Checks whether the emulator container is running. - """ - for container in self.cli.containers(): - if DockerConfig.ALIAS in container['Image']: - return True - - def _docker_image_exists(self): - """ - Check whether the docker image exists already. - If this returns false we'll need to build the image - from the DockerFile. - """ - for image in self.cli.images(): - for tag in image['RepoTags']: - if DockerConfig.ALIAS in tag: - return True - - _image_queue = {} - - def _build_docker_image(self): - """ - Builds the docker container so we can run the android emulator - inside it. - """ - logging.info("Pulling the container from docker hub") - logging.info("Image is roughly 5 GB so be patient") - - logging.info("(Progress output is slow and requires a tty.)") - # we pause briefly to narrow race condition windows of opportunity - sleep(1) - - is_a_tty = os.isatty(sys.stdout.fileno()) - - for output in self.cli.pull( - DockerConfig.CONTAINER, - stream=True, - tag="latest"): - if not is_a_tty: - # run silent, run quick - continue - try: - p = json.loads(output.decode('utf-8')) - p_id = p['id'] - self._image_queue[p_id] = p - t, c, j = 1, 1, 0 - for k in sorted(self._image_queue): - j += 1 - v = self._image_queue[k] - vd = v['progressDetail'] - t += vd['total'] - c += vd['current'] - msg = "\rDownloading: {0}/{1} {2}% [{3} jobs]" - msg = msg.format(c, t, int(c / t * 100), j) - sys.stdout.write(msg) - sys.stdout.flush() - except Exception: - pass - print("\nDONE!\n") - - def _verify_apk_exists(self, full_apk_path): - """ - Verifies that the apk path we have is actually a file. - """ - return os.path.isfile(full_apk_path) - - def init_docker(self): - """ - Perform all the initialization required before a drozer scan. - 1. build the image - 2. run the container - 3. install drozer and enable the service within the app - """ - built = self._docker_image_exists() - - if not built: - self._build_docker_image() - - running = self._container_is_running() - - if not running: - logging.info('Trying to run container...') - try: - check_output(self.Commands.run) - except CalledProcessError as e: - logging.error(( - 'Command "{command}" failed with error code {code}' - .format(command=self.Commands.run, code=e.returncode) - )) - running = self._container_is_running() - - if not running: - logging.info('Trying to start container...') - try: - check_output(self.Commands.start) - except CalledProcessError as e: - logging.error(( - 'Command "{command}" failed with error code {code}' - .format(command=self.Commands.run, code=e.returncode) - )) - running = self._container_is_running() - - if not running: - raise Exception("Running container not found, critical error.") - - containers = self.cli.containers() - - for container in containers: - if DockerConfig.ALIAS in container['Image']: - self.container_id = container['Id'] - n = container['NetworkSettings']['Networks'] - self.ip_address = n['bridge']['IPAddress'] - break - - if not self.container_id or not self.ip_address: - logging.error("No ip address or container id found.") - exit(1) - - if self._verify_apk_install('com.mwr.dz'): - return - - self._install_drozer() - - def clean(self): - """ - Clean up all the containers made by this script. - Should be run after the drozer scan completes. - """ - for container in self.cli.containers(): - if DockerConfig.ALIAS in container['Image']: - logging.info("Removing container {0}".format(container['Id'])) - self.cli.remove_container(container['Id'], force=True) - - def perform_drozer_scan(self, apk_path, app_id): - """ - Entrypoint for scanning an android app. Performs the following steps: - 1. installs an apk on the device - 2. runs a drozer scan - 3. copies the report off the container - 4. uninstalls the apk to save space on the device - """ - self._install_apk(apk_path, app_id) - logging.info("Running the drozer scan.") - self._run_drozer_scan(app_id) - logging.info("Scan finished. Moving the report off the container") - dest = apk_path + '.drozer' - self._copy_from_container('/tmp/drozer_report.log', dest) - self._adb_uninstall_apk(app_id) - - -def main(): - global config, options - - # Parse command line... - parser = ArgumentParser( - usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]" - ) - common.setup_global_opts(parser) - - parser.add_argument( - "app_id", nargs='*', - help=_("applicationId with optional versionCode in the form APPID[:VERCODE]")) - parser.add_argument( - "-l", "--latest", action="store_true", default=False, - help=_("Scan only the latest version of each package")) - parser.add_argument( - "--clean-after", default=False, action='store_true', - help=_("Clean after all scans have finished")) - parser.add_argument( - "--clean-before", default=False, action='store_true', - help=_("Clean before the scans start and rebuild the container")) - parser.add_argument( - "--clean-only", default=False, action='store_true', - help=_("Clean up all containers and then exit")) - parser.add_argument( - "--init-only", default=False, action='store_true', - help=_("Prepare Drozer to run a scan")) - parser.add_argument( - "--repo-path", default="repo", action="store", - help=_("Override path for repo APKs (default: ./repo)")) - - options = parser.parse_args() - config = common.read_config(options) - - if not os.path.isdir(options.repo_path): - sys.stderr.write("repo-path not found: \"" + options.repo_path + "\"") - exit(1) - - # Read all app and srclib metadata - allapps = metadata.read_metadata() - apps = common.read_app_args(options.app_id, allapps, True) - - docker = DockerDriver( - init_only=options.init_only, - fresh_start=options.clean_before, - clean_only=options.clean_only - ) - - if options.clean_before: - docker.clean() - - if options.clean_only: - exit(0) - - for app_id, app in apps.items(): - vercode = 0 - if ':' in app_id: - vercode = app_id.split(':')[1] - for build in reversed(app.builds): - if build.disable: - continue - if options.latest or vercode == 0 or build.versionCode == vercode: - app.builds = [build] - break - continue - continue - - for app_id, app in apps.items(): - for build in app.builds: - apks = [] - for f in os.listdir(options.repo_path): - n = common.get_release_filename(app, build) - if f == n: - apks.append(f) - for apk in sorted(apks): - apk_path = os.path.join(options.repo_path, apk) - docker.perform_drozer_scan(apk_path, app.id) - - if options.clean_after: - docker.clean() - - -if __name__ == "__main__": - main() diff --git a/locale/POTFILES.in b/locale/POTFILES.in index 45480db2..7d31bec0 100644 --- a/locale/POTFILES.in +++ b/locale/POTFILES.in @@ -3,7 +3,6 @@ fdroidserver/btlog.py fdroidserver/build.py fdroidserver/checkupdates.py fdroidserver/common.py -fdroidserver/dscanner.py fdroidserver/import.py fdroidserver/init.py fdroidserver/install.py diff --git a/setup.py b/setup.py index f38759dc..40abdb19 100755 --- a/setup.py +++ b/setup.py @@ -86,7 +86,6 @@ setup(name='fdroidserver', 'qrcode', 'ruamel.yaml >= 0.15', 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', - 'docker-py >= 1.9, < 2.0', ], classifiers=[ 'Development Status :: 4 - Beta', diff --git a/tests/main.TestCase b/tests/main.TestCase index cceafcf4..8621cfcc 100755 --- a/tests/main.TestCase +++ b/tests/main.TestCase @@ -37,7 +37,6 @@ class FdroidTest(unittest.TestCase): 'rewritemeta', 'lint', 'scanner', - 'dscanner', 'stats', 'server', 'signindex',