fdroid-server/.gitlab-ci.yml

variables:
  pip: pip3 --timeout 100 --retries 10
# speed up git checkout phase
  GIT_DEPTH: 1


test:
  image: registry.gitlab.com/fdroid/ci-images-base
  script:
    - $pip install -e .[test]
    # the `fdroid build` test in tests/run-tests needs android-23
    - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platforms;android-23"
    - cd tests
    - ./complete-ci-tests

# Test that the parsing of the .yml metadata format didn't change from last
# released version. This uses the commit ID of the release tags,
# rather than the release tag itself so that contributor forks do not
# need to include the tags in them for this test to work.
#
# The COMMIT_ID should be bumped after each release, so that the list
# of sed hacks needed does not continuously grow.
metadata_v0:
  image: registry.gitlab.com/fdroid/ci-images-base
  variables:
    GIT_DEPTH: 1000
    RELEASE_COMMIT_ID: 37f37ebd88e79ebe93239b72ed5503d5bde13f4b  # 2.0a~
  script:
    - git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID
    - cd tests
    - export GITCOMMIT=`git describe`
    - git checkout $RELEASE_COMMIT_ID
    - cd ..
    - git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git
    - cd fdroiddata
    - ../tests/dump_internal_metadata_format.py
    - cd ..
    - git reset --hard
    - git checkout $GITCOMMIT
    - cd fdroiddata
    - ../tests/dump_internal_metadata_format.py
    - sed -i
          -e '/Liberapay:/d'
          -e '/OpenCollective/d'
          metadata/dump_*/*.yaml
    - diff -uw metadata/dump_*

.apt-template: &apt-template
  variables:
    DEBIAN_FRONTEND: noninteractive
    LANG: C.UTF-8
  before_script:
    - echo Etc/UTC > /etc/timezone
    - echo 'APT::Install-Recommends "0";'
           'APT::Install-Suggests "0";'
           'APT::Acquire::Retries "20";'
           'APT::Get::Assume-Yes "true";'
           'Dpkg::Use-Pty "0";'
           'quiet "1";'
        >> /etc/apt/apt.conf.d/99gitlab
    - apt-get update
    - apt-get dist-upgrade

debian_testing:
  image: debian:testing
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - apt-get install
        aapt
        androguard
        apksigner
        fdroidserver
        git
        gnupg
        python3-defusedxml
        python3-setuptools
        zipalign
    - python3 -c 'import fdroidserver'
    - python3 -c 'import androguard'
    - cd tests
    - ./run-tests

# Test using latest LTS set up with the PPA, including Recommends.
# bionic's apksigner, which comes from Recommends:, requires binfmt
# support in the kernel.
ubuntu_lts_ppa:
  image: ubuntu:latest
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - export ANDROID_HOME=/usr/lib/android-sdk
    - apt-get install gnupg
    - while ! apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done
    - export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1`
    - echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
    - apt-get update
    - apt-get dist-upgrade
    - mount | grep binfmt_misc || mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc
    - apt-get install --install-recommends binfmt-support fdroidserver git python3-defusedxml python3-setuptools
    - ls -l /proc/sys/fs/binfmt_misc || true
    - test -e /proc/sys/fs/binfmt_misc/jarwrapper || apt -qy purge apksigner
    - cd tests
    - ./run-tests

# Test using Xenial LTS with all depends from pypi.  The venv is used
# to isolate the dist tarball generation environment from the clean
# install environment.  Xenial's pip is too old to install all the
# dependencies, so this has to uppgrade pip and setuptools in order to
# run the install.
ubuntu_xenial_pip:
  image: ubuntu:xenial
  <<: *apt-template
  script:
    - apt-get install git default-jdk-headless python3-pip python3-venv rsync zipalign libarchive13
    - rm -rf env
    - pyvenv env
    - . env/bin/activate
    - $pip install --upgrade babel pip setuptools
    # setup venv to act as release build machine
    - python -m venv sdist-env
    - . sdist-env/bin/activate
    - ./setup.py compile_catalog sdist
    - deactivate
    - tar tzf dist/fdroidserver-*.tar.gz | grep locale/de/LC_MESSAGES/fdroidserver.mo
    # back to bare machine to act as user's install machine
    - $pip install --upgrade pip setuptools
    - $pip install dist/fdroidserver-*.tar.gz
    - test -e /usr/share/locale/de/LC_MESSAGES/fdroidserver.mo
    - ./tests/run-tests

# test install process on a bleeding edge distro with pip
arch_pip_install:
  image: archlinux/base
  only:
    - master@fdroid/fdroidserver
  script:
    - pacman --sync --sysupgrade --refresh --noconfirm git grep python-pip python-virtualenv tar
    - pip install -e .
    - fdroid
    - fdroid readmeta
    - fdroid update --help

lint_format_safety_bandit_checks:
  image: alpine:3.10 # cannot upgrade until bandit supports Python 3.8
  variables:
    LANG: C.UTF-8
  script:
    - apk add --no-cache bash build-base dash ca-certificates gcc python3 python3-dev
    - python3 -m ensurepip
    - $pip install Babel 'bandit<1.6.0' pycodestyle pyflakes pylint safety
    - export EXITVALUE=0
    - function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
    - ./hooks/pre-commit || set_error
    - ./tests/test-gradlew-fdroid || set_error
    - bandit
        -ii
        -s B110,B322,B404,B408,B410,B603,B607
        -r $CI_PROJECT_DIR fdroid
        || set_error
    - safety check --full-report || set_error
    - pylint --rcfile=.pylint-rcfile --output-format=colorized --reports=n
            fdroid
            makebuildserver
            setup.py
            fdroidserver/*.py
            tests/*.py
            tests/*.TestCase
        || set_error
    - apk add --no-cache gettext make
    - make -C locale compile || set_error
    - rm -f locale/*/*/*.mo
    - pybabel compile --domain=fdroidserver --directory locale 2>&1 | (grep -F "error:" && exit 1) || true
    - exit $EXITVALUE

fedora_latest:
  image: fedora:latest
  only:
    - master@fdroid/fdroidserver
  script:
    # tricks to hopefully make runs more reliable
    - echo "timeout=600" >> /etc/dnf/dnf.conf
    - echo "retries=50" >> /etc/dnf/dnf.conf
    - echo "keepcache=True" >> /etc/dnf/dnf.conf

    - dnf -y update || dnf -y update
    - dnf -y install @development-tools
                     diffutils
                     findutils
                     git
                     gnupg
                     java-1.8.0-openjdk-devel
                     python3
                     python3-babel
                     python3-matplotlib
                     python3-pip
                     rsync
                     unzip
                     wget
                     which
    - ./setup.py compile_catalog sdist
    - useradd -m -c "test account" --password "fakepassword"  testuser
    - su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz"
    - test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
    - wget --no-verbose -O tools.zip https://dl.google.com/android/repository/tools_r25.2.5-linux.zip
    - unzip -q tools.zip
    - rm tools.zip
    - export BUILD_TOOLS_VERSION=`sed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py`
    - export JAVA_HOME=/etc/alternatives/jre
    - export ANDROID_HOME=`pwd`/android-sdk
    - mkdir $ANDROID_HOME
    - mv tools $ANDROID_HOME/
    - mkdir -p $ANDROID_HOME/licenses/
    - printf "\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e\n24333f8a63b6825ea9c5514f83c2829b004d1fee" > $ANDROID_HOME/licenses/android-sdk-license
    - printf "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
    - printf "\n79120722343a6f314e0719f863036c702b0e6b2a\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license-old
    - mkdir ~/.android
    - touch ~/.android/repositories.cfg
    - echo y | $ANDROID_HOME/tools/bin/sdkmanager "platform-tools"
    - echo y | $ANDROID_HOME/tools/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION"
    - chown -R testuser .
    - cd tests
    - su testuser --login --command
        "cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"

gradle:
  image: alpine:3.7
  variables:
    LANG: C.UTF-8
  script:
    - apk add --no-cache ca-certificates git python3
    # if this is a merge request fork, then only check if makebuildserver changed
    - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
        git fetch https://gitlab.com/fdroid/fdroidserver.git;
        for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
           test "$f" == "makebuildserver" && export CHANGED="yes";
        done;
        test -z "$CHANGED" && exit;
      fi
    - python3 -m ensurepip
    - $pip install beautifulsoup4 requests
    - ./tests/gradle-release-checksums.py

fdroid build:
  image: registry.gitlab.com/fdroid/ci-images-client
  only:
    refs:
      - branches
      - pipelines
    changes:
      - .gitlab-ci.yml
      - buildserver/provision-apt-get-install
      - fdroidserver/build.py
      - fdroidserver/common.py
      - fdroidserver/exception.py
      - fdroidserver/metadata.py
      - fdroidserver/net.py
      - fdroidserver/scanner.py
      - fdroidserver/vmtools.py
  cache:
    key: "$CI_JOB_NAME"
    paths:
      - .gradle
  script:
    - bash buildserver/provision-apt-get-install http://deb.debian.org/debian
    - apt-get dist-upgrade
    - apt-get install -t stretch-backports
         python3-asn1crypto
         python3-pip
         python3-ruamel.yaml
         python3-setuptools
         python3-venv
    - apt-get purge fdroidserver
    - pyvenv env --system-site-packages
    - . env/bin/activate
    - $pip install -e .
    - export PATH="$CI_PROJECT_DIR:$PATH"
    - export PYTHONPATH=$CI_PROJECT_DIR
    - export PYTHONUNBUFFERED=true

    - git clone https://gitlab.com/fdroid/fdroiddata.git --depth 1
    - cd fdroiddata
    - test -d build || mkdir build

    - export GRADLE_USER_HOME=$CI_PROJECT_DIR/.gradle
    # try a user build first
    - fdroid build --verbose --latest org.fdroid.fdroid.privileged
    # each `fdroid build --on-server` run expects sudo, then uninstalls it
    - apt-get install sudo
    - fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid


# test a full update and deploy cycle to gitlab.com
servergitmirrors:
  image: debian
  <<: *apt-template
  only:
    - master@fdroid/fdroidserver
  script:
    - apt-get install
        default-jdk-headless
        git
        openssh-client
        openssl
        python3-pip
        python3-venv
        rsync
        wget
    - python3 -m venv env
    - . env/bin/activate
    - export PYTHONPATH=`pwd`
    - $pip install -e .
    - mkdir /root/.ssh/
    - ./tests/key-tricks.py
    - ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
    - test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
    - cp tests/config.py tests/keystore.jks /tmp/fdroid/
    - cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
    - cd /tmp/fdroid
    - touch fdroid-icon.png
    - printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
    - $PYTHONPATH/fdroid update --verbose --create-metadata
    - $PYTHONPATH/fdroid deploy --verbose
    - export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
    - echo $DLURL
    - wget $DLURL/index-v1.jar
    - diff repo/index-v1.jar index-v1.jar