opensourcemirror
/
fdroid-data
mirror of https://gitlab.com/fdroid/fdroiddata.git
1
0
Fork 0
Code Issues Releases Wiki Activity
fdroid-data/.gitlab-ci.yml

579 lines
20 KiB
YAML
Raw Permalink Blame History

include:
- template: Security/SAST.gitlab-ci.yml
workflow:
rules:
# required to enable merge request pipelines: https://docs.gitlab.com/ee/ci/pipelines/merge_request_pipelines.html#prerequisites
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
# to prevent duplicate pipelines in MRs: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
when: never
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
stages:
- prepare
- build
- test
- report
- deploy
variables:
pip: pip3 --timeout 100 --retries 10
GIT_DEPTH:
description: needs lots of git history since it has to compare the merge request to current master
value: "5000"
CI_ENABLE_MERGE_REQUEST_PIPELINES:
description: push with `git push -o ci.variable="CI_ENABLE_MERGE_REQUEST_PIPELINES=false"` to disable merge request pipelines (e.g., to use self-hosted runners)
value: "true"
.install_fdroid_server: &install_fdroid_server
- rm -rf $fdroidserver
- mkdir $fdroidserver
- git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
- curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz
| tar -xz --directory=$fdroidserver --strip-components=1
- export PATH="$fdroidserver:$PATH"
- export PYTHONPATH="$fdroidserver:$fdroidserver/examples"
- export PYTHONUNBUFFERED=true
.install_fdroid_server_deb: &install_fdroid_server_deb
- apt-get --allow-releaseinfo-change update
- apt-get -qy dist-upgrade
- echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" | debconf-set-selections
- apt-get install -qy --no-install-recommends fdroidserver apksigner mercurial git git-svn brz python3-launchpadlib locales curl
.get_target_source_refs: &get_target_source_refs
- |
if [ "$CI_PROJECT_PATH" = "fdroid/fdroiddata" ] && [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
export TARGET_REF="${CI_COMMIT_BEFORE_SHA}"
export SOURCE_REF="${CI_COMMIT_SHA}"
else
git remote add upstream https://gitlab.com/fdroid/fdroiddata.git || true
git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master}
export TARGET_REF=$(git merge-base HEAD upstream/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master})
export SOURCE_REF="${CI_COMMIT_SHA}"
fi;
.get_changed_builds: &get_changed_builds
- *get_target_source_refs
- echo $TARGET_REF
- |
for f in $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*.yml) $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*/signatures); do
diff=$(git diff "${TARGET_REF}...${SOURCE_REF}" -- "$f")
echo "$diff"
test $(echo "$diff" | perl -wnle '/^[+-](( +-)|( *\w))/ and print' | grep -v -c '^+ *disable:') = 0 && continue
echo "$diff" | grep -E '^\+ *(NoSourceSince|Disabled):' && continue
appid=$(echo "$f" | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p')
export CHANGED="$CHANGED $appid"
done
.get_changed_apps: &get_changed_apps
- *get_target_source_refs
- echo $TARGET_REF
- |
for f in $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*.yml) $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*/signatures); do
diff=$(git diff "${TARGET_REF}...${SOURCE_REF}" -- "$f")
appid=$(echo "$f" | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p')
export CHANGED="$CHANGED $appid"
done
.app_verification_rules:
rules: &app_verification_rules
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
when: never
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# only works for merge request pipelines: https://docs.gitlab.com/ee/ci/jobs/job_control.html#jobs-or-pipelines-run-unexpectedly-when-using-changes
changes:
- metadata/**/*
- srclibs/**/*
- if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
checkupdates:
stage: test
needs: []
image: debian:bookworm-slim
tags:
- saas-linux-medium-amd64
rules: *app_verification_rules
before_script:
- *install_fdroid_server_deb
- export fdroidserver=../fdroidserver
- *install_fdroid_server
script:
- |
set -xe
- *get_changed_builds
- |
if [ -n "$CHANGED" ]; then
fdroid checkupdates --auto -v $CHANGED
git --no-pager diff --color=always --exit-code
fi
git redirect:
stage: test
needs: []
image: debian:bookworm-slim
tags:
- saas-linux-medium-amd64
rules: *app_verification_rules
script:
- apt-get -qy update
- apt-get -qy dist-upgrade
- apt-get -qy install --no-install-recommends ca-certificates git python3 python3-yaml
- *get_changed_apps
- test -n "$CHANGED" || exit 0
- tools/rewrite-git-redirects.py $CHANGED
- git --no-pager diff --color=always --exit-code || {
printf "\x1b[31mERROR git URLs should be direct, not redirects!\x1b[0m\n";
exit 1;
}
fdroid lint:
stage: test
needs: []
image: debian:bookworm-slim
tags:
- saas-linux-medium-amd64
rules: *app_verification_rules
before_script:
- *install_fdroid_server_deb
- export fdroidserver=$PWD/fdroidserver
- *install_fdroid_server
script:
- *get_changed_apps
- export EXITVALUE=0
- function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
- |
if [ -n "$CHANGED" ]; then
fdroid lint -f $CHANGED || {
set_error;
printf "\nThese files have lint issues:\n";
fdroid rewritemeta -l $CHANGED;
printf "\nThese are the formatting issues:\n";
fdroid rewritemeta $CHANGED;
git --no-pager diff --color=always;
}
fi
- exit $EXITVALUE
fdroid lint config:
stage: test
needs: []
image: debian:bookworm-slim
before_script:
- apt-get -qy update
- apt-get -qy dist-upgrade
- apt-get -qy install --no-install-recommends
ca-certificates
curl
git
python3-defusedxml
python3-git
python3-pyasn1-modules
python3-pycountry
python3-qrcode
python3-requests
python3-ruamel.yaml
yamllint
- export fdroidserver=$CI_PROJECT_DIR/fdroidserver
- *install_fdroid_server
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push"
changes:
paths:
- config.yml
- config/*.yml
- config/*/*.yml
script:
- fdroid lint config/*.yml config/*/*.yml
# TODO once config.yml is added to fdroiddata, add it to the line above
- test -e config.yml || exit 0
- fdroid lint config.yml
lint:
stage: test
needs: []
image: debian:bookworm-slim
tags:
- saas-linux-medium-amd64
rules: *app_verification_rules
script:
- apt-get -qy update
- apt-get -qy dist-upgrade
- apt-get -qy install --no-install-recommends exiftool git python3 python3-yaml
- export EXITVALUE=0
- function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
- find metadata/ -name '*.jp*g' -o -name '*.png' | xargs exiftool -all=
- echo "these images have EXIF that must be stripped:"
- git --no-pager diff --stat
- git --no-pager diff --name-only --exit-code || set_error
- ./tools/check-localized-metadata.py || set_error
- ./tools/check-keyalias-collision.py || set_error
- ./tools/check-metadata-summary-whitespace.py || set_error
- ./tools/check-for-unattached-signatures.py || set_error
- ./tools/make-summary-translatable.py || set_error
- exit $EXITVALUE
schema validation:
stage: test
needs: []
image: python
rules: *app_verification_rules
before_script:
- pip install check-jsonschema
script:
- check-jsonschema --check-metaschema schemas/metadata.json || exit 1
- *get_target_source_refs
- git diff --exit-code --name-only "${TARGET_REF}...${SOURCE_REF}" schemas/metadata.json
|| { check-jsonschema --schemafile schemas/metadata.json metadata/*.yml || exit 1; exit; }
- set -xe
- *get_changed_apps
- if [ -z "$CHANGED" ]; then exit; fi
- echo $CHANGED | sed -En 's|([^ ]*)|metadata/\1.yml|gp' | xargs check-jsonschema --schemafile schemas/metadata.json
trigger-issuebot:
stage: prepare
needs: []
when: manual
image: alpine
variables:
GIT_DEPTH: "1"
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- logs/
when: always
expire_in: 1 week
script:
- mkdir logs
- export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
- apk add --no-cache bash curl
- ./tools/trigger-issuebot
schedule-issuebot:
stage: test
needs: []
image: debian:bookworm-slim
only:
variables:
- $SCHEDULE_ISSUEBOT
variables:
GIT_DEPTH: "1"
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- logs/
when: always
expire_in: 1 week
script:
- mkdir logs
- export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
- apt-get update
- apt-get -qy install --no-install-recommends bash curl ca-certificates python3-gitlab
- ./tools/schedule-issuebot.py
checkupdates_runner:
stage: prepare
image: debian:bookworm-slim
variables:
GIT_DEPTH: "1"
parallel: 10
tags:
- fdroid
- pep.security
rules:
- if: $CI_PIPELINE_SOURCE == 'schedule' && $CHECKUPDATES == 'true'
before_script:
- *install_fdroid_server_deb
- export fdroidserver=../fdroidserver
- *install_fdroid_server
- apt-get install -y openssh-client
- git config --global user.email "fdroidci@bubu1.eu"
- git config --global user.name "F-Droid checkupdates bot"
# gitlab.com was still vulnerable to https://terrapin-attack.com/ when this was added
- printf 'Ciphers -chacha20-poly1305@openssh.com,*-cbc\nMACs -*etm*,*-sha1*\n'
> /etc/ssh/ssh_config.d/0-terrapin-workaround.conf
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- cp "${GITLAB_KNOWN_HOSTS}" ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
- eval $(ssh-agent -s)
- echo "${CHECKUPDATES_SSH_DEPLOY_KEY}" | tr -d '\r' | ssh-add -
- url_host=$(echo "${CI_REPOSITORY_URL}" | sed -e 's|https\?://gitlab-ci-token:.*@|ssh://git@|g')
- git remote set-url --push origin "${url_host}"
# reset repo to origin state before adding new commits
- git rebase --abort || true
- rm -fr ".git/rebase-apply"
# https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29187
- rm -f ".git/logs/HEAD.lock"
- git checkout master || true
- git reset --hard origin/master || true
script:
# Get app IDs in this batch
- |
metadata_files=(metadata/*.yml)
metadata_files_count=${#metadata_files[@]}
batch_size=$((metadata_files_count / CI_NODE_TOTAL + 1))
metadata_files_batch=("${metadata_files[@]:$(((CI_NODE_INDEX - 1) * batch_size)):$batch_size}")
declare -a CHECKUPDATES_APPIDS
for file in "${metadata_files_batch[@]}"; do
filename="${file##*/}"
filename_without_extension="${filename%.*}"
CHECKUPDATES_APPIDS=("${CHECKUPDATES_APPIDS[@]}" "$filename_without_extension")
done
export CHECKUPDATES_APPIDS
- echo "$metadata_files_count"
- echo "$batch_size"
- echo "${metadata_files_batch[@]}"
- echo "${#metadata_files_batch[@]}"
- echo "${CHECKUPDATES_APPIDS[@]}"
- echo "${#CHECKUPDATES_APPIDS[@]}"
- fdroid checkupdates --allow-dirty --auto --commit "${CHECKUPDATES_APPIDS[@]}" 2>&1 | tee /tmp/out || true
- git pull --rebase origin master
# when two jobs try to push at the same time they occasionally fail, so try it again
- while ! git push origin HEAD:master; do git pull --rebase origin master; done
- echo "============== Summary ====================="
- 'grep -v "INFO: Processing" /tmp/out || true'
# This job is should be as close as possible to the production
# buildserver. It should not include custom setup or methods, except
# when there is no other way to make this job work. The docker image
# is created using the same provisioning as the producion buildserver,
# via the "docker" job in fdroid/fdroidserver.
#
# This job is tied to the buildserver. Changes must be reviewed by
# the buildserver maintainers.
fdroid build:
stage: build
needs: []
# Some env vars are setup in https://gitlab.com/fdroid/fdroidserver/-/blob/master/buildserver/setup-env-vars
# home_vagrant=/home/vagrant
image: registry.gitlab.com/fdroid/fdroidserver:buildserver-bookworm
tags:
- saas-linux-medium-amd64
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- repo/
- unsigned/
- tmp/
when: always
expire_in: 1 month
rules: *app_verification_rules
cache:
key: "$CI_JOB_NAME"
paths:
- .gradle
variables:
ANDROID_HOME: /opt/android-sdk
ANDROID_SDK_ROOT: ${ANDROID_HOME}
script:
- test -d build || mkdir build
- *get_changed_builds
- test -n "$(printf "$CHANGED" | tr -d '[:space:]')"
|| { echo "no packages need processing, exiting"; exit 0; }
- apt-get update
- apt-get dist-upgrade
- test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh
- rm -rf $ANDROID_HOME/tools # TODO remove once sdkmanager can upgrade installed packages
# These packages are needed to make this env like the production buildserver.
- sdkmanager "tools" "platform-tools" "build-tools;31.0.0"
- *install_fdroid_server
# Set up git config something like the production buildserver. The
# difference is that in production, this is installed on the
# buildserver VM host, while here it is basically in the
# buildserver VM.
# https://gitlab.com/fdroid/fdroid-bootstrap-buildserver/-/merge_requests/30
- curl --silent 'https://gitlab.com/fdroid/fdroid-bootstrap-buildserver/-/raw/master/roles/production_hardening/files/gitconfig' >> $CI_PROJECT_DIR/../.gitconfig
- for d in logs tmp unsigned $home_vagrant/.android $home_vagrant/.gradle $home_vagrant/metadata; do
test -d $d || mkdir $d;
chown -R vagrant $d;
done
# make sure .gradle stores inside $CI_PROJECT_DIR to make it cacheable
- ln -s $home_vagrant/.gradle $CI_PROJECT_DIR/.gradle
- ln -s $CI_PROJECT_DIR/tmp $home_vagrant/tmp
- ln -s $CI_PROJECT_DIR/srclibs $home_vagrant/srclibs
- export GRADLE_USER_HOME=$home_vagrant/.gradle
# each `fdroid build --on-server` run expects sudo, then uninstalls it
- export fdroid="sudo --preserve-env --user vagrant
env PATH=$fdroidserver:$PATH
env PYTHONPATH=$fdroidserver:$fdroidserver/examples
env PYTHONUNBUFFERED=true
env TERM=$TERM
env HOME=$home_vagrant
fdroid"
# use fdroidserver test keystore as placeholder since `fdroid publish` requires it
- printf 'repo_keyalias\x3a sova\n' >> config.yml
- printf 'keystorepass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
- printf 'keypass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
- printf 'keydname\x3a "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"\n' >> config.yml
- keystore=$fdroidserver/tests/keystore.jks
- printf "keystore\x3a $keystore\n" >> config.yml
- APPS="$(./tools/find-changed-builds.py)"
- for build in $APPS; do
set -x;
apt-get install sudo;
cp -R $CI_PROJECT_DIR/build $home_vagrant/build;
appid=${build%:*};
[ -d metadata/$appid ] && cp -R metadata/$appid $home_vagrant/metadata;
cp -R metadata/$appid.yml $home_vagrant/metadata;
chown -R vagrant $home_vagrant $CI_PROJECT_DIR;
pushd $home_vagrant;
ln -s $CI_PROJECT_DIR $home_vagrant/fdroiddata;
ln -s $CI_PROJECT_DIR/../.gitconfig $home_vagrant/.gitconfig;
$fdroid fetchsrclibs $build --verbose;
rm $home_vagrant/fdroiddata $home_vagrant/.gitconfig;
$fdroid build --verbose --test --refresh-scanner --scan-binary --on-server --no-tarball $build;
popd;
rm -rf $home_vagrant/build || true;
rm -rf $ANDROID_HOME/ndk || true;
apt-get install -y openjdk-17-jdk-headless;
update-alternatives --set java /usr/lib/jvm/java-17-openjdk-amd64/bin/java;
./tools/build-contains-signatures.py $build || continue;
apt-get install sudo;
$fdroid publish --verbose --error-on-failed $build;
done
- ./tools/audit-gradle.py $CHANGED;
# issuebot needs secrets to run, so it has to run under the 'fdroid'
# group, therefore needs the trigger without secrets, there would be
# no support for virustotal, github downloads, exodus privacy checks,
# etc. That means it has to be triggered from the lint: job, which
# runs in the fork's CI. The trigger-issuebot job adds the env var
# FROM_CI_MERGE_REQUEST_IID which is required to have this job be
# triggered.
#
# This job has to be called 'pages' because it deploys the JSON API
# to GitLab Pages.
pages:
stage: deploy
image: registry.gitlab.com/fdroid/fdroidserver:buildserver-bookworm
only:
refs:
- branches
variables:
- $FROM_CI_MERGE_REQUEST_IID
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_JOB_ID}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- metadata/
- public/
- repo/index-v1.json
- repo/index.xml
- tmp/apkcache.json
- unsigned/
when: always
script:
- apt-get update
- apt-get dist-upgrade
- apt-get install
python3-github
python3-gitlab
python3-defusedxml
python3-javaproperties
python3-requests-cache
python3-venv
python3-wheel
- test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh
- *install_fdroid_server
- export GRADLE_USER_HOME=$PWD/.gradle
- rm -rf $GRADLE_USER_HOME/fdroid
- mkdir -p $GRADLE_USER_HOME/fdroid
- git ls-remote https://gitlab.com/fdroid/gradle-plugins.git master
- curl --silent https://gitlab.com/fdroid/gradle-plugins/-/archive/master/gradle-plugins-master.tar.gz
| tar -xz --directory=$GRADLE_USER_HOME/fdroid --strip-components=1
- git ls-remote https://gitlab.com/fdroid/issuebot.git master
- curl --silent https://gitlab.com/fdroid/issuebot/-/archive/master/issuebot-master.tar.gz
| tar -xz --strip-components=1
- python3 -m venv --system-site-packages --clear issuebot-env
- . issuebot-env/bin/activate
- ./issuebot.py
# git_stats used to run here, redirect to new location
- echo '<html><head><meta http-equiv="refresh" content="0;URL=https://fdroid.gitlab.io/"></head></html>'
> public/index.html
check_git_repos:
image: debian:bookworm-slim
stage: test
needs: []
only:
refs:
- schedules
variables:
- $CHECK_GIT_REPO == "true"
artifacts:
when: on_failure
expire_in: 1 month
paths:
- public
script:
- apt-get update
- apt-get -qy install --no-install-recommends ca-certificates git python3-colorama python3-yaml
- tools/check-git-repo-availability.py || export EXITVALUE=1
- test -d public || mkdir public
- cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true
- exit $EXITVALUE
sast:
stage: report
tags:
- saas-linux-medium-amd64
needs:
- fdroid build
dependencies:
- fdroid build
variables:
GIT_DEPTH: "1"
MOBSF_API_KEY: key
SAST_EXPERIMENTAL_FEATURES: "true"
SAST_EXCLUDED_ANALYZERS:
bandit, brakeman, eslint, flawfinder, gosec, kubesec,
nodejs-scan, phpcs-security-audit, pmd-apex, security-code-scan, semgrep, sobelow,
spotbugs
# Make sure the apk is searched before the manifest, so only apks are scanned
ANALYZER_TARGET_DIR: $CI_PROJECT_DIR/tmp/
allow_failure: true
after_script:
# GitLab can only display reports at this exact path
- mv $ANALYZER_TARGET_DIR/gl-sast-report.json gl-sast-report.json
mobsf-ios-sast:
rules:
- when: never
mobsf-android-sast:
rules: *app_verification_rules
Reference in New Issue View Git Blame Copy Permalink