1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroiddata.git synced 2024-11-01 21:56:00 +01:00
fdroid-data/.gitlab-ci.yml

568 lines
20 KiB
YAML

workflow:
rules:
# required to enable merge request pipelines: https://docs.gitlab.com/ee/ci/pipelines/merge_request_pipelines.html#prerequisites
- if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
# to prevent duplicate pipelines in MRs: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
when: never
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
stages:
- prepare
- build
- test
- report
- deploy
variables:
pip: pip3 --timeout 100 --retries 10
GIT_DEPTH:
description: needs lots of git history since it has to compare the merge request to current master
value: "5000"
CI_ENABLE_MERGE_REQUEST_PIPELINES:
description: push with `git push -o ci.variable="CI_ENABLE_MERGE_REQUEST_PIPELINES=false"` to disable merge request pipelines (e.g., to use self-hosted runners)
value: "true"
.install_fdroid_server: &install_fdroid_server
- rm -rf $fdroidserver
- mkdir $fdroidserver
- git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
- curl --silent https://gitlab.com/fdroid/fdroidserver/-/archive/master/fdroidserver-master.tar.gz
| tar -xz --directory=$fdroidserver --strip-components=1
- export PATH="$fdroidserver:$PATH"
- export PYTHONPATH="$fdroidserver:$fdroidserver/examples"
- export PYTHONUNBUFFERED=true
.install_fdroid_server_deb: &install_fdroid_server_deb
- apt-get --allow-releaseinfo-change update
- apt-get -qy dist-upgrade
- echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" | debconf-set-selections
- apt-get install -qy --no-install-recommends fdroidserver apksigner mercurial git git-svn brz python3-launchpadlib locales curl
.get_target_source_refs: &get_target_source_refs
- |
if [ "$CI_PROJECT_PATH" = "fdroid/fdroiddata" ] && [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
export TARGET_REF="${CI_COMMIT_BEFORE_SHA}"
export SOURCE_REF="${CI_COMMIT_SHA}"
else
git remote add upstream https://gitlab.com/fdroid/fdroiddata.git || true
git fetch upstream ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master}
export TARGET_REF=$(git merge-base HEAD upstream/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-master})
export SOURCE_REF="${CI_COMMIT_SHA}"
fi;
.get_changed_builds: &get_changed_builds
- *get_target_source_refs
- echo $TARGET_REF
- |
for f in $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*.yml) $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*/signatures); do
diff=$(git diff "${TARGET_REF}...${SOURCE_REF}" -- "$f")
echo "$diff"
test $(echo "$diff" | perl -wnle '/^[+-](( +-)|( *\w))/ and print' | grep -v -c '^+ *disable:') = 0 && continue
echo "$diff" | grep -E '^\+ *(NoSourceSince|Disabled):' && continue
appid=$(echo "$f" | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p')
export CHANGED="$CHANGED $appid"
done
.get_changed_apps: &get_changed_apps
- *get_target_source_refs
- echo $TARGET_REF
- |
for f in $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*.yml) $(git diff --name-only --diff-filter=d "${TARGET_REF}...${SOURCE_REF}" -- metadata/*/signatures); do
diff=$(git diff "${TARGET_REF}...${SOURCE_REF}" -- "$f")
appid=$(echo "$f" | sed -E -n 's,^metadata/([^/][^/]*)(\.yml|/signatures/.*),\1,p')
export CHANGED="$CHANGED $appid"
done
.app_verification_rules:
rules: &app_verification_rules
- if: $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "trigger"
when: never
- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_ENABLE_MERGE_REQUEST_PIPELINES != "false"
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
# only works for merge request pipelines: https://docs.gitlab.com/ee/ci/jobs/job_control.html#jobs-or-pipelines-run-unexpectedly-when-using-changes
changes:
- metadata/**/*
- srclibs/**/*
- if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
checkupdates:
stage: test
needs: []
tags:
- saas-linux-medium-amd64
image: debian:bookworm-slim
rules: *app_verification_rules
before_script:
- *install_fdroid_server_deb
- export fdroidserver=../fdroidserver
- *install_fdroid_server
script:
- |
set -xe
- *get_changed_builds
- |
if [ -n "$CHANGED" ]; then
fdroid checkupdates --auto -v $CHANGED
git --no-pager diff --color=always --exit-code
fi
git redirect:
stage: test
needs: []
image: debian:bookworm-slim
rules: *app_verification_rules
script:
- apt-get -qy update
- apt-get -qy dist-upgrade
- apt-get -qy install --no-install-recommends ca-certificates git python3 python3-yaml
- *get_changed_apps
- test -n "$CHANGED" || exit 0
- tools/rewrite-git-redirects.py $CHANGED
- git --no-pager diff --color=always --exit-code || {
printf "\x1b[31mERROR git URLs should be direct, not redirects!\x1b[0m\n";
exit 1;
}
fdroid lint:
stage: test
needs: []
image: debian:bookworm-slim
rules: *app_verification_rules
before_script:
- *install_fdroid_server_deb
- export fdroidserver=$PWD/fdroidserver
- *install_fdroid_server
script:
- cp tools/lint_config.yml config.yml
- *get_changed_apps
- export EXITVALUE=0
- function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
- |
if [ -n "$CHANGED" ]; then
fdroid rewritemeta $CHANGED;
git diff --exit-code > /dev/null || {
set_error;
printf "\nThese files have lint issues:\n";
git diff --name-only;
printf "\nThese are the formatting issues:\n";
printf "$(git --no-pager diff --color=always --ws-error-highlight=all | sed -e 's/\r/^M/' -e 's/%/%%/g')";
}
fi
- exit $EXITVALUE
fdroid lint config:
stage: test
needs: []
image: debian:bookworm-slim
before_script:
- apt-get -qy update
- apt-get -qy dist-upgrade
- apt-get -qy install --no-install-recommends
ca-certificates
curl
git
python3-asn1crypto
python3-defusedxml
python3-git
python3-pycountry
python3-qrcode
python3-requests
python3-ruamel.yaml
yamllint
- export fdroidserver=$CI_PROJECT_DIR/fdroidserver
- *install_fdroid_server
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
paths:
- .gitlab-ci.yml
- config.yml
- config/*.yml
- config/*/*.yml
# only check pushes to fdroids own repo
- if: $CI_PIPELINE_SOURCE == "push" && $CI_PROJECT_PATH == "fdroid/fdroiddata"
changes:
compare_to: 'refs/heads/master'
paths:
- .gitlab-ci.yml
- config.yml
- config/*.yml
- config/*/*.yml
script:
- fdroid lint config/*.yml config/*/*.yml
# TODO once config.yml is added to fdroiddata, add it to the line above
- test -e config.yml || exit 0
- fdroid lint config.yml
lint:
stage: test
needs: []
image: debian:bookworm-slim
rules: *app_verification_rules
script:
- apt-get -qy update
- apt-get -qy dist-upgrade
- apt-get -qy install --no-install-recommends exiftool git python3 python3-yaml
- export EXITVALUE=0
- function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
- find metadata/ -name '*.jp*g' -o -name '*.png' | xargs exiftool -all=
- echo "these images have EXIF that must be stripped:"
- git --no-pager diff --stat
- git --no-pager diff --name-only --exit-code || set_error
- ./tools/check-localized-metadata.py || set_error
- ./tools/check-keyalias-collision.py || set_error
- ./tools/check-metadata-summary-whitespace.py || set_error
- ./tools/check-for-unattached-signatures.py || set_error
- ./tools/make-summary-translatable.py || set_error
- exit $EXITVALUE
schema validation:
stage: test
needs: []
image: python
rules: *app_verification_rules
before_script:
- pip install check-jsonschema
script:
- check-jsonschema --check-metaschema schemas/metadata.json || exit 1
- *get_target_source_refs
- git diff --exit-code --name-only "${TARGET_REF}...${SOURCE_REF}" schemas/metadata.json
|| { check-jsonschema --schemafile schemas/metadata.json metadata/*.yml || exit 1; exit; }
- set -xe
- *get_changed_apps
- if [ -z "$CHANGED" ]; then exit; fi
- echo $CHANGED | sed -En 's|([^ ]*)|metadata/\1.yml|gp' | xargs check-jsonschema --schemafile schemas/metadata.json
trigger-issuebot:
stage: prepare
needs: []
when: manual
image: alpine
variables:
GIT_DEPTH: "1"
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- logs/
when: always
expire_in: 1 week
script:
- mkdir logs
- export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
- apk add --no-cache bash curl
- ./tools/trigger-issuebot
schedule-issuebot:
stage: test
needs: []
image: debian:bookworm-slim
only:
variables:
- $SCHEDULE_ISSUEBOT
variables:
GIT_DEPTH: "1"
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- logs/
when: always
expire_in: 1 week
script:
- mkdir logs
- export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
- apt-get update
- apt-get -qy install --no-install-recommends bash curl ca-certificates python3-gitlab
- ./tools/schedule-issuebot.py
# This job is should be as close as possible to the production
# buildserver. It should not include custom setup or methods, except
# when there is no other way to make this job work. The docker image
# is created using the same provisioning as the producion buildserver,
# via the "docker" job in fdroid/fdroidserver.
#
# This job is tied to the buildserver. Changes must be reviewed by
# the buildserver maintainers.
fdroid build:
stage: build
needs: []
tags:
- saas-linux-medium-amd64
# Some env vars are setup in https://gitlab.com/fdroid/fdroidserver/-/blob/master/buildserver/setup-env-vars
# home_vagrant=/home/vagrant
image: registry.gitlab.com/fdroid/fdroidserver:buildserver-bookworm
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- repo/
- unsigned/
- tmp/
when: always
expire_in: 1 month
rules: *app_verification_rules
cache:
key: "$CI_JOB_NAME"
paths:
- .gradle
variables:
ANDROID_HOME: /opt/android-sdk
ANDROID_SDK_ROOT: ${ANDROID_HOME}
script:
- test -d build || mkdir build
- *get_changed_builds
- test -n "$(printf "$CHANGED" | tr -d '[:space:]')"
|| { echo "no packages need processing, exiting"; exit 0; }
- apt-get update
- apt-get dist-upgrade
- test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh
# These packages are needed to make this env like the production buildserver.
- sdkmanager "platform-tools" "build-tools;31.0.0"
- *install_fdroid_server
# Set up git config something like the production buildserver. The
# difference is that in production, this is installed on the
# buildserver VM host, while here it is basically in the
# buildserver VM.
# https://gitlab.com/fdroid/fdroid-bootstrap-buildserver/-/merge_requests/30
- curl --silent 'https://gitlab.com/fdroid/fdroid-bootstrap-buildserver/-/raw/master/roles/production_hardening/files/gitconfig' >> $CI_PROJECT_DIR/../.gitconfig
- for d in logs tmp unsigned $home_vagrant/.android $home_vagrant/.gradle $home_vagrant/metadata; do
test -d $d || mkdir $d;
chown -R vagrant $d;
done
# make sure .gradle stores inside $CI_PROJECT_DIR to make it cacheable
- ln -s $home_vagrant/.gradle $CI_PROJECT_DIR/.gradle
- ln -s $CI_PROJECT_DIR/tmp $home_vagrant/tmp
- ln -s $CI_PROJECT_DIR/srclibs $home_vagrant/srclibs
- export GRADLE_USER_HOME=$home_vagrant/.gradle
# each `fdroid build --on-server` run expects sudo, then uninstalls it
- export fdroid="sudo --preserve-env --user vagrant
env PATH=$fdroidserver:$PATH
env PYTHONPATH=$fdroidserver:$fdroidserver/examples
env PYTHONUNBUFFERED=true
env TERM=$TERM
env HOME=$home_vagrant
fdroid"
# use fdroidserver test keystore as placeholder since `fdroid publish` requires it
- printf 'repo_keyalias\x3a sova\n' >> config.yml
- printf 'keystorepass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
- printf 'keypass\x3a r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI=\n' >> config.yml
- printf 'keydname\x3a "CN=Birdman, OU=Cell, O=Alcatraz, L=Alcatraz, S=California, C=US"\n' >> config.yml
- keystore=$fdroidserver/tests/keystore.jks
- printf "keystore\x3a $keystore\n" >> config.yml
- APPS="$(./tools/find-changed-builds.py)"
- for build in $APPS; do
set -x;
apt-get install sudo;
cp -R $CI_PROJECT_DIR/build $home_vagrant/build;
appid=${build%:*};
[ -d metadata/$appid ] && cp -R metadata/$appid $home_vagrant/metadata;
cp -R metadata/$appid.yml $home_vagrant/metadata;
chown -R vagrant $home_vagrant $CI_PROJECT_DIR;
pushd $home_vagrant;
ln -s $CI_PROJECT_DIR $home_vagrant/fdroiddata;
ln -s $CI_PROJECT_DIR/../.gitconfig $home_vagrant/.gitconfig;
$fdroid fetchsrclibs $build --verbose;
rm $home_vagrant/fdroiddata $home_vagrant/.gitconfig;
$fdroid build --verbose --test --refresh-scanner --scan-binary --on-server --no-tarball $build;
apt-get install sudo;
apk="tmp/binaries/${build/:/_}.binary.apk";
[[ -e $apk ]] && $fdroid scanner --verbose --exit-code $apk;
popd;
rm -rf $home_vagrant/build || true;
rm -rf $ANDROID_HOME/ndk || true;
apt-get install -y openjdk-17-jdk-headless;
update-alternatives --set java /usr/lib/jvm/java-17-openjdk-amd64/bin/java;
./tools/build-contains-signatures.py $build || continue;
$fdroid publish --verbose --error-on-failed $build;
done
- ./tools/audit-gradle.py $CHANGED;
# issuebot needs secrets to run, so it has to run under the 'fdroid'
# group, therefore needs the trigger without secrets, there would be
# no support for virustotal, github downloads, exodus privacy checks,
# etc. That means it has to be triggered from the lint: job, which
# runs in the fork's CI. The trigger-issuebot job adds the env var
# FROM_CI_MERGE_REQUEST_IID which is required to have this job be
# triggered.
#
# This job has to be called 'pages' because it deploys the JSON API
# to GitLab Pages.
pages:
stage: deploy
tags:
- saas-linux-medium-amd64
image: registry.gitlab.com/fdroid/fdroidserver:buildserver-bookworm
only:
refs:
- branches
variables:
- $FROM_CI_MERGE_REQUEST_IID
artifacts:
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_JOB_ID}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
paths:
- metadata/
- public/
- repo/index-v1.json
- repo/index.xml
- tmp/apkcache.json
- unsigned/
when: always
script:
- apt-get update
- apt-get dist-upgrade
- apt-get install
python3-github
python3-gitlab
python3-defusedxml
python3-javaproperties
python3-requests-cache
python3-venv
python3-wheel
- test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh
- *install_fdroid_server
- export GRADLE_USER_HOME=$PWD/.gradle
- rm -rf $GRADLE_USER_HOME/fdroid
- mkdir -p $GRADLE_USER_HOME/fdroid
- git ls-remote https://gitlab.com/fdroid/gradle-plugins.git master
- curl --silent https://gitlab.com/fdroid/gradle-plugins/-/archive/master/gradle-plugins-master.tar.gz
| tar -xz --directory=$GRADLE_USER_HOME/fdroid --strip-components=1
- git ls-remote https://gitlab.com/fdroid/issuebot.git master
- curl --silent https://gitlab.com/fdroid/issuebot/-/archive/master/issuebot-master.tar.gz
| tar -xz --strip-components=1
- python3 -m venv --system-site-packages --clear issuebot-env
- . issuebot-env/bin/activate
- ./issuebot.py
# git_stats used to run here, redirect to new location
- echo '<html><head><meta http-equiv="refresh" content="0;URL=https://fdroid.gitlab.io/"></head></html>'
> public/index.html
check_git_repos:
image: debian:bookworm-slim
stage: test
needs: []
only:
refs:
- schedules
variables:
- $CHECK_GIT_REPO == "true"
artifacts:
when: on_failure
expire_in: 1 month
paths:
- public
script:
- apt-get update
- apt-get -qy install --no-install-recommends ca-certificates git python3-colorama python3-yaml
- tools/check-git-repo-availability.py || export EXITVALUE=1
- test -d public || mkdir public
- cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true
- exit $EXITVALUE
# checkupdates should only be allowed to modify app metadata files,
# e.g. metadata/*.yml. Anything else should throw an error here.
checkupdates_filter:
image: debian:bookworm-slim
stage: test
needs: []
rules:
- if: $CI_PROJECT_PATH == 'fdroid/checkupdates-bot-fdroiddata'
script:
- apt-get update
- apt-get -qy install --no-install-recommends ca-certificates git
- git fetch https://gitlab.com/fdroid/fdroiddata.git
- |
if git diff --merge-base FETCH_HEAD HEAD --name-only | grep -v '^metadata/\S*\.yml$'; then
printf "\x1b[31mERROR Modifications to paths checkupdates-bot is not allowed to change! \x1b[0m\n"
exit 1
else
echo "All file modifications match paths checkupdates-bot is allowed to change."
fi
strip EXIF:
stage: test
needs: []
image: debian:bookworm-slim
rules:
- changes:
paths:
- "**.JPEG"
- "**.JPEg"
- "**.JPeG"
- "**.JPeg"
- "**.JpEG"
- "**.JpEg"
- "**.JpeG"
- "**.Jpeg"
- "**.jPEG"
- "**.jPEg"
- "**.jPeG"
- "**.jPeg"
- "**.jpEG"
- "**.jpEg"
- "**.jpeG"
- "**.jpeg"
- "**.JPG"
- "**.JPg"
- "**.JpG"
- "**.Jpg"
- "**.jPG"
- "**.jPg"
- "**.jpG"
- "**.jpg"
- "**.PNG"
- "**.PNg"
- "**.PnG"
- "**.Png"
- "**.pNG"
- "**.pNg"
- "**.pnG"
- "**.png"
script:
- apt-get -qy update
- apt-get -qy dist-upgrade
- apt-get -qy install --no-install-recommends exiftool git
# JPEG
- find . -type f -iname "*.jpg" -o -iname "*.jpeg" -exec chmod a-x {} \;
- find . -type f -iname "*.jpg" -o -iname "*.jpeg" -exec exiftool -quiet -all= {} \;
# PNG
- find . -type f -iname "*.png" -exec chmod a-x {} \;
- find . -type f -iname "*.png" -exec exiftool -quiet -all= {} \;
- git diff --exit-code --color || {
printf "\x1b[31mERROR Image files (JPEG, PNG) should be stripped and not be set executable:\x1b[0m\n";
echo "Try using exiftool, jpegoptim, optipng (e.g. `exiftool -all= foo.png`).";
echo "For more info, see:";
echo "https://umbrella.cisco.com/blog/picture-perfect-how-jpg-exif-data-hides-malware";
echo "https://beinset.medium.com/exif-metadata-a-hidden-door-to-cyber-vulnerabilities-52b0dd2ff4de";
exit 1;
}