309 lines
11 KiB
YAML
309 lines
11 KiB
YAML
|
|
variables:
|
|
pip: pip3 --timeout 100 --retries 10
|
|
|
|
|
|
lint:
|
|
image: registry.gitlab.com/fdroid/ci-images-base
|
|
except:
|
|
- pipelines
|
|
- schedules
|
|
- triggers
|
|
before_script:
|
|
- apt-get update
|
|
- apt-get -qy dist-upgrade
|
|
- rm -rf fdroidserver
|
|
- mkdir fdroidserver
|
|
- git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
|
|
- curl --silent https://gitlab.com/fdroid/fdroidserver/repository/master/archive.tar.gz
|
|
| tar -xz --directory=fdroidserver --strip-components=1
|
|
- export PATH="$PWD/fdroidserver:$PATH"
|
|
- touch config.py
|
|
script:
|
|
# if this is a merge request fork, then only check relevant apps
|
|
- if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
|
|
git fetch https://gitlab.com/fdroid/fdroiddata.git;
|
|
test -d build || mkdir build;
|
|
files=`git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`;
|
|
for f in $files; do
|
|
appid=`echo $f | sed -n -e 's,^metadata/\([^/][^/]*\)\.yml,\1,p'`;
|
|
if [ -n "$appid" ]; then
|
|
export CHANGED="$CHANGED $appid";
|
|
testcmd="git -C build clone `sed -En 's,^Repo\W +(https?://),\1u:p@,p' $f`";
|
|
fi;
|
|
done;
|
|
set -x;
|
|
apt-get install python3-colorama yamllint;
|
|
for f in $files; do [[ $f == *'.yml' ]] && yamllint "$f"; done;
|
|
./tools/check-git-repo-availability.py $files;
|
|
./tools/audit-gradle.py $CHANGED;
|
|
set +x;
|
|
fi
|
|
- export EXITVALUE=0
|
|
- fdroid lint -f $CHANGED || {
|
|
export EXITVALUE=1;
|
|
printf "\nThese files have lint issues:\n";
|
|
fdroid rewritemeta -l $CHANGED;
|
|
printf "\nThese are the formatting issues:\n";
|
|
fdroid rewritemeta $CHANGED;
|
|
git --no-pager diff --color=always;
|
|
}
|
|
- apt-get -qy update
|
|
- apt-get -qy install --no-install-recommends exiftool
|
|
- find metadata/ -name '*.jp*g' -o -name '*.png' | xargs exiftool -all=
|
|
- echo "these images have EXIF that must be stripped:"
|
|
- git --no-pager diff --stat
|
|
- git --no-pager diff --name-only --exit-code || export EXITVALUE=1
|
|
- ./tools/check-localized-metadata.py || export EXITVALUE=1
|
|
- ./tools/check-keyalias-collision.py || export EXITVALUE=1
|
|
- ./tools/check-metadata-summary-whitespace.py || export EXITVALUE=1
|
|
- exit $EXITVALUE
|
|
|
|
trigger-issuebot:
|
|
image: alpine
|
|
except:
|
|
- master@fdroid/fdroiddata
|
|
- pipelines
|
|
- schedules
|
|
- triggers
|
|
variables:
|
|
GIT_DEPTH: "1"
|
|
artifacts:
|
|
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
|
|
paths:
|
|
- logs/
|
|
when: always
|
|
expire_in: 1 week
|
|
script:
|
|
- mkdir logs
|
|
- export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
|
|
- apk add --no-cache bash curl
|
|
- ./tools/trigger-issuebot
|
|
|
|
schedule-issuebot:
|
|
image: alpine
|
|
only:
|
|
variables:
|
|
- $SCHEDULE_ISSUEBOT
|
|
variables:
|
|
GIT_DEPTH: "1"
|
|
artifacts:
|
|
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
|
|
paths:
|
|
- logs/
|
|
when: always
|
|
expire_in: 1 week
|
|
script:
|
|
- mkdir logs
|
|
- export | grep -F CI_ | grep -vFi -e password -e token > logs/export.txt
|
|
- apk add --no-cache bash curl ca-certificates python3
|
|
- python3 -m ensurepip
|
|
- $pip install python-gitlab
|
|
- ./tools/schedule-issuebot.py
|
|
|
|
checkupdates_trigger:
|
|
only:
|
|
refs:
|
|
- schedules
|
|
variables:
|
|
- $CHECKUPDATES == "true"
|
|
image: archlinux:latest
|
|
before_script:
|
|
- pacman -Sy --noconfirm parallel
|
|
script:
|
|
- parallel --verbose --delay 90 -X --jobs 10 'curl -X POST -F "token=$CI_JOB_TOKEN" -F "ref=master" -F "variables[CHECKUPDATES]=true" -F "variables[CHECKUPDATES_APPIDS]= {/.} " https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/trigger/pipeline' ":::" metadata/*.yml
|
|
|
|
checkupdates_runner:
|
|
image: registry.gitlab.com/fdroid/ci-images-checkupdates:latest
|
|
tags:
|
|
- checkupdates-runner
|
|
only:
|
|
refs:
|
|
- pipelines
|
|
variables:
|
|
- $CHECKUPDATES == "true"
|
|
before_script:
|
|
- rm -rf fdroidserver
|
|
- mkdir fdroidserver
|
|
- git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
|
|
- curl --silent https://gitlab.com/fdroid/fdroidserver/repository/master/archive.tar.gz
|
|
| tar -xz --directory=fdroidserver --strip-components=1
|
|
- export PATH="$PWD/fdroidserver:$PATH"
|
|
- touch config.py
|
|
- git config --global user.email "fdroidci@bubu1.eu"
|
|
- git config --global user.name "F-Droid checkupdates bot"
|
|
- mkdir -p ~/.ssh
|
|
- chmod 700 ~/.ssh
|
|
- cp "${GITLAB_KNOWN_HOSTS}" ~/.ssh/known_hosts
|
|
- chmod 644 ~/.ssh/known_hosts
|
|
- eval $(ssh-agent -s)
|
|
- echo "${CHECKUPDATES_SSH_DEPLOY_KEY}" | tr -d '\r' | ssh-add -
|
|
- url_host=$(echo "${CI_REPOSITORY_URL}" | sed -e 's|https\?://gitlab-ci-token:.*@|ssh://git@|g')
|
|
- git remote set-url --push origin "${url_host}"
|
|
script:
|
|
- fdroid checkupdates --allow-dirty --auto --commit ${CHECKUPDATES_APPIDS}
|
|
- git pull --rebase origin master
|
|
# when two jobs try to push at the same time they occasionally fail, so try one more time if it fails
|
|
- git push origin HEAD:master || (git pull --rebase origin master && git push origin HEAD:master)
|
|
|
|
fdroid build:
|
|
image: registry.gitlab.com/fdroid/ci-images-client:latest
|
|
allow_failure: true
|
|
artifacts:
|
|
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
|
|
paths:
|
|
- unsigned/
|
|
when: always
|
|
expire_in: 1 month
|
|
except:
|
|
- master@fdroid/fdroiddata
|
|
- pipelines
|
|
- schedules
|
|
- triggers
|
|
cache:
|
|
key: "$CI_JOB_NAME"
|
|
paths:
|
|
- .gradle
|
|
script:
|
|
- git fetch https://gitlab.com/fdroid/fdroiddata.git;
|
|
- test -d build || mkdir build
|
|
- for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD -- metadata/*.yml`; do
|
|
diff=$(git diff FETCH_HEAD...HEAD -- $f);
|
|
echo "$diff";
|
|
test $(echo "$diff" | grep '^[+-] ' | grep -v '^+ *disable:' | wc -l) = 0 && continue;
|
|
echo "$diff" | grep -E '^\+ *(NoSourceSince|Disabled):' && continue;
|
|
appid=`echo $f | sed -n -e 's,^metadata/\([^/][^/]*\)\.yml,\1,p'`;
|
|
export CHANGED="$CHANGED $appid";
|
|
done;
|
|
- test -n "$(printf "$CHANGED" | tr -d '[:space:]')"
|
|
|| { echo "no packages need processing, exiting"; exit 0; }
|
|
|
|
- test -d fdroidserver || mkdir fdroidserver
|
|
- git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
|
|
- curl --silent https://gitlab.com/fdroid/fdroidserver/repository/master/archive.tar.gz
|
|
| tar -xz --directory=fdroidserver --strip-components=1
|
|
- export PATH="`pwd`/fdroidserver:$PATH"
|
|
- export PYTHONPATH="$CI_PROJECT_DIR/fdroidserver:$CI_PROJECT_DIR/fdroidserver/examples"
|
|
- export PYTHONUNBUFFERED=true
|
|
|
|
- bash fdroidserver/buildserver/setup-env-vars $ANDROID_HOME
|
|
- adduser --disabled-password --gecos "" vagrant
|
|
- ln -s $CI_PROJECT_DIR/fdroidserver /home/vagrant/fdroidserver
|
|
- mkdir -p /vagrant/cache
|
|
- wget -q https://services.gradle.org/distributions/gradle-5.6.2-bin.zip
|
|
--output-document=/vagrant/cache/gradle-5.6.2-bin.zip
|
|
- bash fdroidserver/buildserver/provision-gradle
|
|
- bash fdroidserver/buildserver/provision-apt-get-install http://deb.debian.org/debian
|
|
- source /etc/profile.d/bsenv.sh
|
|
- apt-get dist-upgrade
|
|
|
|
# install fdroidserver from git, with deps from Debian, until fdroidserver
|
|
# is stable enough to include all the things needed here
|
|
- apt-get install -t stretch-backports
|
|
fdroidserver
|
|
python3-asn1crypto
|
|
python3-ruamel.yaml
|
|
yamllint
|
|
- apt-get purge fdroidserver
|
|
|
|
- export GRADLE_USER_HOME=$PWD/.gradle
|
|
# each `fdroid build --on-server` run expects sudo, then uninstalls it
|
|
- for build in `./tools/find-changed-builds.py`; do
|
|
set -x;
|
|
apt-get install sudo;
|
|
fdroid fetchsrclibs $build --verbose;
|
|
fdroid build --verbose --on-server --no-tarball $build;
|
|
done
|
|
|
|
|
|
# issuebot needs secrets to run, so it has to run under the 'fdroid'
|
|
# group, therefore needs the trigger without secrets, there would be
|
|
# no support for virustotal, github downloads, exodus privacy checks,
|
|
# etc. That means it has to be triggered from the lint: job, which
|
|
# runs in the fork's CI. The trigger-issuebot job adds the env var
|
|
# FROM_CI_MERGE_REQUEST_IID which is required to have this job be
|
|
# triggered.
|
|
#
|
|
# This job has to be called 'pages' because it deploys the JSON API
|
|
# to GitLab Pages.
|
|
pages:
|
|
image: registry.gitlab.com/fdroid/ci-images-client:latest
|
|
only:
|
|
refs:
|
|
- branches
|
|
variables:
|
|
- $FROM_CI_MERGE_REQUEST_IID
|
|
artifacts:
|
|
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_JOB_ID}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
|
|
paths:
|
|
- metadata/
|
|
- public/
|
|
- repo/index-v1.json
|
|
- repo/index.xml
|
|
- tmp/apkcache.json
|
|
- unsigned/
|
|
when: always
|
|
# needs lots of git history since it has to compare the merge request to current master
|
|
variables:
|
|
GIT_DEPTH: "5000"
|
|
script:
|
|
- apt-get update
|
|
- apt-get dist-upgrade
|
|
# install fdroidserver from git, with deps from Debian, until fdroidserver
|
|
# is stable enough to include all the things needed here
|
|
- apt-get install -t stretch-backports
|
|
fdroidserver
|
|
python3-asn1crypto
|
|
python3-ruamel.yaml
|
|
python3-venv
|
|
- apt-get purge fdroidserver
|
|
- test -d fdroidserver || mkdir fdroidserver
|
|
- git ls-remote https://gitlab.com/fdroid/fdroidserver.git master
|
|
- curl --silent https://gitlab.com/fdroid/fdroidserver/repository/master/archive.tar.gz
|
|
| tar -xz --directory=fdroidserver --strip-components=1
|
|
- export PATH="`pwd`/fdroidserver:$PATH"
|
|
- export PYTHONPATH="`pwd`/fdroidserver"
|
|
- export PYTHONUNBUFFERED=true
|
|
|
|
- export GRADLE_USER_HOME=$PWD/.gradle
|
|
- rm -rf $GRADLE_USER_HOME/fdroid
|
|
- mkdir -p $GRADLE_USER_HOME/fdroid
|
|
- git ls-remote https://gitlab.com/fdroid/gradle-plugins.git master
|
|
- curl --silent https://gitlab.com/fdroid/gradle-plugins/repository/master/archive.tar.gz
|
|
| tar -xz --directory=$GRADLE_USER_HOME/fdroid --strip-components=1
|
|
|
|
- git ls-remote https://gitlab.com/fdroid/issuebot.git master
|
|
- curl --silent https://gitlab.com/fdroid/issuebot/repository/master/archive.tar.gz
|
|
| tar -xz --strip-components=1
|
|
- pyvenv --system-site-packages --clear issuebot-env
|
|
- . issuebot-env/bin/activate
|
|
- pip3 install wheel # get rid of the bdist_wheel error messages
|
|
- pip3 install python-gitlab pygithub
|
|
- ./issuebot.py
|
|
|
|
# git_stats used to run here, redirect to new location
|
|
- echo '<html><head><meta http-equiv="refresh" content="0;URL=https://fdroid.gitlab.io/"></head></html>'
|
|
> public/index.html
|
|
|
|
|
|
check_git_repos:
|
|
image: debian:buster-slim
|
|
stage: test
|
|
only:
|
|
refs:
|
|
- schedules
|
|
variables:
|
|
- $CHECK_GIT_REPO == "true"
|
|
artifacts:
|
|
when: on_failure
|
|
expire_in: 1 month
|
|
paths:
|
|
- public
|
|
script:
|
|
- apt-get update
|
|
- apt-get -qy install --no-install-recommends ca-certificates git python3-colorama python3-yaml
|
|
- tools/check-git-repo-availability.py || export EXITVALUE=1
|
|
- test -d public || mkdir public
|
|
- cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true
|
|
- exit $EXITVALUE
|