fdroid-server/.gitlab-ci.yml

672 lines
23 KiB
YAML
Raw Permalink Normal View History

---
2016-02-29 20:56:02 +01:00
variables:
pip: pip3 --timeout 100 --retries 10
2021-06-08 17:15:06 +02:00
# speed up git checkout phase
GIT_DEPTH: 1
# Run the whole test suite in an environment that is like the
# buildserver guest VM. This installs python3-babel because that is
# only used by the test suite, and not needed in the buildserver.
#
# Some extra packages are required for this test run that are not
# provided by the buildserver since they are not needed there:
# * python3-babel for compiling localization files
# * gnupg-agent for the full signing setup
# * python3-clint for fancy progress bars for users
# * python3-pycountry for linting config/mirrors.yml
buildserver run-tests:
image: registry.gitlab.com/fdroid/fdroidserver:buildserver
2015-08-28 00:59:46 +02:00
script:
- apt-get update
- apt-get install gnupg-agent python3-babel python3-biplist python3-clint python3-pycountry
- ./tests/run-tests
# make sure that translations do not cause stacktraces
- cd $CI_PROJECT_DIR/locale
- for locale in *; do
test -d $locale || continue;
for cmd in `sed -n 's/.*("\(.*\)", *_.*/\1/p' $CI_PROJECT_DIR/fdroid`; do
LANGUAGE=$locale $CI_PROJECT_DIR/fdroid $cmd --help > /dev/null;
done
done
# Test that the parsing of the .yml metadata format didn't change from last
# released version. This uses the commit ID of the release tags,
# rather than the release tag itself so that contributor forks do not
# need to include the tags in them for this test to work.
#
# The COMMIT_ID should be bumped after each release, so that the list
# of sed hacks needed does not continuously grow.
metadata_v0:
image: registry.gitlab.com/fdroid/fdroidserver:buildserver
variables:
GIT_DEPTH: 1000
RELEASE_COMMIT_ID: 50aa35772b058e76b950c01e16019c072c191b73 # after switching to `git rev-parse`
script:
- git fetch https://gitlab.com/fdroid/fdroidserver.git $RELEASE_COMMIT_ID
- cd tests
- export GITCOMMIT=$(git rev-parse HEAD)
- git checkout $RELEASE_COMMIT_ID
- cd ..
- git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git
- cd fdroiddata
- ../tests/dump_internal_metadata_format.py
- cd ..
- git reset --hard
- git checkout $GITCOMMIT
- cd fdroiddata
- ../tests/dump_internal_metadata_format.py
2023-03-08 10:45:54 +01:00
- sed -i
-e '/ArchivePolicy:/d'
-e '/RequiresRoot:/d'
2023-03-08 10:45:54 +01:00
metadata/dump_*/*.yaml
- diff -uw metadata/dump_*
.apt-template: &apt-template
variables:
DEBIAN_FRONTEND: noninteractive
LANG: C.UTF-8
before_script:
- echo Etc/UTC > /etc/timezone
- echo 'APT::Install-Recommends "0";'
'APT::Install-Suggests "0";'
'APT::Get::Assume-Yes "true";'
'Acquire::Retries "20";'
'Dpkg::Use-Pty "0";'
'quiet "1";'
>> /etc/apt/apt.conf.d/99gitlab
# Ubuntu and other distros often lack https:// support
- grep Debian /etc/issue.net
&& { find /etc/apt/sources.list* -type f | xargs sed -i s,http:,https:, ; }
- echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99nocacertificates
- apt-get update
- apt-get install ca-certificates
- rm /etc/apt/apt.conf.d/99nocacertificates
- apt-get dist-upgrade
2021-04-06 11:52:44 +02:00
# Since F-Droid uses Debian as its default platform, from production
# servers to CI to contributor machines, it is important to know when
# changes in Debian break our stuff. This tests against the latest
# dependencies as they are included in Debian.
debian_testing:
image: debian:testing
<<: *apt-template
only:
- master@fdroid/fdroidserver
script:
- apt-get install
aapt
androguard
apksigner
dexdump
fdroidserver
git
gnupg
ipfs-cid
python3-biplist
python3-defusedxml
python3-pycountry
python3-setuptools
sdkmanager
- python3 -c 'import fdroidserver'
- python3 -c 'import androguard'
- python3 -c 'import sdkmanager'
- cd tests
- ./run-tests
2021-04-06 11:52:44 +02:00
# Test using latest LTS set up with the PPA, including Recommends.
ubuntu_lts_ppa:
image: ubuntu:latest
<<: *apt-template
only:
- master@fdroid/fdroidserver
script:
- export ANDROID_HOME=/usr/lib/android-sdk
- apt-get install gnupg
- while ! apt-key adv --keyserver keyserver.ubuntu.com --recv-key 9AAC253193B65D4DF1D0A13EEC4632C79C5E0151; do sleep 15; done
- export RELEASE=`sed -n 's,^deb [^ ][^ ]* \([a-z]*\).*,\1,p' /etc/apt/sources.list | head -1`
- echo "deb http://ppa.launchpad.net/fdroid/fdroidserver/ubuntu $RELEASE main" >> /etc/apt/sources.list
- apt-get update
- apt-get dist-upgrade
- apt-get install --install-recommends
dexdump
fdroidserver
git
python3-biplist
python3-pycountry
python3-setuptools
sdkmanager
# Test things work with a default branch other than 'master'
- git config --global init.defaultBranch thisisnotmasterormain
- cd tests
- ./run-tests
2021-04-06 11:52:44 +02:00
# Test using Ubuntu/jammy LTS (supported til April, 2027) with depends
# from pypi and sdkmanager. The venv is used to isolate the dist
# tarball generation environment from the clean install environment.
ubuntu_jammy_pip:
image: ubuntu:jammy
<<: *apt-template
script:
- apt-get install git default-jdk-headless python3-pip python3-venv rsync
# setup venv to act as release build machine
- python3 -m venv sdist-env
- . sdist-env/bin/activate
- ./setup.py sdist
- deactivate
- tar tzf dist/fdroidserver-*.tar.gz
# back to bare machine to act as user's install machine
- export ANDROID_HOME=/opt/android-sdk
- $pip install sdkmanager
- sdkmanager 'build-tools;33.0.0'
# Install extras_require.optional from setup.py
- $pip install biplist pycountry
2020-11-28 11:47:26 +01:00
- $pip install dist/fdroidserver-*.tar.gz
2021-01-15 19:57:32 +01:00
- tar xzf dist/fdroidserver-*.tar.gz
- cd fdroidserver-*
2022-11-14 23:25:21 +01:00
- export PATH=$PATH:$ANDROID_HOME/build-tools/33.0.0
- fdroid=`which fdroid` ./tests/run-tests
2020-11-28 11:47:26 +01:00
# check localization was properly installed
- LANGUAGE='de' fdroid --help | grep 'Gültige Befehle sind'
2021-04-06 11:52:44 +02:00
# test installation process on a bleeding edge distro with pip
arch_pip_install:
2021-05-13 15:42:36 +02:00
image: archlinux
only:
- master@fdroid/fdroidserver
2020-11-28 11:47:26 +01:00
script:
- pacman --sync --sysupgrade --refresh --noconfirm gcc git grep python-pip python-virtualenv python-wheel tar
- python -m venv venv
- source venv/bin/activate
- pip install -e .[test]
- fdroid
- fdroid readmeta
- fdroid update --help
2021-04-06 11:52:44 +02:00
# The gradlew-fdroid tests are isolated from the rest of the test
# suite, so they run as their own job.
gradlew-fdroid:
image: debian:bookworm-slim
<<: *apt-template
only:
changes:
- .gitlab-ci.yml
- gradlew-fdroid
- tests/test-gradlew-fdroid
script:
- apt-get install ca-certificates curl default-jdk-headless shellcheck unzip
- shellcheck --severity=error --color gradlew-fdroid tests/test-gradlew-fdroid
- ./tests/test-gradlew-fdroid
2021-04-06 11:52:44 +02:00
# Run all the various linters and static analysis tools.
lint_format_safety_bandit_checks:
image: debian:bookworm-slim
variables:
LANG: C.UTF-8
script:
- apt-get update
- apt-get -y install --no-install-recommends
bash
ca-certificates
dash
gcc
git
make
pycodestyle
pyflakes3
pylint
python3-dev
python3-git
python3-nose
python3-pip
python3-yaml
shellcheck
- $pip install --break-system-packages bandit safety
- export EXITVALUE=0
- function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
- ./hooks/pre-commit || set_error
- bandit
2021-06-18 06:11:39 +02:00
-r
-ii
2021-06-18 06:11:39 +02:00
--ini .bandit
|| set_error
- safety check --full-report || set_error
- pylint --output-format=colorized --reports=n
fdroid
makebuildserver
setup.py
fdroidserver/*.py
tests/*.py
tests/*.TestCase
|| set_error
- shellcheck --exclude SC2046,SC2090 --severity=warning --color tests/run-tests
|| set_error
- exit $EXITVALUE
# Run all the various linters and static analysis tools.
locales:
image: debian:bookworm-slim
variables:
LANG: C.UTF-8
script:
- apt-get update
- apt-get -y install --no-install-recommends
gettext
make
python3-babel
- export EXITVALUE=0
- function set_error() { export EXITVALUE=1; printf "\x1b[31mERROR `history|tail -2|head -1|cut -b 6-500`\x1b[0m\n"; }
- make -C locale compile || set_error
- rm -f locale/*/*/*.mo
- pybabel compile --domain=fdroidserver --directory locale 2>&1 | { grep -F "error:" && exit 1; } || true
- exit $EXITVALUE
2021-04-06 11:52:44 +02:00
black:
image: debian:bookworm-slim
<<: *apt-template
script:
- apt-get install black
- black --check --diff --color $CI_PROJECT_DIR
fedora_latest:
image: fedora:latest
only:
- master@fdroid/fdroidserver
script:
# tricks to hopefully make runs more reliable
- echo "timeout=600" >> /etc/dnf/dnf.conf
- echo "retries=50" >> /etc/dnf/dnf.conf
- echo "keepcache=True" >> /etc/dnf/dnf.conf
- dnf -y update || dnf -y update
- dnf -y install @development-tools
diffutils
findutils
git
gnupg
java-17-openjdk-devel
openssl
python3
python3-babel
python3-matplotlib
python3-pip
python3-pycountry
rsync
which
- $pip install sdkmanager
- ./setup.py sdist
- useradd -m -c "test account" --password "fakepassword" testuser
- su testuser --login --command "cd `pwd`; $pip install --user dist/fdroidserver-*.tar.gz"
- test -e ~testuser/.local/share/locale/de/LC_MESSAGES/fdroidserver.mo
- export BUILD_TOOLS_VERSION=`sed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py`
- export ANDROID_HOME=`pwd`/android-sdk
- mkdir -p $ANDROID_HOME/licenses/
- printf "\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e\n24333f8a63b6825ea9c5514f83c2829b004d1fee" > $ANDROID_HOME/licenses/android-sdk-license
- printf "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
- printf "\n79120722343a6f314e0719f863036c702b0e6b2a\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license-old
- mkdir ~/.android
- touch ~/.android/repositories.cfg
- sdkmanager "platform-tools" "build-tools;$BUILD_TOOLS_VERSION"
- chown -R testuser .
- cd tests
- su testuser --login --command
"cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
2021-04-06 11:52:44 +02:00
macOS:
tags:
- saas-macos-medium-m1
only:
- master@fdroid/fdroidserver
script:
- export HOMEBREW_CURL_RETRIES=10
- brew update > /dev/null
- brew upgrade
- brew install fdroidserver
# Android SDK and Java JDK
- brew install --cask android-commandlinetools temurin # temurin is a JDK
# test suite dependencies
- brew install dash bash coreutils gnu-sed
# TODO port tests/run-tests to POSIX and gsed, it has a couple GNU-isms like du --bytes
- export PATH="$(brew --prefix fdroidserver)/libexec/bin:$(brew --prefix coreutils)/libexec/gnubin:$PATH"
- brew autoremove
- brew info fdroidserver
- export BUILD_TOOLS_VERSION=`gsed -n "s,^MINIMUM_APKSIGNER_BUILD_TOOLS_VERSION\s*=\s*['\"]\(.*\)[['\"],\1,p" fdroidserver/common.py`
- export ANDROID_HOME="$(brew --prefix)/share/android-commandlinetools"
- mkdir -p "$ANDROID_HOME/licenses"
- echo -e "\n8933bad161af4178b1185d1a37fbf41ea5269c55" > "$ANDROID_HOME/licenses/android-sdk-license"
- echo -e "\nd56f5187479451eabf01fb78af6dfcb131a6481e" >> "$ANDROID_HOME/licenses/android-sdk-license"
- echo -e "\n24333f8a63b6825ea9c5514f83c2829b004d1fee" >> "$ANDROID_HOME/licenses/android-sdk-license"
- $(brew --prefix)/bin/sdkmanager "build-tools;$BUILD_TOOLS_VERSION"
- echo "macOS sticks with bash 3.x because of licenses, so avoid new bash syntax"
- /bin/bash --version
- /bin/bash -n gradlew-fdroid tests/run-tests
# TODO remove the packages below once they are included in the Homebrew package
- $(brew --prefix fdroidserver)/libexec/bin/python3 -m pip install biplist pycountry
# test fdroidserver from git with current package's dependencies
- fdroid="$(brew --prefix fdroidserver)/libexec/bin/python3 $PWD/fdroid" ./tests/run-tests
gradle:
image: debian:bookworm-slim
<<: *apt-template
variables:
GIT_DEPTH: 1000
script:
- apt-get install
ca-certificates
git
openssh-client
python3-bs4
python3-colorama
python3-git
python3-gitlab
python3-packaging
python3-requests
# if this is a merge request fork, then only check if relevant files changed
- if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
git fetch https://gitlab.com/fdroid/fdroidserver.git;
for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
test "$f" == "makebuildserver" && export CHANGED="yes";
test "$f" == "gradlew-fdroid" && export CHANGED="yes";
done;
test -z "$CHANGED" && exit;
fi
- ./tests/gradle-release-checksums.py
2021-04-06 11:52:44 +02:00
# Run an actual build in a simple, faked version of the buildserver guest VM.
fdroid build:
image: registry.gitlab.com/fdroid/fdroidserver:buildserver
only:
changes:
- .gitlab-ci.yml
- fdroidserver/build.py
- fdroidserver/common.py
- fdroidserver/exception.py
- fdroidserver/metadata.py
- fdroidserver/net.py
- fdroidserver/scanner.py
- fdroidserver/vmtools.py
cache:
key: "$CI_JOB_NAME"
paths:
- .gradle
script:
- apt-get update
- apt-get dist-upgrade
- apt-get clean
- test -n "$fdroidserver" || source /etc/profile.d/bsenv.sh
- ln -fsv "$CI_PROJECT_DIR" "$fdroidserver"
# TODO remove sdkmanager install once it is included in the buildserver image
- apt-get install sdkmanager
- rm -rf "$ANDROID_HOME/tools" # TODO remove once sdkmanager can upgrade installed packages
- sdkmanager "tools" "platform-tools" "build-tools;31.0.0"
- git ls-remote https://gitlab.com/fdroid/fdroiddata.git master
- git clone --depth 1 https://gitlab.com/fdroid/fdroiddata.git
- cd fdroiddata
- for d in build logs repo tmp unsigned $home_vagrant/.android; do
test -d $d || mkdir $d;
chown -R vagrant $d;
done
- export GRADLE_USER_HOME=$home_vagrant/.gradle
- export fdroid="sudo --preserve-env --user vagrant
env PATH=$fdroidserver:$PATH
env PYTHONPATH=$fdroidserver:$fdroidserver/examples
env PYTHONUNBUFFERED=true
env TERM=$TERM
env HOME=$home_vagrant
fdroid"
- chown -R vagrant $home_vagrant
- chown -R vagrant $fdroidserver/.git
- chown vagrant $fdroidserver/
- chown -R vagrant .git
- chown vagrant .
# try user build
- $fdroid build --verbose --latest org.fdroid.fdroid.privileged
# try on-server build
- $fdroid build --verbose --on-server --no-tarball --latest org.fdroid.fdroid
# each `fdroid build --on-server` run expects sudo, then uninstalls it
- if dpkg --list sudo; then echo "sudo should not be still there"; exit 1; fi
- 'if [ ! -f repo/status/running.json ]; then echo "ERROR: running.json does not exist!"; exit 1; fi'
- 'if [ ! -f repo/status/build.json ]; then echo "ERROR: build.json does not exist!"; exit 1; fi'
2021-04-06 11:52:44 +02:00
# test the plugin API and specifically the fetchsrclibs plugin, which
# is used by the `fdroid build` job. This uses a fixed commit from
# fdroiddata because that one is known to work, and this is a CI job,
# so it should be isolated from the normal churn of fdroiddata.
plugin_fetchsrclibs:
image: debian:bookworm-slim
<<: *apt-template
only:
changes:
- .gitlab-ci.yml
- examples/fdroid_fetchsrclibs.py
- fdroidserver/__main__.py
script:
- apt-get install
curl
git
2021-10-14 13:55:51 +02:00
python3-cffi
python3-matplotlib
gitlab-ci: fix plugin_fetchsrclibs by using Debian packages * https://gitlab.com/eighthave/fdroidserver/-/jobs/1961701458 Collecting pynacl>=1.0.1 (from paramiko->fdroidserver==2.1a0) Downloading https://files.pythonhosted.org/packages/a7/22/27582568be639dfe22ddb3902225f91f2f17ceff88ce80e4db396c8986da/PyNaCl-1.5.0.tar.gz (3.4MB) Installing build dependencies: started Installing build dependencies: finished with status 'error' Complete output from command /builds/eighthave/fdroidserver/env/bin/python3 -m pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-xokvr6uk --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools>=40.8.0 wheel "cffi>=1.4.1; python_implementation != 'PyPy'": Collecting setuptools>=40.8.0 Using cached https://files.pythonhosted.org/packages/eb/53/0dd4c7960579da8be13fa9b2c2591643d37f323e3d79f8bc8b1b6c8e6217/setuptools-60.5.0-py3-none-any.whl Collecting wheel Using cached https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl Collecting cffi>=1.4.1 Downloading https://files.pythonhosted.org/packages/00/9e/92de7e1217ccc3d5f352ba21e52398372525765b2e0c4530e6eb2ba9282a/cffi-1.15.0.tar.gz (484kB) Collecting pycparser (from cffi>=1.4.1) Downloading https://files.pythonhosted.org/packages/62/d5/5f610ebe421e85889f2e55e33b7f9a6795bd982198517d912eb1c76e1a53/pycparser-2.21-py2.py3-none-any.whl (118kB) Building wheels for collected packages: cffi Running setup.py bdist_wheel for cffi: started Running setup.py bdist_wheel for cffi: finished with status 'error' Complete output from command /builds/eighthave/fdroidserver/env/bin/python3 -u -c "import setuptools, tokenize;__file__='/tmp/pip-install-djg9jc8p/cffi/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" bdist_wheel -d /tmp/pip-wheel-d1knhl7l --python-tag cp37: unable to execute 'x86_64-linux-gnu-gcc': No such file or directory unable to execute 'x86_64-linux-gnu-gcc': No such file or directory No working compiler found, or bogus compiler options passed to the compiler from Python's standard "distutils" module. See the error messages above. Likely, the problem is not related to CFFI but generic to the setup.py of any Python package that tries to compile C code. (Hints: on OS/X 10.8, for errors about -mno-fused-madd see http://stackoverflow.com/questions/22313407/ Otherwise, see https://wiki.python.org/moin/CompLangPython or the IRC channel #python on irc.libera.chat.) Trying to continue anyway. If you are trying to install CFFI from a build done in a different context, you can ignore this warning. running bdist_wheel running build running build_py creating build creating build/lib.linux-x86_64-3.7 creating build/lib.linux-x86_64-3.7/cffi copying cffi/backend_ctypes.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/vengine_gen.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/vengine_cpy.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/pkgconfig.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/lock.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/recompiler.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/__init__.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/cparser.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/ffiplatform.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/verifier.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/cffi_opcode.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/setuptools_ext.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/api.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/commontypes.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/error.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/model.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/_cffi_include.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/parse_c_type.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/_embedding.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/_cffi_errors.h -> build/lib.linux-x86_64-3.7/cffi running build_ext building '_cffi_backend' extension creating build/temp.linux-x86_64-3.7 creating build/temp.linux-x86_64-3.7/c x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/ffi -I/usr/include/libffi -I/builds/eighthave/fdroidserver/env/include -I/usr/include/python3.7m -c c/_cffi_backend.c -o build/temp.linux-x86_64-3.7/c/_cffi_backend.o unable to execute 'x86_64-linux-gnu-gcc': No such file or directory error: command 'x86_64-linux-gnu-gcc' failed with exit status 1 ---------------------------------------- Failed building wheel for cffi Running setup.py clean for cffi Failed to build cffi Installing collected packages: setuptools, wheel, pycparser, cffi Running setup.py install for cffi: started Running setup.py install for cffi: finished with status 'error' Complete output from command /builds/eighthave/fdroidserver/env/bin/python3 -u -c "import setuptools, tokenize;__file__='/tmp/pip-install-djg9jc8p/cffi/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-record-ek80c81s/install-record.txt --single-version-externally-managed --prefix /tmp/pip-build-env-xokvr6uk --compile --install-headers /builds/eighthave/fdroidserver/env/include/site/python3.7/cffi: unable to execute 'x86_64-linux-gnu-gcc': No such file or directory unable to execute 'x86_64-linux-gnu-gcc': No such file or directory No working compiler found, or bogus compiler options passed to the compiler from Python's standard "distutils" module. See the error messages above. Likely, the problem is not related to CFFI but generic to the setup.py of any Python package that tries to compile C code. (Hints: on OS/X 10.8, for errors about -mno-fused-madd see http://stackoverflow.com/questions/22313407/ Otherwise, see https://wiki.python.org/moin/CompLangPython or the IRC channel #python on irc.libera.chat.) Trying to continue anyway. If you are trying to install CFFI from a build done in a different context, you can ignore this warning. running install running build running build_py creating build creating build/lib.linux-x86_64-3.7 creating build/lib.linux-x86_64-3.7/cffi copying cffi/backend_ctypes.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/vengine_gen.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/vengine_cpy.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/pkgconfig.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/lock.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/recompiler.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/__init__.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/cparser.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/ffiplatform.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/verifier.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/cffi_opcode.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/setuptools_ext.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/api.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/commontypes.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/error.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/model.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/_cffi_include.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/parse_c_type.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/_embedding.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/_cffi_errors.h -> build/lib.linux-x86_64-3.7/cffi running build_ext building '_cffi_backend' extension creating build/temp.linux-x86_64-3.7 creating build/temp.linux-x86_64-3.7/c x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/ffi -I/usr/include/libffi -I/builds/eighthave/fdroidserver/env/include -I/usr/include/python3.7m -c c/_cffi_backend.c -o build/temp.linux-x86_64-3.7/c/_cffi_backend.o unable to execute 'x86_64-linux-gnu-gcc': No such file or directory error: command 'x86_64-linux-gnu-gcc' failed with exit status 1 ---------------------------------------- Command "/builds/eighthave/fdroidserver/env/bin/python3 -u -c "import setuptools, tokenize;__file__='/tmp/pip-install-djg9jc8p/cffi/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-record-ek80c81s/install-record.txt --single-version-externally-managed --prefix /tmp/pip-build-env-xokvr6uk --compile --install-headers /builds/eighthave/fdroidserver/env/include/site/python3.7/cffi" failed with error code 1 in /tmp/pip-install-djg9jc8p/cffi/ ---------------------------------------- Command "/builds/eighthave/fdroidserver/env/bin/python3 -m pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-xokvr6uk --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools>=40.8.0 wheel "cffi>=1.4.1; python_implementation != 'PyPy'"" failed with error code 1 in None
2021-12-23 21:39:29 +01:00
python3-nacl
python3-paramiko
python3-pil
python3-pip
gitlab-ci: fix plugin_fetchsrclibs by using Debian packages * https://gitlab.com/eighthave/fdroidserver/-/jobs/1961701458 Collecting pynacl>=1.0.1 (from paramiko->fdroidserver==2.1a0) Downloading https://files.pythonhosted.org/packages/a7/22/27582568be639dfe22ddb3902225f91f2f17ceff88ce80e4db396c8986da/PyNaCl-1.5.0.tar.gz (3.4MB) Installing build dependencies: started Installing build dependencies: finished with status 'error' Complete output from command /builds/eighthave/fdroidserver/env/bin/python3 -m pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-xokvr6uk --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools>=40.8.0 wheel "cffi>=1.4.1; python_implementation != 'PyPy'": Collecting setuptools>=40.8.0 Using cached https://files.pythonhosted.org/packages/eb/53/0dd4c7960579da8be13fa9b2c2591643d37f323e3d79f8bc8b1b6c8e6217/setuptools-60.5.0-py3-none-any.whl Collecting wheel Using cached https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl Collecting cffi>=1.4.1 Downloading https://files.pythonhosted.org/packages/00/9e/92de7e1217ccc3d5f352ba21e52398372525765b2e0c4530e6eb2ba9282a/cffi-1.15.0.tar.gz (484kB) Collecting pycparser (from cffi>=1.4.1) Downloading https://files.pythonhosted.org/packages/62/d5/5f610ebe421e85889f2e55e33b7f9a6795bd982198517d912eb1c76e1a53/pycparser-2.21-py2.py3-none-any.whl (118kB) Building wheels for collected packages: cffi Running setup.py bdist_wheel for cffi: started Running setup.py bdist_wheel for cffi: finished with status 'error' Complete output from command /builds/eighthave/fdroidserver/env/bin/python3 -u -c "import setuptools, tokenize;__file__='/tmp/pip-install-djg9jc8p/cffi/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" bdist_wheel -d /tmp/pip-wheel-d1knhl7l --python-tag cp37: unable to execute 'x86_64-linux-gnu-gcc': No such file or directory unable to execute 'x86_64-linux-gnu-gcc': No such file or directory No working compiler found, or bogus compiler options passed to the compiler from Python's standard "distutils" module. See the error messages above. Likely, the problem is not related to CFFI but generic to the setup.py of any Python package that tries to compile C code. (Hints: on OS/X 10.8, for errors about -mno-fused-madd see http://stackoverflow.com/questions/22313407/ Otherwise, see https://wiki.python.org/moin/CompLangPython or the IRC channel #python on irc.libera.chat.) Trying to continue anyway. If you are trying to install CFFI from a build done in a different context, you can ignore this warning. running bdist_wheel running build running build_py creating build creating build/lib.linux-x86_64-3.7 creating build/lib.linux-x86_64-3.7/cffi copying cffi/backend_ctypes.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/vengine_gen.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/vengine_cpy.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/pkgconfig.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/lock.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/recompiler.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/__init__.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/cparser.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/ffiplatform.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/verifier.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/cffi_opcode.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/setuptools_ext.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/api.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/commontypes.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/error.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/model.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/_cffi_include.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/parse_c_type.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/_embedding.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/_cffi_errors.h -> build/lib.linux-x86_64-3.7/cffi running build_ext building '_cffi_backend' extension creating build/temp.linux-x86_64-3.7 creating build/temp.linux-x86_64-3.7/c x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/ffi -I/usr/include/libffi -I/builds/eighthave/fdroidserver/env/include -I/usr/include/python3.7m -c c/_cffi_backend.c -o build/temp.linux-x86_64-3.7/c/_cffi_backend.o unable to execute 'x86_64-linux-gnu-gcc': No such file or directory error: command 'x86_64-linux-gnu-gcc' failed with exit status 1 ---------------------------------------- Failed building wheel for cffi Running setup.py clean for cffi Failed to build cffi Installing collected packages: setuptools, wheel, pycparser, cffi Running setup.py install for cffi: started Running setup.py install for cffi: finished with status 'error' Complete output from command /builds/eighthave/fdroidserver/env/bin/python3 -u -c "import setuptools, tokenize;__file__='/tmp/pip-install-djg9jc8p/cffi/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-record-ek80c81s/install-record.txt --single-version-externally-managed --prefix /tmp/pip-build-env-xokvr6uk --compile --install-headers /builds/eighthave/fdroidserver/env/include/site/python3.7/cffi: unable to execute 'x86_64-linux-gnu-gcc': No such file or directory unable to execute 'x86_64-linux-gnu-gcc': No such file or directory No working compiler found, or bogus compiler options passed to the compiler from Python's standard "distutils" module. See the error messages above. Likely, the problem is not related to CFFI but generic to the setup.py of any Python package that tries to compile C code. (Hints: on OS/X 10.8, for errors about -mno-fused-madd see http://stackoverflow.com/questions/22313407/ Otherwise, see https://wiki.python.org/moin/CompLangPython or the IRC channel #python on irc.libera.chat.) Trying to continue anyway. If you are trying to install CFFI from a build done in a different context, you can ignore this warning. running install running build running build_py creating build creating build/lib.linux-x86_64-3.7 creating build/lib.linux-x86_64-3.7/cffi copying cffi/backend_ctypes.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/vengine_gen.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/vengine_cpy.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/pkgconfig.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/lock.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/recompiler.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/__init__.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/cparser.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/ffiplatform.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/verifier.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/cffi_opcode.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/setuptools_ext.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/api.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/commontypes.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/error.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/model.py -> build/lib.linux-x86_64-3.7/cffi copying cffi/_cffi_include.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/parse_c_type.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/_embedding.h -> build/lib.linux-x86_64-3.7/cffi copying cffi/_cffi_errors.h -> build/lib.linux-x86_64-3.7/cffi running build_ext building '_cffi_backend' extension creating build/temp.linux-x86_64-3.7 creating build/temp.linux-x86_64-3.7/c x86_64-linux-gnu-gcc -pthread -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/include/ffi -I/usr/include/libffi -I/builds/eighthave/fdroidserver/env/include -I/usr/include/python3.7m -c c/_cffi_backend.c -o build/temp.linux-x86_64-3.7/c/_cffi_backend.o unable to execute 'x86_64-linux-gnu-gcc': No such file or directory error: command 'x86_64-linux-gnu-gcc' failed with exit status 1 ---------------------------------------- Command "/builds/eighthave/fdroidserver/env/bin/python3 -u -c "import setuptools, tokenize;__file__='/tmp/pip-install-djg9jc8p/cffi/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /tmp/pip-record-ek80c81s/install-record.txt --single-version-externally-managed --prefix /tmp/pip-build-env-xokvr6uk --compile --install-headers /builds/eighthave/fdroidserver/env/include/site/python3.7/cffi" failed with error code 1 in /tmp/pip-install-djg9jc8p/cffi/ ---------------------------------------- Command "/builds/eighthave/fdroidserver/env/bin/python3 -m pip install --ignore-installed --no-user --prefix /tmp/pip-build-env-xokvr6uk --no-warn-script-location --no-binary :none: --only-binary :none: -i https://pypi.org/simple -- setuptools>=40.8.0 wheel "cffi>=1.4.1; python_implementation != 'PyPy'"" failed with error code 1 in None
2021-12-23 21:39:29 +01:00
python3-pycparser
python3-venv
- python3 -m venv --system-site-packages env
- . env/bin/activate
- export PATH="$CI_PROJECT_DIR:$PATH"
- export PYTHONPATH="$CI_PROJECT_DIR/examples"
# workaround https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1003252
- export SETUPTOOLS_USE_DISTUTILS=stdlib
- $pip install -e .
- fdroid | grep fetchsrclibs
- mkdir fdroiddata
- commitid=b9e9a077d720c86ff6fff4dbb341254cc4370b1a
- curl https://gitlab.com/fdroid/fdroiddata/-/archive/${commitid}/fdroiddata-${commitid}.tar.gz
| tar -xz --directory=fdroiddata --strip-components=1
- cd fdroiddata
2023-11-16 07:58:55 +01:00
- fdroid fetchsrclibs freemap.opentrail:4 --verbose
- test -d build/freemap.opentrail/.git
- test -d build/srclib/andromaps/.git
- test -d build/srclib/freemaplib/.git
- test -d build/srclib/freemaplibProj/.git
- test -d build/srclib/JCoord/.git
- test -d build/srclib/javaproj/.git
# test a full update and deploy cycle to gitlab.com
servergitmirrors:
image: debian:bookworm-slim
<<: *apt-template
only:
- master@fdroid/fdroidserver
script:
- apt-get install
default-jdk-headless
git
openssh-client
openssl
python3-cffi
python3-cryptography
python3-matplotlib
python3-nacl
python3-pil
python3-pip
python3-pycparser
python3-setuptools
python3-venv
rsync
wget
- apt-get install apksigner
- python3 -m venv --system-site-packages env
- . env/bin/activate
- export PYTHONPATH=`pwd`
- export SETUPTOOLS_USE_DISTUTILS=stdlib # https://github.com/pypa/setuptools/issues/2956
- $pip install -e .
- mkdir /root/.ssh/
- ./tests/key-tricks.py
- ssh-keyscan gitlab.com >> /root/.ssh/known_hosts
- test -d /tmp/fdroid/repo || mkdir -p /tmp/fdroid/repo
- cp tests/config.py tests/keystore.jks /tmp/fdroid/
- cp tests/repo/com.politedroid_6.apk /tmp/fdroid/repo/
- cd /tmp/fdroid
- touch fdroid-icon.png
- printf "\nservergitmirrors = 'git@gitlab.com:fdroid/ci-test-servergitmirrors-repo.git'\n" >> config.py
- $PYTHONPATH/fdroid update --verbose --create-metadata
- $PYTHONPATH/fdroid deploy --verbose
- export DLURL=`grep -Eo 'https://gitlab.com/fdroid/ci-test-servergitmirrors-repo[^"]+' repo/index-v1.json`
- echo $DLURL
- wget $DLURL/index-v1.jar
- diff repo/index-v1.jar index-v1.jar
Build documentation:
image: debian:bookworm-slim
2022-10-20 22:08:00 +02:00
<<: *apt-template
script:
- apt-get install make python3-sphinx python3-numpydoc python3-pydata-sphinx-theme pydocstyle fdroidserver
- apt purge fdroidserver
# ignore vendored files
- pydocstyle --verbose --match='(?!apksigcopier|looseversion|setup|test_).*\.py' fdroidserver
- cd docs
- sphinx-apidoc -o ./source ../fdroidserver -M -e
- PYTHONPATH=.. sphinx-autogen -o generated source/*.rst
- PYTHONPATH=.. make html
artifacts:
paths:
2021-05-10 07:50:45 +02:00
- docs/build/html/
2021-06-08 17:15:06 +02:00
# this job will only run in branches called "windows" until the Windows port is complete
2021-06-08 22:04:21 +02:00
Windows:
tags:
- windows
only:
- windows
2021-06-08 22:04:21 +02:00
script:
- Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
- choco install --no-progress -y git --force --params "/GitAndUnixToolsOnPath"
- choco install --no-progress -y python3 --version=3.10
2021-06-08 22:04:21 +02:00
- choco install --no-progress -y jdk8
- choco install --no-progress -y rsync
- refreshenv
- python -m pip install --upgrade babel pip setuptools
- python -m pip install -e .
- $files = @(Get-ChildItem tests\*.TestCase)
- foreach ($f in $files) {
write-output $f;
python $f;
if( $LASTEXITCODE -eq 0 ) {
write-output "SUCCESS $f";
} else {
write-output "ERROR $f failed";
}
}
# these are the tests that must pass
- python tests\checkupdates.TestCase
- python tests\exception.TestCase
- python tests\import_subcommand.TestCase
2021-06-08 22:04:21 +02:00
- python tests\init.TestCase
- python tests\lint.TestCase
- python tests\main.TestCase
- python tests\metadata.TestCase
- python tests\rewritemeta.TestCase
- python tests\vcs.TestCase
after_script:
- Copy-Item C:\ProgramData\chocolatey\logs\chocolatey.log
artifacts:
when: always
paths:
- "*.log"
2021-10-14 13:40:19 +02:00
allow_failure:
exit_codes: 1
2021-06-08 22:04:21 +02:00
2021-05-10 07:50:45 +02:00
pages:
image: alpine:latest
stage: deploy
script:
2021-06-08 17:15:06 +02:00
- cp docs/build/html public -r # GL Pages needs the files in a directory named "public"
2021-05-10 07:53:21 +02:00
artifacts:
2021-05-10 07:50:45 +02:00
paths:
- public
needs: ["Build documentation"]
2021-05-10 08:06:08 +02:00
rules:
2021-06-08 17:15:06 +02:00
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' # only publish pages on default (master) branch
# This job pushes the official CI docker image based on the master
# branch, so in fdroid/fdroidserver, it should only run on the master
# branch. Otherwise, tags or other branches will overwrite the docker
# image which is supposed to be what is in master.
docker:
dependencies:
- fdroid build
only:
changes:
- .gitlab-ci.yml
- makebuildserver
- buildserver/*
variables:
- $CI_COMMIT_BRANCH == "master" || $CI_PROJECT_NAMESPACE != "fdroid"
image: docker:dind
services:
- docker:dind
variables:
RELEASE_IMAGE: $CI_REGISTRY_IMAGE:buildserver
script:
# git ref names can contain many chars that are not allowed in docker tags
- export TEST_IMAGE=$CI_REGISTRY_IMAGE:$(printf $CI_COMMIT_REF_NAME | sed 's,[^a-zA-Z0-9_.-],_,g')
- cd buildserver
- docker build -t $TEST_IMAGE --build-arg GIT_REV_PARSE_HEAD=$(git rev-parse HEAD) .
- docker tag $TEST_IMAGE $RELEASE_IMAGE
2024-01-31 15:36:22 +01:00
- docker tag $TEST_IMAGE ${RELEASE_IMAGE}-bookworm
- echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com
# This avoids filling up gitlab.com free tier accounts with unused docker images.
- if test -z "$FDROID_PUSH_DOCKER_IMAGE"; then
echo "Skipping docker push to save quota on your gitlab namespace.";
echo "If you want to enable the push, set FDROID_PUSH_DOCKER_IMAGE in";
echo "https://gitlab.com/$CI_PROJECT_NAMESPACE/fdroidserver/-/settings/ci_cd#js-cicd-variables-settings";
exit 0;
fi
- docker push $RELEASE_IMAGE
2024-01-31 15:36:22 +01:00
- docker push $RELEASE_IMAGE-bookworm