build: improve gradle experience

This expands the gradle wrapper shell script used by the buildserver for
usage outside the buildserver environment. It also allows downloading
whitelisted versions of gradle if they are not yet deployed to the
buildserver by simply upsating the copy of fdroidserver (in contrast to
having to reprovision the whole buildserver).

We first move the buildserver/gradle shell script to the repo root
as gradlew-fdroid, as it's an fdroid specific gradle wrapper.
We also now sync it inside the build VM before each build.

We then add a list of whitelisted gradle distributions taken from the
makebuildserver script.

The script additionally now reads two env vars which tell it where to
expect installed versions of gradle and where it might store downloaded
gradle .zip files. Both of those are configurable from config.py. As the
first should normally just be a subdir of the second it's not exposed in
the example config.py but only used by the buildserver config.py.

Default config now uses this internal gradle wrapper but a path to a
custom wrapper or specific gradle distribution can still be set from
config.py.

Closes fdroid/fdroidserver#98
Ref: fdroid/fdroidserver#370
This commit is contained in:
Marcus Hoffmann 2018-07-08 17:34:17 +02:00
parent 68cb81f38d
commit 9889a98dea
No known key found for this signature in database
GPG Key ID: ACDF63BC43D5E530
9 changed files with 217 additions and 109 deletions

View File

@ -1,5 +1,4 @@
include buildserver/config.buildserver.py
include buildserver/gradle
include buildserver/provision-android-ndk
include buildserver/provision-android-sdk
include buildserver/provision-apt-get-install
@ -23,6 +22,7 @@ include examples/opensc-fdroid.cfg
include examples/public-read-only-s3-bucket-policy.json
include examples/template.yml
include fdroid
include gradlew-fdroid
include LICENSE
include locale/bo/LC_MESSAGES/fdroidserver.mo
include locale/de/LC_MESSAGES/fdroidserver.mo

View File

@ -75,7 +75,5 @@ Vagrant.configure("2") do |config|
config.vm.provision "shell", path: "provision-pip",
args: ["compare-locales"]
config.vm.provision "shell", path: "provision-gradle"
config.vm.provision "file", source: "gradle",
destination: "/opt/gradle/bin/gradle"
end

View File

@ -12,3 +12,4 @@ ndk_paths = {
java_paths = {
'8': "/usr/lib/jvm/java-8-openjdk-amd64",
}
gradle_version_dir = "/opt/gradle/versions"

View File

@ -1,98 +0,0 @@
#!/bin/bash
bindir="$(dirname $0)"
basedir="$(dirname $bindir)"
verdir="${basedir}/versions"
args=("$@")
run_gradle() {
"${verdir}/${v_found}/bin/gradle" "${args[@]}"
exit $?
}
contains() {
local e
for e in $2; do
[[ $e == $1 ]] && return 0;
done
return 1
}
# key-value pairs of what gradle version (value) each gradle plugin version
# (key) should accept. plugin versions are actually prefixes and catch sub-
# versions as well. Pairs are taken from:
# https://developer.android.com/studio/releases/gradle-plugin.html#updating-gradle
d_plugin_k=(3.1 3.0 2.3 2.2 2.1.3 2.1 2.0 1.5 1.3 1.2 1.1 1.0 0.14 0.13 0.12 0.11 0.10 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2)
d_plugin_v=(4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
# All gradle versions we know about
plugin_v=(4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4)
v_all=${plugin_v[@]}
echo "Available gradle versions: ${v_all[@]}"
# Earliest takes priority
for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do
[[ -f $f ]] || continue
while read l; do
if [[ $l == 'distributionUrl='* ]]; then
wrapper_ver=$(echo -n "$l" | sed "s/.*gradle-\\([0-9\\.\\+]\\+\\).*/\\1/")
fi
done < $f
done
if [[ -n $wrapper_ver ]]; then
v_found=$wrapper_ver
echo "Found $v_found via distributionUrl"
run_gradle
fi
# Earliest takes priority
for f in {.,..}/build.gradle; do
[[ -f $f ]] || continue
while read l; do
if [[ -z "$plugin_pver" && $l == *'com.android.tools.build:gradle:'* ]]; then
plugin_pver=$(echo -n "$l" | sed "s/.*com.android.tools.build:gradle:\\([0-9\\.\\+]\\+\\).*/\\1/")
elif [[ -z "$wrapper_ver" && $l == *'gradleVersion = '* ]]; then
wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion *=* *[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/")
fi
done < $f
done
if [[ -n $wrapper_ver ]]; then
v_found=$wrapper_ver
echo "Found $v_found via gradleVersion"
run_gradle
fi
if [[ -n $plugin_pver ]]; then
i=0
match=false
for k in ${d_plugin_k[@]}; do
if [[ $plugin_pver == ${k}* ]]; then
plugin_ver=${d_plugin_v[$i]}
match=true
break
fi
let i++
done
if $match; then
v_found=$plugin_ver
echo "Found $v_found via gradle plugin version $k"
fi
fi
# Find the highest version available
for v in ${plugin_v[*]}; do
if contains $v "${v_all[*]}"; then
v_def=$v
break
fi
done
if [[ -z $v_found ]]; then
echo "No suitable gradle version found - defaulting to $v_def"
v_found=$v_def
fi
run_gradle

View File

@ -1,6 +1,6 @@
#!/bin/bash
set -e
set -ex
test -e /opt/gradle/versions || mkdir -p /opt/gradle/versions
cd /opt/gradle/versions
@ -15,6 +15,7 @@ done
chmod -R a+rX /opt/gradle
test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin
touch /opt/gradle/bin/gradle
chown vagrant.vagrant /opt/gradle/bin/gradle
chmod 0755 /opt/gradle/bin/gradle
ln -fs /home/vagrant/fdroidserver/gradlew-fdroid /opt/gradle/bin/gradle
chown -h vagrant.vagrant /opt/gradle/bin/gradle
chown vagrant.vagrant /opt/gradle/versions
chmod 0755 /opt/gradle/versions

View File

@ -21,6 +21,10 @@
# 'r17b': None,
# }
# Directory to store downloaded tools in (i.e. gradle versions)
# By default, these are stored in ~/.cache/fdroidserver
# cachedir = cache
# java_paths = {
# '8': "/usr/lib/jvm/java-8-openjdk",
# }
@ -39,6 +43,7 @@
# mvn3 = "mvn"
# Command or path to binary for running Gradle
# Defaults to using an internal gradle wrapper (gradlew-fdroid).
# gradle = "gradle"
# Set the maximum age (in days) of an index that a client should accept from

View File

@ -125,7 +125,9 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
ftp.mkdir('fdroidserver')
ftp.chdir('fdroidserver')
ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid')
ftp.put(os.path.join(serverpath, '..', 'gradlew-fdroid'), 'gradlew-fdroid')
ftp.chmod('fdroid', 0o755)
ftp.chmod('gradlew-fdroid', 0o755)
send_dir(os.path.join(serverpath))
ftp.chdir(homedir)
@ -496,8 +498,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
cmd += ['-P' + kv for kv in build.gradleprops]
cmd += ['clean']
p = FDroidPopen(cmd, cwd=root_dir)
p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']})
elif bmethod == 'buildozer':
pass
@ -720,7 +721,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext
cmd += gradletasks
p = FDroidPopen(cmd, cwd=root_dir)
p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']})
elif bmethod == 'ant':
logging.info("Building Ant project...")

View File

@ -57,6 +57,9 @@ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesEx
BuildException, VerificationException
from .asynchronousfilereader import AsynchronousFileReader
# The path to this fdroidserver distribution
FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
# this is the build-tools version, aapt has a separate version that
# has to be manually set in test_aapt_version()
MINIMUM_AAPT_VERSION = '26.0.0'
@ -87,12 +90,14 @@ default_config = {
'r15c': None,
'r16b': None,
},
'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'),
'build_tools': MINIMUM_AAPT_VERSION,
'force_build_tools': False,
'java_paths': None,
'ant': "ant",
'mvn3': "mvn",
'gradle': 'gradle',
'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'),
'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'),
'accepted_formats': ['txt', 'yml'],
'sync_from_local_copy_dir': False,
'allow_disabled_algorithms': False,

195
gradlew-fdroid Executable file
View File

@ -0,0 +1,195 @@
#!/bin/bash
bindir="$(dirname $0)"
basedir="$(dirname $bindir)"
# Check if GRADLE_VERSION_DIR/CACHEDIR is set from environment
if [ -z "$GRADLE_VERSION_DIR" ]; then
gradle_version_dir="${basedir}/versions"
else
gradle_version_dir="$GRADLE_VERSION_DIR"
fi
if [ -n "$CACHEDIR" ]; then
cachedir="$CACHEDIR"
fi
args=("$@")
run_gradle() {
if [ ! -d "${gradle_version_dir}/${v_found}" ]; then
download_gradle ${v_found}
fi
"${gradle_version_dir}/${v_found}/bin/gradle" "${args[@]}"
exit $?
}
download_gradle() {
URL="https://downloads.gradle.org/distributions/gradle-${1}-bin.zip"
shasum=$(get_sha $1)
if [ $? != 0 ]; then
echo "No hash for gradle version $1! Exiting..."
exit 1
fi
if [ -n "${cachedir}" ] && [ -e "${cachedir}/gradle-$1-bin.zip" ]; then
echo "Using cached ${cachedir}/gradle-$1-bin.zip ..."
gradle_zip="${cachedir}/gradle-$1-bin.zip"
else
echo "Downloading missing gradle version $1"
if [ -n "${cachedir}" ]; then
tmpdir="${cachedir}"
else
tmpdir=$(mktemp -d)
fi
curl -o "${tmpdir}/gradle.zip" --silent --fail --show-error --location "${URL}"
gradle_zip="${tmpdir}/gradle.zip"
fi
echo "${shasum} ${gradle_zip}" | sha256sum -c -
if [ $? != 0 ]; then
echo "gradle download checksum mismatch! Exiting..."
exit 1
fi
mkdir -p "${gradle_version_dir}/"
unzip -q -d "${gradle_version_dir}" "${gradle_zip}"
mv "${gradle_version_dir}/gradle-$1" "${gradle_version_dir}/${v_found}"
}
get_sha() {
declare -A gradle_hashes
gradle_hashes=( ["1.4"]="cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e" \
["1.6"]="de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4" \
["1.7"]="360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30" \
["1.8"]="a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703" \
["1.9"]="097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9" \
["1.10"]="6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe" \
["1.11"]="07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091" \
["1.12"]="8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea" \
["2.1"]="3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878" \
["2.2"]="91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb" \
["2.2.1"]="420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418" \
["2.3"]="010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774" \
["2.4"]="c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f" \
["2.5"]="3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555" \
["2.6"]="18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83" \
["2.7"]="cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06" \
["2.8"]="a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb" \
["2.9"]="c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f" \
["2.10"]="66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13" \
["2.11"]="8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6" \
["2.12"]="e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c" \
["2.13"]="0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664" \
["2.14"]="993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5" \
["2.14.1"]="cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39" \
["3.0"]="39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80" \
["3.1"]="c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc" \
["3.2"]="5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935" \
["3.2.1"]="9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689" \
["3.3"]="c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292" \
["3.4"]="72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205" \
["3.4.1"]="db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2" \
["3.5"]="0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110" \
["3.5.1"]="8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8" \
["4.0"]="56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe" \
["4.0.1"]="d717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4" \
["4.0.2"]="79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22" \
["4.1"]="d55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43" \
["4.2"]="515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae" \
["4.2.1"]="b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0" \
["4.3"]="8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c" \
["4.3.1"]="15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7" \
["4.4"]="fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f" \
["4.4.1"]="e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389" \
["4.5"]="03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3" \
["4.5.1"]="3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e" \
["4.6"]="98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915" \
["4.7"]="fca5087dc8b50c64655c000989635664a73b11b9bd3703c7d6cabd31b7dcdb04" \
["4.8"]="f3e29692a8faa94eb0b02ebf36fa263a642b3ae8694ef806c45c345b8683f1ba" \
["4.8.1"]="af334d994b5e69e439ab55b5d2b7d086da5ea6763d78054f49f147b06370ed71" \
)
[ ! ${gradle_hashes[$1]+abc} ] && exit 1
echo "${gradle_hashes["$1"]}"
}
contains() {
local e
for e in $2; do
[[ $e == $1 ]] && return 0;
done
return 1
}
# key-value pairs of what gradle version (value) each gradle plugin version
# (key) should accept. plugin versions are actually prefixes and catch sub-
# versions as well. Pairs are taken from:
# https://developer.android.com/studio/releases/gradle-plugin.html#updating-gradle
d_plugin_k=(3.1 3.0 2.3 2.2 2.1.3 2.1 2.0 1.5 1.3 1.2 1.1 1.0 0.14 0.13 0.12 0.11 0.10 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2)
d_plugin_v=(4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
# All gradle versions we know about
plugin_v=(4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4)
v_all=${plugin_v[@]}
echo "Available gradle versions: ${v_all[@]}"
# Earliest takes priority
for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do
[[ -f $f ]] || continue
while read l; do
if [[ $l == 'distributionUrl='* ]]; then
wrapper_ver=$(echo -n "$l" | sed "s/.*gradle-\\([0-9\\.\\+]\\+\\).*/\\1/")
fi
done < $f
done
if [[ -n $wrapper_ver ]]; then
v_found=$wrapper_ver
echo "Found $v_found via distributionUrl"
run_gradle
fi
# Earliest takes priority
for f in {.,..}/build.gradle; do
[[ -f $f ]] || continue
while read l; do
if [[ -z "$plugin_pver" && $l == *'com.android.tools.build:gradle:'* ]]; then
plugin_pver=$(echo -n "$l" | sed "s/.*com.android.tools.build:gradle:\\([0-9\\.\\+]\\+\\).*/\\1/")
elif [[ -z "$wrapper_ver" && $l == *'gradleVersion = '* ]]; then
wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion *=* *[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/")
fi
done < $f
done
if [[ -n $wrapper_ver ]]; then
v_found=$wrapper_ver
echo "Found $v_found via gradleVersion"
run_gradle
fi
if [[ -n $plugin_pver ]]; then
i=0
match=false
for k in ${d_plugin_k[@]}; do
if [[ $plugin_pver == ${k}* ]]; then
plugin_ver=${d_plugin_v[$i]}
match=true
break
fi
let i++
done
if $match; then
v_found=$plugin_ver
echo "Found $v_found via gradle plugin version $k"
fi
fi
# Find the highest version available
for v in ${plugin_v[*]}; do
if contains $v "${v_all[*]}"; then
v_def=$v
break
fi
done
if [[ -z $v_found ]]; then
echo "No suitable gradle version found - defaulting to $v_def"
v_found=$v_def
fi
run_gradle