From 9889a98dea9c0497bf06eafd0fbed4adc94f3e47 Mon Sep 17 00:00:00 2001 From: Marcus Hoffmann Date: Sun, 8 Jul 2018 17:34:17 +0200 Subject: [PATCH] build: improve gradle experience This expands the gradle wrapper shell script used by the buildserver for usage outside the buildserver environment. It also allows downloading whitelisted versions of gradle if they are not yet deployed to the buildserver by simply upsating the copy of fdroidserver (in contrast to having to reprovision the whole buildserver). We first move the buildserver/gradle shell script to the repo root as gradlew-fdroid, as it's an fdroid specific gradle wrapper. We also now sync it inside the build VM before each build. We then add a list of whitelisted gradle distributions taken from the makebuildserver script. The script additionally now reads two env vars which tell it where to expect installed versions of gradle and where it might store downloaded gradle .zip files. Both of those are configurable from config.py. As the first should normally just be a subdir of the second it's not exposed in the example config.py but only used by the buildserver config.py. Default config now uses this internal gradle wrapper but a path to a custom wrapper or specific gradle distribution can still be set from config.py. Closes fdroid/fdroidserver#98 Ref: fdroid/fdroidserver#370 --- MANIFEST.in | 2 +- buildserver/Vagrantfile | 2 - buildserver/config.buildserver.py | 1 + buildserver/gradle | 98 --------------- buildserver/provision-gradle | 9 +- examples/config.py | 5 + fdroidserver/build.py | 7 +- fdroidserver/common.py | 7 +- gradlew-fdroid | 195 ++++++++++++++++++++++++++++++ 9 files changed, 217 insertions(+), 109 deletions(-) delete mode 100755 buildserver/gradle create mode 100755 gradlew-fdroid diff --git a/MANIFEST.in b/MANIFEST.in index ec78c166..2b8a51bd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include buildserver/config.buildserver.py -include buildserver/gradle include buildserver/provision-android-ndk include buildserver/provision-android-sdk include buildserver/provision-apt-get-install @@ -23,6 +22,7 @@ include examples/opensc-fdroid.cfg include examples/public-read-only-s3-bucket-policy.json include examples/template.yml include fdroid +include gradlew-fdroid include LICENSE include locale/bo/LC_MESSAGES/fdroidserver.mo include locale/de/LC_MESSAGES/fdroidserver.mo diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 1d1859a6..111df705 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -75,7 +75,5 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", path: "provision-pip", args: ["compare-locales"] config.vm.provision "shell", path: "provision-gradle" - config.vm.provision "file", source: "gradle", - destination: "/opt/gradle/bin/gradle" end diff --git a/buildserver/config.buildserver.py b/buildserver/config.buildserver.py index d2d8cd40..496f83fa 100644 --- a/buildserver/config.buildserver.py +++ b/buildserver/config.buildserver.py @@ -12,3 +12,4 @@ ndk_paths = { java_paths = { '8': "/usr/lib/jvm/java-8-openjdk-amd64", } +gradle_version_dir = "/opt/gradle/versions" diff --git a/buildserver/gradle b/buildserver/gradle deleted file mode 100755 index 3920528e..00000000 --- a/buildserver/gradle +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -bindir="$(dirname $0)" -basedir="$(dirname $bindir)" -verdir="${basedir}/versions" -args=("$@") - -run_gradle() { - "${verdir}/${v_found}/bin/gradle" "${args[@]}" - exit $? -} - -contains() { - local e - for e in $2; do - [[ $e == $1 ]] && return 0; - done - return 1 -} - -# key-value pairs of what gradle version (value) each gradle plugin version -# (key) should accept. plugin versions are actually prefixes and catch sub- -# versions as well. Pairs are taken from: -# https://developer.android.com/studio/releases/gradle-plugin.html#updating-gradle -d_plugin_k=(3.1 3.0 2.3 2.2 2.1.3 2.1 2.0 1.5 1.3 1.2 1.1 1.0 0.14 0.13 0.12 0.11 0.10 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2) -d_plugin_v=(4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4) - -# All gradle versions we know about -plugin_v=(4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4) - -v_all=${plugin_v[@]} -echo "Available gradle versions: ${v_all[@]}" - -# Earliest takes priority -for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do - [[ -f $f ]] || continue - while read l; do - if [[ $l == 'distributionUrl='* ]]; then - wrapper_ver=$(echo -n "$l" | sed "s/.*gradle-\\([0-9\\.\\+]\\+\\).*/\\1/") - fi - done < $f -done - -if [[ -n $wrapper_ver ]]; then - v_found=$wrapper_ver - echo "Found $v_found via distributionUrl" - run_gradle -fi - -# Earliest takes priority -for f in {.,..}/build.gradle; do - [[ -f $f ]] || continue - while read l; do - if [[ -z "$plugin_pver" && $l == *'com.android.tools.build:gradle:'* ]]; then - plugin_pver=$(echo -n "$l" | sed "s/.*com.android.tools.build:gradle:\\([0-9\\.\\+]\\+\\).*/\\1/") - elif [[ -z "$wrapper_ver" && $l == *'gradleVersion = '* ]]; then - wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion *=* *[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/") - fi - done < $f -done - -if [[ -n $wrapper_ver ]]; then - v_found=$wrapper_ver - echo "Found $v_found via gradleVersion" - run_gradle -fi - -if [[ -n $plugin_pver ]]; then - i=0 - match=false - for k in ${d_plugin_k[@]}; do - if [[ $plugin_pver == ${k}* ]]; then - plugin_ver=${d_plugin_v[$i]} - match=true - break - fi - let i++ - done - if $match; then - v_found=$plugin_ver - echo "Found $v_found via gradle plugin version $k" - fi -fi - -# Find the highest version available -for v in ${plugin_v[*]}; do - if contains $v "${v_all[*]}"; then - v_def=$v - break - fi -done - -if [[ -z $v_found ]]; then - echo "No suitable gradle version found - defaulting to $v_def" - v_found=$v_def -fi - -run_gradle diff --git a/buildserver/provision-gradle b/buildserver/provision-gradle index 124b99aa..e385fffe 100644 --- a/buildserver/provision-gradle +++ b/buildserver/provision-gradle @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex test -e /opt/gradle/versions || mkdir -p /opt/gradle/versions cd /opt/gradle/versions @@ -15,6 +15,7 @@ done chmod -R a+rX /opt/gradle test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin -touch /opt/gradle/bin/gradle -chown vagrant.vagrant /opt/gradle/bin/gradle -chmod 0755 /opt/gradle/bin/gradle +ln -fs /home/vagrant/fdroidserver/gradlew-fdroid /opt/gradle/bin/gradle +chown -h vagrant.vagrant /opt/gradle/bin/gradle +chown vagrant.vagrant /opt/gradle/versions +chmod 0755 /opt/gradle/versions diff --git a/examples/config.py b/examples/config.py index b0395fa0..2fd46956 100644 --- a/examples/config.py +++ b/examples/config.py @@ -21,6 +21,10 @@ # 'r17b': None, # } +# Directory to store downloaded tools in (i.e. gradle versions) +# By default, these are stored in ~/.cache/fdroidserver +# cachedir = cache + # java_paths = { # '8': "/usr/lib/jvm/java-8-openjdk", # } @@ -39,6 +43,7 @@ # mvn3 = "mvn" # Command or path to binary for running Gradle +# Defaults to using an internal gradle wrapper (gradlew-fdroid). # gradle = "gradle" # Set the maximum age (in days) of an index that a client should accept from diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 9970a2dd..6fbe925a 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -125,7 +125,9 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): ftp.mkdir('fdroidserver') ftp.chdir('fdroidserver') ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid') + ftp.put(os.path.join(serverpath, '..', 'gradlew-fdroid'), 'gradlew-fdroid') ftp.chmod('fdroid', 0o755) + ftp.chmod('gradlew-fdroid', 0o755) send_dir(os.path.join(serverpath)) ftp.chdir(homedir) @@ -496,8 +498,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext cmd += ['-P' + kv for kv in build.gradleprops] cmd += ['clean'] - - p = FDroidPopen(cmd, cwd=root_dir) + p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']}) elif bmethod == 'buildozer': pass @@ -720,7 +721,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext cmd += gradletasks - p = FDroidPopen(cmd, cwd=root_dir) + p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']}) elif bmethod == 'ant': logging.info("Building Ant project...") diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e3cd165d..1212afca 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -57,6 +57,9 @@ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesEx BuildException, VerificationException from .asynchronousfilereader import AsynchronousFileReader +# The path to this fdroidserver distribution +FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) + # this is the build-tools version, aapt has a separate version that # has to be manually set in test_aapt_version() MINIMUM_AAPT_VERSION = '26.0.0' @@ -87,12 +90,14 @@ default_config = { 'r15c': None, 'r16b': None, }, + 'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'build_tools': MINIMUM_AAPT_VERSION, 'force_build_tools': False, 'java_paths': None, 'ant': "ant", 'mvn3': "mvn", - 'gradle': 'gradle', + 'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'), + 'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'), 'accepted_formats': ['txt', 'yml'], 'sync_from_local_copy_dir': False, 'allow_disabled_algorithms': False, diff --git a/gradlew-fdroid b/gradlew-fdroid new file mode 100755 index 00000000..7f38222e --- /dev/null +++ b/gradlew-fdroid @@ -0,0 +1,195 @@ +#!/bin/bash + +bindir="$(dirname $0)" +basedir="$(dirname $bindir)" +# Check if GRADLE_VERSION_DIR/CACHEDIR is set from environment +if [ -z "$GRADLE_VERSION_DIR" ]; then + gradle_version_dir="${basedir}/versions" +else + gradle_version_dir="$GRADLE_VERSION_DIR" +fi +if [ -n "$CACHEDIR" ]; then + cachedir="$CACHEDIR" +fi +args=("$@") + +run_gradle() { + if [ ! -d "${gradle_version_dir}/${v_found}" ]; then + download_gradle ${v_found} + fi + "${gradle_version_dir}/${v_found}/bin/gradle" "${args[@]}" + exit $? +} + +download_gradle() { + URL="https://downloads.gradle.org/distributions/gradle-${1}-bin.zip" + shasum=$(get_sha $1) + if [ $? != 0 ]; then + echo "No hash for gradle version $1! Exiting..." + exit 1 + fi + if [ -n "${cachedir}" ] && [ -e "${cachedir}/gradle-$1-bin.zip" ]; then + echo "Using cached ${cachedir}/gradle-$1-bin.zip ..." + gradle_zip="${cachedir}/gradle-$1-bin.zip" + else + echo "Downloading missing gradle version $1" + if [ -n "${cachedir}" ]; then + tmpdir="${cachedir}" + else + tmpdir=$(mktemp -d) + fi + curl -o "${tmpdir}/gradle.zip" --silent --fail --show-error --location "${URL}" + gradle_zip="${tmpdir}/gradle.zip" + fi + echo "${shasum} ${gradle_zip}" | sha256sum -c - + if [ $? != 0 ]; then + echo "gradle download checksum mismatch! Exiting..." + exit 1 + fi + mkdir -p "${gradle_version_dir}/" + unzip -q -d "${gradle_version_dir}" "${gradle_zip}" + mv "${gradle_version_dir}/gradle-$1" "${gradle_version_dir}/${v_found}" +} + +get_sha() { + declare -A gradle_hashes + gradle_hashes=( ["1.4"]="cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e" \ + ["1.6"]="de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4" \ + ["1.7"]="360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30" \ + ["1.8"]="a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703" \ + ["1.9"]="097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9" \ + ["1.10"]="6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe" \ + ["1.11"]="07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091" \ + ["1.12"]="8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea" \ + ["2.1"]="3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878" \ + ["2.2"]="91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb" \ + ["2.2.1"]="420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418" \ + ["2.3"]="010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774" \ + ["2.4"]="c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f" \ + ["2.5"]="3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555" \ + ["2.6"]="18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83" \ + ["2.7"]="cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06" \ + ["2.8"]="a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb" \ + ["2.9"]="c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f" \ + ["2.10"]="66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13" \ + ["2.11"]="8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6" \ + ["2.12"]="e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c" \ + ["2.13"]="0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664" \ + ["2.14"]="993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5" \ + ["2.14.1"]="cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39" \ + ["3.0"]="39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80" \ + ["3.1"]="c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc" \ + ["3.2"]="5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935" \ + ["3.2.1"]="9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689" \ + ["3.3"]="c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292" \ + ["3.4"]="72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205" \ + ["3.4.1"]="db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2" \ + ["3.5"]="0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110" \ + ["3.5.1"]="8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8" \ + ["4.0"]="56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe" \ + ["4.0.1"]="d717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4" \ + ["4.0.2"]="79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22" \ + ["4.1"]="d55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43" \ + ["4.2"]="515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae" \ + ["4.2.1"]="b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0" \ + ["4.3"]="8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c" \ + ["4.3.1"]="15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7" \ + ["4.4"]="fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f" \ + ["4.4.1"]="e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389" \ + ["4.5"]="03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3" \ + ["4.5.1"]="3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e" \ + ["4.6"]="98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915" \ + ["4.7"]="fca5087dc8b50c64655c000989635664a73b11b9bd3703c7d6cabd31b7dcdb04" \ + ["4.8"]="f3e29692a8faa94eb0b02ebf36fa263a642b3ae8694ef806c45c345b8683f1ba" \ + ["4.8.1"]="af334d994b5e69e439ab55b5d2b7d086da5ea6763d78054f49f147b06370ed71" \ + ) + [ ! ${gradle_hashes[$1]+abc} ] && exit 1 + echo "${gradle_hashes["$1"]}" +} + +contains() { + local e + for e in $2; do + [[ $e == $1 ]] && return 0; + done + return 1 +} + +# key-value pairs of what gradle version (value) each gradle plugin version +# (key) should accept. plugin versions are actually prefixes and catch sub- +# versions as well. Pairs are taken from: +# https://developer.android.com/studio/releases/gradle-plugin.html#updating-gradle +d_plugin_k=(3.1 3.0 2.3 2.2 2.1.3 2.1 2.0 1.5 1.3 1.2 1.1 1.0 0.14 0.13 0.12 0.11 0.10 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2) +d_plugin_v=(4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4) + +# All gradle versions we know about +plugin_v=(4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4) + +v_all=${plugin_v[@]} +echo "Available gradle versions: ${v_all[@]}" + +# Earliest takes priority +for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do + [[ -f $f ]] || continue + while read l; do + if [[ $l == 'distributionUrl='* ]]; then + wrapper_ver=$(echo -n "$l" | sed "s/.*gradle-\\([0-9\\.\\+]\\+\\).*/\\1/") + fi + done < $f +done + +if [[ -n $wrapper_ver ]]; then + v_found=$wrapper_ver + echo "Found $v_found via distributionUrl" + run_gradle +fi + +# Earliest takes priority +for f in {.,..}/build.gradle; do + [[ -f $f ]] || continue + while read l; do + if [[ -z "$plugin_pver" && $l == *'com.android.tools.build:gradle:'* ]]; then + plugin_pver=$(echo -n "$l" | sed "s/.*com.android.tools.build:gradle:\\([0-9\\.\\+]\\+\\).*/\\1/") + elif [[ -z "$wrapper_ver" && $l == *'gradleVersion = '* ]]; then + wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion *=* *[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/") + fi + done < $f +done + +if [[ -n $wrapper_ver ]]; then + v_found=$wrapper_ver + echo "Found $v_found via gradleVersion" + run_gradle +fi + +if [[ -n $plugin_pver ]]; then + i=0 + match=false + for k in ${d_plugin_k[@]}; do + if [[ $plugin_pver == ${k}* ]]; then + plugin_ver=${d_plugin_v[$i]} + match=true + break + fi + let i++ + done + if $match; then + v_found=$plugin_ver + echo "Found $v_found via gradle plugin version $k" + fi +fi + +# Find the highest version available +for v in ${plugin_v[*]}; do + if contains $v "${v_all[*]}"; then + v_def=$v + break + fi +done + +if [[ -z $v_found ]]; then + echo "No suitable gradle version found - defaulting to $v_def" + v_found=$v_def +fi + +run_gradle