diff --git a/MANIFEST.in b/MANIFEST.in index ec78c166..2b8a51bd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,4 @@ include buildserver/config.buildserver.py -include buildserver/gradle include buildserver/provision-android-ndk include buildserver/provision-android-sdk include buildserver/provision-apt-get-install @@ -23,6 +22,7 @@ include examples/opensc-fdroid.cfg include examples/public-read-only-s3-bucket-policy.json include examples/template.yml include fdroid +include gradlew-fdroid include LICENSE include locale/bo/LC_MESSAGES/fdroidserver.mo include locale/de/LC_MESSAGES/fdroidserver.mo diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile index 1d1859a6..111df705 100644 --- a/buildserver/Vagrantfile +++ b/buildserver/Vagrantfile @@ -75,7 +75,5 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", path: "provision-pip", args: ["compare-locales"] config.vm.provision "shell", path: "provision-gradle" - config.vm.provision "file", source: "gradle", - destination: "/opt/gradle/bin/gradle" end diff --git a/buildserver/config.buildserver.py b/buildserver/config.buildserver.py index d2d8cd40..496f83fa 100644 --- a/buildserver/config.buildserver.py +++ b/buildserver/config.buildserver.py @@ -12,3 +12,4 @@ ndk_paths = { java_paths = { '8': "/usr/lib/jvm/java-8-openjdk-amd64", } +gradle_version_dir = "/opt/gradle/versions" diff --git a/buildserver/gradle b/buildserver/gradle deleted file mode 100755 index 3920528e..00000000 --- a/buildserver/gradle +++ /dev/null @@ -1,98 +0,0 @@ -#!/bin/bash - -bindir="$(dirname $0)" -basedir="$(dirname $bindir)" -verdir="${basedir}/versions" -args=("$@") - -run_gradle() { - "${verdir}/${v_found}/bin/gradle" "${args[@]}" - exit $? -} - -contains() { - local e - for e in $2; do - [[ $e == $1 ]] && return 0; - done - return 1 -} - -# key-value pairs of what gradle version (value) each gradle plugin version -# (key) should accept. plugin versions are actually prefixes and catch sub- -# versions as well. Pairs are taken from: -# https://developer.android.com/studio/releases/gradle-plugin.html#updating-gradle -d_plugin_k=(3.1 3.0 2.3 2.2 2.1.3 2.1 2.0 1.5 1.3 1.2 1.1 1.0 0.14 0.13 0.12 0.11 0.10 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2) -d_plugin_v=(4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4) - -# All gradle versions we know about -plugin_v=(4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4) - -v_all=${plugin_v[@]} -echo "Available gradle versions: ${v_all[@]}" - -# Earliest takes priority -for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do - [[ -f $f ]] || continue - while read l; do - if [[ $l == 'distributionUrl='* ]]; then - wrapper_ver=$(echo -n "$l" | sed "s/.*gradle-\\([0-9\\.\\+]\\+\\).*/\\1/") - fi - done < $f -done - -if [[ -n $wrapper_ver ]]; then - v_found=$wrapper_ver - echo "Found $v_found via distributionUrl" - run_gradle -fi - -# Earliest takes priority -for f in {.,..}/build.gradle; do - [[ -f $f ]] || continue - while read l; do - if [[ -z "$plugin_pver" && $l == *'com.android.tools.build:gradle:'* ]]; then - plugin_pver=$(echo -n "$l" | sed "s/.*com.android.tools.build:gradle:\\([0-9\\.\\+]\\+\\).*/\\1/") - elif [[ -z "$wrapper_ver" && $l == *'gradleVersion = '* ]]; then - wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion *=* *[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/") - fi - done < $f -done - -if [[ -n $wrapper_ver ]]; then - v_found=$wrapper_ver - echo "Found $v_found via gradleVersion" - run_gradle -fi - -if [[ -n $plugin_pver ]]; then - i=0 - match=false - for k in ${d_plugin_k[@]}; do - if [[ $plugin_pver == ${k}* ]]; then - plugin_ver=${d_plugin_v[$i]} - match=true - break - fi - let i++ - done - if $match; then - v_found=$plugin_ver - echo "Found $v_found via gradle plugin version $k" - fi -fi - -# Find the highest version available -for v in ${plugin_v[*]}; do - if contains $v "${v_all[*]}"; then - v_def=$v - break - fi -done - -if [[ -z $v_found ]]; then - echo "No suitable gradle version found - defaulting to $v_def" - v_found=$v_def -fi - -run_gradle diff --git a/buildserver/provision-gradle b/buildserver/provision-gradle index 124b99aa..e385fffe 100644 --- a/buildserver/provision-gradle +++ b/buildserver/provision-gradle @@ -1,6 +1,6 @@ #!/bin/bash -set -e +set -ex test -e /opt/gradle/versions || mkdir -p /opt/gradle/versions cd /opt/gradle/versions @@ -15,6 +15,7 @@ done chmod -R a+rX /opt/gradle test -e /opt/gradle/bin || mkdir -p /opt/gradle/bin -touch /opt/gradle/bin/gradle -chown vagrant.vagrant /opt/gradle/bin/gradle -chmod 0755 /opt/gradle/bin/gradle +ln -fs /home/vagrant/fdroidserver/gradlew-fdroid /opt/gradle/bin/gradle +chown -h vagrant.vagrant /opt/gradle/bin/gradle +chown vagrant.vagrant /opt/gradle/versions +chmod 0755 /opt/gradle/versions diff --git a/examples/config.py b/examples/config.py index b0395fa0..2fd46956 100644 --- a/examples/config.py +++ b/examples/config.py @@ -21,6 +21,10 @@ # 'r17b': None, # } +# Directory to store downloaded tools in (i.e. gradle versions) +# By default, these are stored in ~/.cache/fdroidserver +# cachedir = cache + # java_paths = { # '8': "/usr/lib/jvm/java-8-openjdk", # } @@ -39,6 +43,7 @@ # mvn3 = "mvn" # Command or path to binary for running Gradle +# Defaults to using an internal gradle wrapper (gradlew-fdroid). # gradle = "gradle" # Set the maximum age (in days) of an index that a client should accept from diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 9970a2dd..6fbe925a 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -125,7 +125,9 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): ftp.mkdir('fdroidserver') ftp.chdir('fdroidserver') ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid') + ftp.put(os.path.join(serverpath, '..', 'gradlew-fdroid'), 'gradlew-fdroid') ftp.chmod('fdroid', 0o755) + ftp.chmod('gradlew-fdroid', 0o755) send_dir(os.path.join(serverpath)) ftp.chdir(homedir) @@ -496,8 +498,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext cmd += ['-P' + kv for kv in build.gradleprops] cmd += ['clean'] - - p = FDroidPopen(cmd, cwd=root_dir) + p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']}) elif bmethod == 'buildozer': pass @@ -720,7 +721,7 @@ def build_local(app, build, vcs, build_dir, output_dir, log_dir, srclib_dir, ext cmd += gradletasks - p = FDroidPopen(cmd, cwd=root_dir) + p = FDroidPopen(cmd, cwd=root_dir, envs={"GRADLE_VERSION_DIR": config['gradle_version_dir'], "CACHEDIR": config['cachedir']}) elif bmethod == 'ant': logging.info("Building Ant project...") diff --git a/fdroidserver/common.py b/fdroidserver/common.py index e3cd165d..1212afca 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -57,6 +57,9 @@ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesEx BuildException, VerificationException from .asynchronousfilereader import AsynchronousFileReader +# The path to this fdroidserver distribution +FDROID_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) + # this is the build-tools version, aapt has a separate version that # has to be manually set in test_aapt_version() MINIMUM_AAPT_VERSION = '26.0.0' @@ -87,12 +90,14 @@ default_config = { 'r15c': None, 'r16b': None, }, + 'cachedir': os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'build_tools': MINIMUM_AAPT_VERSION, 'force_build_tools': False, 'java_paths': None, 'ant': "ant", 'mvn3': "mvn", - 'gradle': 'gradle', + 'gradle': os.path.join(FDROID_PATH, 'gradlew-fdroid'), + 'gradle_version_dir': os.path.join(os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver'), 'gradle'), 'accepted_formats': ['txt', 'yml'], 'sync_from_local_copy_dir': False, 'allow_disabled_algorithms': False, diff --git a/gradlew-fdroid b/gradlew-fdroid new file mode 100755 index 00000000..7f38222e --- /dev/null +++ b/gradlew-fdroid @@ -0,0 +1,195 @@ +#!/bin/bash + +bindir="$(dirname $0)" +basedir="$(dirname $bindir)" +# Check if GRADLE_VERSION_DIR/CACHEDIR is set from environment +if [ -z "$GRADLE_VERSION_DIR" ]; then + gradle_version_dir="${basedir}/versions" +else + gradle_version_dir="$GRADLE_VERSION_DIR" +fi +if [ -n "$CACHEDIR" ]; then + cachedir="$CACHEDIR" +fi +args=("$@") + +run_gradle() { + if [ ! -d "${gradle_version_dir}/${v_found}" ]; then + download_gradle ${v_found} + fi + "${gradle_version_dir}/${v_found}/bin/gradle" "${args[@]}" + exit $? +} + +download_gradle() { + URL="https://downloads.gradle.org/distributions/gradle-${1}-bin.zip" + shasum=$(get_sha $1) + if [ $? != 0 ]; then + echo "No hash for gradle version $1! Exiting..." + exit 1 + fi + if [ -n "${cachedir}" ] && [ -e "${cachedir}/gradle-$1-bin.zip" ]; then + echo "Using cached ${cachedir}/gradle-$1-bin.zip ..." + gradle_zip="${cachedir}/gradle-$1-bin.zip" + else + echo "Downloading missing gradle version $1" + if [ -n "${cachedir}" ]; then + tmpdir="${cachedir}" + else + tmpdir=$(mktemp -d) + fi + curl -o "${tmpdir}/gradle.zip" --silent --fail --show-error --location "${URL}" + gradle_zip="${tmpdir}/gradle.zip" + fi + echo "${shasum} ${gradle_zip}" | sha256sum -c - + if [ $? != 0 ]; then + echo "gradle download checksum mismatch! Exiting..." + exit 1 + fi + mkdir -p "${gradle_version_dir}/" + unzip -q -d "${gradle_version_dir}" "${gradle_zip}" + mv "${gradle_version_dir}/gradle-$1" "${gradle_version_dir}/${v_found}" +} + +get_sha() { + declare -A gradle_hashes + gradle_hashes=( ["1.4"]="cd99e85fbcd0ae8b99e81c9992a2f10cceb7b5f009c3720ef3a0078f4f92e94e" \ + ["1.6"]="de3e89d2113923dcc2e0def62d69be0947ceac910abd38b75ec333230183fac4" \ + ["1.7"]="360c97d51621b5a1ecf66748c718594e5f790ae4fbc1499543e0c006033c9d30" \ + ["1.8"]="a342bbfa15fd18e2482287da4959588f45a41b60910970a16e6d97959aea5703" \ + ["1.9"]="097ddc2bcbc9da2bb08cbf6bf8079585e35ad088bafd42e8716bc96405db98e9" \ + ["1.10"]="6e6db4fc595f27ceda059d23693b6f6848583950606112b37dfd0e97a0a0a4fe" \ + ["1.11"]="07e235df824964f0e19e73ea2327ce345c44bcd06d44a0123d29ab287fc34091" \ + ["1.12"]="8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea" \ + ["2.1"]="3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878" \ + ["2.2"]="91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb" \ + ["2.2.1"]="420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418" \ + ["2.3"]="010dd9f31849abc3d5644e282943b1c1c355f8e2635c5789833979ce590a3774" \ + ["2.4"]="c4eaecc621a81f567ded1aede4a5ddb281cc02a03a6a87c4f5502add8fc2f16f" \ + ["2.5"]="3f953e0cb14bb3f9ebbe11946e84071547bf5dfd575d90cfe9cc4e788da38555" \ + ["2.6"]="18a98c560af231dfa0d3f8e0802c20103ae986f12428bb0a6f5396e8f14e9c83" \ + ["2.7"]="cde43b90945b5304c43ee36e58aab4cc6fb3a3d5f9bd9449bb1709a68371cb06" \ + ["2.8"]="a88db9c2f104defdaa8011c58cf6cda6c114298ae3695ecfb8beb30da3a903cb" \ + ["2.9"]="c9159ec4362284c0a38d73237e224deae6139cbde0db4f0f44e1c7691dd3de2f" \ + ["2.10"]="66406247f745fc6f05ab382d3f8d3e120c339f34ef54b86f6dc5f6efc18fbb13" \ + ["2.11"]="8d7437082356c9fd6309a4479c8db307673965546daea445c6c72759cd6b1ed6" \ + ["2.12"]="e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c" \ + ["2.13"]="0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664" \ + ["2.14"]="993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5" \ + ["2.14.1"]="cfc61eda71f2d12a572822644ce13d2919407595c2aec3e3566d2aab6f97ef39" \ + ["3.0"]="39c906941a474444afbddc38144ed44166825acb0a57b0551dddb04bbf157f80" \ + ["3.1"]="c7de3442432253525902f7e8d7eac8b5fd6ce1623f96d76916af6d0e383010fc" \ + ["3.2"]="5321b36837226dc0377047a328f12010f42c7bf88ee4a3b1cee0c11040082935" \ + ["3.2.1"]="9843a3654d3e57dce54db06d05f18b664b95c22bf90c6becccb61fc63ce60689" \ + ["3.3"]="c58650c278d8cf0696cab65108ae3c8d95eea9c1938e0eb8b997095d5ca9a292" \ + ["3.4"]="72d0cd4dcdd5e3be165eb7cd7bbd25cf8968baf400323d9ab1bba622c3f72205" \ + ["3.4.1"]="db1db193d479cc1202be843f17e4526660cfb0b21b57d62f3a87f88c878af9b2" \ + ["3.5"]="0b7450798c190ff76b9f9a3d02e18b33d94553f708ebc08ebe09bdf99111d110" \ + ["3.5.1"]="8dce35f52d4c7b4a4946df73aa2830e76ba7148850753d8b5e94c5dc325ceef8" \ + ["4.0"]="56bd2dde29ba2a93903c557da1745cafd72cdd8b6b0b83c05a40ed7896b79dfe" \ + ["4.0.1"]="d717e46200d1359893f891dab047fdab98784143ac76861b53c50dbd03b44fd4" \ + ["4.0.2"]="79ac421342bd11f6a4f404e0988baa9c1f5fabf07e3c6fa65b0c15c1c31dda22" \ + ["4.1"]="d55dfa9cfb5a3da86a1c9e75bb0b9507f9a8c8c100793ccec7beb6e259f9ed43" \ + ["4.2"]="515dd63d32e55a9c05667809c5e40a947529de3054444ad274b3b75af5582eae" \ + ["4.2.1"]="b551cc04f2ca51c78dd14edb060621f0e5439bdfafa6fd167032a09ac708fbc0" \ + ["4.3"]="8dcbf44eef92575b475dcb1ce12b5f19d38dc79e84c662670248dc8b8247654c" \ + ["4.3.1"]="15ebe098ce0392a2d06d252bff24143cc88c4e963346582c8d88814758d93ac7" \ + ["4.4"]="fa4873ae2c7f5e8c02ec6948ba95848cedced6134772a0169718eadcb39e0a2f" \ + ["4.4.1"]="e7cf7d1853dfc30c1c44f571d3919eeeedef002823b66b6a988d27e919686389" \ + ["4.5"]="03f2a43a314ff0fb843a85ef68078e06d181c4549c1e5fb983f289382b59b5e3" \ + ["4.5.1"]="3e2ea0d8b96605b7c528768f646e0975bd9822f06df1f04a64fd279b1a17805e" \ + ["4.6"]="98bd5fd2b30e070517e03c51cbb32beee3e2ee1a84003a5a5d748996d4b1b915" \ + ["4.7"]="fca5087dc8b50c64655c000989635664a73b11b9bd3703c7d6cabd31b7dcdb04" \ + ["4.8"]="f3e29692a8faa94eb0b02ebf36fa263a642b3ae8694ef806c45c345b8683f1ba" \ + ["4.8.1"]="af334d994b5e69e439ab55b5d2b7d086da5ea6763d78054f49f147b06370ed71" \ + ) + [ ! ${gradle_hashes[$1]+abc} ] && exit 1 + echo "${gradle_hashes["$1"]}" +} + +contains() { + local e + for e in $2; do + [[ $e == $1 ]] && return 0; + done + return 1 +} + +# key-value pairs of what gradle version (value) each gradle plugin version +# (key) should accept. plugin versions are actually prefixes and catch sub- +# versions as well. Pairs are taken from: +# https://developer.android.com/studio/releases/gradle-plugin.html#updating-gradle +d_plugin_k=(3.1 3.0 2.3 2.2 2.1.3 2.1 2.0 1.5 1.3 1.2 1.1 1.0 0.14 0.13 0.12 0.11 0.10 0.9 0.8 0.7 0.6 0.5 0.4 0.3 0.2) +d_plugin_v=(4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4) + +# All gradle versions we know about +plugin_v=(4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4) + +v_all=${plugin_v[@]} +echo "Available gradle versions: ${v_all[@]}" + +# Earliest takes priority +for f in {.,..}/gradle/wrapper/gradle-wrapper.properties; do + [[ -f $f ]] || continue + while read l; do + if [[ $l == 'distributionUrl='* ]]; then + wrapper_ver=$(echo -n "$l" | sed "s/.*gradle-\\([0-9\\.\\+]\\+\\).*/\\1/") + fi + done < $f +done + +if [[ -n $wrapper_ver ]]; then + v_found=$wrapper_ver + echo "Found $v_found via distributionUrl" + run_gradle +fi + +# Earliest takes priority +for f in {.,..}/build.gradle; do + [[ -f $f ]] || continue + while read l; do + if [[ -z "$plugin_pver" && $l == *'com.android.tools.build:gradle:'* ]]; then + plugin_pver=$(echo -n "$l" | sed "s/.*com.android.tools.build:gradle:\\([0-9\\.\\+]\\+\\).*/\\1/") + elif [[ -z "$wrapper_ver" && $l == *'gradleVersion = '* ]]; then + wrapper_ver=$(echo -n "$l" | sed "s/.*gradleVersion *=* *[\"']\\([0-9\\.]\\+\\)[\"'].*/\\1/") + fi + done < $f +done + +if [[ -n $wrapper_ver ]]; then + v_found=$wrapper_ver + echo "Found $v_found via gradleVersion" + run_gradle +fi + +if [[ -n $plugin_pver ]]; then + i=0 + match=false + for k in ${d_plugin_k[@]}; do + if [[ $plugin_pver == ${k}* ]]; then + plugin_ver=${d_plugin_v[$i]} + match=true + break + fi + let i++ + done + if $match; then + v_found=$plugin_ver + echo "Found $v_found via gradle plugin version $k" + fi +fi + +# Find the highest version available +for v in ${plugin_v[*]}; do + if contains $v "${v_all[*]}"; then + v_def=$v + break + fi +done + +if [[ -z $v_found ]]; then + echo "No suitable gradle version found - defaulting to $v_def" + v_found=$v_def +fi + +run_gradle