remove NDK download handling in favor of fdroid/sdkmanager

This commit is contained in:
Hans-Christoph Steiner 2022-01-14 12:50:44 +01:00
parent 66d750e5fa
commit 0dd5a7db64
No known key found for this signature in database
GPG Key ID: 3E177817BA1B9BFA
6 changed files with 172 additions and 565 deletions

View File

@ -276,7 +276,6 @@ black:
tests/key-tricks.py
tests/lint.TestCase
tests/metadata.TestCase
tests/ndk-release-checksums.py
tests/nightly.TestCase
tests/rewritemeta.TestCase
tests/scanner.TestCase
@ -329,7 +328,7 @@ fedora_latest:
"cd `pwd`; export ANDROID_HOME=$ANDROID_HOME; fdroid=~testuser/.local/bin/fdroid ./run-tests"
gradle/ndk:
gradle:
image: debian:bullseye
<<: *apt-template
variables:
@ -350,12 +349,10 @@ gradle/ndk:
for f in `git diff --name-only --diff-filter=d FETCH_HEAD...HEAD`; do
test "$f" == "makebuildserver" && export CHANGED="yes";
test "$f" == "gradlew-fdroid" && export CHANGED="yes";
test "$f" == "fdroidserver/common.py" && export CHANGED="yes";
done;
test -z "$CHANGED" && exit;
fi
- ./tests/gradle-release-checksums.py
- ./tests/ndk-release-checksums.py
# Run an actual build in a simple, faked version of the buildserver guest VM.

View File

@ -29,6 +29,7 @@
# libraries here as they will become a requirement for all commands.
import git
import glob
import io
import os
import sys
@ -36,7 +37,6 @@ import re
import ast
import gzip
import shutil
import glob
import stat
import subprocess
import time
@ -64,7 +64,6 @@ from pyasn1.codec.der import decoder, encoder
from pyasn1_modules import rfc2315
from pyasn1.error import PyAsn1Error
from . import net
import fdroidserver.metadata
import fdroidserver.lint
from fdroidserver import _
@ -310,13 +309,6 @@ def fill_config_defaults(thisconfig):
if version not in ndk_paths:
ndk_paths[version] = ndk
for k in list(ndk_paths.keys()):
if not re.match(r'r[1-9][0-9]*[a-z]?', k):
for ndkdict in NDKS:
if k == ndkdict.get('revision'):
ndk_paths[ndkdict['release']] = ndk_paths.pop(k)
break
if 'cachedir_scanner' not in thisconfig:
thisconfig['cachedir_scanner'] = str(Path(thisconfig['cachedir']) / 'scanner')
if 'gradle_version_dir' not in thisconfig:
@ -4226,6 +4218,8 @@ def auto_install_ndk(build):
they are packaged differently.
"""
import sdkmanager
global config
if build.get('disable'):
return
@ -4233,8 +4227,10 @@ def auto_install_ndk(build):
if not ndk:
return
if isinstance(ndk, str):
sdkmanager.build_package_list(use_net=True)
_install_ndk(ndk)
elif isinstance(ndk, list):
sdkmanager.build_package_list(use_net=True)
for n in ndk:
_install_ndk(n)
else:
@ -4251,295 +4247,16 @@ def _install_ndk(ndk):
The NDK version to install, either in "release" form (r21e) or
"revision" form (21.4.7075529).
"""
if re.match(r'[1-9][0-9.]+[0-9]', ndk):
for ndkdict in NDKS:
if ndk == ndkdict.get('revision'):
ndk = ndkdict['release']
break
import sdkmanager
ndk_path = config.get(ndk)
if ndk_path and os.path.isdir(ndk_path):
return
for ndkdict in NDKS:
if ndk == ndkdict['release']:
url = ndkdict['url']
sha256 = ndkdict['sha256']
break
else:
raise FDroidException("NDK %s not found" % ndk)
ndk_base = os.path.join(config['sdk_path'], 'ndk')
logging.info(_('Downloading %s') % url)
with tempfile.TemporaryDirectory(prefix='android-ndk-') as ndk_dir:
zipball = os.path.join(
ndk_dir,
os.path.basename(url)
sdk_path = config['sdk_path']
sdkmanager.install(f'ndk;{ndk}', sdk_path)
for found in glob.glob(f'{sdk_path}/ndk/*'):
version = get_ndk_version(found)
if 'ndk_paths' not in config:
config['ndk_paths'] = dict()
config['ndk_paths'][ndk] = found
config['ndk_paths'][version] = found
logging.info(
_('Set NDK {release} ({version}) up').format(release=ndk, version=version)
)
net.download_file(url, zipball)
calced = sha256sum(zipball)
if sha256 != calced:
raise FDroidException('SHA-256 %s does not match expected for %s (%s)' % (calced, url, sha256))
logging.info(_('Unzipping to %s') % ndk_base)
with zipfile.ZipFile(zipball) as zipfp:
for info in zipfp.infolist():
permbits = info.external_attr >> 16
if stat.S_ISLNK(permbits):
link = os.path.join(ndk_base, info.filename)
link_target = zipfp.read(info).decode()
link_dir = os.path.dirname(link)
os.makedirs(link_dir, 0o755, True) # ensure intermediate directories are created
os.symlink(link_target, link)
real_target = os.path.realpath(link)
if not real_target.startswith(ndk_base):
os.remove(link)
logging.error(_('Unexpected symlink target: {link} -> {target}')
.format(link=link, target=real_target))
elif stat.S_ISDIR(permbits) or stat.S_IXUSR & permbits:
zipfp.extract(info.filename, path=ndk_base)
os.chmod(os.path.join(ndk_base, info.filename), 0o755) # nosec bandit B103
else:
zipfp.extract(info.filename, path=ndk_base)
os.chmod(os.path.join(ndk_base, info.filename), 0o644) # nosec bandit B103
os.remove(zipball)
for extracted in glob.glob(os.path.join(ndk_base, '*')):
version = get_ndk_version(extracted)
if os.path.basename(extracted) != version:
ndk_dir = os.path.join(ndk_base, version)
os.rename(extracted, ndk_dir)
if 'ndk_paths' not in config:
config['ndk_paths'] = dict()
config['ndk_paths'][ndk] = ndk_dir
logging.info(_('Set NDK {release} ({version}) up')
.format(release=ndk, version=version))
"""Derived from https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json"""
NDKS = [
{
"release": "r10e",
"sha256": "ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c",
"url": "https://dl.google.com/android/repository/android-ndk-r10e-linux-x86_64.zip"
},
{
"release": "r11",
"revision": "11.0.2655954",
"sha256": "59ab44f7ee6201df4381844736fdc456134c7f7660151003944a3017a0dcce97",
"url": "https://dl.google.com/android/repository/android-ndk-r11-linux-x86_64.zip"
},
{
"release": "r11b",
"revision": "11.1.2683735",
"sha256": "51d429bfda8bbe038683ed7ae7acc03b39604b84711901b555fe18c698867e53",
"url": "https://dl.google.com/android/repository/android-ndk-r11b-linux-x86_64.zip"
},
{
"release": "r11c",
"revision": "11.2.2725575",
"sha256": "ba85dbe4d370e4de567222f73a3e034d85fc3011b3cbd90697f3e8dcace3ad94",
"url": "https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip"
},
{
"release": "r12",
"revision": "12.0.2931149",
"sha256": "7876e3b99f3596a3215ecf4e9f152d24b82dfdf2bbe7d3a38c423ae6a3edee79",
"url": "https://dl.google.com/android/repository/android-ndk-r12-linux-x86_64.zip"
},
{
"release": "r12b",
"revision": "12.1.2977051",
"sha256": "eafae2d614e5475a3bcfd7c5f201db5b963cc1290ee3e8ae791ff0c66757781e",
"url": "https://dl.google.com/android/repository/android-ndk-r12b-linux-x86_64.zip"
},
{
"release": "r13",
"revision": "13.0.3315539",
"sha256": "0a1dbd216386399e2979c17a48f65b962bf7ddc0c2311ef35d902b90c298c400",
"url": "https://dl.google.com/android/repository/android-ndk-r13-linux-x86_64.zip"
},
{
"release": "r13b",
"revision": "13.1.3345770",
"sha256": "3524d7f8fca6dc0d8e7073a7ab7f76888780a22841a6641927123146c3ffd29c",
"url": "https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip"
},
{
"release": "r14",
"revision": "14.0.3770861",
"sha256": "3e622c2c9943964ea44cd56317d0769ed4c811bb4b40dc45b1f6965e4db9aa44",
"url": "https://dl.google.com/android/repository/android-ndk-r14-linux-x86_64.zip"
},
{
"release": "r14b",
"revision": "14.1.3816874",
"sha256": "0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024",
"url": "https://dl.google.com/android/repository/android-ndk-r14b-linux-x86_64.zip"
},
{
"release": "r15",
"revision": "15.0.4075724",
"sha256": "078eb7d28c3fcf45841f5baf6e6582e7fd5b73d8e8c4e0101df490f51abd37b6",
"url": "https://dl.google.com/android/repository/android-ndk-r15-linux-x86_64.zip"
},
{
"release": "r15b",
"revision": "15.1.4119039",
"sha256": "d1ce63f68cd806b5a992d4e5aa60defde131c243bf523cdfc5b67990ef0ee0d3",
"url": "https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip"
},
{
"release": "r15c",
"revision": "15.2.4203891",
"sha256": "f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c",
"url": "https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip"
},
{
"release": "r16",
"revision": "16.0.4442984",
"sha256": "a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e",
"url": "https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip"
},
{
"release": "r16b",
"revision": "16.1.4479499",
"sha256": "bcdea4f5353773b2ffa85b5a9a2ae35544ce88ec5b507301d8cf6a76b765d901",
"url": "https://dl.google.com/android/repository/android-ndk-r16b-linux-x86_64.zip"
},
{
"release": "r17",
"revision": "17.0.4754217",
"sha256": "ba3d813b47de75bc32a2f3de087f72599c6cb36fdc9686b96f517f5492ff43ca",
"url": "https://dl.google.com/android/repository/android-ndk-r17-linux-x86_64.zip"
},
{
"release": "r17b",
"revision": "17.1.4828580",
"sha256": "5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd",
"url": "https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip"
},
{
"release": "r17c",
"revision": "17.2.4988734",
"sha256": "3f541adbd0330a9205ba12697f6d04ec90752c53d6b622101a2a8a856e816589",
"url": "https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip"
},
{
"release": "r18b",
"revision": "18.1.5063045",
"sha256": "4f61cbe4bbf6406aa5ef2ae871def78010eed6271af72de83f8bd0b07a9fd3fd",
"url": "https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip"
},
{
"release": "r19",
"revision": "19.0.5232133",
"sha256": "c0a2425206191252197b97ea5fcc7eab9f693a576e69ef4773a9ed1690feed53",
"url": "https://dl.google.com/android/repository/android-ndk-r19-linux-x86_64.zip"
},
{
"release": "r19b",
"revision": "19.1.5304403",
"sha256": "0fbb1645d0f1de4dde90a4ff79ca5ec4899c835e729d692f433fda501623257a",
"url": "https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip"
},
{
"release": "r19c",
"revision": "19.2.5345600",
"sha256": "4c62514ec9c2309315fd84da6d52465651cdb68605058f231f1e480fcf2692e1",
"url": "https://dl.google.com/android/repository/android-ndk-r19c-linux-x86_64.zip"
},
{
"release": "r20",
"revision": "20.0.5594570",
"sha256": "57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0",
"url": "https://dl.google.com/android/repository/android-ndk-r20-linux-x86_64.zip"
},
{
"release": "r20b",
"revision": "20.1.5948944",
"sha256": "8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c",
"url": "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip"
},
{
"release": "r21",
"revision": "21.0.6113669",
"sha256": "b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b",
"url": "https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip"
},
{
"release": "r21b",
"revision": "21.1.6352462",
"sha256": "0c7af5dd23c5d2564915194e71b1053578438ac992958904703161c7672cbed7",
"url": "https://dl.google.com/android/repository/android-ndk-r21b-linux-x86_64.zip"
},
{
"release": "r21c",
"revision": "21.2.6472646",
"sha256": "214ebfcfa5108ba78f5b2cc8db4d575068f9c973ac7f27d2fa1987dfdb76c9e7",
"url": "https://dl.google.com/android/repository/android-ndk-r21c-linux-x86_64.zip"
},
{
"release": "r21d",
"revision": "21.3.6528147",
"sha256": "dd6dc090b6e2580206c64bcee499bc16509a5d017c6952dcd2bed9072af67cbd",
"url": "https://dl.google.com/android/repository/android-ndk-r21d-linux-x86_64.zip"
},
{
"release": "r21e",
"revision": "21.4.7075529",
"sha256": "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e",
"url": "https://dl.google.com/android/repository/android-ndk-r21e-linux-x86_64.zip"
},
{
"release": "r22",
"revision": "22.0.7026061",
"sha256": "d37fc69cd81e5660234a686e20adef39bc0244086e4d66525a40af771c020718",
"url": "https://dl.google.com/android/repository/android-ndk-r22-linux-x86_64.zip"
},
{
"release": "r22b",
"revision": "22.1.7171670",
"sha256": "ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b",
"url": "https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip"
},
{
"release": "r23",
"revision": "23.0.7599858",
"sha256": "e3eacf80016b91d4cd2c8ca9f34eebd32df912bb799c859cc5450b6b19277b4f",
"url": "https://dl.google.com/android/repository/android-ndk-r23-linux.zip"
},
{
"release": "r23b",
"revision": "23.1.7779620",
"sha256": "c6e97f9c8cfe5b7be0a9e6c15af8e7a179475b7ded23e2d1c1fa0945d6fb4382",
"url": "https://dl.google.com/android/repository/android-ndk-r23b-linux.zip"
},
{
"release": "r23c",
"revision": "23.2.8568313",
"sha256": "6ce94604b77d28113ecd588d425363624a5228d9662450c48d2e4053f8039242",
"url": "https://dl.google.com/android/repository/android-ndk-r23c-linux.zip"
},
{
"release": "r24",
"revision": "24.0.8215888",
"sha256": "caac638f060347c9aae994e718ba00bb18413498d8e0ad4e12e1482964032997",
"url": "https://dl.google.com/android/repository/android-ndk-r24-linux.zip"
},
{
"release": "r25",
"revision": "25.0.8775105",
"sha256": "cd661aeda5d9b7cfb6e64bd80737c274d7c1c0d026df2f85be3bf3327b25e545",
"url": "https://dl.google.com/android/repository/android-ndk-r25-linux.zip"
},
{
"release": "r25b",
"revision": "25.1.8937393",
"sha256": "403ac3e3020dd0db63a848dcaba6ceb2603bf64de90949d5c4361f848e44b005",
"url": "https://dl.google.com/android/repository/android-ndk-r25b-linux.zip"
},
{
"release": "r25c",
"revision": "25.2.9519653",
"sha256": "769ee342ea75f80619d985c2da990c48b3d8eaf45f48783a2d48870d04b46108",
"url": "https://dl.google.com/android/repository/android-ndk-r25c-linux.zip"
}
]

View File

@ -105,6 +105,7 @@ setup(
'qrcode',
'ruamel.yaml >= 0.15',
'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0',
'sdkmanager >= 0.6.4',
'yamllint',
],
extras_require={

View File

@ -12,7 +12,7 @@ import tempfile
import textwrap
import unittest
import yaml
import zipfile
from pathlib import Path
from unittest import mock
localmodule = os.path.realpath(
@ -214,6 +214,7 @@ class BuildTest(unittest.TestCase):
self.assertEqual(versionCode, vc)
self.assertEqual(versionName, vn)
@mock.patch('sdkmanager.build_package_list', lambda use_net: None)
def test_build_local_ndk(self):
"""Test if `fdroid build` detects installed NDKs and auto-installs when missing"""
with tempfile.TemporaryDirectory() as testdir, TmpCwd(
@ -235,6 +236,8 @@ class BuildTest(unittest.TestCase):
build.versionCode = 1
build.versionName = '1.0'
build.ndk = 'r21e' # aka 21.4.7075529
ndk_version = '21.4.7075529'
ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version
vcs = mock.Mock()
def make_fake_apk(output, build):
@ -242,13 +245,12 @@ class BuildTest(unittest.TestCase):
fp.write('APK PLACEHOLDER')
return output
def fake_download_file(_ignored, local_filename):
_ignored # silence the linters
with zipfile.ZipFile(local_filename, 'x') as zipfp:
zipfp.writestr(
'android-ndk-r21e/source.properties',
'Pkg.Revision = 21.4.7075529\n',
)
# pylint: disable=unused-argument
def fake_sdkmanager_install(to_install, android_home=None):
ndk_dir.mkdir(parents=True)
self.assertNotEqual(ndk_version, to_install) # converts r21e to version
with (ndk_dir / 'source.properties').open('w') as fp:
fp.write('Pkg.Revision = %s\n' % ndk_version)
# use "as _ignored" just to make a pretty layout
with mock.patch(
@ -258,8 +260,6 @@ class BuildTest(unittest.TestCase):
) as _ignored, mock.patch(
'fdroidserver.common.get_apk_id',
return_value=(app.id, build.versionCode, build.versionName),
) as _ignored, mock.patch(
'fdroidserver.common.is_apk_and_debuggable', return_value=False
) as _ignored, mock.patch(
'fdroidserver.common.sha256sum',
return_value='ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
@ -268,7 +268,7 @@ class BuildTest(unittest.TestCase):
) as _ignored, mock.patch(
'fdroidserver.build.FDroidPopen', FakeProcess
) as _ignored, mock.patch(
'fdroidserver.net.download_file', wraps=fake_download_file
'sdkmanager.install', wraps=fake_sdkmanager_install
) as _ignored:
_ignored # silence the linters
with self.assertRaises(
@ -290,7 +290,10 @@ class BuildTest(unittest.TestCase):
refresh=False,
)
# now run `fdroid build --onserver`
self.assertTrue('r21e' not in config['ndk_paths'])
print('now run `fdroid build --onserver`')
self.assertFalse(ndk_dir.exists())
self.assertFalse('r21e' in config['ndk_paths'])
self.assertFalse(ndk_version in config['ndk_paths'])
fdroidserver.build.build_local(
app,
build,
@ -305,7 +308,89 @@ class BuildTest(unittest.TestCase):
onserver=True,
refresh=False,
)
self.assertTrue(os.path.exists(config['ndk_paths']['r21e']))
self.assertTrue(ndk_dir.exists())
self.assertTrue(os.path.exists(config['ndk_paths'][ndk_version]))
# All paths in the config must be strings, never pathlib.Path instances
self.assertIsInstance(config['ndk_paths'][ndk_version], str)
@mock.patch('sdkmanager.build_package_list', lambda use_net: None)
@mock.patch('fdroidserver.build.FDroidPopen', FakeProcess)
@mock.patch('fdroidserver.common.get_native_code', lambda _ignored: 'x86')
@mock.patch('fdroidserver.common.is_apk_and_debuggable', lambda _ignored: False)
@mock.patch(
'fdroidserver.common.sha256sum',
lambda f: 'ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e',
)
def test_build_local_ndk_some_installed(self):
"""Test if `fdroid build` detects installed NDKs and auto-installs when missing"""
with tempfile.TemporaryDirectory() as testdir, TmpCwd(
testdir
), tempfile.TemporaryDirectory() as sdk_path:
ndk_r24 = os.path.join(sdk_path, 'ndk', '24.0.8215888')
os.makedirs(ndk_r24)
with open(os.path.join(ndk_r24, 'source.properties'), 'w') as fp:
fp.write('Pkg.Revision = 24.0.8215888\n')
config = {'ndk_paths': {'r24': ndk_r24}, 'sdk_path': sdk_path}
fdroidserver.common.config = config
fdroidserver.build.config = config
fdroidserver.build.options = mock.Mock()
fdroidserver.build.options.scan_binary = False
fdroidserver.build.options.notarball = True
fdroidserver.build.options.skipscan = True
app = fdroidserver.metadata.App()
app.id = 'mocked.app.id'
build = fdroidserver.metadata.Build()
build.commit = '1.0'
build.output = app.id + '.apk'
build.versionCode = 1
build.versionName = '1.0'
build.ndk = 'r21e' # aka 21.4.7075529
ndk_version = '21.4.7075529'
ndk_dir = Path(config['sdk_path']) / 'ndk' / ndk_version
vcs = mock.Mock()
def make_fake_apk(output, build):
with open(build.output, 'w') as fp:
fp.write('APK PLACEHOLDER')
return output
# pylint: disable=unused-argument
def fake_sdkmanager_install(to_install, android_home=None):
ndk_dir.mkdir(parents=True)
self.assertNotEqual(ndk_version, to_install) # converts r21e to version
with (ndk_dir / 'source.properties').open('w') as fp:
fp.write('Pkg.Revision = %s\n' % ndk_version)
# use "as _ignored" just to make a pretty layout
with mock.patch(
'fdroidserver.common.replace_build_vars', wraps=make_fake_apk
) as _ignored, mock.patch(
'fdroidserver.common.get_apk_id',
return_value=(app.id, build.versionCode, build.versionName),
) as _ignored, mock.patch(
'sdkmanager.install', wraps=fake_sdkmanager_install
) as _ignored:
_ignored # silence the linters
self.assertFalse(ndk_dir.exists())
self.assertFalse('r21e' in config['ndk_paths'])
self.assertFalse(ndk_version in config['ndk_paths'])
fdroidserver.build.build_local(
app,
build,
vcs,
build_dir=testdir,
output_dir=testdir,
log_dir=os.getcwd(),
srclib_dir=None,
extlib_dir=None,
tmp_dir=None,
force=False,
onserver=True,
refresh=False,
)
self.assertTrue(ndk_dir.exists())
self.assertTrue(os.path.exists(config['ndk_paths'][ndk_version]))
def test_build_local_clean(self):
"""Test if `fdroid build` cleans ant and gradle build products"""

View File

@ -5,6 +5,7 @@
import difflib
import git
import glob
import importlib
import inspect
import json
import logging
@ -20,9 +21,8 @@ import unittest
import textwrap
import yaml
import gzip
import stat
from distutils.version import LooseVersion
from zipfile import BadZipFile, ZipFile, ZipInfo
from zipfile import BadZipFile, ZipFile
from unittest import mock
from pathlib import Path
@ -2164,7 +2164,7 @@ class CommonTest(unittest.TestCase):
)
self.assertEqual(
{'com.example': [12345, 67890], 'org.fdroid.fdroid': [1]},
fdroidserver.common.read_pkg_args(appid_versionCode_pairs, allow_vercodes)
fdroidserver.common.read_pkg_args(appid_versionCode_pairs, allow_vercodes),
)
appid_versionCode_pairs = (
'com.example:67890',
@ -2210,42 +2210,79 @@ class CommonTest(unittest.TestCase):
fdroidserver.common.metadata_find_developer_signing_files(appid, vc),
)
@mock.patch('sdkmanager.build_package_list', lambda use_net: None)
def test_auto_install_ndk(self):
"""Test all possible field data types for build.ndk"""
fdroidserver.common.config = {'sdk_path': self.testdir}
sdk_path = self.testdir
build = fdroidserver.metadata.Build()
none_entry = mock.Mock()
with mock.patch('fdroidserver.common._install_ndk', none_entry):
with mock.patch('sdkmanager.install', none_entry):
fdroidserver.common.auto_install_ndk(build)
none_entry.assert_not_called()
empty_list = mock.Mock()
build.ndk = []
with mock.patch('fdroidserver.common._install_ndk', empty_list):
with mock.patch('sdkmanager.install', empty_list):
fdroidserver.common.auto_install_ndk(build)
empty_list.assert_not_called()
release_entry = mock.Mock()
build.ndk = 'r21e'
with mock.patch('fdroidserver.common._install_ndk', release_entry):
with mock.patch('sdkmanager.install', release_entry):
fdroidserver.common.auto_install_ndk(build)
release_entry.assert_called_once_with('r21e')
release_entry.assert_called_once_with('ndk;r21e', sdk_path)
revision_entry = mock.Mock()
build.ndk = '21.4.7075529'
with mock.patch('fdroidserver.common._install_ndk', revision_entry):
with mock.patch('sdkmanager.install', revision_entry):
fdroidserver.common.auto_install_ndk(build)
revision_entry.assert_called_once_with('21.4.7075529')
revision_entry.assert_called_once_with('ndk;21.4.7075529', sdk_path)
list_entry = mock.Mock()
calls = []
build.ndk = ['r10e', '11.0.2655954', 'r12b', 'r21e']
for n in build.ndk:
calls.append(mock.call(n))
with mock.patch('fdroidserver.common._install_ndk', list_entry):
calls.append(mock.call(f'ndk;{n}', sdk_path))
with mock.patch('sdkmanager.install', list_entry):
fdroidserver.common.auto_install_ndk(build)
list_entry.assert_has_calls(calls)
@unittest.skipIf(importlib.util.find_spec('sdkmanager') is None, 'needs sdkmanager')
@mock.patch('sdkmanager.build_package_list', lambda use_net: None)
@mock.patch('sdkmanager._install_zipball_from_cache', lambda a, b: None)
@mock.patch('sdkmanager._generate_package_xml', lambda a, b, c: None)
def test_auto_install_ndk_mock_dl(self):
"""Test NDK installs by actually calling sdkmanager"""
import sdkmanager
import pkg_resources
sdkmanager_version = LooseVersion(
pkg_resources.get_distribution('sdkmanager').version
)
if sdkmanager_version < LooseVersion('0.6.4'):
raise unittest.SkipTest('needs fdroid sdkmanager >= 0.6.4')
fdroidserver.common.config = {'sdk_path': 'placeholder'}
build = fdroidserver.metadata.Build()
url = 'https://dl.google.com/android/repository/android-ndk-r24-linux.zip'
path = sdkmanager.get_cachedir() / os.path.basename(url)
sdkmanager.packages = {
('ndk', '24.0.8215888'): url,
('ndk', 'r24'): url,
}
build.ndk = 'r24'
firstrun = mock.Mock()
with mock.patch('sdkmanager.download_file', firstrun):
fdroidserver.common.auto_install_ndk(build)
firstrun.assert_called_once_with(url, path)
build.ndk = '24.0.8215888'
secondrun = mock.Mock()
with mock.patch('sdkmanager.download_file', secondrun):
fdroidserver.common.auto_install_ndk(build)
secondrun.assert_called_once_with(url, path)
@unittest.skip("This test downloads and unzips a 1GB file.")
def test_install_ndk(self):
"""NDK r10e is a special case since its missing source.properties"""
@ -2257,101 +2294,6 @@ class CommonTest(unittest.TestCase):
fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r10e': r10e}, config['ndk_paths'])
def test_install_ndk_versions(self):
"""Test whether NDK version parsing is working properly"""
def fake_download(url, zipball):
print(url, zipball)
with ZipFile(zipball, 'w') as zipfp:
zipfp.writestr(os.path.basename(url), url)
with tempfile.TemporaryDirectory(
prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir
) as sdk_path:
config = {'sdk_path': sdk_path}
fdroidserver.common.config = config
for r, sha256 in (
(
'r10e',
'ee5f405f3b57c4f5c3b3b8b5d495ae12b660e03d2112e4ed5c728d349f1e520c',
),
('r20', '57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0'),
(
'23.0.7599858',
'e3eacf80016b91d4cd2c8ca9f34eebd32df912bb799c859cc5450b6b19277b4f',
),
):
with mock.patch(
'fdroidserver.net.download_file', side_effect=fake_download
) as _ignored, mock.patch(
'fdroidserver.common.get_ndk_version', return_value=r
) as _ignored, mock.patch(
'fdroidserver.common.sha256sum', return_value=sha256
):
_ignored # silence the linters
fdroidserver.common._install_ndk(r)
def test_install_ndk_with_symlinks(self):
"""Some NDK zipballs might have symlinks in them."""
def fake_download(url, zipball):
print(url, zipball)
unix_st_mode = (
stat.S_IFLNK
| stat.S_IRUSR
| stat.S_IWUSR
| stat.S_IXUSR
| stat.S_IRGRP
| stat.S_IWGRP
| stat.S_IXGRP
| stat.S_IROTH
| stat.S_IWOTH
| stat.S_IXOTH
)
with ZipFile(zipball, 'w') as zipfp:
zipfp.writestr('ndk/' + os.path.basename(url), url)
zipInfo = ZipInfo('ndk/basename')
zipInfo.create_system = 3
zipInfo.external_attr = unix_st_mode << 16
zipfp.writestr(zipInfo, os.path.basename(url))
zipInfo = ZipInfo('ndk/bad_abs_link')
zipInfo.create_system = 3
zipInfo.external_attr = unix_st_mode << 16
zipfp.writestr(zipInfo, '/etc/passwd')
zipInfo = ZipInfo('ndk/bad_rel_link')
zipInfo.create_system = 3
zipInfo.external_attr = unix_st_mode << 16
zipfp.writestr(zipInfo, '../../../../../../../etc/passwd')
zipInfo = ZipInfo('ndk/bad_rel_link2')
zipInfo.create_system = 3
zipInfo.external_attr = unix_st_mode << 16
zipfp.writestr(zipInfo, 'foo/../../../../../../../../../etc/passwd')
config = {'sdk_path': self.tmpdir}
fdroidserver.common.config = config
r = 'r20'
sha256 = '57435158f109162f41f2f43d5563d2164e4d5d0364783a9a6fab3ef12cb06ce0'
with mock.patch(
'fdroidserver.net.download_file', side_effect=fake_download
) as _ignored, mock.patch(
'fdroidserver.common.get_ndk_version', return_value=r
) as _ignored, mock.patch(
'fdroidserver.common.sha256sum', return_value=sha256
):
_ignored # silence the linters
fdroidserver.common._install_ndk(r)
self.assertTrue(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20')))
self.assertTrue(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'basename')))
self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_abs_link')))
self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_rel_link')))
self.assertFalse(os.path.exists(os.path.join(self.tmpdir, 'ndk', 'r20', 'bad_rel_link2')))
os.system('ls -l ' + os.path.join(self.tmpdir, 'ndk', 'r20'))
def test_fill_config_defaults(self):
"""Test the auto-detection of NDKs installed in standard paths"""
ndk_bundle = os.path.join(self.tmpdir, 'ndk-bundle')
@ -2360,7 +2302,7 @@ class CommonTest(unittest.TestCase):
fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 17.2.4988734\n')
config = {'sdk_path': self.tmpdir}
fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r17c': ndk_bundle}, config['ndk_paths'])
self.assertEqual({'17.2.4988734': ndk_bundle}, config['ndk_paths'])
r21e = os.path.join(self.tmpdir, 'ndk', '21.4.7075529')
os.makedirs(r21e)
@ -2368,7 +2310,10 @@ class CommonTest(unittest.TestCase):
fp.write('Pkg.Desc = Android NDK\nPkg.Revision = 21.4.7075529\n')
config = {'sdk_path': self.tmpdir}
fdroidserver.common.fill_config_defaults(config)
self.assertEqual({'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths'])
self.assertEqual(
{'17.2.4988734': ndk_bundle, '21.4.7075529': r21e},
config['ndk_paths'],
)
r10e = os.path.join(self.tmpdir, 'ndk', 'r10e')
os.makedirs(r10e)
@ -2377,7 +2322,8 @@ class CommonTest(unittest.TestCase):
config = {'sdk_path': self.tmpdir}
fdroidserver.common.fill_config_defaults(config)
self.assertEqual(
{'r10e': r10e, 'r17c': ndk_bundle, 'r21e': r21e}, config['ndk_paths']
{'r10e': r10e, '17.2.4988734': ndk_bundle, '21.4.7075529': r21e},
config['ndk_paths'],
)
@unittest.skipIf(not os.path.isdir('/usr/lib/jvm/default-java'), 'uses Debian path')
@ -2411,7 +2357,7 @@ class CommonTest(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
repo = git.Repo.init(Path.cwd())
f = Path("test")
date = 10**9
date = 10 ** 9
for tag in tags:
date += 1
f.write_text(tag)

View File

@ -1,139 +0,0 @@
#!/usr/bin/env python3
import git
import gitlab
import json
import os
import re
import requests
import subprocess
import sys
from colorama import Fore, Style
checksums = None
versions = dict()
while not checksums:
r = requests.get(
'https://gitlab.com/fdroid/android-sdk-transparency-log/-/raw/master/checksums.json',
timeout=300,
)
if r.status_code == 200:
checksums = r.json()
with open('fdroidserver/common.py') as fp:
common_py = fp.read()
to_compile = re.search(r'\nNDKS = [^\]]+\]', common_py).group()
if not to_compile:
sys.exit(1)
code = compile(to_compile, '<string>', 'exec')
config = {}
exec(code, None, config) # nosec this is just a CI script
ndks = []
errors = 0
release = None
revision = None
for k, entries in checksums.items():
if k.endswith('.zip') and k.startswith(
'https://dl.google.com/android/repository/android-ndk'
):
m = re.search(r'-(r[1-9][0-9]?[a-z]?)-linux', k)
if m:
d = {'url': k, 'release': m.group(1), 'sha256': checksums[k][0]['sha256']}
for entry in entries:
if 'source.properties' in entry:
n = re.search(
r'[1-9][0-9]\.[0-9]\.[0-9]{7}', entry['source.properties']
)
if n:
d['revision'] = n.group()
ndks.append(d)
for d in config['NDKS']:
if k == d['url']:
sha256 = d['sha256']
found = False
for entry in entries:
if sha256 == entry['sha256']:
found = True
if not found:
print(
Fore.RED
+ (
'ERROR: checksum mismatch: %s != %s'
% (sha256, entry['sha256'])
)
+ Style.RESET_ALL
)
errors += 1
with open('fdroidserver/common.py', 'w') as fp:
fp.write(
common_py.replace(
to_compile, '\nNDKS = ' + json.dumps(ndks, indent=4, sort_keys=True)
)
)
if os.getenv('CI_PROJECT_NAMESPACE') != 'fdroid':
p = subprocess.run(['git', '--no-pager', 'diff'])
print(p.stdout)
sys.exit(errors)
# This only runs after commits are pushed to fdroid/fdroidserver
git_repo = git.repo.Repo('.')
modified = git_repo.git().ls_files(modified=True).split()
if git_repo.is_dirty() and 'fdroidserver/common.py' in modified:
branch = git_repo.create_head(os.path.basename(__file__), force=True)
branch.checkout()
git_repo.index.add(['fdroidserver/common.py'])
author = git.Actor('fdroid-bot', 'fdroid-bot@f-droid.org')
git_repo.index.commit('Android NDK %s (%s)' % (release, revision), author=author)
project_path = 'fdroid-bot/' + os.getenv('CI_PROJECT_NAME')
url = 'https://gitlab-ci-token:%s@%s/%s.git' % (
os.getenv('PERSONAL_ACCESS_TOKEN'),
os.getenv('CI_SERVER_HOST'),
project_path,
)
remote_name = 'fdroid-bot'
try:
remote = git_repo.create_remote(remote_name, url)
# See https://github.com/PyCQA/pylint/issues/2856 .
# pylint: disable-next=no-member
except git.exc.GitCommandError:
remote = git.remote.Remote(git_repo, remote_name)
remote.set_url(url)
remote.push(force=True)
git.remote.Remote.rm(git_repo, remote_name)
private_token = os.getenv('PERSONAL_ACCESS_TOKEN')
if not private_token:
print(
Fore.RED
+ 'ERROR: GitLab Token not found in PERSONAL_ACCESS_TOKEN!'
+ Style.RESET_ALL
)
sys.exit(1)
gl = gitlab.Gitlab(
os.getenv('CI_SERVER_URL'), api_version=4, private_token=private_token
)
project = gl.projects.get(project_path, lazy=True)
description = (
'see <https://gitlab.com/fdroid/android-sdk-transparency-log/-/blob/master/checksums.json>'
'\n\n<p><small>generated by <a href="%s/-/jobs/%s">GitLab CI Job #%s</a></small></p>'
% (os.getenv('CI_PROJECT_URL'), os.getenv('CI_JOB_ID'), os.getenv('CI_JOB_ID'))
)
mr = project.mergerequests.create(
{
'source_branch': branch.name,
'target_project_id': 36527, # fdroid/fdroidserver
'target_branch': 'master',
'title': 'update NDK',
'description': description,
'labels': ['fdroid-bot', 'buildserver'],
'remove_source_branch': True,
}
)
mr.save()