convert apkcache from pickle to JSON
pickle can serialize executable code, while JSON is only ever pure data. The APK cache is only ever pure data, so no need for the security risks of pickle. For example, if some malicious thing gets write access on the `fdroid update` machine, it can write out a custom tmp/apkcache which would then be executed. That is not possible with JSON. This does just ignore any existing cache and rebuilds from scratch. That is so we don't need to maintain pickle anywhere, and to ensure there are no glitches from a conversion from pickle to JSON. closes #163
This commit is contained in:
Hans-Christoph Steiner
parent
74776e026f
commit
3011953d0e
|
|
@ -533,7 +533,7 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
|
|||
old_permissions = set()
|
||||
sorted_permissions = sorted(apk['uses-permission'])
|
||||
for perm in sorted_permissions:
|
||||
perm_name = perm.name
|
||||
perm_name = perm[0]
|
||||
if perm_name.startswith("android.permission."):
|
||||
perm_name = perm_name[19:]
|
||||
old_permissions.add(perm_name)
|
||||
|
|
@ -541,15 +541,15 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
|
|||
|
||||
for permission in sorted_permissions:
|
||||
permel = doc.createElement('uses-permission')
|
||||
permel.setAttribute('name', permission.name)
|
||||
if permission.maxSdkVersion is not None:
|
||||
permel.setAttribute('maxSdkVersion', '%d' % permission.maxSdkVersion)
|
||||
permel.setAttribute('name', permission[0])
|
||||
if permission[1] is not None:
|
||||
permel.setAttribute('maxSdkVersion', '%d' % permission[1])
|
||||
apkel.appendChild(permel)
|
||||
for permission_sdk_23 in sorted(apk['uses-permission-sdk-23']):
|
||||
permel = doc.createElement('uses-permission-sdk-23')
|
||||
permel.setAttribute('name', permission_sdk_23.name)
|
||||
if permission_sdk_23.maxSdkVersion is not None:
|
||||
permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23.maxSdkVersion)
|
||||
permel.setAttribute('name', permission_sdk_23[0])
|
||||
if permission_sdk_23[1] is not None:
|
||||
permel.setAttribute('maxSdkVersion', '%d' % permission_sdk_23[1])
|
||||
apkel.appendChild(permel)
|
||||
if 'nativecode' in apk:
|
||||
addElement('nativecode', ','.join(sorted(apk['nativecode'])), doc, apkel)
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ import re
|
|||
import socket
|
||||
import zipfile
|
||||
import hashlib
|
||||
import pickle # nosec TODO
|
||||
import json
|
||||
import time
|
||||
import copy
|
||||
from datetime import datetime
|
||||
|
|
@ -46,7 +46,7 @@ from . import metadata
|
|||
from .common import SdkToolsPopen
|
||||
from .exception import BuildException, FDroidException
|
||||
|
||||
METADATA_VERSION = 19
|
||||
METADATA_VERSION = 20
|
||||
|
||||
# less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
|
||||
UNSET_VERSION_CODE = -0x100000000
|
||||
|
|
@ -56,8 +56,7 @@ APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
|
|||
APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
|
||||
APK_LABEL_ICON_PAT = re.compile(r".*\s+label='(.*)'\s+icon='(.*?)'")
|
||||
APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
|
||||
APK_PERMISSION_PAT = \
|
||||
re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
|
||||
APK_PERMISSION_PAT = re.compile(".*(name='(.*)')(.*maxSdkVersion='(.*)')?.*")
|
||||
APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
|
||||
|
||||
screen_densities = ['65534', '640', '480', '320', '240', '160', '120']
|
||||
|
|
@ -438,7 +437,7 @@ def getsig(apkpath):
|
|||
|
||||
|
||||
def get_cache_file():
|
||||
return os.path.join('tmp', 'apkcache')
|
||||
return os.path.join('tmp', 'apkcache.json')
|
||||
|
||||
|
||||
def get_cache():
|
||||
|
|
@ -460,27 +459,46 @@ def get_cache():
|
|||
apkcachefile = get_cache_file()
|
||||
ada = options.allow_disabled_algorithms or config['allow_disabled_algorithms']
|
||||
if not options.clean and os.path.exists(apkcachefile):
|
||||
with open(apkcachefile, 'rb') as cf:
|
||||
apkcache = pickle.load(cf, encoding='utf-8') # nosec TODO
|
||||
with open(apkcachefile) as fp:
|
||||
apkcache = json.load(fp, object_pairs_hook=collections.OrderedDict)
|
||||
if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
|
||||
or apkcache.get('allow_disabled_algorithms') != ada:
|
||||
apkcache = {}
|
||||
apkcache = collections.OrderedDict()
|
||||
else:
|
||||
apkcache = {}
|
||||
apkcache = collections.OrderedDict()
|
||||
|
||||
apkcache["METADATA_VERSION"] = METADATA_VERSION
|
||||
apkcache['allow_disabled_algorithms'] = ada
|
||||
|
||||
for k, v in apkcache.items():
|
||||
if not isinstance(v, dict):
|
||||
continue
|
||||
if 'antiFeatures' in v:
|
||||
v['antiFeatures'] = set(v['antiFeatures'])
|
||||
if 'added' in v:
|
||||
v['added'] = datetime.fromtimestamp(v['added'])
|
||||
|
||||
return apkcache
|
||||
|
||||
|
||||
def write_cache(apkcache):
|
||||
class Encoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, set):
|
||||
return ['SET'] + list(obj)
|
||||
elif isinstance(obj, datetime):
|
||||
return obj.timestamp()
|
||||
return super().default(obj)
|
||||
|
||||
apkcachefile = get_cache_file()
|
||||
cache_path = os.path.dirname(apkcachefile)
|
||||
if not os.path.exists(cache_path):
|
||||
os.makedirs(cache_path)
|
||||
with open(apkcachefile, 'wb') as cf:
|
||||
pickle.dump(apkcache, cf)
|
||||
for k, v in apkcache.items():
|
||||
if isinstance(k, bytes):
|
||||
print('BYTES: ' + str(k) + ' ' + str(v))
|
||||
with open(apkcachefile, 'w') as fp:
|
||||
json.dump(apkcache, fp, cls=Encoder, indent=2)
|
||||
|
||||
|
||||
def get_icon_bytes(apkzip, iconsrc):
|
||||
|
|
@ -948,16 +966,16 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
|||
|
||||
cachechanged = False
|
||||
repo_files = []
|
||||
repodir = repodir.encode('utf-8')
|
||||
repodir = repodir.encode()
|
||||
for name in os.listdir(repodir):
|
||||
file_extension = common.get_file_extension(name)
|
||||
if file_extension == 'apk' or file_extension == 'obb':
|
||||
continue
|
||||
filename = os.path.join(repodir, name)
|
||||
name_utf8 = name.decode('utf-8')
|
||||
name_utf8 = name.decode()
|
||||
if filename.endswith(b'_src.tar.gz'):
|
||||
logging.debug(_('skipping source tarball: {path}')
|
||||
.format(path=filename.decode('utf-8')))
|
||||
.format(path=filename.decode()))
|
||||
continue
|
||||
if not common.is_repo_file(filename):
|
||||
continue
|
||||
|
|
@ -968,15 +986,8 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
|||
|
||||
shasum = sha256sum(filename)
|
||||
usecache = False
|
||||
if name in apkcache:
|
||||
repo_file = apkcache[name]
|
||||
# added time is cached as tuple but used here as datetime instance
|
||||
if 'added' in repo_file:
|
||||
a = repo_file['added']
|
||||
if isinstance(a, datetime):
|
||||
repo_file['added'] = a
|
||||
else:
|
||||
repo_file['added'] = datetime(*a[:6])
|
||||
if name_utf8 in apkcache:
|
||||
repo_file = apkcache[name_utf8]
|
||||
if repo_file.get('hash') == shasum:
|
||||
logging.debug(_("Reading {apkfilename} from cache")
|
||||
.format(apkfilename=name_utf8))
|
||||
|
|
@ -1004,10 +1015,10 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
|
|||
repo_file['versionCode'] = int(m.group(2))
|
||||
srcfilename = name + b'_src.tar.gz'
|
||||
if os.path.exists(os.path.join(repodir, srcfilename)):
|
||||
repo_file['srcname'] = srcfilename.decode('utf-8')
|
||||
repo_file['srcname'] = srcfilename.decode()
|
||||
repo_file['size'] = stat.st_size
|
||||
|
||||
apkcache[name] = repo_file
|
||||
apkcache[name_utf8] = repo_file
|
||||
cachechanged = True
|
||||
|
||||
if use_date_from_file:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"repo": {
|
||||
"timestamp": 1502845383782,
|
||||
"version": 19,
|
||||
"version": 20,
|
||||
"name": "My First F-Droid Repo Demo",
|
||||
"icon": "fdroid-icon.png",
|
||||
"address": "https://MyFirstFDroidRepo.org/fdroid/repo",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<fdroid>
|
||||
<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="19">
|
||||
<repo icon="fdroid-icon.png" name="My First F-Droid Repo Demo" pubkey="308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a" timestamp="1480431575" url="https://MyFirstFDroidRepo.org/fdroid/repo" version="20">
|
||||
<description>This is a repository of apps to be used with F-Droid. Applications in this repository are either official binaries built by the original application developers, or are binaries built from source by the admin of f-droid.org using the tools on https://gitlab.com/u/fdroid. </description>
|
||||
<mirror>http://foobarfoobarfoobar.onion/fdroid/repo</mirror>
|
||||
<mirror>https://foo.bar/fdroid/repo</mirror>
|
||||
|
|
|
|||
|
|
@ -177,8 +177,8 @@ if which zipalign || ls -1 $ANDROID_HOME/build-tools/*/zipalign; then
|
|||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
test -L urzip.apk
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
fi
|
||||
|
|
@ -808,8 +808,8 @@ else
|
|||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
export ANDROID_HOME=$STORED_ANDROID_HOME
|
||||
fi
|
||||
|
||||
|
|
@ -860,8 +860,8 @@ $fdroid readmeta
|
|||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
|
||||
|
||||
|
|
@ -890,8 +890,8 @@ $fdroid readmeta
|
|||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
|
||||
|
||||
|
|
@ -917,8 +917,8 @@ $fdroid readmeta
|
|||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
|
||||
|
||||
|
|
@ -1009,8 +1009,8 @@ $fdroid readmeta
|
|||
test -e repo/index.xml
|
||||
test -e repo/index.jar
|
||||
test -e repo/index-v1.jar
|
||||
test -e tmp/apkcache
|
||||
! test -z tmp/apkcache
|
||||
test -e tmp/apkcache.json
|
||||
! test -z tmp/apkcache.json
|
||||
grep -F '<application id=' repo/index.xml > /dev/null
|
||||
|
||||
# now set fake repo_keyalias
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
# http://www.drdobbs.com/testing/unit-testing-with-python/240165163
|
||||
|
||||
import git
|
||||
import glob
|
||||
import inspect
|
||||
import logging
|
||||
import optparse
|
||||
|
|
@ -289,6 +290,45 @@ class UpdateTest(unittest.TestCase):
|
|||
self.assertIsNone(apk.get('obbMainFile'))
|
||||
self.assertIsNone(apk.get('obbPatchFile'))
|
||||
|
||||
def test_apkcache_json(self):
|
||||
"""test the migration from pickle to json"""
|
||||
os.chdir(os.path.join(localmodule, 'tests'))
|
||||
if os.path.basename(os.getcwd()) != 'tests':
|
||||
raise Exception('This test must be run in the "tests/" subdir')
|
||||
|
||||
config = dict()
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
config['ndk_paths'] = dict()
|
||||
config['accepted_formats'] = ['json', 'txt', 'yml']
|
||||
fdroidserver.common.config = config
|
||||
fdroidserver.update.config = config
|
||||
|
||||
fdroidserver.update.options = type('', (), {})()
|
||||
fdroidserver.update.options.clean = True
|
||||
fdroidserver.update.options.delete_unknown = True
|
||||
fdroidserver.update.options.rename_apks = False
|
||||
fdroidserver.update.options.allow_disabled_algorithms = False
|
||||
|
||||
fdroidserver.metadata.read_metadata(xref=True)
|
||||
knownapks = fdroidserver.common.KnownApks()
|
||||
apkcache = fdroidserver.update.get_cache()
|
||||
self.assertEqual(2, len(apkcache))
|
||||
self.assertEqual(fdroidserver.update.METADATA_VERSION, apkcache["METADATA_VERSION"])
|
||||
self.assertEqual(fdroidserver.update.options.allow_disabled_algorithms,
|
||||
apkcache['allow_disabled_algorithms'])
|
||||
apks, cachechanged = fdroidserver.update.process_apks(apkcache, 'repo', knownapks, False)
|
||||
fdroidserver.update.write_cache(apkcache)
|
||||
|
||||
fdroidserver.update.options.clean = False
|
||||
read_from_json = fdroidserver.update.get_cache()
|
||||
self.assertEqual(16, len(read_from_json))
|
||||
for f in glob.glob('repo/*.apk'):
|
||||
self.assertTrue(os.path.basename(f) in read_from_json)
|
||||
|
||||
fdroidserver.update.options.clean = True
|
||||
reset = fdroidserver.update.get_cache()
|
||||
self.assertEqual(2, len(reset))
|
||||
|
||||
def test_scan_apk(self):
|
||||
config = dict()
|
||||
fdroidserver.common.fill_config_defaults(config)
|
||||
|
|
|
|||
Loading…
Reference in New Issue