opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

scanner: implement caching rules for suss

This commit is contained in:
Michael Pöhn 2022-09-30 04:44:14 +02:00
parent bfcc30b854
commit a8bcaa3d70
4 changed files with 78 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fdroidserver/build.py
View File

@ -987,7 +987,7 @@ def main():
if not options.appid and not options.all:
parser.error("option %s: If you really want to build all the apps, use --all" % "all")
config = common.read_config(options)
config = common.read_config(opts=options)
if config['build_server_always']:
options.server = True

13
fdroidserver/common.py
View File

@ -324,19 +324,26 @@ def fill_config_defaults(thisconfig):
thisconfig['gradle_version_dir'] = str(Path(thisconfig['cachedir']) / 'gradle')
def get_config(options=None):
def get_config(opts=None):
"""
helper function for getting access to commons.config while safely
initializing if it wasn't initialized yet.
"""
global config
global config, options
if config is not None:
return config
config = {}
common.fill_config_defaults(config)
common.read_config(options)
common.read_config(opts=opts)
# make sure these values are available in common.py even if they didn't
# declare global in a scope
common.config = config
if opts is not None:
common.options = opts
return config

99
fdroidserver/scanner.py
View File

@ -141,6 +141,10 @@ class SignatureDataOutdatedException(Exception):
pass
class SignatureDataCacheMissException(Exception):
pass
class SignatureDataVersionMismatchException(Exception):
pass
@ -151,7 +155,7 @@ class SignatureDataController:
self.filename = filename
self.url = url
# by default we assume cache is valid indefinitely
self.cache_outdated_interval = timedelta(days=999999)
self.cache_duration = timedelta(days=999999)
self.data = {}
def check_data_version(self):
@ -162,63 +166,65 @@ class SignatureDataController:
'''
NOTE: currently not in use
Checks if the timestamp value is ok. Raises an exception if something
is not ok.
Checks if the last_updated value is ok. Raises an exception if
it's expired or inaccessible.
:raises SignatureDataMalformedException: when timestamp value is
inaccessible or not parse-able
:raises SignatureDataOutdatedException: when timestamp is older then
`self.cache_outdated_interval`
`self.cache_duration`
'''
timestamp = self.data.get("timestamp")
if not timestamp:
raise SignatureDataMalformedException()
try:
timestamp = datetime.fromisoformat(timestamp)
except ValueError as e:
raise SignatureDataMalformedException() from e
except TypeError as e:
raise SignatureDataMalformedException() from e
if (timestamp + self.cache_outdated_interval) < scanner._datetime_now():
raise SignatureDataOutdatedException()
last_updated = self.data.get("last_updated", None)
if last_updated:
try:
last_updated = datetime.fromisoformat(last_updated)
except ValueError as e:
raise SignatureDataMalformedException() from e
except TypeError as e:
raise SignatureDataMalformedException() from e
delta = (last_updated + self.cache_duration) - scanner._datetime_now()
if delta > timedelta(seconds=0):
logging.debug(_('next {name} cache update due in {time}').format(
name=self.filename, time=delta
))
else:
raise SignatureDataOutdatedException()
def fetch(self):
try:
self.load_from_cache()
self.verify_data()
self.check_last_updated()
except (
SignatureDataMalformedException,
SignatureDataVersionMismatchException,
SignatureDataOutdatedException
):
try:
self.fetch_signatures_from_web()
except AttributeError:
# just load from defaults if fetch_signatures_from_web is not
# implemented
self.load_from_defaults()
self.fetch_signatures_from_web()
self.write_to_cache()
except Exception as e:
raise Exception(_("downloading scanner signatures from '{}' failed").format(self.url)) from e
def load(self):
try:
self.load_from_cache()
self.verify_data()
except (SignatureDataMalformedException, SignatureDataVersionMismatchException):
self.load_from_defaults()
try:
self.load_from_cache()
self.verify_data()
self.check_last_updated()
except SignatureDataCacheMissException:
self.load_from_defaults()
except SignatureDataOutdatedException:
self.fetch_signatures_from_web()
self.write_to_cache()
except (SignatureDataMalformedException, SignatureDataVersionMismatchException) as e:
logging.critical(_("scanner cache is malformed! You can clear it with: '{clear}'").format(
clear='rm -r {}'.format(common.get_config()['cachedir_scanner'])
))
raise e
def load_from_defaults(self):
sig_file = (Path(__file__).parent / 'data' / 'scanner' / self.filename).resolve()
with open(sig_file) as f:
self.data = json.load(f)
self.set_data(json.load(f))
def load_from_cache(self):
sig_file = scanner._scanner_cachedir() / self.filename
if not sig_file.exists():
raise SignatureDataMalformedException()
raise SignatureDataCacheMissException()
with open(sig_file) as f:
self.data = json.load(f)
self.set_data(json.load(f))
def write_to_cache(self):
sig_file = scanner._scanner_cachedir() / self.filename
@ -231,29 +237,36 @@ class SignatureDataController:
cleans and validates and cleans `self.data`
'''
self.check_data_version()
valid_keys = ['timestamp', 'version', 'signatures']
valid_keys = ['timestamp', 'last_updated', 'version', 'signatures', 'cache_duration']
for k in list(self.data.keys()):
if k not in valid_keys:
del self.data[k]
def set_data(self, new_data):
self.data = new_data
if 'cache_duration' in new_data:
self.cache_duration = timedelta(seconds=new_data['cache_duration'])
def fetch_signatures_from_web(self):
logging.debug(_("downloading '{}'").format(self.url))
with urllib.request.urlopen(self.url) as f:
self.data = json.load(f)
self.set_data(json.load(f))
self.data['last_updated'] = scanner._datetime_now().isoformat()
class ExodusSignatureDataController(SignatureDataController):
def __init__(self):
super().__init__('Exodus signatures', 'exodus.yml', 'https://reports.exodus-privacy.eu.org/api/trackers')
self.cache_outdated_interval = timedelta(days=1) # refresh exodus cache after one day
self.cache_duration = timedelta(days=1) # refresh exodus cache after one day
def fetch_signatures_from_web(self):
logging.debug(_("downloading '{}'").format(self.url))
self.data = {
data = {
"signatures": {},
"timestamp": scanner._datetime_now().isoformat(),
"last_updated": scanner._datetime_now().isoformat(),
"version": SCANNER_CACHE_VERSION,
}
@ -261,7 +274,7 @@ class ExodusSignatureDataController(SignatureDataController):
d = json.load(f)
for tracker in d["trackers"].values():
if tracker.get('code_signature'):
self.data["signatures"][tracker["name"]] = {
data["signatures"][tracker["name"]] = {
"name": tracker["name"],
"warn_code_signatures": [tracker["code_signature"]],
# exodus also provides network signatures, unused atm.
@ -273,6 +286,7 @@ class ExodusSignatureDataController(SignatureDataController):
# etc. might be listed by exodus
# too.
}
self.set_data(data)
class ScannerTool():
@ -316,6 +330,7 @@ class ScannerTool():
def refresh(self):
for sdc in self.sdcs:
sdc.fetch_signatures_from_web()
sdc.write_to_cache()
def add(self, new_controller: SignatureDataController):
self.sdcs.append(new_controller)
@ -684,7 +699,7 @@ def main():
logging.getLogger().setLevel(logging.ERROR)
# initialize/load configuration values
common.get_config(options)
common.get_config(opts=options)
if options.refresh:
scanner._get_tool().refresh()

21
tests/scanner.TestCase
View File

@ -502,39 +502,38 @@ class Test_SignatureDataController(unittest.TestCase):
sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
self.assertEqual(sdc.name, 'nnn')
self.assertEqual(sdc.filename, 'fff.yml')
self.assertEqual(sdc.cache_outdated_interval, timedelta(999999))
self.assertEqual(sdc.cache_duration, timedelta(999999))
self.assertDictEqual(sdc.data, {})
# check_last_updated
def test_check_last_updated_ok(self):
sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
sdc.data['timestamp'] = datetime.now().astimezone().isoformat()
sdc.data['last_updated'] = datetime.now().astimezone().isoformat()
sdc.check_last_updated()
def test_check_last_updated_exception_cache_outdated(self):
sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
sdc.cache_outdated_interval = timedelta(days=7)
sdc.data['timestamp'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat()
sdc.cache_duration = timedelta(days=7)
sdc.data['last_updated'] = (datetime.now().astimezone() - timedelta(days=30)).isoformat()
with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException):
sdc.check_last_updated()
def test_check_last_updated_exception_missing_timestamp_value(self):
sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
sdc.check_last_updated()
def test_check_last_updated_exception_not_string(self):
sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
sdc.data['timestamp'] = 12345
sdc.data['last_updated'] = 12345
with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
sdc.check_last_updated()
def test_check_last_updated_exception_not_iso_formatted_string(self):
sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
sdc.data['timestamp'] = '01/09/2002 10:11'
sdc.data['last_updated'] = '01/09/2002 10:11'
with self.assertRaises(fdroidserver.scanner.SignatureDataMalformedException):
sdc.check_last_updated()
def test_check_last_updated_no_exception_missing_when_last_updated_not_set(self):
sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')
sdc.check_last_updated()
# check_data_version
def test_check_data_version_ok(self):
sdc = fdroidserver.scanner.SignatureDataController('nnn', 'fff.yml', 'https://example.com/test.json')