diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py index b1be88f8..ac838593 100644 --- a/fdroidserver/metadata.py +++ b/fdroidserver/metadata.py @@ -41,6 +41,10 @@ from fdroidserver.exception import MetaDataException, FDroidException srclibs = None warnings_action = None +# validates usernames based on a loose collection of rules from GitHub, GitLab, +# Liberapay and issuehunt. This is mostly to block abuse. +VALID_USERNAME_REGEX = re.compile(r'^[a-z\d](?:[a-z\d/._-]){0,38}$', re.IGNORECASE) + def warn_or_exception(value, cause=None): '''output warning or Exception depending on -W''' @@ -455,7 +459,7 @@ valuetypes = { ['LiberapayID']), FieldValidator("Open Collective", - r'^[0-9a-zA-Z_-]+$', + VALID_USERNAME_REGEX, ['OpenCollective']), FieldValidator("HTTP link", diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 3b7c5951..c83e3f21 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -31,10 +31,15 @@ import zipfile import hashlib import json import time +import yaml import copy from datetime import datetime from argparse import ArgumentParser from base64 import urlsafe_b64encode +try: + from yaml import CSafeLoader as SafeLoader +except ImportError: + from yaml import SafeLoader import collections from binascii import hexlify @@ -854,6 +859,118 @@ def _get_base_hash_extension(f): return base, None, extension +def sanitize_funding_yml_entry(entry): + """FUNDING.yml comes from upstream repos, entries must be sanitized""" + if type(entry) not in (bytes, int, float, list, str): + return + if isinstance(entry, bytes): + entry = entry.decode() + elif isinstance(entry, list): + if entry: + entry = entry[0] + else: + return + try: + entry = str(entry) + except (TypeError, ValueError): + return + if len(entry) > 2048: + logging.warning(_('Ignoring FUNDING.yml entry longer than 2048: %s') % entry[:2048]) + return + if '\n' in entry: + return + return entry.strip() + + +def sanitize_funding_yml_name(name): + """Sanitize usernames that come from FUNDING.yml""" + entry = sanitize_funding_yml_entry(name) + if entry: + m = metadata.VALID_USERNAME_REGEX.match(entry) + if m: + return m.group() + return + + +def insert_funding_yml_donation_links(apps): + """include donation links from FUNDING.yml in app's source repo + + GitHub made a standard file format for declaring donation + links. This parses that format from upstream repos to include in + metadata here. GitHub supports mostly proprietary services, so + this logic adds proprietary services only as Donate: links. + + FUNDING.yml can be either in the root of the project, or in the + ".github" subdir. + + https://help.github.com/en/articles/displaying-a-sponsor-button-in-your-repository#about-funding-files + + """ + + if not os.path.isdir('build'): + return # nothing to do + for packageName, app in apps.items(): + sourcedir = os.path.join('build', packageName) + if not os.path.isdir(sourcedir): + continue + for f in ([os.path.join(sourcedir, 'FUNDING.yml'), ] + + glob.glob(os.path.join(sourcedir, '.github', 'FUNDING.yml'))): + if not os.path.isfile(f): + continue + data = None + try: + with open(f) as fp: + data = yaml.load(fp, Loader=SafeLoader) + except yaml.YAMLError as e: + logging.error(_('Found bad funding file "{path}" for "{name}":') + .format(path=f, name=packageName)) + logging.error(e) + if not data or type(data) != dict: + continue + if not app.get('OpenCollective') and 'open_collective' in data: + s = sanitize_funding_yml_name(data['open_collective']) + if s: + app['OpenCollective'] = s + if not app.get('Donate'): + del(data['liberapay']) + del(data['open_collective']) + # this tuple provides a preference ordering + for k in ('custom', 'github', 'patreon', 'community_bridge', 'ko_fi', 'issuehunt'): + v = data.get(k) + if not v: + continue + if k == 'custom': + s = sanitize_funding_yml_entry(v) + if s: + app['Donate'] = s + break + elif k == 'community_bridge': + s = sanitize_funding_yml_name(v) + if s: + app['Donate'] = 'https://funding.communitybridge.org/projects/' + s + break + elif k == 'github': + s = sanitize_funding_yml_name(v) + if s: + app['Donate'] = 'https://github.com/sponsors/' + s + break + elif k == 'issuehunt': + s = sanitize_funding_yml_name(v) + if s: + app['Donate'] = 'https://issuehunt.io/r/' + s + break + elif k == 'ko_fi': + s = sanitize_funding_yml_name(v) + if s: + app['Donate'] = 'https://ko-fi.com/' + s + break + elif k == 'patreon': + s = sanitize_funding_yml_name(v) + if s: + app['Donate'] = 'https://patreon.com/' + s + break + + def copy_triple_t_store_metadata(apps): """Include store metadata from the app's source repo @@ -2179,6 +2296,7 @@ def main(): else: logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.')) + insert_funding_yml_donation_links(apps) copy_triple_t_store_metadata(apps) insert_obbs(repodirs[0], apps, apks) insert_localized_app_metadata(apps) diff --git a/tests/funding-usernames.yaml b/tests/funding-usernames.yaml new file mode 100644 index 00000000..04c08a13 --- /dev/null +++ b/tests/funding-usernames.yaml @@ -0,0 +1,197 @@ +bad: + - "Robert'); DROP TABLE Students; --" + - '' + - -a-b + - '1234567890123456789012345678901234567890' + - ~derp@darp---++asdf + - foo@bar.com + - me++ + - --me +bitcoin: + - 3Lbz4vdt15Fsa4wVD3Yk8uGf6ugKKY4zSc +community_bridge: [] +custom: + - bc1qvll2mp5ndwd4sgycu4ad2ken4clhjac7mdlcaj + - http://www.roguetemple.com/z/donate.php + - https://donate.openfoodfacts.org + - https://email.faircode.eu/donate/ + - https://etchdroid.depau.eu/donate/ + - https://f-droid.org/about/ + - https://flattr.com/github/bk138 + - https://gultsch.de/donate.html + - https://jahir.dev/donate + - https://kodi.tv/contribute/donate + - https://link.xbrowsersync.org/cryptos + - https://manyver.se/donate + - https://paypal.me/DanielQuahShaoHian + - https://paypal.me/deletescape + - https://paypal.me/freaktechnik + - https://paypal.me/hpoul + - https://paypal.me/imkosh + - https://paypal.me/paphonb + - https://paypal.me/vocabletrainer + - https://pendulums.io/donation.html + - https://play.google.com/store/apps/details?id=de.dennisguse.opentracks.playstore + - https://play.google.com/store/apps/details?id=eu.faircode.email + - https://raw.githubusercontent.com/Blankj/AndroidUtilCode/master/art/donate.png + - https://raw.githubusercontent.com/CarGuo/GSYGithubAppFlutter/master/thanks.jpg + - https://raw.githubusercontent.com/GanZhiXiong/GZXTaoBaoAppFlutter/blob/master/preview_images/thanks.png + - https://seriesgui.de/whypay + - https://transportr.app/donate/ + - https://www.bountysource.com/teams/nextcloud/issues?tracker_ids=38838206 + - https://www.donationalerts.com/r/blidingmage835 + - https://www.hellotux.com/f-droid + - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=8UH5MBVYM3J36 + - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=E2FCXCT6837GL + - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=FMLNN8GXZKJEE + - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=K7HVLE6J7SXXA + - https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=ZD39ZE7MGEGBL&source=url + - https://www.paypal.me/SimpleMobileTools + - https://www.paypal.me/TheAlphamerc/ + - https://www.paypal.me/avirias + - https://www.paypal.me/btimofeev + - https://www.paypal.me/enricocid + - https://www.paypal.me/gsnathan + - https://www.paypal.me/nikita36078 + - https://www.paypal.me/sahdeep + - https://www.paypal.me/saulhenriquez + - https://www.simplemobiletools.com/donate + - https://www.youtube.com/watch?v=ZmrNc1ZhBkQ + - paypal.me/amangautam1 + - paypal.me/pools/c/8lCZfNnU0u + - paypal.me/psoffritti +github: + - 00-Evan + - adrcotfas + - afollestad + - ar- + - BarnabyShearer + - CarGuo + - cketti + - eighthave + - emansih + - GanZhiXiong + - gpeal + - hpoul + - i-- + - inorichi + - inputmice + - jahirfiquitiva + - johnjohndoe + - kaloudis + - kiwix + - ligi + - M66B + - mikepenz + - Mygod + - paroj + - PerfectSlayer + - sschueller + - tateisu + - tibbi + - westnordost + - x1unix + - xn--nding-jua + - zenorogue +issuehunt: + - bk138/multivnc +ko_fi: + - afollestad + - fennifith + - inorichi + - mastalab + - psoffritti +liberapay: + - ActivityDiary + - AndStatus + - BM835 + - Briar + - DAVx5 + - F-Droid-Data + - Feeel + - Fruit-Radar-Development + - Gadgetbridge + - GuardianProject + - Hocuri + - KOReader + - Kanedias + - Kunzisoft + - MaxK + - NovaVideoPlayer + - Phie + - Rudloff + - Schoumi + - Syncthing-Fork + - TeamNewPipe + - Telegram-FOSS + - Transportr + - Varlorg + - Wesnoth + - ZiiS + - ar- + - bk138 + - btimofeev + - bubblineyuri + - dennis.guse + - developerfromjokela + - devgianlu + - eneiluj + - experiment322 + - fdossena + - fennifith + - freaktechnik + - gsantner + - hisname + - hsn6 + - iNPUTmice + - inputmice + - k9mail + - matrixdotorg + - mmarif + - moezbhatti + - proninyaroslav + - quite + - renyuneyun + - rocketnine.space + - sanskritbscs + - sschueller + - sschueller/donate + - stefan-niedermann + - tasks + - teamkodi + - thermatk + - tom79 + - wallabag + - westnordost + - whyorean + - wilko + - xbrowsersync + - yeriomin + - zeh +open_collective: + - avirias + - curl + - libsodium + - manyverse + - mastalab + - tusky +otechie: [] +patreon: + - BaldPhone + - Bm835 + - FastHub + - Teamkodi + - andrestaltz + - bk138 + - depau + - iamSahdeep + - ligi + - ogre1 + - orhunp + - tiborkaputa + - tom79 + - westnordost + - xbrowsersync + - yairm210 + - zenorogue +tidelift: [] diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase index 3799e48c..5a574702 100755 --- a/tests/metadata.TestCase +++ b/tests/metadata.TestCase @@ -100,6 +100,21 @@ class MetadataTest(unittest.TestCase): self.assertRaises(fdroidserver.exception.MetaDataException, validator.check, 'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id') + def test_valid_funding_yml_regex(self): + """Check the regex can find all the cases""" + with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp: + data = yaml.safe_load(fp) + + for k, entries in data.items(): + for entry in entries: + m = fdroidserver.metadata.VALID_USERNAME_REGEX.match(entry) + if k == 'custom': + pass + elif k == 'bad': + self.assertIsNone(m, 'this is an invalid %s username: {%s}' % (k, entry)) + else: + self.assertIsNotNone(m, 'this is a valid %s username: {%s}' % (k, entry)) + def test_read_metadata(self): def _build_yaml_representer(dumper, data): diff --git a/tests/update.TestCase b/tests/update.TestCase index 1465ad57..35932489 100755 --- a/tests/update.TestCase +++ b/tests/update.TestCase @@ -8,6 +8,7 @@ import inspect import logging import optparse import os +import random import shutil import subprocess import sys @@ -33,6 +34,12 @@ import fdroidserver.update from fdroidserver.common import FDroidPopen +DONATION_FIELDS = ( + 'Donate', + 'OpenCollective', +) + + class UpdateTest(unittest.TestCase): '''fdroid update''' @@ -972,6 +979,84 @@ class UpdateTest(unittest.TestCase): 'VercodeOperation': '', 'WebSite': ''}) + def test_insert_funding_yml_donation_links(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + os.mkdir('build') + content = textwrap.dedent(""" + community_bridge: '' + custom: [LINK1, LINK2] + github: USERNAME + issuehunt: USERNAME + ko_fi: USERNAME + liberapay: USERNAME + open_collective: USERNAME + otechie: USERNAME + patreon: USERNAME + """) + app = fdroidserver.metadata.App() + app.id = 'fake.app.id' + apps = {app.id: app} + os.mkdir(os.path.join('build', app.id)) + fdroidserver.update.insert_funding_yml_donation_links(apps) + for field in DONATION_FIELDS: + self.assertFalse(app.get(field)) + with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp: + fp.write(content) + + fdroidserver.update.insert_funding_yml_donation_links(apps) + for field in DONATION_FIELDS: + self.assertIsNotNone(app.get(field), field) + self.assertEqual('LINK1', app.get('Donate')) + self.assertEqual('USERNAME', app.get('OpenCollective')) + + app['Donate'] = 'keepme' + app['OpenCollective'] = 'keepme' + fdroidserver.update.insert_funding_yml_donation_links(apps) + for field in DONATION_FIELDS: + self.assertEqual('keepme', app.get(field)) + + def test_insert_funding_yml_donation_links_with_corrupt_file(self): + testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir) + os.chdir(testdir) + os.mkdir('build') + app = fdroidserver.metadata.App() + app.id = 'fake.app.id' + apps = {app.id: app} + os.mkdir(os.path.join('build', app.id)) + with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp: + fp.write(textwrap.dedent(""" + opencollective: foo + custom: [] + liberapay: : + """)) + fdroidserver.update.insert_funding_yml_donation_links(apps) + for field in DONATION_FIELDS: + self.assertIsNone(app.get(field)) + + def test_sanitize_funding_yml(self): + with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp: + data = yaml.safe_load(fp) + for k, entries in data.items(): + for entry in entries: + if k in 'custom': + m = fdroidserver.update.sanitize_funding_yml_entry(entry) + else: + m = fdroidserver.update.sanitize_funding_yml_name(entry) + if k == 'bad': + self.assertIsNone(m) + else: + self.assertIsNotNone(m) + self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry('foo\nbar')) + self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry( + ''.join(chr(random.randint(65, 90)) for _ in range(2049)))) + + # not recommended but valid entries + self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(12345)) + self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(5.0)) + self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(' WhyIncludeWhitespace ')) + self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(['first', 'second'])) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))