update: insert donation links based on FUNDING.yml

GitHub has specified FUNDING.yml, a file to include in a git repo for
pointing people to donation links.  Since F-Droid also points people
to donation links, this parses them to fill out Donate:
and OpenCollective:.  Specifying those in the metadata file takes
precedence over the FUNDING.yml.  This follows the same pattern as how
`fdroid update` includes Fastlane/Triple-T metadata.  This lets the
git repo maintain those specific donations links themselves.

https://help.github.com/en/articles/displaying-a-sponsor-button-in-your-repository#about-funding-files

The test file was generated using:

```python
import os, re, yaml

found = dict()
for root, dirs, files in os.walk('.'):
    for f in files:
        if f == 'FUNDING.yml':
            with open(os.path.join(root, f)) as fp:
                data = yaml.safe_load(fp)
            for k, v in data.items():
                if k not in found:
                    found[k] = set()
                if not v:
                    continue
                if isinstance(v, list):
                    for i in v:
                        found[k].add(i)
                else:
                    found[k].add(v)

            with open('gather-funding-names.yaml', 'w') as fp:
                output = dict()
                for k, v in found.items():
                    output[k] = sorted(v)
                yaml.dump(output, fp, default_flow_style=False)
```
This commit is contained in:
Hans-Christoph Steiner 2019-11-06 09:03:27 +01:00
parent 8d517d4583
commit 0183592526
No known key found for this signature in database
GPG Key ID: 3E177817BA1B9BFA
5 changed files with 420 additions and 1 deletions

View File

@ -41,6 +41,10 @@ from fdroidserver.exception import MetaDataException, FDroidException
srclibs = None
warnings_action = None
# validates usernames based on a loose collection of rules from GitHub, GitLab,
# Liberapay and issuehunt. This is mostly to block abuse.
VALID_USERNAME_REGEX = re.compile(r'^[a-z\d](?:[a-z\d/._-]){0,38}$', re.IGNORECASE)
def warn_or_exception(value, cause=None):
'''output warning or Exception depending on -W'''
@ -455,7 +459,7 @@ valuetypes = {
['LiberapayID']),
FieldValidator("Open Collective",
r'^[0-9a-zA-Z_-]+$',
VALID_USERNAME_REGEX,
['OpenCollective']),
FieldValidator("HTTP link",

View File

@ -31,10 +31,15 @@ import zipfile
import hashlib
import json
import time
import yaml
import copy
from datetime import datetime
from argparse import ArgumentParser
from base64 import urlsafe_b64encode
try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
import collections
from binascii import hexlify
@ -854,6 +859,118 @@ def _get_base_hash_extension(f):
return base, None, extension
def sanitize_funding_yml_entry(entry):
"""FUNDING.yml comes from upstream repos, entries must be sanitized"""
if type(entry) not in (bytes, int, float, list, str):
return
if isinstance(entry, bytes):
entry = entry.decode()
elif isinstance(entry, list):
if entry:
entry = entry[0]
else:
return
try:
entry = str(entry)
except (TypeError, ValueError):
return
if len(entry) > 2048:
logging.warning(_('Ignoring FUNDING.yml entry longer than 2048: %s') % entry[:2048])
return
if '\n' in entry:
return
return entry.strip()
def sanitize_funding_yml_name(name):
"""Sanitize usernames that come from FUNDING.yml"""
entry = sanitize_funding_yml_entry(name)
if entry:
m = metadata.VALID_USERNAME_REGEX.match(entry)
if m:
return m.group()
return
def insert_funding_yml_donation_links(apps):
"""include donation links from FUNDING.yml in app's source repo
GitHub made a standard file format for declaring donation
links. This parses that format from upstream repos to include in
metadata here. GitHub supports mostly proprietary services, so
this logic adds proprietary services only as Donate: links.
FUNDING.yml can be either in the root of the project, or in the
".github" subdir.
https://help.github.com/en/articles/displaying-a-sponsor-button-in-your-repository#about-funding-files
"""
if not os.path.isdir('build'):
return # nothing to do
for packageName, app in apps.items():
sourcedir = os.path.join('build', packageName)
if not os.path.isdir(sourcedir):
continue
for f in ([os.path.join(sourcedir, 'FUNDING.yml'), ]
+ glob.glob(os.path.join(sourcedir, '.github', 'FUNDING.yml'))):
if not os.path.isfile(f):
continue
data = None
try:
with open(f) as fp:
data = yaml.load(fp, Loader=SafeLoader)
except yaml.YAMLError as e:
logging.error(_('Found bad funding file "{path}" for "{name}":')
.format(path=f, name=packageName))
logging.error(e)
if not data or type(data) != dict:
continue
if not app.get('OpenCollective') and 'open_collective' in data:
s = sanitize_funding_yml_name(data['open_collective'])
if s:
app['OpenCollective'] = s
if not app.get('Donate'):
del(data['liberapay'])
del(data['open_collective'])
# this tuple provides a preference ordering
for k in ('custom', 'github', 'patreon', 'community_bridge', 'ko_fi', 'issuehunt'):
v = data.get(k)
if not v:
continue
if k == 'custom':
s = sanitize_funding_yml_entry(v)
if s:
app['Donate'] = s
break
elif k == 'community_bridge':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://funding.communitybridge.org/projects/' + s
break
elif k == 'github':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://github.com/sponsors/' + s
break
elif k == 'issuehunt':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://issuehunt.io/r/' + s
break
elif k == 'ko_fi':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://ko-fi.com/' + s
break
elif k == 'patreon':
s = sanitize_funding_yml_name(v)
if s:
app['Donate'] = 'https://patreon.com/' + s
break
def copy_triple_t_store_metadata(apps):
"""Include store metadata from the app's source repo
@ -2179,6 +2296,7 @@ def main():
else:
logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
insert_funding_yml_donation_links(apps)
copy_triple_t_store_metadata(apps)
insert_obbs(repodirs[0], apps, apks)
insert_localized_app_metadata(apps)

View File

@ -0,0 +1,197 @@
bad:
- "Robert'); DROP TABLE Students; --"
- ''
- -a-b
- '1234567890123456789012345678901234567890'
- ~derp@darp---++asdf
- foo@bar.com
- me++
- --me
bitcoin:
- 3Lbz4vdt15Fsa4wVD3Yk8uGf6ugKKY4zSc
community_bridge: []
custom:
- bc1qvll2mp5ndwd4sgycu4ad2ken4clhjac7mdlcaj
- http://www.roguetemple.com/z/donate.php
- https://donate.openfoodfacts.org
- https://email.faircode.eu/donate/
- https://etchdroid.depau.eu/donate/
- https://f-droid.org/about/
- https://flattr.com/github/bk138
- https://gultsch.de/donate.html
- https://jahir.dev/donate
- https://kodi.tv/contribute/donate
- https://link.xbrowsersync.org/cryptos
- https://manyver.se/donate
- https://paypal.me/DanielQuahShaoHian
- https://paypal.me/deletescape
- https://paypal.me/freaktechnik
- https://paypal.me/hpoul
- https://paypal.me/imkosh
- https://paypal.me/paphonb
- https://paypal.me/vocabletrainer
- https://pendulums.io/donation.html
- https://play.google.com/store/apps/details?id=de.dennisguse.opentracks.playstore
- https://play.google.com/store/apps/details?id=eu.faircode.email
- https://raw.githubusercontent.com/Blankj/AndroidUtilCode/master/art/donate.png
- https://raw.githubusercontent.com/CarGuo/GSYGithubAppFlutter/master/thanks.jpg
- https://raw.githubusercontent.com/GanZhiXiong/GZXTaoBaoAppFlutter/blob/master/preview_images/thanks.png
- https://seriesgui.de/whypay
- https://transportr.app/donate/
- https://www.bountysource.com/teams/nextcloud/issues?tracker_ids=38838206
- https://www.donationalerts.com/r/blidingmage835
- https://www.hellotux.com/f-droid
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=8UH5MBVYM3J36
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=E2FCXCT6837GL
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=FMLNN8GXZKJEE
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=K7HVLE6J7SXXA
- https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=ZD39ZE7MGEGBL&source=url
- https://www.paypal.me/SimpleMobileTools
- https://www.paypal.me/TheAlphamerc/
- https://www.paypal.me/avirias
- https://www.paypal.me/btimofeev
- https://www.paypal.me/enricocid
- https://www.paypal.me/gsnathan
- https://www.paypal.me/nikita36078
- https://www.paypal.me/sahdeep
- https://www.paypal.me/saulhenriquez
- https://www.simplemobiletools.com/donate
- https://www.youtube.com/watch?v=ZmrNc1ZhBkQ
- paypal.me/amangautam1
- paypal.me/pools/c/8lCZfNnU0u
- paypal.me/psoffritti
github:
- 00-Evan
- adrcotfas
- afollestad
- ar-
- BarnabyShearer
- CarGuo
- cketti
- eighthave
- emansih
- GanZhiXiong
- gpeal
- hpoul
- i--
- inorichi
- inputmice
- jahirfiquitiva
- johnjohndoe
- kaloudis
- kiwix
- ligi
- M66B
- mikepenz
- Mygod
- paroj
- PerfectSlayer
- sschueller
- tateisu
- tibbi
- westnordost
- x1unix
- xn--nding-jua
- zenorogue
issuehunt:
- bk138/multivnc
ko_fi:
- afollestad
- fennifith
- inorichi
- mastalab
- psoffritti
liberapay:
- ActivityDiary
- AndStatus
- BM835
- Briar
- DAVx5
- F-Droid-Data
- Feeel
- Fruit-Radar-Development
- Gadgetbridge
- GuardianProject
- Hocuri
- KOReader
- Kanedias
- Kunzisoft
- MaxK
- NovaVideoPlayer
- Phie
- Rudloff
- Schoumi
- Syncthing-Fork
- TeamNewPipe
- Telegram-FOSS
- Transportr
- Varlorg
- Wesnoth
- ZiiS
- ar-
- bk138
- btimofeev
- bubblineyuri
- dennis.guse
- developerfromjokela
- devgianlu
- eneiluj
- experiment322
- fdossena
- fennifith
- freaktechnik
- gsantner
- hisname
- hsn6
- iNPUTmice
- inputmice
- k9mail
- matrixdotorg
- mmarif
- moezbhatti
- proninyaroslav
- quite
- renyuneyun
- rocketnine.space
- sanskritbscs
- sschueller
- sschueller/donate
- stefan-niedermann
- tasks
- teamkodi
- thermatk
- tom79
- wallabag
- westnordost
- whyorean
- wilko
- xbrowsersync
- yeriomin
- zeh
open_collective:
- avirias
- curl
- libsodium
- manyverse
- mastalab
- tusky
otechie: []
patreon:
- BaldPhone
- Bm835
- FastHub
- Teamkodi
- andrestaltz
- bk138
- depau
- iamSahdeep
- ligi
- ogre1
- orhunp
- tiborkaputa
- tom79
- westnordost
- xbrowsersync
- yairm210
- zenorogue
tidelift: []

View File

@ -100,6 +100,21 @@ class MetadataTest(unittest.TestCase):
self.assertRaises(fdroidserver.exception.MetaDataException, validator.check,
'tb1qw5r8drrejxrrg4y5rrrrrraryrrrrwrkxrjrsx', 'fake.app.id')
def test_valid_funding_yml_regex(self):
"""Check the regex can find all the cases"""
with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
data = yaml.safe_load(fp)
for k, entries in data.items():
for entry in entries:
m = fdroidserver.metadata.VALID_USERNAME_REGEX.match(entry)
if k == 'custom':
pass
elif k == 'bad':
self.assertIsNone(m, 'this is an invalid %s username: {%s}' % (k, entry))
else:
self.assertIsNotNone(m, 'this is a valid %s username: {%s}' % (k, entry))
def test_read_metadata(self):
def _build_yaml_representer(dumper, data):

View File

@ -8,6 +8,7 @@ import inspect
import logging
import optparse
import os
import random
import shutil
import subprocess
import sys
@ -33,6 +34,12 @@ import fdroidserver.update
from fdroidserver.common import FDroidPopen
DONATION_FIELDS = (
'Donate',
'OpenCollective',
)
class UpdateTest(unittest.TestCase):
'''fdroid update'''
@ -972,6 +979,84 @@ class UpdateTest(unittest.TestCase):
'VercodeOperation': '',
'WebSite': ''})
def test_insert_funding_yml_donation_links(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
os.mkdir('build')
content = textwrap.dedent("""
community_bridge: ''
custom: [LINK1, LINK2]
github: USERNAME
issuehunt: USERNAME
ko_fi: USERNAME
liberapay: USERNAME
open_collective: USERNAME
otechie: USERNAME
patreon: USERNAME
""")
app = fdroidserver.metadata.App()
app.id = 'fake.app.id'
apps = {app.id: app}
os.mkdir(os.path.join('build', app.id))
fdroidserver.update.insert_funding_yml_donation_links(apps)
for field in DONATION_FIELDS:
self.assertFalse(app.get(field))
with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
fp.write(content)
fdroidserver.update.insert_funding_yml_donation_links(apps)
for field in DONATION_FIELDS:
self.assertIsNotNone(app.get(field), field)
self.assertEqual('LINK1', app.get('Donate'))
self.assertEqual('USERNAME', app.get('OpenCollective'))
app['Donate'] = 'keepme'
app['OpenCollective'] = 'keepme'
fdroidserver.update.insert_funding_yml_donation_links(apps)
for field in DONATION_FIELDS:
self.assertEqual('keepme', app.get(field))
def test_insert_funding_yml_donation_links_with_corrupt_file(self):
testdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name, dir=self.tmpdir)
os.chdir(testdir)
os.mkdir('build')
app = fdroidserver.metadata.App()
app.id = 'fake.app.id'
apps = {app.id: app}
os.mkdir(os.path.join('build', app.id))
with open(os.path.join('build', app.id, 'FUNDING.yml'), 'w') as fp:
fp.write(textwrap.dedent("""
opencollective: foo
custom: []
liberapay: :
"""))
fdroidserver.update.insert_funding_yml_donation_links(apps)
for field in DONATION_FIELDS:
self.assertIsNone(app.get(field))
def test_sanitize_funding_yml(self):
with open(os.path.join(self.basedir, 'funding-usernames.yaml')) as fp:
data = yaml.safe_load(fp)
for k, entries in data.items():
for entry in entries:
if k in 'custom':
m = fdroidserver.update.sanitize_funding_yml_entry(entry)
else:
m = fdroidserver.update.sanitize_funding_yml_name(entry)
if k == 'bad':
self.assertIsNone(m)
else:
self.assertIsNotNone(m)
self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry('foo\nbar'))
self.assertIsNone(fdroidserver.update.sanitize_funding_yml_entry(
''.join(chr(random.randint(65, 90)) for _ in range(2049))))
# not recommended but valid entries
self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(12345))
self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(5.0))
self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(' WhyIncludeWhitespace '))
self.assertIsNotNone(fdroidserver.update.sanitize_funding_yml_entry(['first', 'second']))
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))