opensourcemirror
/
fdroid-server
mirror of https://gitlab.com/fdroid/fdroidserver.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'repomaker-api-fixes' into 'master' 

repomaker/issuebot API fixes

See merge request fdroid/fdroidserver!807
This commit is contained in:
Hans-Christoph Steiner 2020-10-08 06:06:20 +00:00
parent 7adba093e4 602cf30c1e
commit 2952e74b71
4 changed files with 110 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
fdroidserver/index.py
View File

@ -61,7 +61,7 @@ def make(apps, apks, repodir, archive):
common.assert_config_keystore(common.config)
# Historically the index has been sorted by App Name, so we enforce this ordering here
sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper())
sortedids = sorted(apps, key=lambda appid: apps[appid]['Name'].upper())
sortedapps = collections.OrderedDict()
for appid in sortedids:
sortedapps[appid] = apps[appid]
@ -593,8 +593,21 @@ def make_v0(apps, apks, repodir, repodict, requestsdict, fdroid_signing_key_fing
# Copy the repo icon into the repo directory...
icon_dir = os.path.join(repodir, 'icons')
iconfilename = os.path.join(icon_dir, os.path.basename(common.config['repo_icon']))
shutil.copyfile(common.config['repo_icon'], iconfilename)
repo_icon = common.config.get('repo_icon', common.default_config['repo_icon'])
iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
if os.path.exists(repo_icon):
shutil.copyfile(common.config['repo_icon'], iconfilename)
else:
logging.warning(_('repo_icon %s does not exist, generating placeholder.')
% repo_icon)
os.makedirs(os.path.dirname(iconfilename), exist_ok=True)
try:
import qrcode
qrcode.make(common.config['repo_url']).save(iconfilename)
except Exception:
exampleicon = os.path.join(common.get_examples_dir(),
common.default_config['repo_icon'])
shutil.copy(exampleicon, iconfilename)
def extract_pubkey():

85
fdroidserver/scanner.py
View File

@ -40,6 +40,47 @@ json_per_build = DEFAULT_JSON_PER_BUILD
MAVEN_URL_REGEX = re.compile(r"""\smaven\s*{.*?(?:setUrl|url)\s*=?\s*(?:uri)?\(?\s*["']?([^\s"']+)["']?[^}]*}""",
re.DOTALL)
CODE_SIGNATURES = {
# The `apkanalyzer dex packages` output looks like this:
# M d 1 1 93 <packagename> <other stuff>
# The first column has P/C/M/F for package, class, method or field
# The second column has x/k/r/d for removed, kept, referenced and defined.
# We already filter for defined only in the apkanalyzer call. 'r' will be
# for things referenced but not distributed in the apk.
exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [
r'(com\.google\.firebase[^\s]*)',
r'(com\.google\.android\.gms[^\s]*)',
r'(com\.google\.tagmanager[^\s]*)',
r'(com\.google\.analytics[^\s]*)',
r'(com\.android\.billing[^\s]*)',
]
}
# Common known non-free blobs (always lower case):
NON_FREE_GRADLE_LINES = {
exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
r'flurryagent',
r'paypal.*mpl',
r'admob.*sdk.*android',
r'google.*ad.*view',
r'google.*admob',
r'google.*play.*services',
r'crittercism',
r'heyzap',
r'jpct.*ae',
r'youtube.*android.*player.*api',
r'bugsense',
r'crashlytics',
r'ouya.*sdk',
r'libspen23',
r'firebase',
r'''["']com.facebook.android['":]''',
r'cloudrail',
r'com.tencent.bugly',
r'appcenter-push',
]
}
def get_gradle_compile_commands(build):
compileCommands = ['compile',
@ -59,25 +100,10 @@ def get_gradle_compile_commands(build):
def scan_binary(apkfile):
usual_suspects = {
# The `apkanalyzer dex packages` output looks like this:
# M d 1 1 93 <packagename> <other stuff>
# The first column has P/C/M/F for package, class, method or field
# The second column has x/k/r/d for removed, kept, referenced and defined.
# We already filter for defined only in the apkanalyzer call. 'r' will be
# for things referenced but not distributed in the apk.
exp: re.compile(r'.[\s]*d[\s]*[0-9]*[\s]*[0-9*][\s]*[0-9]*[\s]*' + exp, re.IGNORECASE) for exp in [
r'(com\.google\.firebase[^\s]*)',
r'(com\.google\.android\.gms[^\s]*)',
r'(com\.google\.tagmanager[^\s]*)',
r'(com\.google\.analytics[^\s]*)',
r'(com\.android\.billing[^\s]*)',
]
}
logging.info("Scanning APK for known non-free classes.")
result = common.SdkToolsPopen(["apkanalyzer", "dex", "packages", "--defined-only", apkfile], output=False)
problems = 0
for suspect, regexp in usual_suspects.items():
for suspect, regexp in CODE_SIGNATURES.items():
matches = regexp.findall(result.output)
if matches:
for m in set(matches):
@ -95,31 +121,6 @@ def scan_source(build_dir, build=metadata.Build()):
count = 0
# Common known non-free blobs (always lower case):
usual_suspects = {
exp: re.compile(r'.*' + exp, re.IGNORECASE) for exp in [
r'flurryagent',
r'paypal.*mpl',
r'admob.*sdk.*android',
r'google.*ad.*view',
r'google.*admob',
r'google.*play.*services',
r'crittercism',
r'heyzap',
r'jpct.*ae',
r'youtube.*android.*player.*api',
r'bugsense',
r'crashlytics',
r'ouya.*sdk',
r'libspen23',
r'firebase',
r'''["']com.facebook.android['":]''',
r'cloudrail',
r'com.tencent.bugly',
r'appcenter-push',
]
}
whitelisted = [
'firebase-jobdispatcher', # https://github.com/firebase/firebase-jobdispatcher-android/blob/master/LICENSE
'com.firebaseui', # https://github.com/firebase/FirebaseUI-Android/blob/master/LICENSE
@ -130,7 +131,7 @@ def scan_source(build_dir, build=metadata.Build()):
return any(wl in s for wl in whitelisted)
def suspects_found(s):
for n, r in usual_suspects.items():
for n, r in NON_FREE_GRADLE_LINES.items():
if r.match(s) and not is_whitelisted(s):
yield n

32
fdroidserver/update.py
View File

@ -123,7 +123,9 @@ def get_all_icon_dirs(repodir):
def disabled_algorithms_allowed():
return options.allow_disabled_algorithms or config['allow_disabled_algorithms']
return ((options is not None and options.allow_disabled_algorithms)
or (config is not None and config['allow_disabled_algorithms'])
or common.default_config['allow_disabled_algorithms'])
def status_update_json(apps, apks):
@ -147,7 +149,7 @@ def status_update_json(apps, apks):
antiFeatures = output['antiFeatures'] # JSON camelCase
if af not in antiFeatures:
antiFeatures[af] = dict()
if appid not in antiFeatures[af]:
if 'apps' not in antiFeatures[af]:
antiFeatures[af]['apps'] = set()
antiFeatures[af]['apps'].add(appid)
@ -521,7 +523,7 @@ def get_cache():
"""
apkcachefile = get_cache_file()
ada = disabled_algorithms_allowed()
if not options.clean and os.path.exists(apkcachefile):
if options is not None and not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile) as fp:
apkcache = json.load(fp, object_pairs_hook=collections.OrderedDict)
if apkcache.get("METADATA_VERSION") != METADATA_VERSION \
@ -1778,7 +1780,7 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
for icon_dir in get_all_icon_dirs(repodir):
if os.path.exists(icon_dir):
if options.clean:
if options is not None and options.clean:
shutil.rmtree(icon_dir)
os.makedirs(icon_dir)
else:
@ -1971,26 +1973,26 @@ def apply_info_from_latest_apk(apps, apks):
bestver = apk['versionCode']
bestapk = apk
if app.NoSourceSince:
if app['NoSourceSince']:
apk['antiFeatures'].add('NoSourceSince')
if not app.added:
if not app['added']:
logging.debug("Don't know when " + appid + " was added")
if not app.lastUpdated:
if not app['lastUpdated']:
logging.debug("Don't know when " + appid + " was last updated")
if bestver == UNSET_VERSION_CODE:
if app.Name is None:
app.Name = app.AutoName or appid
app.icon = None
if app['Name'] is None:
app['Name'] = app['AutoName'] or appid
app['icon'] = None
logging.debug("Application " + appid + " has no packages")
else:
if app.Name is None:
app.Name = bestapk['name']
app.icon = bestapk['icon'] if 'icon' in bestapk else None
if app.CurrentVersionCode is None:
app.CurrentVersionCode = str(bestver)
if app['Name'] is None:
app['Name'] = bestapk['name']
app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
if app['CurrentVersionCode'] is None:
app['CurrentVersionCode'] = str(bestver)
def make_categories_txt(repodir, categories):

35
tests/index.TestCase
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import datetime
import inspect
import logging
import optparse
@ -29,17 +30,25 @@ from testcommon import TmpCwd
GP_FINGERPRINT = 'B7C2EEFD8DAC7806AF67DFCD92EB18126BC08312A7F2D6F3862E46013C7A6135'
class Options:
nosign = True
pretty = False
verbose = False
class IndexTest(unittest.TestCase):
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
os.chmod(os.path.join(self.basedir, 'config.py'), 0o600)
self.tmpdir = os.path.abspath(os.path.join(self.basedir, '..', '.testfiles'))
if not os.path.exists(self.tmpdir):
os.makedirs(self.tmpdir)
os.chdir(self.basedir)
fdroidserver.common.config = None
fdroidserver.common.options = Options
config = fdroidserver.common.read_config(fdroidserver.common.options)
config['jarsigner'] = fdroidserver.common.find_sdk_tools_cmd('jarsigner')
fdroidserver.common.config = config
@ -215,6 +224,29 @@ class IndexTest(unittest.TestCase):
self.maxDiff = None
self.assertEqual(json.dumps(i, indent=2), json.dumps(o, indent=2))
def test_make_v0(self):
tmptestsdir = tempfile.mkdtemp(prefix=inspect.currentframe().f_code.co_name,
dir=self.tmpdir)
os.chdir(tmptestsdir)
os.mkdir('repo')
repo_icons_dir = os.path.join('repo', 'icons')
self.assertFalse(os.path.isdir(repo_icons_dir))
repodict = {
'address': 'https://example.com/fdroid/repo',
'description': 'This is just a test',
'icon': 'blahblah',
'name': 'test',
'timestamp': datetime.datetime.now(),
'version': 12,
}
requestsdict = {'install': [], 'uninstall': []}
fdroidserver.common.config['repo_pubkey'] = 'ffffffffffffffffffffffffffffffffff'
fdroidserver.index.make_v0({}, [], 'repo', repodict, requestsdict, [])
self.assertTrue(os.path.isdir(repo_icons_dir))
self.assertTrue(os.path.exists(os.path.join(repo_icons_dir,
fdroidserver.common.default_config['repo_icon'])))
self.assertTrue(os.path.exists(os.path.join('repo', 'index.xml')))
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
@ -222,7 +254,8 @@ if __name__ == "__main__":
parser = optparse.OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
(fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
(options, args) = parser.parse_args()
Options.verbose = options.verbose
newSuite = unittest.TestSuite()
newSuite.addTest(unittest.makeSuite(IndexTest))